diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 72ec17297..d1cad71b8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -46,7 +46,7 @@ jobs: # these are checks that should be run on pull-request and merges to main. # we do NOT want to kick off a release if these have not been verified on main. # Please see the validations.yaml workflow for the names that should be used here. - checks: '["Acceptance tests (Linux)", "Acceptance tests (Mac)", "Build snapshot artifacts", "CLI tests (Linux)", "Integration tests", "Static analysis", "Unit tests"]' + checks: '["Acceptance tests (Linux)", "Acceptance tests (Mac)", "Build snapshot artifacts", "CLI tests (Linux)", "Containers-storage tests (Podman)", "Containers-storage tests (Buildah)", "Integration tests", "Static analysis", "Unit tests"]' release: needs: [check-gate, version-available] diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index 942ab1eb2..998ba78ef 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -177,6 +177,72 @@ jobs: path: snapshot/ retention-days: 30 + Containers-Storage-Podman: + name: "Containers-storage tests (Podman)" + needs: [Build-Snapshot-Artifacts] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1 + with: + name: snapshot + path: snapshot + + # go-make's setup only (not ./.github/actions/bootstrap): the test only needs make + the snapshot binary + - name: Setup go + go-make tooling + uses: anchore/go-make/.github/actions/setup@fa5421b3bf24c9d18f11a736c5eb4172f2b7d683 # v0.8.1 + with: + # keep in sync with the go-version default in ./.github/actions/bootstrap + go-version: "1.26.2" + + - name: Install Podman + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y jq podman uidmap + + - name: Scan image from Podman's local store + run: make containers-storage-test + env: + BUILDER: podman + + Containers-Storage-Buildah: + name: "Containers-storage tests (Buildah)" + needs: [Build-Snapshot-Artifacts] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1 + with: + name: snapshot + path: snapshot + + # go-make's setup only (not ./.github/actions/bootstrap): the test only needs make + the snapshot binary + - name: Setup go + go-make tooling + uses: anchore/go-make/.github/actions/setup@fa5421b3bf24c9d18f11a736c5eb4172f2b7d683 # v0.8.1 + with: + # keep in sync with the go-version default in ./.github/actions/bootstrap + go-version: "1.26.2" + + - name: Install Buildah + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y buildah jq uidmap + + - name: Scan image from Buildah's local store + run: make containers-storage-test + env: + BUILDER: buildah + Acceptance-Linux: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "Acceptance tests (Linux)" diff --git a/.golangci.yaml b/.golangci.yaml index 5aa2a4905..bdb4e0a22 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,6 +1,10 @@ version: "2" run: tests: false + # keep in sync with buildTags in .make/main.go + build-tags: + - containers_image_openpgp + - exclude_graphdriver_btrfs linters: default: none enable: diff --git a/.goreleaser.yaml b/.goreleaser.yaml index a06e04762..4f4470432 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -15,6 +15,7 @@ builds: binary: syft goos: [linux] goarch: [amd64, arm64, ppc64le, riscv64, s390x] + tags: [containers_image_openpgp, exclude_graphdriver_btrfs] mod_timestamp: &build-timestamp '{{ .CommitTimestamp }}' ldflags: &build-ldflags | -w diff --git a/.make/main.go b/.make/main.go index dc18a31d7..73f398454 100644 --- a/.make/main.go +++ b/.make/main.go @@ -16,6 +16,12 @@ import ( "github.com/anchore/go-make/tasks/gotest" ) +// buildTags must match the linux build in .goreleaser.yaml so tests and lint compile the same code that ships +// (containers_image_openpgp compiles in stereoscope's real containers-storage provider instead of its stub). +// exclude_graphdriver_btrfs drops the cgo-only btrfs driver, which needs libbtrfs headers whenever cgo is on (e.g. +// under -race); release builds are CGO_ENABLED=0 and never include it anyway. +const buildTags = "containers_image_openpgp,exclude_graphdriver_btrfs" + func main() { Makefile( // shared anchore tasks @@ -29,6 +35,7 @@ func main() { gotest.Name("unit"), gotest.ExcludeGlob("**/test/**"), gotest.CoverageThreshold(62), + gotest.Tags(buildTags), race(), ), @@ -52,7 +59,7 @@ func main() { raceFlag = " -race" } Run( - "go test -count=1 -timeout=30m"+raceFlag+" ./cmd/syft/internal/test/integration/...", + "go test -count=1 -timeout=30m -tags="+buildTags+raceFlag+" ./cmd/syft/internal/test/integration/...", run.Env("GODEBUG", "dontfreezetheworld=1"), ) }, @@ -66,7 +73,7 @@ func main() { Log("race detector disabled (RACE=false); skipping race smoke") return } - Run("go run -race cmd/syft/main.go anchore/test_images:grype-quality-dotnet-69f15d2") + Run("go run -race -tags="+buildTags+" cmd/syft/main.go anchore/test_images:grype-quality-dotnet-69f15d2") }, }, @@ -96,6 +103,24 @@ func main() { }, }, + // containers-storage tests: build a fixture image with BUILDER (podman or buildah) into an isolated local store + // and scan it with the snapshot binary. Linux only, and not hooked into "test" since it needs the builder installed. + Task{ + Name: "containers-storage-test", + Description: "Run containers-storage tests (BUILDER=podman|buildah)", + Run: func() { + bin := snapshotBinPath() + if !file.Exists(bin) { + Log("snapshot binary not found at %s; building single-target snapshot", bin) + Run("make snapshot:single-target") + } + Run( + "bash test/containers-storage/source-test.sh "+config.Env("BUILDER", "podman"), + run.Env("SYFT_BINARY_LOCATION", bin), + ) + }, + }, + // default validation pipeline (replaces Taskfile `default`/`pr-validations`/`validations`). Task{ Name: "default", diff --git a/README.md b/README.md index d864e1428..9ffd2affa 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,18 @@ syft -o cyclonedx-json syft -o spdx-json=./spdx.json -o cyclonedx-json=./cdx.json ``` +### Local containers-storage images + +On Linux, syft can scan images from a local containers-storage store (e.g. built with Buildah or Podman) when asked explicitly; plain image references never look there: + +```bash +syft --from containers-storage localhost/myimage:latest + +# rootless stores must be read from inside the builder's user namespace +podman unshare syft --from containers-storage localhost/myimage:latest +``` + +This is included in the Linux release binaries. When building from source, add `-tags containers_image_openpgp,exclude_graphdriver_btrfs`. > [!TIP] > **Check out the [Getting Started guide](https://oss.anchore.com/docs/guides/sbom/getting-started/)** to explore all of the capabilities and features. diff --git a/cmd/syft/internal/options/source.go b/cmd/syft/internal/options/source.go index 76cae4580..2c517946b 100644 --- a/cmd/syft/internal/options/source.go +++ b/cmd/syft/internal/options/source.go @@ -36,7 +36,7 @@ var _ clio.PostLoader = (*imageSource)(nil) func (o *sourceConfig) DescribeFields(descriptions clio.FieldDescriptionSet) { descriptions.Add(&o.File.Digests, `the file digest algorithms to use on the scanned file (options: "md5", "sha1", "sha224", "sha256", "sha384", "sha512")`) descriptions.Add(&o.Image.DefaultPullSource, `allows users to specify which image source should be used to generate the sbom -valid values are: registry, docker, podman`) +valid values are: registry, docker, podman, containers-storage`) } type imageSource struct { @@ -74,7 +74,7 @@ func (c *imageSource) PostLoad() error { return checkDefaultSourceValues(c.DefaultPullSource) } -var validDefaultSourceValues = []string{"registry", "docker", "podman", ""} +var validDefaultSourceValues = []string{"registry", "docker", "podman", "containers-storage", ""} func checkDefaultSourceValues(source string) error { validValues := strset.New(validDefaultSourceValues...) diff --git a/syft/get_source_config.go b/syft/get_source_config.go index 7a163f1d1..735dba516 100644 --- a/syft/get_source_config.go +++ b/syft/get_source_config.go @@ -3,6 +3,7 @@ package syft import ( "crypto" "fmt" + "slices" "github.com/anchore/go-collections" "github.com/anchore/stereoscope/pkg/image" @@ -10,6 +11,11 @@ import ( "github.com/anchore/syft/syft/source/sourceproviders" ) +// GetSourceConfig controls which source providers are tried, and in what order, when resolving user input to a source. +// +// The "containers-storage" provider (local Buildah / Podman store) is never part of automatic resolution: it is only +// used when named in Sources or DefaultImagePullSource. It is also only functional when built with the +// containers_image_openpgp build tag (as syft's Linux release binaries are); otherwise it always returns an error. type GetSourceConfig struct { // SourceProviderConfig may optionally be provided to be used when constructing the default set of source providers, unused if All specified SourceProviderConfig *sourceproviders.Config @@ -64,6 +70,12 @@ func (c *GetSourceConfig) WithDefaultImagePullSource(defaultImagePullSource stri func (c *GetSourceConfig) getProviders(userInput string) ([]source.Provider, error) { providers := collections.TaggedValueSet[source.Provider]{}.Join(sourceproviders.All(userInput, c.SourceProviderConfig)...) + // opening a containers-storage store runs full graph driver init (mounts, locks, possible store writes), which is + // too invasive for an "is this image local?" probe on every plain image reference. Only use it when asked by name. + if !c.requestsSource(image.ContainersStorageSource) { + providers = providers.Remove(image.ContainersStorageSource) + } + // if the "default image pull source" is set, we move this as the first pull source if c.DefaultImagePullSource != "" { base := providers.Remove(sourceproviders.PullTag) @@ -84,6 +96,10 @@ func (c *GetSourceConfig) getProviders(userInput string) ([]source.Provider, err return providers.Values(), nil } +func (c *GetSourceConfig) requestsSource(name string) bool { + return c.DefaultImagePullSource == name || slices.Contains(c.Sources, name) +} + func DefaultGetSourceConfig() *GetSourceConfig { return &GetSourceConfig{ SourceProviderConfig: sourceproviders.DefaultConfig(), diff --git a/syft/get_source_config_test.go b/syft/get_source_config_test.go index 912131494..0d12c3cf7 100644 --- a/syft/get_source_config_test.go +++ b/syft/get_source_config_test.go @@ -1,9 +1,15 @@ package syft import ( + "slices" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/anchore/stereoscope" + "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source/sourceproviders" ) @@ -17,8 +23,9 @@ func TestGetProviders_DefaultImagePullSource(t *testing.T) { t.Errorf("Expected no error for DefaultImagePullSource parameter, got: %v", err) } - if len(providers) != len(allSourceProviders) { - t.Errorf("Expected %d providers, got %d", len(allSourceProviders), len(providers)) + // everything except containers-storage, which is only used when requested by name + if len(providers) != len(allSourceProviders)-1 { + t.Errorf("Expected %d providers, got %d", len(allSourceProviders)-1, len(providers)) } } @@ -36,3 +43,55 @@ func TestGetProviders_Sources(t *testing.T) { t.Errorf("Expected 2 providers, got %d", len(providers)) } } + +func TestGetProviders_ContainersStorageOnlyWhenRequested(t *testing.T) { + const storage = image.ContainersStorageSource + + tests := []struct { + name string + cfg *GetSourceConfig + wantFound bool + // if set, containers-storage must be ordered ahead of this provider + wantBefore string + }{ + { + name: "excluded from automatic resolution", + cfg: DefaultGetSourceConfig(), + }, + { + name: "excluded when selecting all pull sources", + cfg: DefaultGetSourceConfig().WithSources(sourceproviders.PullTag), + }, + { + name: "excluded when a different default pull source is set", + cfg: DefaultGetSourceConfig().WithDefaultImagePullSource(stereoscope.RegistryTag), + }, + { + name: "included when explicitly selected", + cfg: DefaultGetSourceConfig().WithSources(storage), + wantFound: true, + }, + { + name: "included ahead of other pull sources when it is the default pull source", + cfg: DefaultGetSourceConfig().WithDefaultImagePullSource(storage), + wantFound: true, + wantBefore: image.DockerDaemonSource, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + providers, err := tt.cfg.getProviders("localhost/myimage:latest") + require.NoError(t, err) + require.NotEmpty(t, providers) + + assert.Equal(t, tt.wantFound, providerIndex(providers, storage) >= 0) + if tt.wantBefore != "" { + assert.Less(t, providerIndex(providers, storage), providerIndex(providers, tt.wantBefore)) + } + }) + } +} + +func providerIndex(providers []source.Provider, name string) int { + return slices.IndexFunc(providers, func(p source.Provider) bool { return p.Name() == name }) +} diff --git a/test/containers-storage/source-test.sh b/test/containers-storage/source-test.sh new file mode 100755 index 000000000..cf4d98607 --- /dev/null +++ b/test/containers-storage/source-test.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# build an image with podman or buildah into an isolated containers-storage store, then verify syft can scan it out +# of that store (for both the vfs and overlay drivers) and that plain image references never touch the store. +# +# usage: SYFT_BINARY_LOCATION=/path/to/syft source-test.sh podman|buildah +set -euo pipefail + +builder="${1:?expected podman or buildah}" +case "$builder" in + podman|buildah) ;; + *) + printf 'unsupported image builder: %s\n' "$builder" >&2 + exit 2 + ;; +esac + +syft_binary="${SYFT_BINARY_LOCATION:?expected SYFT_BINARY_LOCATION to point at a linux syft binary built with containers-storage support}" +if [[ ! -f "$syft_binary" ]]; then + printf 'syft binary not found at %s\n' "$syft_binary" >&2 + exit 1 +fi +chmod +x "$syft_binary" + +test_dir="$(mktemp -d)" +# the overlay driver bind-mounts its graphroot onto itself (in the builder's user namespace when rootless), which must +# be unmounted before the store can be removed. Cleanup is best-effort so it never masks the test result. +cleanup_store() { + umount -l "$1"/*/graphroot/overlay 2>/dev/null || true + rm -rf "$1" +} +cleanup() { + if [[ "$(id -u)" -eq 0 ]]; then + cleanup_store "$test_dir" + else + "$builder" unshare bash -c "$(declare -f cleanup_store); cleanup_store \"\$1\"" _ "$test_dir" + fi || printf 'warning: unable to fully clean up %s\n' "$test_dir" >&2 +} +trap cleanup EXIT + +# a rootless store can only be opened from inside the builder's user namespace (as podman/buildah/skopeo do for +# themselves), so rootless users must run syft under " unshare". Root opens the store directly. +run_syft() { + if [[ "$(id -u)" -eq 0 ]]; then + "$syft_binary" "$@" + else + "$builder" unshare "$syft_binary" "$@" + fi +} + +build_context="$test_dir/context" +mkdir -p "$build_context" +cat > "$build_context/package.json" <<'EOF' +{"name":"syft-cs-fixture","version":"1.2.3"} +EOF +cat > "$build_context/Containerfile" <<'EOF' +FROM scratch +COPY package.json /app/node_modules/syft-cs-fixture/package.json +EOF + +# note: the image name deliberately avoids the string "containers-storage" so stderr greps below are unambiguous +image_ref="localhost/syft-cs-fixture:latest" +missing_ref="localhost/syft-cs-missing:latest" + +for driver in vfs overlay; do + store_dir="$test_dir/$driver" + mkdir -p "$store_dir" + export CONTAINERS_STORAGE_CONF="$store_dir/storage.conf" + cat > "$CONTAINERS_STORAGE_CONF" < "$out.json" 2> "$out.stderr" + ;; + scheme) + run_syft -vv "containers-storage:$image_ref" --output json > "$out.json" 2> "$out.stderr" + ;; + default-pull-source) + SYFT_SOURCE_IMAGE_DEFAULT_PULL_SOURCE=containers-storage \ + run_syft -vv "$image_ref" --output json > "$out.json" 2> "$out.stderr" + ;; + esac + + # prove the image came out of the store, not from some other provider that happened to answer + if ! grep -q 'copied image from containers-storage' "$out.stderr"; then + printf 'Expected %s scan to resolve via containers-storage\n' "$scan_mode" >&2 + cat "$out.stderr" >&2 + exit 1 + fi + jq -e --arg image_ref "$image_ref" ' + .source.type == "image" and + .source.metadata.userInput == $image_ref and + any(.artifacts[]; .name == "syft-cs-fixture" and .version == "1.2.3" and .type == "npm") + ' "$out.json" >/dev/null + done + + printf 'Testing plain reference skips containers-storage with %s (%s driver)\n' "$builder" "$driver" + # the image only exists in the local store, so automatic resolution (daemons, then registry) must fail without + # ever trying containers-storage + if run_syft "$image_ref" --output json > "$store_dir/plain.json" 2> "$store_dir/plain.stderr"; then + printf 'Expected plain reference scan to fail since the image only exists in containers-storage\n' >&2 + exit 1 + fi + if grep -q 'containers-storage' "$store_dir/plain.stderr"; then + printf 'Plain reference unexpectedly attempted containers-storage resolution\n' >&2 + cat "$store_dir/plain.stderr" >&2 + exit 1 + fi + + printf 'Testing missing image with %s (%s driver)\n' "$builder" "$driver" + if run_syft --from containers-storage "$missing_ref" --output json > "$store_dir/missing.json" 2> "$store_dir/missing.stderr"; then + printf 'Expected the missing containers-storage image scan to fail\n' >&2 + exit 1 + fi + if ! grep -q 'does not resolve to an image ID' "$store_dir/missing.stderr"; then + printf 'Expected a containers-storage "image not found" error\n' >&2 + cat "$store_dir/missing.stderr" >&2 + exit 1 + fi +done