From 7f4e8ab97de64e7197571f90b5422ccdb7d3731e Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 4 Jan 2021 16:41:45 -0500 Subject: [PATCH] Fix symlink resolutions for constituent paths (#304) * bump stereoscope to pull in content API refactors Signed-off-by: Alex Goodman * incorporate symlink fixes Signed-off-by: Alex Goodman * with filetree.File() adjustments Signed-off-by: Alex Goodman * regress all-layers scope to not include dead-links + default tests to squashed scope Signed-off-by: Alex Goodman * restore all layers resolver glob behavior (custom + lazy link resolution) Signed-off-by: Alex Goodman * incorporate filetree link resolution options and restore no-follow dead link option for resolvers Signed-off-by: Alex Goodman * removed path from lower-level FileTree.File() calls Signed-off-by: Alex Goodman * bump stereoscope to pull in latest link resolution fixes Signed-off-by: Alex Goodman * bump doublestar to v2 for directory resolver Signed-off-by: Alex Goodman --- Makefile | 4 +- go.mod | 4 +- go.sum | 12 ++---- syft/cataloger/deb/cataloger_test.go | 2 +- .../python/package_cataloger_test.go | 3 +- syft/presenter/cyclonedx/presenter_test.go | 11 ++++-- .../TestCycloneDxDirsPresenter.golden | 4 +- .../TestCycloneDxImgsPresenter.golden | 4 +- syft/presenter/json/presenter_test.go | 11 ++++-- .../snapshot/TestJsonImgsPresenter.golden | 2 +- syft/presenter/table/presenter_test.go | 9 ++++- syft/presenter/text/presenter_test.go | 11 ++++-- syft/source/all_layers_resolver.go | 39 ++++++++++++------- syft/source/all_layers_resolver_test.go | 38 +++++------------- syft/source/directory_resolver.go | 3 +- syft/source/directory_resolver_test.go | 15 ++++--- syft/source/image_squash_resolver.go | 27 +++++++------ syft/source/image_squash_resolver_test.go | 36 ++++++++++++----- syft/source/location.go | 4 +- .../test-fixtures/image-symlinks/Dockerfile | 10 ++++- .../image-symlinks/new-file-4.txt | 1 + .../image-symlinks/parent/file-4.txt | 1 + test/integration/distro_test.go | 2 +- test/integration/document_import_test.go | 2 +- test/integration/json_schema_test.go | 4 +- test/integration/pkg_coverage_test.go | 4 +- 26 files changed, 153 insertions(+), 110 deletions(-) create mode 100644 syft/source/test-fixtures/image-symlinks/new-file-4.txt create mode 100644 syft/source/test-fixtures/image-symlinks/parent/file-4.txt diff --git a/Makefile b/Makefile index 19448ce20..d9ec8a8ec 100644 --- a/Makefile +++ b/Makefile @@ -177,7 +177,7 @@ $(SNAPSHOTDIR): ## Build snapshot release binaries and packages $(TEMPDIR)/goreleaser release --skip-publish --rm-dist --snapshot --config $(TEMPDIR)/goreleaser.yaml .PHONY: acceptance-mac -acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac) +acceptance-mac: clean-snapshot $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac) $(call title,Running acceptance test: Run on Mac) $(ACC_DIR)/mac.sh \ $(SNAPSHOTDIR) \ @@ -186,7 +186,7 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie $(RESULTSDIR) .PHONY: acceptance-linux -acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux) +acceptance-linux: clean-snapshot acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux) # note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests .PHONY: compare-fingerprint diff --git a/go.mod b/go.mod index 62c133861..7346e4220 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,9 @@ require ( github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b - github.com/anchore/stereoscope v0.0.0-20201219143203-af397ece87bf + github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255 github.com/antihax/optional v1.0.0 - github.com/bmatcuk/doublestar v1.3.3 + github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible github.com/dustin/go-humanize v1.0.0 github.com/facebookincubator/nvdtools v0.1.4 diff --git a/go.sum b/go.sum index 8f297a26f..b1aa41dad 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74 h1:9kkKTIyXJC+/syUcY6KWxFoJZJ+GWwrIscF+gBY067k= -github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk= github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238 h1:/iI+1cj1a27ow0wj378pPJIm8sCSy6I21Tz6oLbLDQY= github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk= github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ= @@ -136,10 +134,8 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= -github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e h1:vHUqHTvH9/oxdDDh1fxS9Ls9gWGytKO7XbbzcQ9MBwI= -github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e/go.mod h1:/dHAFjYflH/1tzhdHAcnMCjprMch+YzHJKi59m/1KCM= -github.com/anchore/stereoscope v0.0.0-20201219143203-af397ece87bf h1:4sN/HJ6whcrK/HxORFGAQUWM58Q7EFiPmoxRKcEs76A= -github.com/anchore/stereoscope v0.0.0-20201219143203-af397ece87bf/go.mod h1:/dHAFjYflH/1tzhdHAcnMCjprMch+YzHJKi59m/1KCM= +github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255 h1:Ng7BDr9PQTCztANogjfEdEjjWUylhlPyZPhtarIGo00= +github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -170,8 +166,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4= github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/bmatcuk/doublestar v1.3.3 h1:pVP1d49CcQQaNOl+PI6sPybIrIOD/6sux31PFdmhTH0= -github.com/bmatcuk/doublestar v1.3.3/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= +github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI= +github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw= github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U= github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6abzXN4Pg= github.com/bombsimon/wsl/v3 v3.0.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= diff --git a/syft/cataloger/deb/cataloger_test.go b/syft/cataloger/deb/cataloger_test.go index 9a3e390c7..993407674 100644 --- a/syft/cataloger/deb/cataloger_test.go +++ b/syft/cataloger/deb/cataloger_test.go @@ -54,7 +54,7 @@ func TestDpkgCataloger(t *testing.T) { img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg") defer cleanup() - s, err := source.NewFromImage(img, source.AllLayersScope, "") + s, err := source.NewFromImage(img, source.SquashedScope, "") if err != nil { t.Fatal(err) } diff --git a/syft/cataloger/python/package_cataloger_test.go b/syft/cataloger/python/package_cataloger_test.go index 2cc320855..0c4d99f96 100644 --- a/syft/cataloger/python/package_cataloger_test.go +++ b/syft/cataloger/python/package_cataloger_test.go @@ -3,9 +3,8 @@ package python import ( "testing" - "github.com/anchore/syft/syft/source" - "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" "github.com/go-test/deep" ) diff --git a/syft/presenter/cyclonedx/presenter_test.go b/syft/presenter/cyclonedx/presenter_test.go index a1c8413f7..c7909e22f 100644 --- a/syft/presenter/cyclonedx/presenter_test.go +++ b/syft/presenter/cyclonedx/presenter_test.go @@ -6,6 +6,8 @@ import ( "regexp" "testing" + "github.com/anchore/stereoscope/pkg/filetree" + "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/go-testutils" @@ -94,12 +96,15 @@ func TestCycloneDxImgsPresenter(t *testing.T) { img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple") defer cleanup() + _, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks) + _, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks) + // populate catalog with test data catalog.Add(pkg.Package{ Name: "package1", Version: "1.0.1", Locations: []source.Location{ - source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), + source.NewLocationFromImage(*ref1, img), }, Type: pkg.RpmPkg, FoundBy: "the-cataloger-1", @@ -109,7 +114,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { Name: "package2", Version: "2.0.1", Locations: []source.Location{ - source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), + source.NewLocationFromImage(*ref2, img), }, Type: pkg.RpmPkg, FoundBy: "the-cataloger-2", @@ -120,7 +125,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) { PURL: "the-purl-2", }) - s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input") + s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input") if err != nil { t.Fatal(err) } diff --git a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden index a6200b9b6..5c3da3632 100644 --- a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden +++ b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden @@ -1,7 +1,7 @@ - + - 2020-12-01T22:19:00-05:00 + 2020-12-28T13:56:29-05:00 anchore diff --git a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden index 9740019be..fd72a5ce8 100644 --- a/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden +++ b/syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden @@ -1,7 +1,7 @@ - + - 2020-12-01T22:19:00-05:00 + 2020-12-28T13:56:29-05:00 anchore diff --git a/syft/presenter/json/presenter_test.go b/syft/presenter/json/presenter_test.go index a26148a0d..585bad978 100644 --- a/syft/presenter/json/presenter_test.go +++ b/syft/presenter/json/presenter_test.go @@ -5,6 +5,8 @@ import ( "flag" "testing" + "github.com/anchore/stereoscope/pkg/filetree" + "github.com/anchore/go-testutils" "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/distro" @@ -106,12 +108,15 @@ func TestJsonImgsPresenter(t *testing.T) { catalog := pkg.NewCatalog() img := imagetest.GetGoldenFixtureImage(t, testImage) + _, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks) + _, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks) + // populate catalog with test data catalog.Add(pkg.Package{ Name: "package-1", Version: "1.0.1", Locations: []source.Location{ - source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), + source.NewLocationFromImage(*ref1, img), }, Type: pkg.PythonPkg, FoundBy: "the-cataloger-1", @@ -131,7 +136,7 @@ func TestJsonImgsPresenter(t *testing.T) { Name: "package-2", Version: "2.0.1", Locations: []source.Location{ - source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), + source.NewLocationFromImage(*ref2, img), }, Type: pkg.DebPkg, FoundBy: "the-cataloger-2", @@ -149,7 +154,7 @@ func TestJsonImgsPresenter(t *testing.T) { // this is a hard coded value that is not given by the fixture helper and must be provided manually img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368" - s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input") + s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input") var d *distro.Distro pres := NewPresenter(catalog, s.Metadata, d) diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden index 6983afac0..4d0a023b7 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden +++ b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden @@ -71,7 +71,7 @@ "stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7" ], "imageSize": 65, - "scope": "AllLayers", + "scope": "Squashed", "layers": [ { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", diff --git a/syft/presenter/table/presenter_test.go b/syft/presenter/table/presenter_test.go index 02dd2f5d4..481d4261c 100644 --- a/syft/presenter/table/presenter_test.go +++ b/syft/presenter/table/presenter_test.go @@ -5,6 +5,8 @@ import ( "flag" "testing" + "github.com/anchore/stereoscope/pkg/filetree" + "github.com/go-test/deep" "github.com/anchore/go-testutils" @@ -25,12 +27,15 @@ func TestTablePresenter(t *testing.T) { img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", testImage) defer cleanup() + _, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks) + _, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks) + // populate catalog with test data catalog.Add(pkg.Package{ Name: "package-1", Version: "1.0.1", Locations: []source.Location{ - source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), + source.NewLocationFromImage(*ref1, img), }, Type: pkg.DebPkg, }) @@ -38,7 +43,7 @@ func TestTablePresenter(t *testing.T) { Name: "package-2", Version: "2.0.1", Locations: []source.Location{ - source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), + source.NewLocationFromImage(*ref2, img), }, Type: pkg.DebPkg, }) diff --git a/syft/presenter/text/presenter_test.go b/syft/presenter/text/presenter_test.go index 4ca2f756b..cf92fd3b3 100644 --- a/syft/presenter/text/presenter_test.go +++ b/syft/presenter/text/presenter_test.go @@ -5,6 +5,8 @@ import ( "flag" "testing" + "github.com/anchore/stereoscope/pkg/filetree" + "github.com/anchore/go-testutils" "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft/pkg" @@ -70,12 +72,15 @@ func TestTextImgPresenter(t *testing.T) { img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple") defer cleanup() + _, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks) + _, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks) + // populate catalog with test data catalog.Add(pkg.Package{ Name: "package-1", Version: "1.0.1", Locations: []source.Location{ - source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img), + source.NewLocationFromImage(*ref1, img), }, FoundBy: "dpkg", Type: pkg.DebPkg, @@ -84,7 +89,7 @@ func TestTextImgPresenter(t *testing.T) { Name: "package-2", Version: "2.0.1", Locations: []source.Location{ - source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img), + source.NewLocationFromImage(*ref2, img), }, FoundBy: "dpkg", Metadata: PackageInfo{Name: "package-2", Version: "1.0.2"}, @@ -97,7 +102,7 @@ func TestTextImgPresenter(t *testing.T) { l.Metadata.Digest = "sha256:ad8ecdc058976c07e7e347cb89fa9ad86a294b5ceaae6d09713fb035f84115abf3c4a2388a4af3aa60f13b94f4c6846930bdf53" } - s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input") + s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input") if err != nil { t.Fatal(err) } diff --git a/syft/source/all_layers_resolver.go b/syft/source/all_layers_resolver.go index 6cfb0b5ef..757183127 100644 --- a/syft/source/all_layers_resolver.go +++ b/syft/source/all_layers_resolver.go @@ -5,6 +5,10 @@ import ( "fmt" "io" + "github.com/anchore/stereoscope/pkg/filetree" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" ) @@ -71,19 +75,22 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) { for _, path := range paths { for idx, layerIdx := range r.layers { tree := r.img.Layers[layerIdx].Tree - ref := tree.File(file.Path(path)) + _, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks, filetree.DoNotFollowDeadBasenameLinks) + if err != nil { + return nil, err + } if ref == nil { // no file found, keep looking through layers continue } // don't consider directories (special case: there is no path information for /) - if ref.Path == "/" { + if ref.RealPath == "/" { continue } else if r.img.FileCatalog.Exists(*ref) { metadata, err := r.img.FileCatalog.Get(*ref) if err != nil { - return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err) + return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err) } if metadata.Metadata.IsDir { continue @@ -110,31 +117,31 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) for _, pattern := range patterns { for idx, layerIdx := range r.layers { - refs, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern) + results, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern, filetree.DoNotFollowDeadBasenameLinks) if err != nil { return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err) } - for _, ref := range refs { + for _, result := range results { // don't consider directories (special case: there is no path information for /) - if ref.Path == "/" { + if result.RealPath == "/" { continue - } else if r.img.FileCatalog.Exists(ref) { - metadata, err := r.img.FileCatalog.Get(ref) + } else if r.img.FileCatalog.Exists(result.Reference) { + metadata, err := r.img.FileCatalog.Get(result.Reference) if err != nil { - return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err) + return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err) } if metadata.Metadata.IsDir { continue } } - results, err := r.fileByRef(ref, uniqueFileIDs, idx) + refResults, err := r.fileByRef(result.Reference, uniqueFileIDs, idx) if err != nil { return nil, err } - for _, result := range results { - uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img)) + for _, refResult := range refResults { + uniqueLocations = append(uniqueLocations, NewLocationFromImage(refResult, r.img)) } } } @@ -151,8 +158,12 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) * return nil } - relativeRef := entry.Layer.SquashedTree.File(file.Path(path)) - if relativeRef == nil { + exists, relativeRef, err := entry.Layer.SquashedTree.File(file.Path(path), filetree.FollowBasenameLinks) + if err != nil { + log.Errorf("failed to find path=%q in squash: %+w", path, err) + return nil + } + if !exists && relativeRef == nil { return nil } diff --git a/syft/source/all_layers_resolver_test.go b/syft/source/all_layers_resolver_test.go index c10dd47cf..a1aeebd5f 100644 --- a/syft/source/all_layers_resolver_test.go +++ b/syft/source/all_layers_resolver_test.go @@ -41,10 +41,6 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) { name: "link with overridden data", linkPath: "/link-2", resolutions: []resolution{ - { - layer: 3, - path: "/link-2", - }, { layer: 4, path: "/file-2.txt", @@ -70,14 +66,9 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) { }, }, { - name: "dead link", - linkPath: "/link-dead", - resolutions: []resolution{ - { - layer: 8, - path: "/link-dead", - }, - }, + name: "dead link", + linkPath: "/link-dead", + resolutions: []resolution{}, }, { name: "ignore directories", @@ -132,7 +123,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) { }{ { name: "link with previous data", - glob: "**ink-1", + glob: "**/*ink-1", resolutions: []resolution{ { layer: 1, @@ -142,7 +133,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) { }, { name: "link with in layer data", - glob: "**nk-within", + glob: "**/*nk-within", resolutions: []resolution{ { layer: 5, @@ -152,12 +143,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) { }, { name: "link with overridden data", - glob: "**ink-2", + glob: "**/*ink-2", resolutions: []resolution{ - { - layer: 3, - path: "/link-2", - }, { layer: 4, path: "/file-2.txt", @@ -170,7 +157,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) { }, { name: "indirect link (with overridden data)", - glob: "**nk-indirect", + glob: "**/*nk-indirect", resolutions: []resolution{ { layer: 4, @@ -183,14 +170,9 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) { }, }, { - name: "dead link", - glob: "**k-dead", - resolutions: []resolution{ - { - layer: 8, - path: "/link-dead", - }, - }, + name: "dead link", + glob: "**/*k-dead", + resolutions: []resolution{}, }, { name: "ignore directories", diff --git a/syft/source/directory_resolver.go b/syft/source/directory_resolver.go index 5c6f473d1..ccdd5ba8e 100644 --- a/syft/source/directory_resolver.go +++ b/syft/source/directory_resolver.go @@ -8,9 +8,8 @@ import ( "path/filepath" "github.com/anchore/stereoscope/pkg/file" - "github.com/anchore/syft/internal/log" - "github.com/bmatcuk/doublestar" + "github.com/bmatcuk/doublestar/v2" ) var _ Resolver = (*DirectoryResolver)(nil) diff --git a/syft/source/directory_resolver_test.go b/syft/source/directory_resolver_test.go index fdc82971f..8cec50f61 100644 --- a/syft/source/directory_resolver_test.go +++ b/syft/source/directory_resolver_test.go @@ -171,8 +171,9 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) { t.Fatalf("could not use resolver: %+v, %+v", err, refs) } - if len(refs) != 2 { - t.Errorf("unexpected number of refs: %d != 2", len(refs)) + expected := 2 + if len(refs) != expected { + t.Errorf("unexpected number of refs: %d != %d", len(refs), expected) } }) @@ -187,8 +188,9 @@ func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) { t.Fatalf("could not use resolver: %+v, %+v", err, refs) } - if len(refs) != 4 { - t.Errorf("unexpected number of refs: %d != 4", len(refs)) + expected := 6 + if len(refs) != expected { + t.Errorf("unexpected number of refs: %d != %d", len(refs), expected) } }) @@ -202,8 +204,9 @@ func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) { t.Fatalf("could not use resolver: %+v, %+v", err, refs) } - if len(refs) != 1 { - t.Errorf("unexpected number of refs: %d != 1", len(refs)) + expected := 1 + if len(refs) != expected { + t.Errorf("unexpected number of refs: %d != %d", len(refs), expected) } }) diff --git a/syft/source/image_squash_resolver.go b/syft/source/image_squash_resolver.go index dcd5c065a..6ed0d7bb3 100644 --- a/syft/source/image_squash_resolver.go +++ b/syft/source/image_squash_resolver.go @@ -4,6 +4,8 @@ import ( "fmt" "io" + "github.com/anchore/stereoscope/pkg/filetree" + "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" ) @@ -30,19 +32,22 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) { for _, path := range paths { tree := r.img.SquashedTree() - ref := tree.File(file.Path(path)) + _, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks) + if err != nil { + return nil, err + } if ref == nil { // no file found, keep looking through layers continue } // don't consider directories (special case: there is no path information for /) - if ref.Path == "/" { + if ref.RealPath == "/" { continue } else if r.img.FileCatalog.Exists(*ref) { metadata, err := r.img.FileCatalog.Get(*ref) if err != nil { - return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err) + return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err) } if metadata.Metadata.IsDir { continue @@ -70,28 +75,28 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error uniqueLocations := make([]Location, 0) for _, pattern := range patterns { - refs, err := r.img.SquashedTree().FilesByGlob(pattern) + results, err := r.img.SquashedTree().FilesByGlob(pattern) if err != nil { return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err) } - for _, ref := range refs { + for _, result := range results { // don't consider directories (special case: there is no path information for /) - if ref.Path == "/" { + if result.MatchPath == "/" { continue - } else if r.img.FileCatalog.Exists(ref) { - metadata, err := r.img.FileCatalog.Get(ref) + } else if r.img.FileCatalog.Exists(result.Reference) { + metadata, err := r.img.FileCatalog.Get(result.Reference) if err != nil { - return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err) + return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err) } if metadata.Metadata.IsDir { continue } } - resolvedLocations, err := r.FilesByPath(string(ref.Path)) + resolvedLocations, err := r.FilesByPath(string(result.MatchPath)) if err != nil { - return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err) + return nil, fmt.Errorf("failed to find files by path (result=%+v): %w", result, err) } for _, resolvedLocation := range resolvedLocations { if !uniqueFileIDs.Contains(resolvedLocation.ref) { diff --git a/syft/source/image_squash_resolver_test.go b/syft/source/image_squash_resolver_test.go index ef7ab75c5..8b9d10a0f 100644 --- a/syft/source/image_squash_resolver_test.go +++ b/syft/source/image_squash_resolver_test.go @@ -41,13 +41,19 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) { name: "dead link", linkPath: "/link-dead", resolveLayer: 8, - resolvePath: "/link-dead", + resolvePath: "", }, { name: "ignore directories", linkPath: "/bin", resolvePath: "", }, + { + name: "parent is a link (with overridden data)", + linkPath: "/parent-link/file-4.txt", + resolveLayer: 11, + resolvePath: "/parent/file-4.txt", + }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -105,39 +111,51 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) { }{ { name: "link with previous data", - glob: "**link-1", + glob: "**/link-1", resolveLayer: 1, resolvePath: "/file-1.txt", }, { name: "link with in layer data", - glob: "**link-within", + glob: "**/link-within", resolveLayer: 5, resolvePath: "/file-3.txt", }, { name: "link with overridden data", - glob: "**link-2", + glob: "**/link-2", resolveLayer: 7, resolvePath: "/file-2.txt", }, { name: "indirect link (with overridden data)", - glob: "**link-indirect", + glob: "**/link-indirect", resolveLayer: 7, resolvePath: "/file-2.txt", }, { - name: "dead link", - glob: "**link-dead", - resolveLayer: 8, - resolvePath: "/link-dead", + name: "dead link", + glob: "**/link-dead", + // dead links are dead! they shouldn't match on globs + resolvePath: "", }, { name: "ignore directories", glob: "**/bin", resolvePath: "", }, + { + name: "parent without link", + glob: "**/parent/*.txt", + resolveLayer: 11, + resolvePath: "/parent/file-4.txt", + }, + { + name: "parent is a link (override)", + glob: "**/parent-link/file-4.txt", + resolveLayer: 11, + resolvePath: "/parent/file-4.txt", + }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { diff --git a/syft/source/location.go b/syft/source/location.go index b017a442e..b4aae7500 100644 --- a/syft/source/location.go +++ b/syft/source/location.go @@ -27,13 +27,13 @@ func NewLocationFromImage(ref file.Reference, img *image.Image) Location { if err != nil { log.Warnf("unable to find file catalog entry for ref=%+v", ref) return Location{ - Path: string(ref.Path), + Path: string(ref.RealPath), ref: ref, } } return Location{ - Path: string(ref.Path), + Path: string(ref.RealPath), FileSystemID: entry.Layer.Metadata.Digest, ref: ref, } diff --git a/syft/source/test-fixtures/image-symlinks/Dockerfile b/syft/source/test-fixtures/image-symlinks/Dockerfile index c73d100a1..ba637cd0d 100644 --- a/syft/source/test-fixtures/image-symlinks/Dockerfile +++ b/syft/source/test-fixtures/image-symlinks/Dockerfile @@ -21,4 +21,12 @@ RUN ln -s ./link-2 link-indirect ADD new-file-2.txt file-2.txt # LAYER 8: dead link -RUN ln -s ./i-dont-exist.txt link-dead \ No newline at end of file +RUN ln -s ./i-dont-exist.txt link-dead + +# LAYER 9: add the parent dir +ADD parent /parent +# LAYER 10: parent is a symlink +RUN ln -s /parent parent-link + +# LAYER 11: parent is a symlink and the child target is overridden +COPY new-file-4.txt /parent-link/file-4.txt \ No newline at end of file diff --git a/syft/source/test-fixtures/image-symlinks/new-file-4.txt b/syft/source/test-fixtures/image-symlinks/new-file-4.txt new file mode 100644 index 000000000..6e6239258 --- /dev/null +++ b/syft/source/test-fixtures/image-symlinks/new-file-4.txt @@ -0,0 +1 @@ +override file 4! \ No newline at end of file diff --git a/syft/source/test-fixtures/image-symlinks/parent/file-4.txt b/syft/source/test-fixtures/image-symlinks/parent/file-4.txt new file mode 100644 index 000000000..75cda1588 --- /dev/null +++ b/syft/source/test-fixtures/image-symlinks/parent/file-4.txt @@ -0,0 +1 @@ +the 4th file! \ No newline at end of file diff --git a/test/integration/distro_test.go b/test/integration/distro_test.go index 126a1d7bd..51d888e75 100644 --- a/test/integration/distro_test.go +++ b/test/integration/distro_test.go @@ -16,7 +16,7 @@ func TestDistroImage(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - _, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) + _, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } diff --git a/test/integration/document_import_test.go b/test/integration/document_import_test.go index 301b6ee64..416fca062 100644 --- a/test/integration/document_import_test.go +++ b/test/integration/document_import_test.go @@ -31,7 +31,7 @@ func TestCatalogFromJSON(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture) defer cleanup() - expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) + expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } diff --git a/test/integration/json_schema_test.go b/test/integration/json_schema_test.go index f60f09ce0..022c9db54 100644 --- a/test/integration/json_schema_test.go +++ b/test/integration/json_schema_test.go @@ -60,7 +60,7 @@ func TestJsonSchemaImg(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) + src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } @@ -87,7 +87,7 @@ func TestJsonSchemaImg(t *testing.T) { } func TestJsonSchemaDirs(t *testing.T) { - src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope) + src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope) if err != nil { t.Errorf("unable to create source from dir: %+v", err) } diff --git a/test/integration/pkg_coverage_test.go b/test/integration/pkg_coverage_test.go index cc25f391a..d676c7644 100644 --- a/test/integration/pkg_coverage_test.go +++ b/test/integration/pkg_coverage_test.go @@ -18,7 +18,7 @@ func TestPkgCoverageImage(t *testing.T) { tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) defer cleanup() - _, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope) + _, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } @@ -100,7 +100,7 @@ func TestPkgCoverageImage(t *testing.T) { } func TestPkgCoverageDirectory(t *testing.T) { - _, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope) + _, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope) if err != nil { t.Errorf("unable to create source from dir: %+v", err)