Merge pull request #312 from anchore/omit-empty-packages

Omit empty packages

.golangci.yaml

@@ -31,7 +31,6 @@ linters:
     - misspell
     - nakedret
     - nolintlint
-    - prealloc
     - rowserrcheck
     - scopelint
     - staticcheck
@@ -55,5 +54,6 @@ linters:
 #    - lll         # without a way to specify per-line exception cases, this is not usable
 #    - maligned    # this is an excellent linter, but tricky to optimize and we are not sensitive to memory layout optimizations
 #    - nestif
+#    - prealloc    # following this rule isn't consistently a good idea, as it sometimes forces unnecessary allocations that result in less idiomatic code
 #    - testpackage
-#    - wsl
+#    - wsl

Makefile

@@ -16,6 +16,9 @@ TITLE := $(BOLD)$(PURPLE)
 SUCCESS := $(BOLD)$(GREEN)
 # the quality gate lower threshold for unit test total % coverage (by function statements)
 COVERAGE_THRESHOLD := 68
+# CI cache busting values; change these if you want CI to not use previous stored cache
+COMPARE_CACHE_BUSTER="f7e689d76a9"
+INTEGRATION_CACHE_BUSTER="789bacdf"
 
 ## Build variables
 DISTDIR=./dist
@@ -140,7 +143,7 @@ integration: ## Run integration tests
 
 # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
 integration-fingerprint:
-	find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint
+	find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint
 
 .PHONY: java-packages-fingerprint
 java-packages-fingerprint:
@@ -173,6 +176,7 @@ $(SNAPSHOTDIR): ## Build snapshot release binaries and packages
 	BUILD_GIT_TREE_STATE=$(GITTREESTATE) \
 	$(TEMPDIR)/goreleaser release --skip-publish --rm-dist --snapshot --config $(TEMPDIR)/goreleaser.yaml
 
+# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
 .PHONY: acceptance-mac
 acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac)
 	$(call title,Running acceptance test: Run on Mac)
@@ -182,13 +186,14 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie
 			$(ACC_TEST_IMAGE) \
 			$(RESULTSDIR)
 
+# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
 .PHONY: acceptance-linux
 acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux)
 
 # note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests
 .PHONY: compare-fingerprint
 compare-fingerprint:
-	find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint
+	find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint && echo "$(COMPARE_CACHE_BUSTER)" >> test/inline-compare/inline-compare.fingerprint
 
 .PHONY: compare-snapshot
 compare-snapshot: $(SNAPSHOTDIR) ## Compare the reports of a run of a snapshot build of syft against inline-scan

go.mod

@@ -9,9 +9,9 @@ require (
 	github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
-	github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e
+	github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5
 	github.com/antihax/optional v1.0.0
-	github.com/bmatcuk/doublestar v1.3.3
+	github.com/bmatcuk/doublestar/v2 v2.0.4
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
 	github.com/dustin/go-humanize v1.0.0
 	github.com/facebookincubator/nvdtools v0.1.4

go.sum

@@ -126,8 +126,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74 h1:9kkKTIyXJC+/syUcY6KWxFoJZJ+GWwrIscF+gBY067k=
-github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
 github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238 h1:/iI+1cj1a27ow0wj378pPJIm8sCSy6I21Tz6oLbLDQY=
 github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
 github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ=
@@ -136,8 +134,12 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
-github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e h1:vHUqHTvH9/oxdDDh1fxS9Ls9gWGytKO7XbbzcQ9MBwI=
-github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e/go.mod h1:/dHAFjYflH/1tzhdHAcnMCjprMch+YzHJKi59m/1KCM=
+github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255 h1:Ng7BDr9PQTCztANogjfEdEjjWUylhlPyZPhtarIGo00=
+github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
+github.com/anchore/stereoscope v0.0.0-20210105000809-428eda0b2ec6 h1:JWpsV/8x1fuCYjJmNjT43cVFblLTpO/ISDnePukiTNw=
+github.com/anchore/stereoscope v0.0.0-20210105000809-428eda0b2ec6/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
+github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5 h1:NGRfS6BZKElgiMbqdoH9iQn+6oxT7CJdZYrqgwvGkWY=
+github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -168,8 +170,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/bmatcuk/doublestar v1.3.3 h1:pVP1d49CcQQaNOl+PI6sPybIrIOD/6sux31PFdmhTH0=
-github.com/bmatcuk/doublestar v1.3.3/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
+github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI=
+github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw=
 github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U=
 github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6abzXN4Pg=
 github.com/bombsimon/wsl/v3 v3.0.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=

internal/anchore/import_package_sbom_test.go

@@ -61,7 +61,7 @@ func TestPackageSbomToModel(t *testing.T) {
 		FoundBy: "foundBy",
 		Locations: []source.Location{
 			{
-				Path:         "path",
+				RealPath:     "path",
 				FileSystemID: "layerID",
 			},
 		},
@@ -139,7 +139,7 @@ func TestPackageSbomImport(t *testing.T) {
 		FoundBy: "foundBy",
 		Locations: []source.Location{
 			{
-				Path:         "path",
+				RealPath:     "path",
 				FileSystemID: "layerID",
 			},
 		},

syft/cataloger/common/generic_cataloger.go

@@ -4,7 +4,7 @@ Package common provides generic utilities used by multiple catalogers.
 package common
 
 import (
-	"strings"
+	"io"
 
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg"
@@ -89,7 +89,7 @@ func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []source.Lo
 }
 
 // catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
-func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.Package, error) {
+func (c *GenericCataloger) catalog(contents map[source.Location]io.ReadCloser) ([]pkg.Package, error) {
 	defer c.clear()
 
 	packages := make([]pkg.Package, 0)
@@ -101,7 +101,7 @@ func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.P
 			continue
 		}
 
-		entries, err := parser(location.Path, strings.NewReader(content))
+		entries, err := parser(location.RealPath, content)
 		if err != nil {
 			// TODO: should we fail? or only log?
 			log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)

syft/cataloger/common/generic_cataloger_test.go

@@ -4,28 +4,32 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"strings"
 	"testing"
 
-	"github.com/anchore/syft/syft/source"
-
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
 )
 
 type testResolverMock struct {
-	contents map[source.Location]string
+	contents map[source.Location]io.ReadCloser
 }
 
 func newTestResolver() *testResolverMock {
 	return &testResolverMock{
-		contents: make(map[source.Location]string),
+		contents: make(map[source.Location]io.ReadCloser),
 	}
 }
 
-func (r *testResolverMock) FileContentsByLocation(_ source.Location) (string, error) {
-	return "", fmt.Errorf("not implemented")
+func (r testResolverMock) HasPath(path string) bool {
+	panic("not implemented")
 }
 
-func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]string, error) {
+func (r *testResolverMock) FileContentsByLocation(_ source.Location) (io.ReadCloser, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]io.ReadCloser, error) {
 	return r.contents, nil
 }
 
@@ -34,7 +38,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
 
 	for idx, p := range paths {
 		results[idx] = source.NewLocation(p)
-		r.contents[results[idx]] = fmt.Sprintf("%s file contents!", p)
+		r.contents[results[idx]] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", p)))
 	}
 
 	return results, nil
@@ -43,7 +47,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
 func (r *testResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) {
 	path := "/a-path.txt"
 	location := source.NewLocation(path)
-	r.contents[location] = fmt.Sprintf("%s file contents!", path)
+	r.contents[location] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", path)))
 	return []source.Location{location}, nil
 }
 
@@ -97,7 +101,7 @@ func TestGenericCataloger(t *testing.T) {
 
 	for _, p := range actualPkgs {
 		ref := p.Locations[0]
-		exP, ok := expectedPkgs[ref.Path]
+		exP, ok := expectedPkgs[ref.RealPath]
 		if !ok {
 			t.Errorf("missing expected pkg: ref=%+v", ref)
 			continue

syft/cataloger/deb/cataloger.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"path"
 	"path/filepath"
-	"strings"
 
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
@@ -47,9 +46,9 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 			return nil, err
 		}
 
-		pkgs, err = parseDpkgStatus(strings.NewReader(dbContents))
+		pkgs, err = parseDpkgStatus(dbContents)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err)
+			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err)
 		}
 
 		md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs)
@@ -57,7 +56,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 			return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err)
 		}
 
-		copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
+		copyrightContentsByName, copyrightLocationByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
 		if err != nil {
 			return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err)
 		}
@@ -91,7 +90,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 				p.Licenses = parseLicensesFromCopyright(copyrightReader)
 
 				// keep a record of the file where this was discovered
-				if ref, ok := copyrightRefsByName[p.Name]; ok {
+				if ref, ok := copyrightLocationByName[p.Name]; ok {
 					p.Locations = append(p.Locations, ref)
 				}
 			}
@@ -105,20 +104,17 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
 
 	var md5FileMatches []source.Location
 	var nameByRef = make(map[source.Location]string)
-	parentPath := filepath.Dir(dbLocation.Path)
+	parentPath := filepath.Dir(dbLocation.RealPath)
 
 	for _, p := range pkgs {
 		// look for /var/lib/dpkg/info/NAME:ARCH.md5sums
 		name := md5Key(p)
-		md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
-		md5SumLocation := resolver.RelativeFileByPath(dbLocation, md5sumPath)
+		md5SumLocation := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", name+md5sumsExt))
 
 		if md5SumLocation == nil {
 			// the most specific key did not work, fallback to just the name
 			// look for /var/lib/dpkg/info/NAME.md5sums
-			name := p.Name
-			md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
-			md5SumLocation = resolver.RelativeFileByPath(dbLocation, md5sumPath)
+			md5SumLocation = resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", p.Name+md5sumsExt))
 		}
 		// we should have at least one reference
 		if md5SumLocation != nil {
@@ -135,14 +131,14 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
 
 	// organize content results and refs by a combination of name and architecture
 	var contentsByName = make(map[string]io.Reader)
-	var refsByName = make(map[string]source.Location)
+	var locationByName = make(map[string]source.Location)
 	for location, contents := range md5ContentsByLocation {
 		name := nameByRef[location]
-		contentsByName[name] = strings.NewReader(contents)
-		refsByName[name] = location
+		contentsByName[name] = contents
+		locationByName[name] = location
 	}
 
-	return contentsByName, refsByName, nil
+	return contentsByName, locationByName, nil
 }
 
 func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
@@ -174,7 +170,7 @@ func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location
 	var refsByName = make(map[string]source.Location)
 	for location, contents := range copyrightContentsByLocation {
 		name := nameByLocation[location]
-		contentsByName[name] = strings.NewReader(contents)
+		contentsByName[name] = contents
 		refsByName[name] = location
 	}
 

syft/cataloger/deb/cataloger_test.go

@@ -54,7 +54,7 @@ func TestDpkgCataloger(t *testing.T) {
 			img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg")
 			defer cleanup()
 
-			s, err := source.NewFromImage(img, source.AllLayersScope, "")
+			s, err := source.NewFromImage(img, source.SquashedScope, "")
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -79,7 +79,7 @@ func TestDpkgCataloger(t *testing.T) {
 				// we will test the sources separately
 				var sourcesList = make([]string, len(a.Locations))
 				for i, s := range a.Locations {
-					sourcesList[i] = s.Path
+					sourcesList[i] = s.RealPath
 				}
 				a.Locations = nil
 

syft/cataloger/javascript/parse_package_json.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"regexp"
 
+	"github.com/anchore/syft/internal/log"
+
 	"github.com/anchore/syft/internal"
 
 	"github.com/mitchellh/mapstructure"
@@ -172,6 +174,11 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
 			return nil, fmt.Errorf("failed to parse package.json file: %w", err)
 		}
 
+		if !p.hasNameAndVersionValues() {
+			log.Debug("encountered package.json file without a name and/or version field, ignoring this file")
+			return nil, nil
+		}
+
 		licenses, err := licensesFromJSON(p)
 		if err != nil {
 			return nil, fmt.Errorf("failed to parse package.json file: %w", err)
@@ -195,3 +202,7 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
 
 	return packages, nil
 }
+
+func (p PackageJSON) hasNameAndVersionValues() bool {
+	return p.Name != "" && p.Version != ""
+}

syft/cataloger/javascript/parse_package_json_test.go

@@ -142,3 +142,20 @@ func TestParsePackageJSON(t *testing.T) {
 		})
 	}
 }
+
+func TestParsePackageJSON_Partial(t *testing.T) { // see https://github.com/anchore/syft/issues/311
+	const fixtureFile = "test-fixtures/pkg-json/package-partial.json"
+	fixture, err := os.Open(fixtureFile)
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parsePackageJSON("", fixture)
+	if err != nil {
+		t.Fatalf("failed to parse package-lock.json: %+v", err)
+	}
+
+	if actualCount := len(actual); actualCount != 0 {
+		t.Errorf("no packages should've been returned (but got %d packages)", actualCount)
+	}
+}

syft/cataloger/javascript/test-fixtures/pkg-json/package-partial.json

@@ -0,0 +1,5 @@
+{
+  "sideEffects": false,
+  "module": "../../esm/fp/isSaturday/index.js",
+  "typings": "../../typings.d.ts"
+}

syft/cataloger/python/package_cataloger.go

@@ -3,7 +3,6 @@ package python
 import (
 	"bufio"
 	"fmt"
-	"strings"
 
 	"github.com/anchore/syft/syft/pkg"
 
@@ -39,7 +38,7 @@ func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, err
 	for _, entry := range entries {
 		p, err := c.catalogEggOrWheel(entry)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog python package=%+v: %w", entry.Metadata.Location.Path, err)
+			return nil, fmt.Errorf("unable to catalog python package=%+v: %w", entry.Metadata.Location.RealPath, err)
 		}
 		if p != nil {
 			packages = append(packages, *p)
@@ -91,6 +90,12 @@ func (c *PackageCataloger) catalogEggOrWheel(entry *packageEntry) (*pkg.Package,
 		return nil, err
 	}
 
+	// This can happen for Python 2.7 where it is reported from an egg-info, but Python is
+	// the actual runtime, it isn't a "package". The special-casing here allows to skip it
+	if metadata.Name == "Python" {
+		return nil, nil
+	}
+
 	var licenses []string
 	if metadata.License != "" {
 		licenses = []string{metadata.License}
@@ -113,7 +118,7 @@ func (c *PackageCataloger) catalogEggOrWheel(entry *packageEntry) (*pkg.Package,
 func (c *PackageCataloger) assembleEggOrWheelMetadata(entry *packageEntry) (*pkg.PythonPackageMetadata, []source.Location, error) {
 	var sources = []source.Location{entry.Metadata.Location}
 
-	metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.Path, strings.NewReader(entry.Metadata.Contents))
+	metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.RealPath, entry.Metadata.Contents)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -147,7 +152,7 @@ func (c *PackageCataloger) processRecordFiles(entry *source.FileData) (files []p
 		sources = append(sources, entry.Location)
 
 		// parse the record contents
-		records, err := parseWheelOrEggRecord(strings.NewReader(entry.Contents))
+		records, err := parseWheelOrEggRecord(entry.Contents)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -165,7 +170,7 @@ func (c *PackageCataloger) processTopLevelPackages(entry *source.FileData) (pkgs
 
 	sources = append(sources, entry.Location)
 
-	scanner := bufio.NewScanner(strings.NewReader(entry.Contents))
+	scanner := bufio.NewScanner(entry.Contents)
 	for scanner.Scan() {
 		pkgs = append(pkgs, scanner.Text())
 	}

syft/cataloger/python/package_cataloger_test.go

@@ -1,154 +1,27 @@
 package python
 
 import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"strings"
 	"testing"
 
-	"github.com/anchore/syft/internal/file"
-
-	"github.com/anchore/syft/syft/source"
-
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
 	"github.com/go-test/deep"
 )
 
-// TODO: make this generic (based on maps of source.FileData) and make a generic mock to move to the source pkg
-type pythonTestResolverMock struct {
-	metadataReader io.Reader
-	recordReader   io.Reader
-	topLevelReader io.Reader
-	metadataRef    *source.Location
-	recordRef      *source.Location
-	topLevelRef    *source.Location
-	contents       map[source.Location]string
-}
-
-func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMock {
-	metadataReader, err := os.Open(metaPath)
-	if err != nil {
-		panic(fmt.Errorf("failed to open metadata: %+v", err))
-	}
-
-	var recordReader io.Reader
-	if recordPath != "" {
-		recordReader, err = os.Open(recordPath)
-		if err != nil {
-			panic(fmt.Errorf("failed to open record: %+v", err))
-		}
-	}
-
-	var topLevelReader io.Reader
-	if topPath != "" {
-		topLevelReader, err = os.Open(topPath)
-		if err != nil {
-			panic(fmt.Errorf("failed to open top level: %+v", err))
-		}
-	}
-
-	var recordRef *source.Location
-	if recordReader != nil {
-		ref := source.NewLocation("test-fixtures/dist-info/RECORD")
-		recordRef = &ref
-	}
-	var topLevelRef *source.Location
-	if topLevelReader != nil {
-		ref := source.NewLocation("test-fixtures/dist-info/top_level.txt")
-		topLevelRef = &ref
-	}
-	metadataRef := source.NewLocation("test-fixtures/dist-info/METADATA")
-	return &pythonTestResolverMock{
-		recordReader:   recordReader,
-		metadataReader: metadataReader,
-		topLevelReader: topLevelReader,
-		metadataRef:    &metadataRef,
-		recordRef:      recordRef,
-		topLevelRef:    topLevelRef,
-		contents:       make(map[source.Location]string),
-	}
-}
-
-func (r *pythonTestResolverMock) FileContentsByLocation(location source.Location) (string, error) {
-	switch {
-	case r.topLevelRef != nil && location.Path == r.topLevelRef.Path:
-		b, err := ioutil.ReadAll(r.topLevelReader)
-		if err != nil {
-			return "", err
-		}
-		return string(b), nil
-	case location.Path == r.metadataRef.Path:
-		b, err := ioutil.ReadAll(r.metadataReader)
-		if err != nil {
-			return "", err
-		}
-		return string(b), nil
-	case location.Path == r.recordRef.Path:
-		b, err := ioutil.ReadAll(r.recordReader)
-		if err != nil {
-			return "", err
-		}
-		return string(b), nil
-	}
-	return "", fmt.Errorf("invalid value given")
-}
-
-func (r *pythonTestResolverMock) MultipleFileContentsByLocation(locations []source.Location) (map[source.Location]string, error) {
-	var results = make(map[source.Location]string)
-	var err error
-	for _, l := range locations {
-		results[l], err = r.FileContentsByLocation(l)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return results, nil
-}
-
-func (r *pythonTestResolverMock) FilesByPath(_ ...string) ([]source.Location, error) {
-	return nil, fmt.Errorf("not implemented")
-}
-
-func (r *pythonTestResolverMock) FilesByGlob(patterns ...string) ([]source.Location, error) {
-	var results []source.Location
-	for _, pattern := range patterns {
-		for _, l := range []*source.Location{r.topLevelRef, r.metadataRef, r.recordRef} {
-			if l == nil {
-				continue
-			}
-			if file.GlobMatch(pattern, l.Path) {
-				results = append(results, *l)
-			}
-		}
-	}
-	return results, nil
-}
-func (r *pythonTestResolverMock) RelativeFileByPath(_ source.Location, path string) *source.Location {
-	switch {
-	case strings.Contains(path, "RECORD"):
-		return r.recordRef
-	case strings.Contains(path, "top_level.txt"):
-		return r.topLevelRef
-	default:
-		panic(fmt.Errorf("invalid RelativeFileByPath value given: %q", path))
-	}
-}
-
 func TestPythonPackageWheelCataloger(t *testing.T) {
 	tests := []struct {
-		MetadataFixture string
-		RecordFixture   string
-		TopLevelFixture string
-		ExpectedPackage pkg.Package
+		name            string
+		fixtures        []string
+		expectedPackage pkg.Package
 	}{
 		{
-			MetadataFixture: "test-fixtures/egg-info/PKG-INFO",
-			RecordFixture:   "test-fixtures/egg-info/RECORD",
-			TopLevelFixture: "test-fixtures/egg-info/top_level.txt",
-			ExpectedPackage: pkg.Package{
+			name: "egg-info directory",
+			fixtures: []string{
+				"test-fixtures/egg-info/PKG-INFO",
+				"test-fixtures/egg-info/RECORD",
+				"test-fixtures/egg-info/top_level.txt",
+			},
+			expectedPackage: pkg.Package{
 				Name:         "requests",
 				Version:      "2.22.0",
 				Type:         pkg.PythonPkg,
@@ -177,10 +50,13 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 			},
 		},
 		{
-			MetadataFixture: "test-fixtures/dist-info/METADATA",
-			RecordFixture:   "test-fixtures/dist-info/RECORD",
-			TopLevelFixture: "test-fixtures/dist-info/top_level.txt",
-			ExpectedPackage: pkg.Package{
+			name: "dist-info directory",
+			fixtures: []string{
+				"test-fixtures/dist-info/METADATA",
+				"test-fixtures/dist-info/RECORD",
+				"test-fixtures/dist-info/top_level.txt",
+			},
+			expectedPackage: pkg.Package{
 				Name:         "Pygments",
 				Version:      "2.6.1",
 				Type:         pkg.PythonPkg,
@@ -210,8 +86,9 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 		{
 			// in cases where the metadata file is available and the record is not we should still record there is a package
 			// additionally empty top_level.txt files should not result in an error
-			MetadataFixture: "test-fixtures/partial.dist-info/METADATA",
-			ExpectedPackage: pkg.Package{
+			name:     "partial dist-info directory",
+			fixtures: []string{"test-fixtures/partial.dist-info/METADATA"},
+			expectedPackage: pkg.Package{
 				Name:         "Pygments",
 				Version:      "2.6.1",
 				Type:         pkg.PythonPkg,
@@ -231,8 +108,9 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 			},
 		},
 		{
-			MetadataFixture: "test-fixtures/test.egg-info",
-			ExpectedPackage: pkg.Package{
+			name:     "egg-info regular file",
+			fixtures: []string{"test-fixtures/test.egg-info"},
+			expectedPackage: pkg.Package{
 				Name:         "requests",
 				Version:      "2.22.0",
 				Type:         pkg.PythonPkg,
@@ -254,19 +132,15 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		t.Run(test.MetadataFixture, func(t *testing.T) {
-			resolver := newTestResolver(test.MetadataFixture, test.RecordFixture, test.TopLevelFixture)
+		t.Run(test.name, func(t *testing.T) {
+			resolver := source.NewMockResolverForPaths(test.fixtures...)
 
-			// note that the source is the record ref created by the resolver mock... attach the expected values
-			test.ExpectedPackage.Locations = []source.Location{*resolver.metadataRef}
-			if resolver.recordRef != nil {
-				test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.recordRef)
+			locations, err := resolver.FilesByPath(test.fixtures...)
+			if err != nil {
+				t.Fatal(err)
 			}
 
-			if resolver.topLevelRef != nil {
-				test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.topLevelRef)
-			}
-			// end patching expected values with runtime data...
+			test.expectedPackage.Locations = locations
 
 			actual, err := NewPythonPackageCataloger().Catalog(resolver)
 			if err != nil {
@@ -274,13 +148,37 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 			}
 
 			if len(actual) != 1 {
-				t.Fatalf("unexpected length: %d", len(actual))
+				t.Fatalf("unexpected number of packages: %d", len(actual))
 			}
 
-			for _, d := range deep.Equal(actual[0], test.ExpectedPackage) {
+			for _, d := range deep.Equal(actual[0], test.expectedPackage) {
 				t.Errorf("diff: %+v", d)
 			}
 		})
 	}
+}
 
+func TestIgnorePackage(t *testing.T) {
+	tests := []struct {
+		MetadataFixture string
+	}{
+		{
+			MetadataFixture: "test-fixtures/Python-2.7.egg-info",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.MetadataFixture, func(t *testing.T) {
+			resolver := source.NewMockResolverForPaths(test.MetadataFixture)
+
+			actual, err := NewPythonPackageCataloger().Catalog(resolver)
+			if err != nil {
+				t.Fatalf("failed to catalog python package: %+v", err)
+			}
+
+			if len(actual) != 0 {
+				t.Fatalf("Expected 0 packages but found: %d", len(actual))
+			}
+		})
+	}
 }

syft/cataloger/python/package_entry.go

@@ -19,11 +19,11 @@ func newPackageEntry(resolver source.FileResolver, metadataLocation source.Locat
 	// to reconcile the RECORD path to the same layer (or a lower layer). The same is true with the top_level.txt file.
 
 	// lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure)
-	recordPath := filepath.Join(filepath.Dir(metadataLocation.Path), "RECORD")
+	recordPath := filepath.Join(filepath.Dir(metadataLocation.RealPath), "RECORD")
 	recordLocation := resolver.RelativeFileByPath(metadataLocation, recordPath)
 
 	// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
-	parentDir := filepath.Dir(metadataLocation.Path)
+	parentDir := filepath.Dir(metadataLocation.RealPath)
 	topLevelPath := filepath.Join(parentDir, "top_level.txt")
 	topLevelLocation := resolver.RelativeFileByPath(metadataLocation, topLevelPath)
 

syft/cataloger/python/parse_wheel_egg_metadata.go

@@ -7,6 +7,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/anchore/syft/internal/file"
+
 	"github.com/mitchellh/mapstructure"
 
 	"github.com/anchore/syft/syft/pkg"
@@ -70,11 +72,28 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
 
 	// add additional metadata not stored in the egg/wheel metadata file
 
-	metadata.SitePackagesRootPath = filepath.Clean(filepath.Join(filepath.Dir(path), ".."))
+	metadata.SitePackagesRootPath = determineSitePackagesRootPath(path)
 
 	return metadata, nil
 }
 
+// isEggRegularFile determines if the specified path is the regular file variant
+// of egg metadata (as opposed to a directory that contains more metadata
+// files).
+func isEggRegularFile(path string) bool {
+	return file.GlobMatch(eggFileMetadataGlob, path)
+}
+
+// determineSitePackagesRootPath returns the path of the site packages root,
+// given the egg metadata file or directory specified in the path.
+func determineSitePackagesRootPath(path string) string {
+	if isEggRegularFile(path) {
+		return filepath.Clean(filepath.Dir(path))
+	}
+
+	return filepath.Clean(filepath.Dir(filepath.Dir(path)))
+}
+
 // handleFieldBodyContinuation returns the updated value for the specified field after processing the specified line.
 // If the continuation cannot be processed, it returns an error.
 func handleFieldBodyContinuation(key, line string, fields map[string]string) (string, error) {

syft/cataloger/python/parse_wheel_egg_metadata_test.go

@@ -56,5 +56,64 @@ func TestParseWheelEggMetadata(t *testing.T) {
 			}
 		})
 	}
+}
 
+func TestIsRegularEggFile(t *testing.T) {
+	cases := []struct {
+		path     string
+		expected bool
+	}{
+		{
+			"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.egg-info",
+			true,
+		},
+		{
+			"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.egg-info/PKG-INFO",
+			false,
+		},
+		{
+			"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.dist-info/METADATA",
+			false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.path, func(t *testing.T) {
+			actual := isEggRegularFile(c.path)
+
+			if actual != c.expected {
+				t.Errorf("expected %t but got %t", c.expected, actual)
+			}
+		})
+	}
+}
+
+func TestDetermineSitePackagesRootPath(t *testing.T) {
+	cases := []struct {
+		inputPath string
+		expected  string
+	}{
+		{
+			inputPath: "/usr/lib64/python2.6/site-packages/ethtool-0.6-py2.6.egg-info",
+			expected:  "/usr/lib64/python2.6/site-packages",
+		},
+		{
+			inputPath: "/usr/lib/python2.7/dist-packages/configobj-5.0.6.egg-info/top_level.txt",
+			expected:  "/usr/lib/python2.7/dist-packages",
+		},
+		{
+			inputPath: "/usr/lib/python2.7/dist-packages/six-1.10.0.egg-info/PKG-INFO",
+			expected:  "/usr/lib/python2.7/dist-packages",
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.inputPath, func(t *testing.T) {
+			actual := determineSitePackagesRootPath(c.inputPath)
+
+			if actual != c.expected {
+				t.Errorf("expected %s but got %s", c.expected, actual)
+			}
+		})
+	}
 }

syft/cataloger/python/test-fixtures/Python-2.7.egg-info

@@ -0,0 +1,31 @@
+Metadata-Version: 1.1
+Name: Python
+Version: 2.7.13
+Summary: A high-level object-oriented programming language
+Home-page: http://www.python.org/2.7
+Author: Guido van Rossum and the Python community
+Author-email: python-dev@python.org
+License: PSF license
+Description: Python is an interpreted, interactive, object-oriented programming
+        language. It is often compared to Tcl, Perl, Scheme or Java.
+
+        Python combines remarkable power with very clear syntax. It has
+        modules, classes, exceptions, very high level dynamic data types, and
+        dynamic typing. There are interfaces to many system calls and
+        libraries, as well as to various windowing systems (X11, Motif, Tk,
+        Mac, MFC). New built-in modules are easily written in C or C++. Python
+        is also usable as an extension language for applications that need a
+        programmable interface.
+
+        The Python implementation is portable: it runs on many brands of UNIX,
+        on Windows, DOS, OS/2, Mac, Amiga... If your favorite system isn't
+        listed here, it may still be supported, if there's a C compiler for
+        it. Ask around on comp.lang.python -- or just try compiling Python
+        yourself.
+Platform: Many
+Classifier: Development Status :: 6 - Mature
+Classifier: License :: OSI Approved :: Python Software Foundation License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: C
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development

syft/cataloger/rpmdb/cataloger.go

@@ -5,7 +5,6 @@ package rpmdb
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
@@ -37,14 +36,14 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 
 	var pkgs []pkg.Package
 	for _, location := range fileMatches {
-		dbContents, err := resolver.FileContentsByLocation(location)
+		dbContentReader, err := resolver.FileContentsByLocation(location)
 		if err != nil {
 			return nil, err
 		}
 
-		pkgs, err = parseRpmDB(resolver, location, strings.NewReader(dbContents))
+		pkgs, err = parseRpmDB(resolver, location, dbContentReader)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.Path, err)
+			return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err)
 		}
 	}
 	return pkgs, nil

syft/cataloger/rpmdb/parse_rpmdb.go

@@ -45,11 +45,6 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 	allPkgs := make([]pkg.Package, 0)
 
 	for _, entry := range pkgList {
-		records, err := extractRpmdbFileRecords(resolver, entry)
-		if err != nil {
-			return nil, err
-		}
-
 		p := pkg.Package{
 			Name:         entry.Name,
 			Version:      fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does, instead of fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch)
@@ -67,7 +62,7 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 				Vendor:    entry.Vendor,
 				License:   entry.License,
 				Size:      entry.Size,
-				Files:     records,
+				Files:     extractRpmdbFileRecords(resolver, entry),
 			},
 		}
 
@@ -77,25 +72,19 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 	return allPkgs, nil
 }
 
-func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) {
+func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) []pkg.RpmdbFileRecord {
 	var records = make([]pkg.RpmdbFileRecord, 0)
 
 	for _, record := range entry.Files {
-		refs, err := resolver.FilesByPath(record.Path)
-		if err != nil {
-			return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err)
-		}
 		//only persist RPMDB file records which exist in the image/directory, otherwise ignore them
-		if len(refs) == 0 {
-			continue
+		if resolver.HasPath(record.Path) {
+			records = append(records, pkg.RpmdbFileRecord{
+				Path:   record.Path,
+				Mode:   pkg.RpmdbFileMode(record.Mode),
+				Size:   int(record.Size),
+				SHA256: record.SHA256,
+			})
 		}
-
-		records = append(records, pkg.RpmdbFileRecord{
-			Path:   record.Path,
-			Mode:   pkg.RpmdbFileMode(record.Mode),
-			Size:   int(record.Size),
-			SHA256: record.SHA256,
-		})
 	}
-	return records, nil
+	return records
 }

syft/cataloger/rpmdb/parse_rpmdb_test.go

@@ -21,6 +21,10 @@ func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock {
 	}
 }
 
+func (r rpmdbTestFileResolverMock) HasPath(path string) bool {
+	return !r.ignorePaths
+}
+
 func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...string) ([]source.Location, error) {
 	if r.ignorePaths {
 		// act as if no paths exist

syft/cataloger/ruby/catalogers.go

@@ -19,7 +19,7 @@ func NewGemFileLockCataloger() *common.GenericCataloger {
 // NewGemSpecCataloger returns a new Bundler cataloger object tailored for detecting installations of gems (e.g. Gemspec).
 func NewGemSpecCataloger() *common.GenericCataloger {
 	globParsers := map[string]common.ParserFn{
-		"**/specifications/*.gemspec": parseGemSpecEntries,
+		"**/specifications/**/*.gemspec": parseGemSpecEntries,
 	}
 
 	return common.NewGenericCataloger(nil, globParsers, "ruby-gemspec-cataloger")

syft/distro/identify.go

@@ -1,6 +1,7 @@
 package distro
 
 import (
+	"io/ioutil"
 	"regexp"
 	"strings"
 
@@ -16,28 +17,28 @@ type parseEntry struct {
 	fn   parseFunc
 }
 
+var identityFiles = []parseEntry{
+	{
+		// most distros provide a link at this location
+		path: "/etc/os-release",
+		fn:   parseOsRelease,
+	},
+	{
+		// standard location for rhel & debian distros
+		path: "/usr/lib/os-release",
+		fn:   parseOsRelease,
+	},
+	{
+		// check for busybox (important to check this last since other distros contain the busybox binary)
+		path: "/bin/busybox",
+		fn:   parseBusyBox,
+	},
+}
+
 // Identify parses distro-specific files to determine distro metadata like version and release.
 func Identify(resolver source.Resolver) *Distro {
 	var distro *Distro
 
-	identityFiles := []parseEntry{
-		{
-			// most distros provide a link at this location
-			path: "/etc/os-release",
-			fn:   parseOsRelease,
-		},
-		{
-			// standard location for rhel & debian distros
-			path: "/usr/lib/os-release",
-			fn:   parseOsRelease,
-		},
-		{
-			// check for busybox (important to check this last since other distros contain the busybox binary)
-			path: "/bin/busybox",
-			fn:   parseBusyBox,
-		},
-	}
-
 identifyLoop:
 	for _, entry := range identityFiles {
 		locations, err := resolver.FilesByPath(entry.path)
@@ -52,19 +53,25 @@ identifyLoop:
 		}
 
 		for _, location := range locations {
-			content, err := resolver.FileContentsByLocation(location)
+			contentReader, err := resolver.FileContentsByLocation(location)
 
 			if err != nil {
 				log.Debugf("unable to get contents from %s: %s", entry.path, err)
 				continue
 			}
 
-			if content == "" {
+			content, err := ioutil.ReadAll(contentReader)
+			if err != nil {
+				log.Errorf("unable to read %q: %+v", location.RealPath, err)
+				break
+			}
+
+			if len(content) == 0 {
 				log.Debugf("no contents in file, skipping: %s", entry.path)
 				continue
 			}
 
-			if candidateDistro := entry.fn(content); candidateDistro != nil {
+			if candidateDistro := entry.fn(string(content)); candidateDistro != nil {
 				distro = candidateDistro
 				break identifyLoop
 			}

syft/presenter/cyclonedx/presenter_test.go

@@ -6,6 +6,8 @@ import (
 	"regexp"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/anchore/stereoscope/pkg/imagetest"
 
 	"github.com/anchore/go-testutils"
@@ -28,7 +30,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-1",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Metadata: pkg.DpkgMetadata{
 			Package:      "package1",
@@ -42,7 +44,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-2",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Licenses: []string{
 			"MIT",
@@ -94,12 +96,15 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 	img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
 	defer cleanup()
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type:    pkg.RpmPkg,
 		FoundBy: "the-cataloger-1",
@@ -109,7 +114,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 		Name:    "package2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type:    pkg.RpmPkg,
 		FoundBy: "the-cataloger-2",
@@ -120,7 +125,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 		PURL: "the-purl-2",
 	})
 
-	s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
+	s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
 	if err != nil {
 		t.Fatal(err)
 	}

syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:553b2601-a7b0-4461-bce9-fd56e2e2a4be">
+<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:544b9f08-b39d-4d0c-8723-32609567e8ed">
   <metadata>
-    <timestamp>2020-12-01T22:19:00-05:00</timestamp>
+    <timestamp>2020-12-28T13:56:29-05:00</timestamp>
     <tools>
       <tool>
         <vendor>anchore</vendor>

syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:4a54c180-9469-4513-a1cb-af3fe1e83738">
+<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:e1515688-2fd6-422a-acb2-97405a44cdfe">
   <metadata>
-    <timestamp>2020-12-01T22:19:00-05:00</timestamp>
+    <timestamp>2020-12-28T13:56:29-05:00</timestamp>
     <tools>
       <tool>
         <vendor>anchore</vendor>

syft/presenter/json/presenter_test.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/anchore/go-testutils"
 	"github.com/anchore/stereoscope/pkg/imagetest"
 	"github.com/anchore/syft/syft/distro"
@@ -34,7 +36,7 @@ func TestJsonDirsPresenter(t *testing.T) {
 		Type:    pkg.PythonPkg,
 		FoundBy: "the-cataloger-1",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Language:     pkg.Python,
 		MetadataType: pkg.PythonPackageMetadataType,
@@ -54,7 +56,7 @@ func TestJsonDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-2",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		MetadataType: pkg.DpkgMetadataType,
 		Metadata: pkg.DpkgMetadata{
@@ -106,12 +108,15 @@ func TestJsonImgsPresenter(t *testing.T) {
 	catalog := pkg.NewCatalog()
 	img := imagetest.GetGoldenFixtureImage(t, testImage)
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type:         pkg.PythonPkg,
 		FoundBy:      "the-cataloger-1",
@@ -131,7 +136,7 @@ func TestJsonImgsPresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type:         pkg.DebPkg,
 		FoundBy:      "the-cataloger-2",
@@ -149,7 +154,7 @@ func TestJsonImgsPresenter(t *testing.T) {
 	// this is a hard coded value that is not given by the fixture helper and must be provided manually
 	img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
 
-	s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
+	s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
 	var d *distro.Distro
 	pres := NewPresenter(catalog, s.Metadata, d)
 

syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden

@@ -71,7 +71,7 @@
     "stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7"
    ],
    "imageSize": 65,
-   "scope": "AllLayers",
+   "scope": "Squashed",
    "layers": [
     {
      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",

syft/presenter/table/presenter_test.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/go-test/deep"
 
 	"github.com/anchore/go-testutils"
@@ -25,12 +27,15 @@ func TestTablePresenter(t *testing.T) {
 	img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", testImage)
 	defer cleanup()
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type: pkg.DebPkg,
 	})
@@ -38,7 +43,7 @@ func TestTablePresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type: pkg.DebPkg,
 	})

syft/presenter/text/presenter_test.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/anchore/go-testutils"
 	"github.com/anchore/stereoscope/pkg/imagetest"
 	"github.com/anchore/syft/syft/pkg"
@@ -70,12 +72,15 @@ func TestTextImgPresenter(t *testing.T) {
 	img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
 	defer cleanup()
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		FoundBy: "dpkg",
 		Type:    pkg.DebPkg,
@@ -84,7 +89,7 @@ func TestTextImgPresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		FoundBy:  "dpkg",
 		Metadata: PackageInfo{Name: "package-2", Version: "1.0.2"},
@@ -97,7 +102,7 @@ func TestTextImgPresenter(t *testing.T) {
 		l.Metadata.Digest = "sha256:ad8ecdc058976c07e7e347cb89fa9ad86a294b5ceaae6d09713fb035f84115abf3c4a2388a4af3aa60f13b94f4c6846930bdf53"
 	}
 
-	s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
+	s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
 	if err != nil {
 		t.Fatal(err)
 	}

syft/source/all_layers_resolver.go

@@ -2,10 +2,15 @@ package source
 
 import (
 	"archive/tar"
+	"bytes"
 	"fmt"
+	"io"
+	"io/ioutil"
 
 	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/image"
+	"github.com/anchore/syft/internal/log"
 )
 
 var _ Resolver = (*AllLayersResolver)(nil)
@@ -32,6 +37,18 @@ func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) {
 	}, nil
 }
 
+// HasPath indicates if the given path exists in the underlying source.
+func (r *AllLayersResolver) HasPath(path string) bool {
+	p := file.Path(path)
+	for _, layerIdx := range r.layers {
+		tree := r.img.Layers[layerIdx].Tree
+		if tree.HasPath(p) {
+			return true
+		}
+	}
+	return false
+}
+
 func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) {
 	uniqueFiles := make([]file.Reference, 0)
 
@@ -70,19 +87,22 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
 	for _, path := range paths {
 		for idx, layerIdx := range r.layers {
 			tree := r.img.Layers[layerIdx].Tree
-			ref := tree.File(file.Path(path))
+			_, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks, filetree.DoNotFollowDeadBasenameLinks)
+			if err != nil {
+				return nil, err
+			}
 			if ref == nil {
 				// no file found, keep looking through layers
 				continue
 			}
 
 			// don't consider directories (special case: there is no path information for /)
-			if ref.Path == "/" {
+			if ref.RealPath == "/" {
 				continue
 			} else if r.img.FileCatalog.Exists(*ref) {
 				metadata, err := r.img.FileCatalog.Get(*ref)
 				if err != nil {
-					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
 				}
 				if metadata.Metadata.IsDir {
 					continue
@@ -94,7 +114,7 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
 				return nil, err
 			}
 			for _, result := range results {
-				uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
+				uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, result, r.img))
 			}
 		}
 	}
@@ -109,31 +129,31 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error)
 
 	for _, pattern := range patterns {
 		for idx, layerIdx := range r.layers {
-			refs, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern)
+			results, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern, filetree.DoNotFollowDeadBasenameLinks)
 			if err != nil {
 				return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
 			}
 
-			for _, ref := range refs {
+			for _, result := range results {
 				// don't consider directories (special case: there is no path information for /)
-				if ref.Path == "/" {
+				if result.RealPath == "/" {
 					continue
-				} else if r.img.FileCatalog.Exists(ref) {
-					metadata, err := r.img.FileCatalog.Get(ref)
+				} else if r.img.FileCatalog.Exists(result.Reference) {
+					metadata, err := r.img.FileCatalog.Get(result.Reference)
 					if err != nil {
-						return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+						return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err)
 					}
 					if metadata.Metadata.IsDir {
 						continue
 					}
 				}
 
-				results, err := r.fileByRef(ref, uniqueFileIDs, idx)
+				refResults, err := r.fileByRef(result.Reference, uniqueFileIDs, idx)
 				if err != nil {
 					return nil, err
 				}
-				for _, result := range results {
-					uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
+				for _, refResult := range refResults {
+					uniqueLocations = append(uniqueLocations, NewLocationFromImage(string(result.MatchPath), refResult, r.img))
 				}
 			}
 		}
@@ -150,37 +170,41 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *
 		return nil
 	}
 
-	relativeRef := entry.Source.SquashedTree.File(file.Path(path))
-	if relativeRef == nil {
+	exists, relativeRef, err := entry.Layer.SquashedTree.File(file.Path(path), filetree.FollowBasenameLinks)
+	if err != nil {
+		log.Errorf("failed to find path=%q in squash: %+w", path, err)
+		return nil
+	}
+	if !exists && relativeRef == nil {
 		return nil
 	}
 
-	relativeLocation := NewLocationFromImage(*relativeRef, r.img)
+	relativeLocation := NewLocationFromImage(path, *relativeRef, r.img)
 
 	return &relativeLocation
 }
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
 // file.Reference is a path relative to a particular layer.
-func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
+func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
 	return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
 }
 
 // FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
 // If the path does not exist an error is returned.
-func (r *AllLayersResolver) FileContentsByLocation(location Location) (string, error) {
+func (r *AllLayersResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
 	return r.img.FileContentsByRef(location.ref)
 }
 
-type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]string, error)
+type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]io.ReadCloser, error)
 
-func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]string, error) {
+func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]io.ReadCloser, error) {
 	var fileRefs = make([]file.Reference, len(locations))
-	var locationByRefs = make(map[file.Reference]Location)
-	var results = make(map[Location]string)
+	var locationByRefs = make(map[file.Reference][]Location)
+	var results = make(map[Location]io.ReadCloser)
 
 	for i, location := range locations {
-		locationByRefs[location.ref] = location
+		locationByRefs[location.ref] = append(locationByRefs[location.ref], location)
 		fileRefs[i] = location.ref
 	}
 
@@ -189,8 +213,26 @@ func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Lo
 		return nil, err
 	}
 
+	// TODO: this is not tested, we need a test case that covers a mapLocationRefs which has multiple Locations with the same reference in the request. The io.Reader should be copied.
 	for ref, content := range contentsByRef {
-		results[locationByRefs[ref]] = content
+		mappedLocations := locationByRefs[ref]
+		switch {
+		case len(mappedLocations) > 1:
+			// TODO: fixme... this can lead to lots of unexpected memory usage in unusual circumstances (cache is not leveraged for large files).
+			// stereoscope wont duplicate content requests if the caller asks for the same file multiple times... thats up to the caller
+			contentsBytes, err := ioutil.ReadAll(content)
+			if err != nil {
+				return nil, fmt.Errorf("unable to read ref=%+v :%w", ref, err)
+			}
+			for _, loc := range mappedLocations {
+				results[loc] = ioutil.NopCloser(bytes.NewReader(contentsBytes))
+			}
+
+		case len(mappedLocations) == 1:
+			results[locationByRefs[ref][0]] = content
+		default:
+			return nil, fmt.Errorf("unexpected ref-location count=%d for ref=%v", len(mappedLocations), ref)
+		}
 	}
 	return results, nil
 }

syft/source/all_layers_resolver_test.go

@@ -13,9 +13,10 @@ type resolution struct {
 
 func TestAllLayersResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name        string
-		linkPath    string
-		resolutions []resolution
+		name                 string
+		linkPath             string
+		resolutions          []resolution
+		forcePositiveHasPath bool
 	}{
 		{
 			name:     "link with previous data",
@@ -41,10 +42,6 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			name:     "link with overridden data",
 			linkPath: "/link-2",
 			resolutions: []resolution{
-				{
-					layer: 3,
-					path:  "/link-2",
-				},
 				{
 					layer: 4,
 					path:  "/file-2.txt",
@@ -70,19 +67,17 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			},
 		},
 		{
-			name:     "dead link",
-			linkPath: "/link-dead",
-			resolutions: []resolution{
-				{
-					layer: 8,
-					path:  "/link-dead",
-				},
-			},
+			name:                 "dead link",
+			linkPath:             "/link-dead",
+			resolutions:          []resolution{},
+			forcePositiveHasPath: true,
 		},
 		{
 			name:        "ignore directories",
 			linkPath:    "/bin",
 			resolutions: []resolution{},
+			// directories don't resolve BUT do exist
+			forcePositiveHasPath: true,
 		},
 	}
 	for _, c := range cases {
@@ -95,6 +90,17 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("could not create resolver: %+v", err)
 			}
 
+			hasPath := resolver.HasPath(c.linkPath)
+			if !c.forcePositiveHasPath {
+				if len(c.resolutions) > 0 && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if len(c.resolutions) == 0 && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.linkPath)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v", err)
@@ -107,8 +113,12 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			for idx, actual := range refs {
 				expected := c.resolutions[idx]
 
-				if actual.Path != expected.path {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				if string(actual.ref.RealPath) != expected.path {
+					t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
+				}
+
+				if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
+					t.Errorf("we should always prefer real paths over ones with links")
 				}
 
 				entry, err := img.FileCatalog.Get(actual.ref)
@@ -116,8 +126,8 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 					t.Fatalf("failed to get metadata: %+v", err)
 				}
 
-				if entry.Source.Metadata.Index != expected.layer {
-					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				if entry.Layer.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
 				}
 			}
 		})
@@ -132,7 +142,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 	}{
 		{
 			name: "link with previous data",
-			glob: "**ink-1",
+			glob: "**/*ink-1",
 			resolutions: []resolution{
 				{
 					layer: 1,
@@ -142,7 +152,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "link with in layer data",
-			glob: "**nk-within",
+			glob: "**/*nk-within",
 			resolutions: []resolution{
 				{
 					layer: 5,
@@ -152,12 +162,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "link with overridden data",
-			glob: "**ink-2",
+			glob: "**/*ink-2",
 			resolutions: []resolution{
-				{
-					layer: 3,
-					path:  "/link-2",
-				},
 				{
 					layer: 4,
 					path:  "/file-2.txt",
@@ -170,7 +176,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "indirect link (with overridden data)",
-			glob: "**nk-indirect",
+			glob: "**/*nk-indirect",
 			resolutions: []resolution{
 				{
 					layer: 4,
@@ -183,14 +189,9 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 			},
 		},
 		{
-			name: "dead link",
-			glob: "**k-dead",
-			resolutions: []resolution{
-				{
-					layer: 8,
-					path:  "/link-dead",
-				},
-			},
+			name:        "dead link",
+			glob:        "**/*k-dead",
+			resolutions: []resolution{},
 		},
 		{
 			name:        "ignore directories",
@@ -220,8 +221,12 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 			for idx, actual := range refs {
 				expected := c.resolutions[idx]
 
-				if actual.Path != expected.path {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				if string(actual.ref.RealPath) != expected.path {
+					t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
+				}
+
+				if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
+					t.Errorf("we should always prefer real paths over ones with links")
 				}
 
 				entry, err := img.FileCatalog.Get(actual.ref)
@@ -229,8 +234,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 					t.Fatalf("failed to get metadata: %+v", err)
 				}
 
-				if entry.Source.Metadata.Index != expected.layer {
-					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				if entry.Layer.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
 				}
 			}
 		})

syft/source/content_requester_test.go

@@ -1,6 +1,7 @@
 package source
 
 import (
+	"io/ioutil"
 	"testing"
 
 	"github.com/anchore/stereoscope/pkg/imagetest"
@@ -53,16 +54,20 @@ func TestContentRequester(t *testing.T) {
 			}
 
 			for _, entry := range data {
-				if expected, ok := test.expectedContents[entry.Location.Path]; ok {
-					for expected != entry.Contents {
-						t.Errorf("mismatched contents for %q", entry.Location.Path)
+				if expected, ok := test.expectedContents[entry.Location.RealPath]; ok {
+					actualBytes, err := ioutil.ReadAll(entry.Contents)
+					if err != nil {
+						t.Fatalf("could not read %q: %+v", entry.Location.RealPath, err)
+					}
+					for expected != string(actualBytes) {
+						t.Errorf("mismatched contents for %q", entry.Location.RealPath)
 						dmp := diffmatchpatch.New()
-						diffs := dmp.DiffMain(expected, entry.Contents, true)
+						diffs := dmp.DiffMain(expected, string(actualBytes), true)
 						t.Errorf("diff: %s", dmp.DiffPrettyText(diffs))
 					}
 					continue
 				}
-				t.Errorf("could not find %q", entry.Location.Path)
+				t.Errorf("could not find %q", entry.Location.RealPath)
 			}
 		})
 	}

syft/source/directory_resolver.go

@@ -2,13 +2,14 @@ package source
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
 
+	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/internal/log"
-	"github.com/bmatcuk/doublestar"
+	"github.com/bmatcuk/doublestar/v2"
 )
 
 var _ Resolver = (*DirectoryResolver)(nil)
@@ -18,27 +19,37 @@ type DirectoryResolver struct {
 	Path string
 }
 
+func (r DirectoryResolver) requestPath(userPath string) string {
+	fullPath := userPath
+	if filepath.IsAbs(fullPath) {
+		// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
+		fullPath = path.Join(r.Path, fullPath)
+	}
+	return fullPath
+}
+
+// HasPath indicates if the given path exists in the underlying source.
+func (r *DirectoryResolver) HasPath(userPath string) bool {
+	_, err := os.Stat(r.requestPath(userPath))
+	return !os.IsNotExist(err)
+}
+
 // Stringer to represent a directory path data source
-func (s DirectoryResolver) String() string {
-	return fmt.Sprintf("dir:%s", s.Path)
+func (r DirectoryResolver) String() string {
+	return fmt.Sprintf("dir:%s", r.Path)
 }
 
 // FilesByPath returns all file.References that match the given paths from the directory.
-func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
+func (r DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
 	var references = make([]Location, 0)
 
 	for _, userPath := range userPaths {
-		userStrPath := userPath
-
-		if filepath.IsAbs(userStrPath) {
-			// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
-			userStrPath = path.Join(s.Path, userStrPath)
-		}
+		userStrPath := r.requestPath(userPath)
 		fileMeta, err := os.Stat(userStrPath)
 		if os.IsNotExist(err) {
 			continue
 		} else if err != nil {
-			log.Errorf("path (%s) is not valid: %v", userStrPath, err)
+			log.Errorf("path (%r) is not valid: %v", userStrPath, err)
 		}
 
 		// don't consider directories
@@ -53,11 +64,11 @@ func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
 }
 
 // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
-func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
+func (r DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 	result := make([]Location, 0)
 
 	for _, pattern := range patterns {
-		pathPattern := path.Join(s.Path, pattern)
+		pathPattern := path.Join(r.Path, pattern)
 		pathMatches, err := doublestar.Glob(pathPattern)
 		if err != nil {
 			return nil, err
@@ -83,8 +94,8 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
 // This is helpful when attempting to find a file that is in the same layer or lower as another file. For the
 // DirectoryResolver, this is a simple path lookup.
-func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location {
-	paths, err := s.FilesByPath(path)
+func (r *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location {
+	paths, err := r.FilesByPath(path)
 	if err != nil {
 		return nil
 	}
@@ -96,35 +107,16 @@ func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Locatio
 }
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative a directory.
-func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
-	refContents := make(map[Location]string)
+func (r DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
+	refContents := make(map[Location]io.ReadCloser)
 	for _, location := range locations {
-		contents, err := fileContents(location.Path)
-
-		if err != nil {
-			return nil, fmt.Errorf("could not read contents of file: %s", location.Path)
-		}
-		refContents[location] = string(contents)
+		refContents[location] = file.NewDeferredReadCloser(location.RealPath)
 	}
 	return refContents, nil
 }
 
 // FileContentsByLocation fetches file contents for a single file reference relative to a directory.
 // If the path does not exist an error is returned.
-func (s DirectoryResolver) FileContentsByLocation(location Location) (string, error) {
-	contents, err := fileContents(location.Path)
-	if err != nil {
-		return "", fmt.Errorf("could not read contents of file: %s", location.Path)
-	}
-
-	return string(contents), nil
-}
-
-func fileContents(path string) ([]byte, error) {
-	contents, err := ioutil.ReadFile(path)
-
-	if err != nil {
-		return nil, err
-	}
-	return contents, nil
+func (r DirectoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
+	return file.NewDeferredReadCloser(location.RealPath), nil
 }

syft/source/directory_resolver_test.go

@@ -6,11 +6,12 @@ import (
 
 func TestDirectoryResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name     string
-		root     string
-		input    string
-		expected string
-		refCount int
+		name                 string
+		root                 string
+		input                string
+		expected             string
+		refCount             int
+		forcePositiveHasPath bool
 	}{
 		{
 			name:     "finds a file (relative)",
@@ -47,15 +48,28 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
 			refCount: 1,
 		},
 		{
-			name:     "directories ignored",
-			root:     "./test-fixtures/",
-			input:    "/image-symlinks",
-			refCount: 0,
+			name:                 "directories ignored",
+			root:                 "./test-fixtures/",
+			input:                "/image-symlinks",
+			refCount:             0,
+			forcePositiveHasPath: true,
 		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			resolver := DirectoryResolver{c.root}
+
+			hasPath := resolver.HasPath(c.input)
+			if !c.forcePositiveHasPath {
+				if c.refCount != 0 && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if c.refCount == 0 && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.input)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v, %+v", err, refs)
@@ -66,8 +80,8 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
 			}
 
 			for _, actual := range refs {
-				if actual.Path != c.expected {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.expected)
+				if actual.RealPath != c.expected {
+					t.Errorf("bad resolve path: '%s'!='%s'", actual.RealPath, c.expected)
 				}
 			}
 		})
@@ -171,8 +185,9 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 2 {
-			t.Errorf("unexpected number of refs: %d != 2", len(refs))
+		expected := 2
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})
@@ -187,8 +202,9 @@ func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 4 {
-			t.Errorf("unexpected number of refs: %d != 4", len(refs))
+		expected := 6
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})
@@ -202,8 +218,9 @@ func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 1 {
-			t.Errorf("unexpected number of refs: %d != 1", len(refs))
+		expected := 1
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})

syft/source/file_data.go

@@ -1,6 +1,8 @@
 package source
 
+import "io"
+
 type FileData struct {
 	Location Location
-	Contents string
+	Contents io.ReadCloser
 }

syft/source/image_squash_resolver.go

@@ -2,8 +2,10 @@ package source
 
 import (
 	"fmt"
+	"io"
 
 	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/image"
 )
 
@@ -22,6 +24,11 @@ func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) {
 	return &ImageSquashResolver{img: img}, nil
 }
 
+// HasPath indicates if the given path exists in the underlying source.
+func (r *ImageSquashResolver) HasPath(path string) bool {
+	return r.img.SquashedTree().HasPath(file.Path(path))
+}
+
 // FilesByPath returns all file.References that match the given paths within the squashed representation of the image.
 func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 	uniqueFileIDs := file.NewFileReferenceSet()
@@ -29,19 +36,22 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 
 	for _, path := range paths {
 		tree := r.img.SquashedTree()
-		ref := tree.File(file.Path(path))
+		_, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks)
+		if err != nil {
+			return nil, err
+		}
 		if ref == nil {
 			// no file found, keep looking through layers
 			continue
 		}
 
 		// don't consider directories (special case: there is no path information for /)
-		if ref.Path == "/" {
+		if ref.RealPath == "/" {
 			continue
 		} else if r.img.FileCatalog.Exists(*ref) {
 			metadata, err := r.img.FileCatalog.Get(*ref)
 			if err != nil {
-				return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+				return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
 			}
 			if metadata.Metadata.IsDir {
 				continue
@@ -56,7 +66,7 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 
 		if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
 			uniqueFileIDs.Add(*resolvedRef)
-			uniqueLocations = append(uniqueLocations, NewLocationFromImage(*resolvedRef, r.img))
+			uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, *resolvedRef, r.img))
 		}
 	}
 
@@ -69,28 +79,28 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error
 	uniqueLocations := make([]Location, 0)
 
 	for _, pattern := range patterns {
-		refs, err := r.img.SquashedTree().FilesByGlob(pattern)
+		results, err := r.img.SquashedTree().FilesByGlob(pattern)
 		if err != nil {
 			return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
 		}
 
-		for _, ref := range refs {
+		for _, result := range results {
 			// don't consider directories (special case: there is no path information for /)
-			if ref.Path == "/" {
+			if result.MatchPath == "/" {
 				continue
-			} else if r.img.FileCatalog.Exists(ref) {
-				metadata, err := r.img.FileCatalog.Get(ref)
+			} else if r.img.FileCatalog.Exists(result.Reference) {
+				metadata, err := r.img.FileCatalog.Get(result.Reference)
 				if err != nil {
-					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err)
 				}
 				if metadata.Metadata.IsDir {
 					continue
 				}
 			}
 
-			resolvedLocations, err := r.FilesByPath(string(ref.Path))
+			resolvedLocations, err := r.FilesByPath(string(result.MatchPath))
 			if err != nil {
-				return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err)
+				return nil, fmt.Errorf("failed to find files by path (result=%+v): %w", result, err)
 			}
 			for _, resolvedLocation := range resolvedLocations {
 				if !uniqueFileIDs.Contains(resolvedLocation.ref) {
@@ -121,12 +131,12 @@ func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Locat
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
 // file.Reference is a path relative to a particular layer, in this case only from the squashed representation.
-func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
+func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
 	return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
 }
 
 // FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
 // If the path does not exist an error is returned.
-func (r *ImageSquashResolver) FileContentsByLocation(location Location) (string, error) {
+func (r *ImageSquashResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
 	return r.img.FileContentsByRef(location.ref)
 }

syft/source/image_squash_resolver_test.go

@@ -8,10 +8,11 @@ import (
 
 func TestImageSquashResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name         string
-		linkPath     string
-		resolveLayer uint
-		resolvePath  string
+		name                 string
+		linkPath             string
+		resolveLayer         uint
+		resolvePath          string
+		forcePositiveHasPath bool
 	}{
 		{
 			name:         "link with previous data",
@@ -41,12 +42,22 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 			name:         "dead link",
 			linkPath:     "/link-dead",
 			resolveLayer: 8,
-			resolvePath:  "/link-dead",
+			resolvePath:  "",
+			// the path should exist, even if the link is dead
+			forcePositiveHasPath: true,
 		},
 		{
 			name:        "ignore directories",
 			linkPath:    "/bin",
 			resolvePath: "",
+			// the path should exist, even if we ignore it
+			forcePositiveHasPath: true,
+		},
+		{
+			name:         "parent is a link (with overridden data)",
+			linkPath:     "/parent-link/file-4.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
 		},
 	}
 	for _, c := range cases {
@@ -59,6 +70,17 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("could not create resolver: %+v", err)
 			}
 
+			hasPath := resolver.HasPath(c.linkPath)
+			if !c.forcePositiveHasPath {
+				if c.resolvePath != "" && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if c.resolvePath == "" && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.linkPath)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v", err)
@@ -80,8 +102,12 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 
 			actual := refs[0]
 
-			if actual.Path != c.resolvePath {
-				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			if string(actual.ref.RealPath) != c.resolvePath {
+				t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
+			}
+
+			if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
+				t.Errorf("we should always prefer real paths over ones with links")
 			}
 
 			entry, err := img.FileCatalog.Get(actual.ref)
@@ -89,8 +115,8 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("failed to get metadata: %+v", err)
 			}
 
-			if entry.Source.Metadata.Index != c.resolveLayer {
-				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			if entry.Layer.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
 			}
 		})
 	}
@@ -105,39 +131,51 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 	}{
 		{
 			name:         "link with previous data",
-			glob:         "**link-1",
+			glob:         "**/link-1",
 			resolveLayer: 1,
 			resolvePath:  "/file-1.txt",
 		},
 		{
 			name:         "link with in layer data",
-			glob:         "**link-within",
+			glob:         "**/link-within",
 			resolveLayer: 5,
 			resolvePath:  "/file-3.txt",
 		},
 		{
 			name:         "link with overridden data",
-			glob:         "**link-2",
+			glob:         "**/link-2",
 			resolveLayer: 7,
 			resolvePath:  "/file-2.txt",
 		},
 		{
 			name:         "indirect link (with overridden data)",
-			glob:         "**link-indirect",
+			glob:         "**/link-indirect",
 			resolveLayer: 7,
 			resolvePath:  "/file-2.txt",
 		},
 		{
-			name:         "dead link",
-			glob:         "**link-dead",
-			resolveLayer: 8,
-			resolvePath:  "/link-dead",
+			name: "dead link",
+			glob: "**/link-dead",
+			// dead links are dead! they shouldn't match on globs
+			resolvePath: "",
 		},
 		{
 			name:        "ignore directories",
 			glob:        "**/bin",
 			resolvePath: "",
 		},
+		{
+			name:         "parent without link",
+			glob:         "**/parent/*.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
+		},
+		{
+			name:         "parent is a link (override)",
+			glob:         "**/parent-link/file-4.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
@@ -170,8 +208,12 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 
 			actual := refs[0]
 
-			if actual.Path != c.resolvePath {
-				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			if string(actual.ref.RealPath) != c.resolvePath {
+				t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
+			}
+
+			if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
+				t.Errorf("we should always prefer real paths over ones with links")
 			}
 
 			entry, err := img.FileCatalog.Get(actual.ref)
@@ -179,8 +221,8 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 				t.Fatalf("failed to get metadata: %+v", err)
 			}
 
-			if entry.Source.Metadata.Index != c.resolveLayer {
-				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			if entry.Layer.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
 			}
 		})
 	}

syft/source/location.go

@@ -7,9 +7,11 @@ import (
 	"github.com/anchore/stereoscope/pkg/image"
 )
 
-// Location represents a path relative to a particular filesystem.
+// Location represents a path relative to a particular filesystem resolved to a specific file.Reference. This struct is used as a key
+// in content fetching to uniquely identify a file relative to a request (the VirtualPath).
 type Location struct {
-	Path         string         `json:"path"`              // The string path of the location (e.g. /etc/hosts)
+	RealPath     string         `json:"path"`              // The path where all path ancestors have no hardlinks / symlinks
+	VirtualPath  string         `json:"-"`                 // The path to the file which may or may not have hardlinks / symlinks
 	FileSystemID string         `json:"layerID,omitempty"` // An ID representing the filesystem. For container images this is a layer digest, directories or root filesystem this is blank.
 	ref          file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
 }
@@ -17,24 +19,26 @@ type Location struct {
 // NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference.
 func NewLocation(path string) Location {
 	return Location{
-		Path: path,
+		RealPath: path,
 	}
 }
 
 // NewLocationFromImage creates a new Location representing the given path (extracted from the ref) relative to the given image.
-func NewLocationFromImage(ref file.Reference, img *image.Image) Location {
+func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Image) Location {
 	entry, err := img.FileCatalog.Get(ref)
 	if err != nil {
 		log.Warnf("unable to find file catalog entry for ref=%+v", ref)
 		return Location{
-			Path: string(ref.Path),
-			ref:  ref,
+			VirtualPath: virtualPath,
+			RealPath:    string(ref.RealPath),
+			ref:         ref,
 		}
 	}
 
 	return Location{
-		Path:         string(ref.Path),
-		FileSystemID: entry.Source.Metadata.Digest,
+		VirtualPath:  virtualPath,
+		RealPath:     string(ref.RealPath),
+		FileSystemID: entry.Layer.Metadata.Digest,
 		ref:          ref,
 	}
 }

syft/source/mock_resolver.go

@@ -0,0 +1,112 @@
+package source
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/anchore/syft/internal/file"
+)
+
+var _ Resolver = (*MockResolver)(nil)
+
+// MockResolver implements the Resolver interface and is intended for use *only in test code*.
+// It provides an implementation that can resolve local filesystem paths using only a provided discrete list of file
+// paths, which are typically paths to test fixtures.
+type MockResolver struct {
+	Locations []Location
+}
+
+// NewMockResolverForPaths creates a new MockResolver, where the only resolvable
+// files are those specified by the supplied paths.
+func NewMockResolverForPaths(paths ...string) *MockResolver {
+	var locations []Location
+	for _, p := range paths {
+		locations = append(locations, NewLocation(p))
+	}
+
+	return &MockResolver{Locations: locations}
+}
+
+// HasPath indicates if the given path exists in the underlying source.
+func (r MockResolver) HasPath(path string) bool {
+	for _, l := range r.Locations {
+		if l.RealPath == path {
+			return true
+		}
+	}
+	return false
+}
+
+// String returns the string representation of the MockResolver.
+func (r MockResolver) String() string {
+	return fmt.Sprintf("mock:(%s,...)", r.Locations[0].RealPath)
+}
+
+// FileContentsByLocation fetches file contents for a single location. If the
+// path does not exist, an error is returned.
+func (r MockResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
+	for _, l := range r.Locations {
+		if l == location {
+			return os.Open(location.RealPath)
+		}
+	}
+
+	return nil, fmt.Errorf("no file for location: %v", location)
+}
+
+// MultipleFileContentsByLocation returns the file contents for all specified Locations.
+func (r MockResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
+	results := make(map[Location]io.ReadCloser)
+	for _, l := range locations {
+		contents, err := r.FileContentsByLocation(l)
+		if err != nil {
+			return nil, err
+		}
+		results[l] = contents
+	}
+
+	return results, nil
+}
+
+// FilesByPath returns all Locations that match the given paths.
+func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) {
+	var results []Location
+	for _, p := range paths {
+		for _, location := range r.Locations {
+			if p == location.RealPath {
+				results = append(results, NewLocation(p))
+			}
+		}
+	}
+
+	return results, nil
+}
+
+// FilesByGlob returns all Locations that match the given path glob pattern.
+func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) {
+	var results []Location
+	for _, pattern := range patterns {
+		for _, location := range r.Locations {
+			if file.GlobMatch(pattern, location.RealPath) {
+				results = append(results, location)
+			}
+		}
+	}
+
+	return results, nil
+}
+
+// RelativeFileByPath returns a single Location for the given path.
+func (r MockResolver) RelativeFileByPath(_ Location, path string) *Location {
+	paths, err := r.FilesByPath(path)
+	if err != nil {
+		return nil
+	}
+
+	if len(paths) < 1 {
+		return nil
+	}
+
+	return &paths[0]
+}

syft/source/resolver.go

@@ -2,6 +2,7 @@ package source
 
 import (
 	"fmt"
+	"io"
 
 	"github.com/anchore/stereoscope/pkg/image"
 )
@@ -14,13 +15,15 @@ type Resolver interface {
 
 // ContentResolver knows how to get file content for given file.References
 type ContentResolver interface {
-	FileContentsByLocation(Location) (string, error)
-	MultipleFileContentsByLocation([]Location) (map[Location]string, error)
-	// TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering).
+	FileContentsByLocation(Location) (io.ReadCloser, error)
+	// TODO: it is possible to be given duplicate locations that will be overridden in the map (key), a subtle problem that coule easily be misued.
+	MultipleFileContentsByLocation([]Location) (map[Location]io.ReadCloser, error)
 }
 
-// FileResolver knows how to get file.References for given string paths and globs
+// FileResolver knows how to get a Location for given string paths and globs
 type FileResolver interface {
+	// HasPath indicates if the given path exists in the underlying source.
+	HasPath(path string) bool
 	// FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches)
 	FilesByPath(paths ...string) ([]Location, error)
 	// FilesByGlob fetches a set of file references which the given glob matches

syft/source/source_test.go

@@ -1,6 +1,7 @@
 package source
 
 import (
+	"io/ioutil"
 	"os"
 	"testing"
 
@@ -137,9 +138,14 @@ func TestMultipleFileContentsByLocation(t *testing.T) {
 			location := locations[0]
 
 			contents, err := p.Resolver.MultipleFileContentsByLocation([]Location{location})
-			content := contents[location]
+			contentReader := contents[location]
 
-			if content != test.expected {
+			content, err := ioutil.ReadAll(contentReader)
+			if err != nil {
+				t.Fatalf("cannot read contents: %+v", err)
+			}
+
+			if string(content) != test.expected {
 				t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected)
 			}
 

syft/source/test-fixtures/image-symlinks/Dockerfile

@@ -21,4 +21,12 @@ RUN ln -s ./link-2 link-indirect
 ADD new-file-2.txt file-2.txt
 
 # LAYER 8: dead link
-RUN ln -s ./i-dont-exist.txt link-dead
+RUN ln -s ./i-dont-exist.txt link-dead
+
+# LAYER 9: add the parent dir
+ADD parent /parent
+# LAYER 10: parent is a symlink
+RUN ln -s /parent parent-link
+
+# LAYER 11: parent is a symlink and the child target is overridden
+COPY new-file-4.txt /parent-link/file-4.txt

syft/source/test-fixtures/image-symlinks/new-file-4.txt

@@ -0,0 +1 @@
+override file 4!

syft/source/test-fixtures/image-symlinks/parent/file-4.txt

@@ -0,0 +1 @@
+the 4th file!

test/integration/distro_test.go

@@ -16,7 +16,7 @@ func TestDistroImage(t *testing.T) {
 	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
 	defer cleanup()
 
-	_, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
+	_, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
 	if err != nil {
 		t.Fatalf("failed to catalog image: %+v", err)
 	}

test/integration/document_import_test.go

@@ -31,7 +31,7 @@ func TestCatalogFromJSON(t *testing.T) {
 			tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture)
 			defer cleanup()
 
-			expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
+			expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
 			if err != nil {
 				t.Fatalf("failed to catalog image: %+v", err)
 			}
@@ -81,6 +81,12 @@ func TestCatalogFromJSON(t *testing.T) {
 					e.Metadata = metadata
 				}
 
+				// ignore the virtual path on the location for now
+				for l := range a.Locations {
+					a.Locations[l].VirtualPath = ""
+					e.Locations[l].VirtualPath = ""
+				}
+
 				for _, d := range deep.Equal(a, e) {
 					// ignore errors for empty collections vs nil for select fields
 					// TODO: this is brittle, but not dangerously so. We should still find a better way to do this.

test/integration/json_schema_test.go

@@ -60,7 +60,7 @@ func TestJsonSchemaImg(t *testing.T) {
 	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
 	defer cleanup()
 
-	src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
+	src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
 	if err != nil {
 		t.Fatalf("failed to catalog image: %+v", err)
 	}
@@ -87,7 +87,7 @@ func TestJsonSchemaImg(t *testing.T) {
 }
 
 func TestJsonSchemaDirs(t *testing.T) {
-	src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope)
+	src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope)
 	if err != nil {
 		t.Errorf("unable to create source from dir: %+v", err)
 	}

test/integration/pkg_cases_test.go

@@ -15,7 +15,10 @@ var imageOnlyTestCases = []testCase{
 		pkgType:     pkg.GemPkg,
 		pkgLanguage: pkg.Ruby,
 		pkgInfo: map[string]string{
+			// specifications in the root specification directory
 			"bundler": "2.1.4",
+			// specifications in named directories
+			"unbundler": "3.1.4",
 		},
 	},
 	{

test/integration/pkg_coverage_test.go

@@ -18,7 +18,7 @@ func TestPkgCoverageImage(t *testing.T) {
 	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
 	defer cleanup()
 
-	_, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
+	_, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
 	if err != nil {
 		t.Fatalf("failed to catalog image: %+v", err)
 	}
@@ -100,7 +100,7 @@ func TestPkgCoverageImage(t *testing.T) {
 }
 
 func TestPkgCoverageDirectory(t *testing.T) {
-	_, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope)
+	_, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope)
 
 	if err != nil {
 		t.Errorf("unable to create source from dir: %+v", err)

test/integration/regression_test.go

@@ -24,7 +24,7 @@ func TestRegression212ApkBufferSize(t *testing.T) {
 		t.Fatalf("failed to catalog image: %+v", err)
 	}
 
-	expectedPkgs := 57
+	expectedPkgs := 58
 	actualPkgs := 0
 	for range catalog.Enumerate(pkg.ApkPkg) {
 		actualPkgs += 1

test/integration/test-fixtures/image-large-apk-data/Dockerfile

@@ -1,2 +1,5 @@
-FROM alpine:latest
-RUN apk add tzdata vim alpine-sdk
+FROM alpine@sha256:d9a7354e3845ea8466bb00b22224d9116b183e594527fb5b6c3d30bc01a20378
+RUN apk add --no-cache \
+            tzdata=2020f-r0 \
+            vim=8.2.2320-r0 \
+            alpine-sdk=1.0-r0

test/integration/test-fixtures/image-pkg-coverage/ruby/specifications/default/unbundler.gemspec

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# -*- encoding: utf-8 -*-                                                                                                                                         
+# stub: unbundler 2.1.4 ruby lib                                                                                                                                    
+                                                                                                                                                                  
+Gem::Specification.new do |s|                                                                                                                                     
+    s.name = "unbundler".freeze
+    s.version = "3.1.4"
+                                                                                                                                                                    
+    s.required_rubygems_version = Gem::Requirement.new(">= 2.5.2".freeze) if s.respond_to? :required_rubygems_version=                                              
+    s.require_paths = ["lib".freeze]                                                                                                                                
+    s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".f
+    s.bindir = "exe".freeze                                                                                                                                         
+    s.date = "2020-01-05"                                                                                                                                           
+    s.description = "Bundler manages an application's dependencies through its entire life, across many machines, systematically and repeatably".freeze             
+    s.email = ["team@unbundler.io".freeze]                                                                                                                            
+    s.executables = ["unbundle".freeze, "unbundler".freeze]                                                                                                             
+    s.files = ["exe/unbundle".freeze, "exe/unbundler".freeze]                                                                                                           
+    s.homepage = "https://unbundler.io".freeze                                                                                                                        
+    s.licenses = ["MIT".freeze]                                                                                                                                     
+    s.required_ruby_version = Gem::Requirement.new(">= 2.3.0".freeze)                                                                                               
+    s.rubygems_version = "3.1.2".freeze                                                                                                                             
+    s.summary = "The best way to manage your application's dependencies".freeze                                                                                     
+                                                                                                                                                                    
+    s.installed_by_version = "3.1.2" if s.respond_to? :installed_by_version                                                                                         
+  end