Merge pull request #308 from anchore/prefer-real-paths-on-glob-results

Prefer real paths over those with links for glob results

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
-	github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255
+	github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5
 	github.com/antihax/optional v1.0.0
 	github.com/bmatcuk/doublestar/v2 v2.0.4
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible

go.sum

@@ -136,6 +136,10 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
 github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255 h1:Ng7BDr9PQTCztANogjfEdEjjWUylhlPyZPhtarIGo00=
 github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
+github.com/anchore/stereoscope v0.0.0-20210105000809-428eda0b2ec6 h1:JWpsV/8x1fuCYjJmNjT43cVFblLTpO/ISDnePukiTNw=
+github.com/anchore/stereoscope v0.0.0-20210105000809-428eda0b2ec6/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
+github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5 h1:NGRfS6BZKElgiMbqdoH9iQn+6oxT7CJdZYrqgwvGkWY=
+github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=

internal/anchore/import_package_sbom_test.go

@@ -61,7 +61,7 @@ func TestPackageSbomToModel(t *testing.T) {
 		FoundBy: "foundBy",
 		Locations: []source.Location{
 			{
-				Path:         "path",
+				RealPath:     "path",
 				FileSystemID: "layerID",
 			},
 		},
@@ -139,7 +139,7 @@ func TestPackageSbomImport(t *testing.T) {
 		FoundBy: "foundBy",
 		Locations: []source.Location{
 			{
-				Path:         "path",
+				RealPath:     "path",
 				FileSystemID: "layerID",
 			},
 		},

syft/cataloger/common/generic_cataloger.go

@@ -101,7 +101,7 @@ func (c *GenericCataloger) catalog(contents map[source.Location]io.ReadCloser) (
 			continue
 		}
 
-		entries, err := parser(location.Path, content)
+		entries, err := parser(location.RealPath, content)
 		if err != nil {
 			// TODO: should we fail? or only log?
 			log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)

syft/cataloger/common/generic_cataloger_test.go

@@ -21,6 +21,10 @@ func newTestResolver() *testResolverMock {
 	}
 }
 
+func (r testResolverMock) HasPath(path string) bool {
+	panic("not implemented")
+}
+
 func (r *testResolverMock) FileContentsByLocation(_ source.Location) (io.ReadCloser, error) {
 	return nil, fmt.Errorf("not implemented")
 }
@@ -97,7 +101,7 @@ func TestGenericCataloger(t *testing.T) {
 
 	for _, p := range actualPkgs {
 		ref := p.Locations[0]
-		exP, ok := expectedPkgs[ref.Path]
+		exP, ok := expectedPkgs[ref.RealPath]
 		if !ok {
 			t.Errorf("missing expected pkg: ref=%+v", ref)
 			continue

syft/cataloger/deb/cataloger.go

@@ -48,7 +48,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 
 		pkgs, err = parseDpkgStatus(dbContents)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err)
+			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err)
 		}
 
 		md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs)
@@ -56,7 +56,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 			return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err)
 		}
 
-		copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
+		copyrightContentsByName, copyrightLocationByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
 		if err != nil {
 			return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err)
 		}
@@ -90,7 +90,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 				p.Licenses = parseLicensesFromCopyright(copyrightReader)
 
 				// keep a record of the file where this was discovered
-				if ref, ok := copyrightRefsByName[p.Name]; ok {
+				if ref, ok := copyrightLocationByName[p.Name]; ok {
 					p.Locations = append(p.Locations, ref)
 				}
 			}
@@ -104,20 +104,17 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
 
 	var md5FileMatches []source.Location
 	var nameByRef = make(map[source.Location]string)
-	parentPath := filepath.Dir(dbLocation.Path)
+	parentPath := filepath.Dir(dbLocation.RealPath)
 
 	for _, p := range pkgs {
 		// look for /var/lib/dpkg/info/NAME:ARCH.md5sums
 		name := md5Key(p)
-		md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
-		md5SumLocation := resolver.RelativeFileByPath(dbLocation, md5sumPath)
+		md5SumLocation := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", name+md5sumsExt))
 
 		if md5SumLocation == nil {
 			// the most specific key did not work, fallback to just the name
 			// look for /var/lib/dpkg/info/NAME.md5sums
-			name := p.Name
-			md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
-			md5SumLocation = resolver.RelativeFileByPath(dbLocation, md5sumPath)
+			md5SumLocation = resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", p.Name+md5sumsExt))
 		}
 		// we should have at least one reference
 		if md5SumLocation != nil {
@@ -134,14 +131,14 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
 
 	// organize content results and refs by a combination of name and architecture
 	var contentsByName = make(map[string]io.Reader)
-	var refsByName = make(map[string]source.Location)
+	var locationByName = make(map[string]source.Location)
 	for location, contents := range md5ContentsByLocation {
 		name := nameByRef[location]
 		contentsByName[name] = contents
-		refsByName[name] = location
+		locationByName[name] = location
 	}
 
-	return contentsByName, refsByName, nil
+	return contentsByName, locationByName, nil
 }
 
 func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {

syft/cataloger/deb/cataloger_test.go

@@ -79,7 +79,7 @@ func TestDpkgCataloger(t *testing.T) {
 				// we will test the sources separately
 				var sourcesList = make([]string, len(a.Locations))
 				for i, s := range a.Locations {
-					sourcesList[i] = s.Path
+					sourcesList[i] = s.RealPath
 				}
 				a.Locations = nil
 

syft/cataloger/python/package_cataloger.go

@@ -38,7 +38,7 @@ func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, err
 	for _, entry := range entries {
 		p, err := c.catalogEggOrWheel(entry)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog python package=%+v: %w", entry.Metadata.Location.Path, err)
+			return nil, fmt.Errorf("unable to catalog python package=%+v: %w", entry.Metadata.Location.RealPath, err)
 		}
 		if p != nil {
 			packages = append(packages, *p)
@@ -118,7 +118,7 @@ func (c *PackageCataloger) catalogEggOrWheel(entry *packageEntry) (*pkg.Package,
 func (c *PackageCataloger) assembleEggOrWheelMetadata(entry *packageEntry) (*pkg.PythonPackageMetadata, []source.Location, error) {
 	var sources = []source.Location{entry.Metadata.Location}
 
-	metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.Path, entry.Metadata.Contents)
+	metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.RealPath, entry.Metadata.Contents)
 	if err != nil {
 		return nil, nil, err
 	}

syft/cataloger/python/package_entry.go

@@ -19,11 +19,11 @@ func newPackageEntry(resolver source.FileResolver, metadataLocation source.Locat
 	// to reconcile the RECORD path to the same layer (or a lower layer). The same is true with the top_level.txt file.
 
 	// lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure)
-	recordPath := filepath.Join(filepath.Dir(metadataLocation.Path), "RECORD")
+	recordPath := filepath.Join(filepath.Dir(metadataLocation.RealPath), "RECORD")
 	recordLocation := resolver.RelativeFileByPath(metadataLocation, recordPath)
 
 	// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
-	parentDir := filepath.Dir(metadataLocation.Path)
+	parentDir := filepath.Dir(metadataLocation.RealPath)
 	topLevelPath := filepath.Join(parentDir, "top_level.txt")
 	topLevelLocation := resolver.RelativeFileByPath(metadataLocation, topLevelPath)
 

syft/cataloger/rpmdb/cataloger.go

@@ -43,7 +43,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 
 		pkgs, err = parseRpmDB(resolver, location, dbContentReader)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.Path, err)
+			return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err)
 		}
 	}
 	return pkgs, nil

syft/cataloger/rpmdb/parse_rpmdb.go

@@ -45,11 +45,6 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 	allPkgs := make([]pkg.Package, 0)
 
 	for _, entry := range pkgList {
-		records, err := extractRpmdbFileRecords(resolver, entry)
-		if err != nil {
-			return nil, err
-		}
-
 		p := pkg.Package{
 			Name:         entry.Name,
 			Version:      fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does, instead of fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch)
@@ -67,7 +62,7 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 				Vendor:    entry.Vendor,
 				License:   entry.License,
 				Size:      entry.Size,
-				Files:     records,
+				Files:     extractRpmdbFileRecords(resolver, entry),
 			},
 		}
 
@@ -77,25 +72,19 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 	return allPkgs, nil
 }
 
-func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) {
+func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) []pkg.RpmdbFileRecord {
 	var records = make([]pkg.RpmdbFileRecord, 0)
 
 	for _, record := range entry.Files {
-		refs, err := resolver.FilesByPath(record.Path)
-		if err != nil {
-			return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err)
-		}
 		//only persist RPMDB file records which exist in the image/directory, otherwise ignore them
-		if len(refs) == 0 {
-			continue
+		if resolver.HasPath(record.Path) {
+			records = append(records, pkg.RpmdbFileRecord{
+				Path:   record.Path,
+				Mode:   pkg.RpmdbFileMode(record.Mode),
+				Size:   int(record.Size),
+				SHA256: record.SHA256,
+			})
 		}
-
-		records = append(records, pkg.RpmdbFileRecord{
-			Path:   record.Path,
-			Mode:   pkg.RpmdbFileMode(record.Mode),
-			Size:   int(record.Size),
-			SHA256: record.SHA256,
-		})
 	}
-	return records, nil
+	return records
 }

syft/cataloger/rpmdb/parse_rpmdb_test.go

@@ -21,6 +21,10 @@ func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock {
 	}
 }
 
+func (r rpmdbTestFileResolverMock) HasPath(path string) bool {
+	return !r.ignorePaths
+}
+
 func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...string) ([]source.Location, error) {
 	if r.ignorePaths {
 		// act as if no paths exist

syft/cataloger/ruby/catalogers.go

@@ -19,7 +19,7 @@ func NewGemFileLockCataloger() *common.GenericCataloger {
 // NewGemSpecCataloger returns a new Bundler cataloger object tailored for detecting installations of gems (e.g. Gemspec).
 func NewGemSpecCataloger() *common.GenericCataloger {
 	globParsers := map[string]common.ParserFn{
-		"**/specifications/*.gemspec": parseGemSpecEntries,
+		"**/specifications/**/*.gemspec": parseGemSpecEntries,
 	}
 
 	return common.NewGenericCataloger(nil, globParsers, "ruby-gemspec-cataloger")

syft/distro/identify.go

@@ -62,7 +62,7 @@ identifyLoop:
 
 			content, err := ioutil.ReadAll(contentReader)
 			if err != nil {
-				log.Errorf("unable to read %q: %+v", location.Path, err)
+				log.Errorf("unable to read %q: %+v", location.RealPath, err)
 				break
 			}
 

syft/presenter/cyclonedx/presenter_test.go

@@ -30,7 +30,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-1",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Metadata: pkg.DpkgMetadata{
 			Package:      "package1",
@@ -44,7 +44,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-2",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Licenses: []string{
 			"MIT",
@@ -104,7 +104,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 		Name:    "package1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*ref1, img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type:    pkg.RpmPkg,
 		FoundBy: "the-cataloger-1",
@@ -114,7 +114,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 		Name:    "package2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*ref2, img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type:    pkg.RpmPkg,
 		FoundBy: "the-cataloger-2",

syft/presenter/json/presenter_test.go

@@ -36,7 +36,7 @@ func TestJsonDirsPresenter(t *testing.T) {
 		Type:    pkg.PythonPkg,
 		FoundBy: "the-cataloger-1",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Language:     pkg.Python,
 		MetadataType: pkg.PythonPackageMetadataType,
@@ -56,7 +56,7 @@ func TestJsonDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-2",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		MetadataType: pkg.DpkgMetadataType,
 		Metadata: pkg.DpkgMetadata{
@@ -116,7 +116,7 @@ func TestJsonImgsPresenter(t *testing.T) {
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*ref1, img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type:         pkg.PythonPkg,
 		FoundBy:      "the-cataloger-1",
@@ -136,7 +136,7 @@ func TestJsonImgsPresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*ref2, img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type:         pkg.DebPkg,
 		FoundBy:      "the-cataloger-2",

syft/presenter/table/presenter_test.go

@@ -35,7 +35,7 @@ func TestTablePresenter(t *testing.T) {
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*ref1, img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type: pkg.DebPkg,
 	})
@@ -43,7 +43,7 @@ func TestTablePresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*ref2, img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type: pkg.DebPkg,
 	})

syft/presenter/text/presenter_test.go

@@ -80,7 +80,7 @@ func TestTextImgPresenter(t *testing.T) {
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*ref1, img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		FoundBy: "dpkg",
 		Type:    pkg.DebPkg,
@@ -89,7 +89,7 @@ func TestTextImgPresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*ref2, img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		FoundBy:  "dpkg",
 		Metadata: PackageInfo{Name: "package-2", Version: "1.0.2"},

syft/source/all_layers_resolver.go

@@ -2,15 +2,15 @@ package source
 
 import (
 	"archive/tar"
+	"bytes"
 	"fmt"
 	"io"
-
-	"github.com/anchore/stereoscope/pkg/filetree"
-
-	"github.com/anchore/syft/internal/log"
+	"io/ioutil"
 
 	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/image"
+	"github.com/anchore/syft/internal/log"
 )
 
 var _ Resolver = (*AllLayersResolver)(nil)
@@ -37,6 +37,18 @@ func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) {
 	}, nil
 }
 
+// HasPath indicates if the given path exists in the underlying source.
+func (r *AllLayersResolver) HasPath(path string) bool {
+	p := file.Path(path)
+	for _, layerIdx := range r.layers {
+		tree := r.img.Layers[layerIdx].Tree
+		if tree.HasPath(p) {
+			return true
+		}
+	}
+	return false
+}
+
 func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) {
 	uniqueFiles := make([]file.Reference, 0)
 
@@ -102,7 +114,7 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
 				return nil, err
 			}
 			for _, result := range results {
-				uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
+				uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, result, r.img))
 			}
 		}
 	}
@@ -141,7 +153,7 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error)
 					return nil, err
 				}
 				for _, refResult := range refResults {
-					uniqueLocations = append(uniqueLocations, NewLocationFromImage(refResult, r.img))
+					uniqueLocations = append(uniqueLocations, NewLocationFromImage(string(result.MatchPath), refResult, r.img))
 				}
 			}
 		}
@@ -167,7 +179,7 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *
 		return nil
 	}
 
-	relativeLocation := NewLocationFromImage(*relativeRef, r.img)
+	relativeLocation := NewLocationFromImage(path, *relativeRef, r.img)
 
 	return &relativeLocation
 }
@@ -188,11 +200,11 @@ type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]io.Rea
 
 func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]io.ReadCloser, error) {
 	var fileRefs = make([]file.Reference, len(locations))
-	var locationByRefs = make(map[file.Reference]Location)
+	var locationByRefs = make(map[file.Reference][]Location)
 	var results = make(map[Location]io.ReadCloser)
 
 	for i, location := range locations {
-		locationByRefs[location.ref] = location
+		locationByRefs[location.ref] = append(locationByRefs[location.ref], location)
 		fileRefs[i] = location.ref
 	}
 
@@ -201,8 +213,26 @@ func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Lo
 		return nil, err
 	}
 
+	// TODO: this is not tested, we need a test case that covers a mapLocationRefs which has multiple Locations with the same reference in the request. The io.Reader should be copied.
 	for ref, content := range contentsByRef {
-		results[locationByRefs[ref]] = content
+		mappedLocations := locationByRefs[ref]
+		switch {
+		case len(mappedLocations) > 1:
+			// TODO: fixme... this can lead to lots of unexpected memory usage in unusual circumstances (cache is not leveraged for large files).
+			// stereoscope wont duplicate content requests if the caller asks for the same file multiple times... thats up to the caller
+			contentsBytes, err := ioutil.ReadAll(content)
+			if err != nil {
+				return nil, fmt.Errorf("unable to read ref=%+v :%w", ref, err)
+			}
+			for _, loc := range mappedLocations {
+				results[loc] = ioutil.NopCloser(bytes.NewReader(contentsBytes))
+			}
+
+		case len(mappedLocations) == 1:
+			results[locationByRefs[ref][0]] = content
+		default:
+			return nil, fmt.Errorf("unexpected ref-location count=%d for ref=%v", len(mappedLocations), ref)
+		}
 	}
 	return results, nil
 }

syft/source/all_layers_resolver_test.go

@@ -13,9 +13,10 @@ type resolution struct {
 
 func TestAllLayersResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name        string
-		linkPath    string
-		resolutions []resolution
+		name                 string
+		linkPath             string
+		resolutions          []resolution
+		forcePositiveHasPath bool
 	}{
 		{
 			name:     "link with previous data",
@@ -66,14 +67,17 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			},
 		},
 		{
-			name:        "dead link",
-			linkPath:    "/link-dead",
-			resolutions: []resolution{},
+			name:                 "dead link",
+			linkPath:             "/link-dead",
+			resolutions:          []resolution{},
+			forcePositiveHasPath: true,
 		},
 		{
 			name:        "ignore directories",
 			linkPath:    "/bin",
 			resolutions: []resolution{},
+			// directories don't resolve BUT do exist
+			forcePositiveHasPath: true,
 		},
 	}
 	for _, c := range cases {
@@ -86,6 +90,17 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("could not create resolver: %+v", err)
 			}
 
+			hasPath := resolver.HasPath(c.linkPath)
+			if !c.forcePositiveHasPath {
+				if len(c.resolutions) > 0 && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if len(c.resolutions) == 0 && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.linkPath)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v", err)
@@ -98,8 +113,12 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			for idx, actual := range refs {
 				expected := c.resolutions[idx]
 
-				if actual.Path != expected.path {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				if string(actual.ref.RealPath) != expected.path {
+					t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
+				}
+
+				if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
+					t.Errorf("we should always prefer real paths over ones with links")
 				}
 
 				entry, err := img.FileCatalog.Get(actual.ref)
@@ -202,8 +221,12 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 			for idx, actual := range refs {
 				expected := c.resolutions[idx]
 
-				if actual.Path != expected.path {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				if string(actual.ref.RealPath) != expected.path {
+					t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
+				}
+
+				if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
+					t.Errorf("we should always prefer real paths over ones with links")
 				}
 
 				entry, err := img.FileCatalog.Get(actual.ref)

syft/source/content_requester_test.go

@@ -54,20 +54,20 @@ func TestContentRequester(t *testing.T) {
 			}
 
 			for _, entry := range data {
-				if expected, ok := test.expectedContents[entry.Location.Path]; ok {
+				if expected, ok := test.expectedContents[entry.Location.RealPath]; ok {
 					actualBytes, err := ioutil.ReadAll(entry.Contents)
 					if err != nil {
-						t.Fatalf("could not read %q: %+v", entry.Location.Path, err)
+						t.Fatalf("could not read %q: %+v", entry.Location.RealPath, err)
 					}
 					for expected != string(actualBytes) {
-						t.Errorf("mismatched contents for %q", entry.Location.Path)
+						t.Errorf("mismatched contents for %q", entry.Location.RealPath)
 						dmp := diffmatchpatch.New()
 						diffs := dmp.DiffMain(expected, string(actualBytes), true)
 						t.Errorf("diff: %s", dmp.DiffPrettyText(diffs))
 					}
 					continue
 				}
-				t.Errorf("could not find %q", entry.Location.Path)
+				t.Errorf("could not find %q", entry.Location.RealPath)
 			}
 		})
 	}

syft/source/directory_resolver.go

@@ -19,27 +19,37 @@ type DirectoryResolver struct {
 	Path string
 }
 
+func (r DirectoryResolver) requestPath(userPath string) string {
+	fullPath := userPath
+	if filepath.IsAbs(fullPath) {
+		// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
+		fullPath = path.Join(r.Path, fullPath)
+	}
+	return fullPath
+}
+
+// HasPath indicates if the given path exists in the underlying source.
+func (r *DirectoryResolver) HasPath(userPath string) bool {
+	_, err := os.Stat(r.requestPath(userPath))
+	return !os.IsNotExist(err)
+}
+
 // Stringer to represent a directory path data source
-func (s DirectoryResolver) String() string {
-	return fmt.Sprintf("dir:%s", s.Path)
+func (r DirectoryResolver) String() string {
+	return fmt.Sprintf("dir:%s", r.Path)
 }
 
 // FilesByPath returns all file.References that match the given paths from the directory.
-func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
+func (r DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
 	var references = make([]Location, 0)
 
 	for _, userPath := range userPaths {
-		userStrPath := userPath
-
-		if filepath.IsAbs(userStrPath) {
-			// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
-			userStrPath = path.Join(s.Path, userStrPath)
-		}
+		userStrPath := r.requestPath(userPath)
 		fileMeta, err := os.Stat(userStrPath)
 		if os.IsNotExist(err) {
 			continue
 		} else if err != nil {
-			log.Errorf("path (%s) is not valid: %v", userStrPath, err)
+			log.Errorf("path (%r) is not valid: %v", userStrPath, err)
 		}
 
 		// don't consider directories
@@ -54,11 +64,11 @@ func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
 }
 
 // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
-func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
+func (r DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 	result := make([]Location, 0)
 
 	for _, pattern := range patterns {
-		pathPattern := path.Join(s.Path, pattern)
+		pathPattern := path.Join(r.Path, pattern)
 		pathMatches, err := doublestar.Glob(pathPattern)
 		if err != nil {
 			return nil, err
@@ -84,8 +94,8 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
 // This is helpful when attempting to find a file that is in the same layer or lower as another file. For the
 // DirectoryResolver, this is a simple path lookup.
-func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location {
-	paths, err := s.FilesByPath(path)
+func (r *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location {
+	paths, err := r.FilesByPath(path)
 	if err != nil {
 		return nil
 	}
@@ -97,16 +107,16 @@ func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Locatio
 }
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative a directory.
-func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
+func (r DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
 	refContents := make(map[Location]io.ReadCloser)
 	for _, location := range locations {
-		refContents[location] = file.NewDeferredReadCloser(location.Path)
+		refContents[location] = file.NewDeferredReadCloser(location.RealPath)
 	}
 	return refContents, nil
 }
 
 // FileContentsByLocation fetches file contents for a single file reference relative to a directory.
 // If the path does not exist an error is returned.
-func (s DirectoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
-	return file.NewDeferredReadCloser(location.Path), nil
+func (r DirectoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
+	return file.NewDeferredReadCloser(location.RealPath), nil
 }

syft/source/directory_resolver_test.go

@@ -6,11 +6,12 @@ import (
 
 func TestDirectoryResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name     string
-		root     string
-		input    string
-		expected string
-		refCount int
+		name                 string
+		root                 string
+		input                string
+		expected             string
+		refCount             int
+		forcePositiveHasPath bool
 	}{
 		{
 			name:     "finds a file (relative)",
@@ -47,15 +48,28 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
 			refCount: 1,
 		},
 		{
-			name:     "directories ignored",
-			root:     "./test-fixtures/",
-			input:    "/image-symlinks",
-			refCount: 0,
+			name:                 "directories ignored",
+			root:                 "./test-fixtures/",
+			input:                "/image-symlinks",
+			refCount:             0,
+			forcePositiveHasPath: true,
 		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			resolver := DirectoryResolver{c.root}
+
+			hasPath := resolver.HasPath(c.input)
+			if !c.forcePositiveHasPath {
+				if c.refCount != 0 && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if c.refCount == 0 && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.input)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v, %+v", err, refs)
@@ -66,8 +80,8 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
 			}
 
 			for _, actual := range refs {
-				if actual.Path != c.expected {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.expected)
+				if actual.RealPath != c.expected {
+					t.Errorf("bad resolve path: '%s'!='%s'", actual.RealPath, c.expected)
 				}
 			}
 		})

syft/source/image_squash_resolver.go

@@ -4,9 +4,8 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/anchore/stereoscope/pkg/filetree"
-
 	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/image"
 )
 
@@ -25,6 +24,11 @@ func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) {
 	return &ImageSquashResolver{img: img}, nil
 }
 
+// HasPath indicates if the given path exists in the underlying source.
+func (r *ImageSquashResolver) HasPath(path string) bool {
+	return r.img.SquashedTree().HasPath(file.Path(path))
+}
+
 // FilesByPath returns all file.References that match the given paths within the squashed representation of the image.
 func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 	uniqueFileIDs := file.NewFileReferenceSet()
@@ -62,7 +66,7 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 
 		if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
 			uniqueFileIDs.Add(*resolvedRef)
-			uniqueLocations = append(uniqueLocations, NewLocationFromImage(*resolvedRef, r.img))
+			uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, *resolvedRef, r.img))
 		}
 	}
 

syft/source/image_squash_resolver_test.go

@@ -8,10 +8,11 @@ import (
 
 func TestImageSquashResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name         string
-		linkPath     string
-		resolveLayer uint
-		resolvePath  string
+		name                 string
+		linkPath             string
+		resolveLayer         uint
+		resolvePath          string
+		forcePositiveHasPath bool
 	}{
 		{
 			name:         "link with previous data",
@@ -42,11 +43,15 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 			linkPath:     "/link-dead",
 			resolveLayer: 8,
 			resolvePath:  "",
+			// the path should exist, even if the link is dead
+			forcePositiveHasPath: true,
 		},
 		{
 			name:        "ignore directories",
 			linkPath:    "/bin",
 			resolvePath: "",
+			// the path should exist, even if we ignore it
+			forcePositiveHasPath: true,
 		},
 		{
 			name:         "parent is a link (with overridden data)",
@@ -65,6 +70,17 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("could not create resolver: %+v", err)
 			}
 
+			hasPath := resolver.HasPath(c.linkPath)
+			if !c.forcePositiveHasPath {
+				if c.resolvePath != "" && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if c.resolvePath == "" && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.linkPath)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v", err)
@@ -86,8 +102,12 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 
 			actual := refs[0]
 
-			if actual.Path != c.resolvePath {
-				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			if string(actual.ref.RealPath) != c.resolvePath {
+				t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
+			}
+
+			if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
+				t.Errorf("we should always prefer real paths over ones with links")
 			}
 
 			entry, err := img.FileCatalog.Get(actual.ref)
@@ -188,8 +208,12 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 
 			actual := refs[0]
 
-			if actual.Path != c.resolvePath {
-				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			if string(actual.ref.RealPath) != c.resolvePath {
+				t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
+			}
+
+			if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
+				t.Errorf("we should always prefer real paths over ones with links")
 			}
 
 			entry, err := img.FileCatalog.Get(actual.ref)

syft/source/location.go

@@ -7,9 +7,11 @@ import (
 	"github.com/anchore/stereoscope/pkg/image"
 )
 
-// Location represents a path relative to a particular filesystem.
+// Location represents a path relative to a particular filesystem resolved to a specific file.Reference. This struct is used as a key
+// in content fetching to uniquely identify a file relative to a request (the VirtualPath).
 type Location struct {
-	Path         string         `json:"path"`              // The string path of the location (e.g. /etc/hosts)
+	RealPath     string         `json:"path"`              // The path where all path ancestors have no hardlinks / symlinks
+	VirtualPath  string         `json:"-"`                 // The path to the file which may or may not have hardlinks / symlinks
 	FileSystemID string         `json:"layerID,omitempty"` // An ID representing the filesystem. For container images this is a layer digest, directories or root filesystem this is blank.
 	ref          file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
 }
@@ -17,23 +19,25 @@ type Location struct {
 // NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference.
 func NewLocation(path string) Location {
 	return Location{
-		Path: path,
+		RealPath: path,
 	}
 }
 
 // NewLocationFromImage creates a new Location representing the given path (extracted from the ref) relative to the given image.
-func NewLocationFromImage(ref file.Reference, img *image.Image) Location {
+func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Image) Location {
 	entry, err := img.FileCatalog.Get(ref)
 	if err != nil {
 		log.Warnf("unable to find file catalog entry for ref=%+v", ref)
 		return Location{
-			Path: string(ref.RealPath),
-			ref:  ref,
+			VirtualPath: virtualPath,
+			RealPath:    string(ref.RealPath),
+			ref:         ref,
 		}
 	}
 
 	return Location{
-		Path:         string(ref.RealPath),
+		VirtualPath:  virtualPath,
+		RealPath:     string(ref.RealPath),
 		FileSystemID: entry.Layer.Metadata.Digest,
 		ref:          ref,
 	}

syft/source/mock_resolver.go

@@ -28,9 +28,19 @@ func NewMockResolverForPaths(paths ...string) *MockResolver {
 	return &MockResolver{Locations: locations}
 }
 
+// HasPath indicates if the given path exists in the underlying source.
+func (r MockResolver) HasPath(path string) bool {
+	for _, l := range r.Locations {
+		if l.RealPath == path {
+			return true
+		}
+	}
+	return false
+}
+
 // String returns the string representation of the MockResolver.
 func (r MockResolver) String() string {
-	return fmt.Sprintf("mock:(%s,...)", r.Locations[0].Path)
+	return fmt.Sprintf("mock:(%s,...)", r.Locations[0].RealPath)
 }
 
 // FileContentsByLocation fetches file contents for a single location. If the
@@ -38,7 +48,7 @@ func (r MockResolver) String() string {
 func (r MockResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
 	for _, l := range r.Locations {
 		if l == location {
-			return os.Open(location.Path)
+			return os.Open(location.RealPath)
 		}
 	}
 
@@ -64,7 +74,7 @@ func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) {
 	var results []Location
 	for _, p := range paths {
 		for _, location := range r.Locations {
-			if p == location.Path {
+			if p == location.RealPath {
 				results = append(results, NewLocation(p))
 			}
 		}
@@ -78,7 +88,7 @@ func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 	var results []Location
 	for _, pattern := range patterns {
 		for _, location := range r.Locations {
-			if file.GlobMatch(pattern, location.Path) {
+			if file.GlobMatch(pattern, location.RealPath) {
 				results = append(results, location)
 			}
 		}

syft/source/resolver.go

@@ -16,12 +16,14 @@ type Resolver interface {
 // ContentResolver knows how to get file content for given file.References
 type ContentResolver interface {
 	FileContentsByLocation(Location) (io.ReadCloser, error)
+	// TODO: it is possible to be given duplicate locations that will be overridden in the map (key), a subtle problem that coule easily be misued.
 	MultipleFileContentsByLocation([]Location) (map[Location]io.ReadCloser, error)
-	// TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering).
 }
 
-// FileResolver knows how to get file.References for given string paths and globs
+// FileResolver knows how to get a Location for given string paths and globs
 type FileResolver interface {
+	// HasPath indicates if the given path exists in the underlying source.
+	HasPath(path string) bool
 	// FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches)
 	FilesByPath(paths ...string) ([]Location, error)
 	// FilesByGlob fetches a set of file references which the given glob matches

test/integration/document_import_test.go

@@ -81,6 +81,12 @@ func TestCatalogFromJSON(t *testing.T) {
 					e.Metadata = metadata
 				}
 
+				// ignore the virtual path on the location for now
+				for l := range a.Locations {
+					a.Locations[l].VirtualPath = ""
+					e.Locations[l].VirtualPath = ""
+				}
+
 				for _, d := range deep.Equal(a, e) {
 					// ignore errors for empty collections vs nil for select fields
 					// TODO: this is brittle, but not dangerously so. We should still find a better way to do this.

test/integration/pkg_cases_test.go

@@ -15,7 +15,10 @@ var imageOnlyTestCases = []testCase{
 		pkgType:     pkg.GemPkg,
 		pkgLanguage: pkg.Ruby,
 		pkgInfo: map[string]string{
+			// specifications in the root specification directory
 			"bundler": "2.1.4",
+			// specifications in named directories
+			"unbundler": "3.1.4",
 		},
 	},
 	{

test/integration/test-fixtures/image-pkg-coverage/ruby/specifications/default/unbundler.gemspec

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# -*- encoding: utf-8 -*-                                                                                                                                         
+# stub: unbundler 2.1.4 ruby lib                                                                                                                                    
+                                                                                                                                                                  
+Gem::Specification.new do |s|                                                                                                                                     
+    s.name = "unbundler".freeze
+    s.version = "3.1.4"
+                                                                                                                                                                    
+    s.required_rubygems_version = Gem::Requirement.new(">= 2.5.2".freeze) if s.respond_to? :required_rubygems_version=                                              
+    s.require_paths = ["lib".freeze]                                                                                                                                
+    s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".f
+    s.bindir = "exe".freeze                                                                                                                                         
+    s.date = "2020-01-05"                                                                                                                                           
+    s.description = "Bundler manages an application's dependencies through its entire life, across many machines, systematically and repeatably".freeze             
+    s.email = ["team@unbundler.io".freeze]                                                                                                                            
+    s.executables = ["unbundle".freeze, "unbundler".freeze]                                                                                                             
+    s.files = ["exe/unbundle".freeze, "exe/unbundler".freeze]                                                                                                           
+    s.homepage = "https://unbundler.io".freeze                                                                                                                        
+    s.licenses = ["MIT".freeze]                                                                                                                                     
+    s.required_ruby_version = Gem::Requirement.new(">= 2.3.0".freeze)                                                                                               
+    s.rubygems_version = "3.1.2".freeze                                                                                                                             
+    s.summary = "The best way to manage your application's dependencies".freeze                                                                                     
+                                                                                                                                                                    
+    s.installed_by_version = "3.1.2" if s.respond_to? :installed_by_version                                                                                         
+  end