Merge pull request #63 from anchore/add-symlink-suport

Add symlink support

.gitignore

@@ -1,3 +1,5 @@
+.vscode/
+
 *.tar
 .idea/
 *.log
@@ -16,4 +18,4 @@ coverage.txt
 *.test
 
 # Output of the go coverage tool, specifically when used with LiteIDE
-*.out
+*.out

go.mod

@@ -5,10 +5,10 @@ go 1.14
 require (
 	github.com/adrg/xdg v0.2.1
 	github.com/anchore/go-testutils v0.0.0-20200520222037-edc2bf1864fe
-	github.com/anchore/stereoscope v0.0.0-20200604133300-7e63b350b6d6
+	github.com/anchore/stereoscope v0.0.0-20200616152009-189722bdb61b
 	github.com/aquasecurity/go-dep-parser v0.0.0-20200123140603-4dc0125084da
 	github.com/go-test/deep v1.0.6
-	github.com/google/go-containerregistry v0.1.0 // indirect
+	github.com/google/go-containerregistry v0.1.1 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/hashicorp/go-version v1.2.0
@@ -20,8 +20,9 @@ require (
 	github.com/spf13/viper v1.7.0
 	go.uber.org/zap v1.15.0
 	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
-	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect
+	golang.org/x/sys v0.0.0-20200610111108-226ff32320da // indirect
 	google.golang.org/appengine v1.6.6
+	google.golang.org/genproto v0.0.0-20200615140333-fd031eab31e7 // indirect
 	google.golang.org/protobuf v1.24.0 // indirect
 	gopkg.in/ini.v1 v1.57.0 // indirect
 	gopkg.in/yaml.v2 v2.3.0

go.sum

@@ -111,6 +111,10 @@ github.com/anchore/stereoscope v0.0.0-20200602123205-6c2ce3c0b2d5 h1:eViCIr4O1e4
 github.com/anchore/stereoscope v0.0.0-20200602123205-6c2ce3c0b2d5/go.mod h1:OeCrFeSu8+p02qC7u9/u8wBOh50VQa8eHJjXVuANvLo=
 github.com/anchore/stereoscope v0.0.0-20200604133300-7e63b350b6d6 h1:Fu779yw004jyFH1UkQD8lTf0GmGRfrOQIK5QiqmIwU8=
 github.com/anchore/stereoscope v0.0.0-20200604133300-7e63b350b6d6/go.mod h1:eQ2/Al6XDA7RFk3FVfpjyGRErITRjNciUPIWixHc7kQ=
+github.com/anchore/stereoscope v0.0.0-20200612195212-342a44f79c65 h1:wghtT1rUItLg/gx/LhMx6fYKJwnUGpfXvcA8WGWM/co=
+github.com/anchore/stereoscope v0.0.0-20200612195212-342a44f79c65/go.mod h1:eQ2/Al6XDA7RFk3FVfpjyGRErITRjNciUPIWixHc7kQ=
+github.com/anchore/stereoscope v0.0.0-20200616152009-189722bdb61b h1:LmFKsQi4oj2VJjch7JhQNzJg1A56FjwHqWZz1ZZKgIw=
+github.com/anchore/stereoscope v0.0.0-20200616152009-189722bdb61b/go.mod h1:eQ2/Al6XDA7RFk3FVfpjyGRErITRjNciUPIWixHc7kQ=
 github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ=
 github.com/apex/log v1.3.0/go.mod h1:jd8Vpsr46WAe3EZSQ/IUMs2qQD/GOycT5rPWCO1yGcs=
 github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
@@ -154,6 +158,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
+github.com/containerd/containerd v1.3.0 h1:xjvXQWABwS2uiv3TWgQt5Uth60Gu86LTGZXMJkjc7rY=
 github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.3.4 h1:3o0smo5SKY7H6AJCmJhsnCjR2/V2T8VmiHt7seN2/kI=
@@ -331,6 +336,8 @@ github.com/google/go-containerregistry v0.0.0-20200430153450-5cbd060f5c92/go.mod
 github.com/google/go-containerregistry v0.0.0-20200601195303-96cf69f03a3c/go.mod h1:npTSyywOeILcgWqd+rvtzGWflIPPcBQhYoOONaY4ltM=
 github.com/google/go-containerregistry v0.1.0 h1:hL5mVw7cTX3SBr64Arpv+cJH93L+Z9Q6WjckImYLB3g=
 github.com/google/go-containerregistry v0.1.0/go.mod h1:npTSyywOeILcgWqd+rvtzGWflIPPcBQhYoOONaY4ltM=
+github.com/google/go-containerregistry v0.1.1 h1:AG8FSAfXglim2l5qSrqp5VK2Xl03PiBf25NiTGGamws=
+github.com/google/go-containerregistry v0.1.1/go.mod h1:npTSyywOeILcgWqd+rvtzGWflIPPcBQhYoOONaY4ltM=
 github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
@@ -863,6 +870,8 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200610111108-226ff32320da h1:bGb80FudwxpeucJUjPYJXuJ8Hk91vNtfvrymzwiei38=
+golang.org/x/sys v0.0.0-20200610111108-226ff32320da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1006,6 +1015,8 @@ google.golang.org/genproto v0.0.0-20200603110839-e855014d5736 h1:+IE3xTD+6Eb7QWG
 google.golang.org/genproto v0.0.0-20200603110839-e855014d5736/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
 google.golang.org/genproto v0.0.0-20200604104852-0b0486081ffb h1:ek2py5bOqzR7MR/6obzk0rXUgYCLmjyLnaO9ssT+l6w=
 google.golang.org/genproto v0.0.0-20200604104852-0b0486081ffb/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200615140333-fd031eab31e7 h1:1N7l1PuXZwEK7OhHdmKQROOM75PnUjABGwvVRbLBgFk=
+google.golang.org/genproto v0.0.0-20200615140333-fd031eab31e7/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

imgbom/cataloger/bundler/cataloger.go

@@ -3,8 +3,8 @@ package bundler
 import (
 	"github.com/anchore/imgbom/imgbom/cataloger/common"
 	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/imgbom/scope"
 	"github.com/anchore/stereoscope/pkg/file"
-	"github.com/anchore/stereoscope/pkg/tree"
 )
 
 type Cataloger struct {
@@ -25,8 +25,8 @@ func (a *Cataloger) Name() string {
 	return "bundler-cataloger"
 }
 
-func (a *Cataloger) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
+func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
-	return a.cataloger.SelectFiles(trees)
+	return a.cataloger.SelectFiles(resolver)
 }
 
 func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {

imgbom/cataloger/cataloger.go

@@ -2,14 +2,14 @@ package cataloger
 
 import (
 	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/imgbom/scope"
 	"github.com/anchore/stereoscope/pkg/file"
-	"github.com/anchore/stereoscope/pkg/tree"
 )
 
 type Cataloger interface {
 	Name() string
 	// TODO: add ID / Name for catalog for uniquely identifying this cataloger type
-	SelectFiles([]tree.FileTreeReader) []file.Reference
+	SelectFiles(scope.FileResolver) []file.Reference
 	// NOTE: one of the errors which is returned is "IterationNeeded", which indicates to the driver to
 	// continue with another Select/Catalog pass
 	Catalog(map[file.Reference]string) ([]pkg.Package, error)

imgbom/cataloger/common/generic_cataloger.go

@@ -4,13 +4,11 @@ import (
 	"strings"
 
 	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/imgbom/scope"
 	"github.com/anchore/imgbom/internal/log"
 	"github.com/anchore/stereoscope/pkg/file"
-	"github.com/anchore/stereoscope/pkg/tree"
 )
 
-// TODO: put under test...
-
 // GenericCataloger implements the Catalog interface and is responsible for dispatching the proper parser function for
 // a given path or glob pattern. This is intended to be reusable across many package cataloger types.
 type GenericCataloger struct {
@@ -45,25 +43,26 @@ func (a *GenericCataloger) clear() {
 }
 
 // SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging
-func (a *GenericCataloger) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
+func (a *GenericCataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
-	for _, t := range trees {
+	// select by exact path
-		// select by exact path
+	for path, parser := range a.pathParsers {
-		for path, parser := range a.pathParsers {
+		files, err := resolver.FilesByPath(file.Path(path))
-			f := t.File(file.Path(path))
+		if err != nil {
-			if f != nil {
+			log.Errorf("cataloger failed to select files by path: %w", err)
-				a.register([]file.Reference{*f}, parser)
-			}
 		}
+		if files != nil {
+			a.register(files, parser)
+		}
+	}
 
-		// select by pattern
+	// select by glob pattern
-		for globPattern, parser := range a.globParsers {
+	for globPattern, parser := range a.globParsers {
-			fileMatches, err := t.FilesByGlob(globPattern)
+		fileMatches, err := resolver.FilesByGlob(globPattern)
-			if err != nil {
+		if err != nil {
-				log.Errorf("failed to find files by glob: %s", globPattern)
+			log.Errorf("failed to find files by glob: %s", globPattern)
-			}
+		}
-			if fileMatches != nil {
+		if fileMatches != nil {
-				a.register(fileMatches, parser)
+			a.register(fileMatches, parser)
-			}
 		}
 	}
 

imgbom/cataloger/common/generic_cataloger_test.go

@@ -0,0 +1,117 @@
+package common
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"testing"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/internal"
+	"github.com/anchore/stereoscope/pkg/file"
+)
+
+type testResolver struct {
+	contents map[file.Reference]string
+}
+
+func newTestResolver() *testResolver {
+	return &testResolver{
+		contents: make(map[file.Reference]string),
+	}
+}
+
+func (r *testResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+	results := make([]file.Reference, len(paths))
+
+	for idx, p := range paths {
+		results[idx] = file.NewFileReference(p)
+		r.contents[results[idx]] = fmt.Sprintf("%s file contents!", p)
+	}
+
+	return results, nil
+}
+
+func (r *testResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+	path := "/a-path.txt"
+	ref := file.NewFileReference(file.Path(path))
+	r.contents[ref] = fmt.Sprintf("%s file contents!", path)
+	return []file.Reference{ref}, nil
+}
+
+func parser(reader io.Reader) ([]pkg.Package, error) {
+	contents, err := ioutil.ReadAll(reader)
+	if err != nil {
+		panic(err)
+	}
+	return []pkg.Package{
+		{
+			Name: string(contents),
+		},
+	}, nil
+}
+
+func TestGenericCataloger(t *testing.T) {
+
+	globParsers := map[string]ParserFn{
+		"**a-path.txt": parser,
+	}
+	pathParsers := map[string]ParserFn{
+		"/another-path.txt": parser,
+		"/last/path.txt":    parser,
+	}
+
+	resolver := newTestResolver()
+	cataloger := NewGenericCataloger(pathParsers, globParsers)
+
+	selected := cataloger.SelectFiles(resolver)
+
+	if len(selected) != 3 {
+		t.Fatalf("unexpected selection length: %d", len(selected))
+	}
+
+	expectedSelection := internal.NewStringSetFromSlice([]string{"/last/path.txt", "/another-path.txt", "/a-path.txt"})
+	selectionByPath := make(map[string]file.Reference)
+	for _, s := range selected {
+		if !expectedSelection.Contains(string(s.Path)) {
+			t.Errorf("unexpected selection path: %+v", s.Path)
+		}
+		selectionByPath[string(s.Path)] = s
+	}
+
+	upstream := "some-other-cataloger"
+	expectedPkgs := make(map[file.Reference]pkg.Package)
+	for path, ref := range selectionByPath {
+		expectedPkgs[ref] = pkg.Package{
+			FoundBy: upstream,
+			Source:  []file.Reference{ref},
+			Name:    fmt.Sprintf("%s file contents!", path),
+		}
+	}
+
+	actualPkgs, err := cataloger.Catalog(resolver.contents, upstream)
+	if err != nil {
+		t.Fatalf("cataloger catalog action failed: %+v", err)
+	}
+
+	if len(actualPkgs) != len(expectedPkgs) {
+		t.Fatalf("unexpected packages len: %d", len(actualPkgs))
+	}
+
+	for _, p := range actualPkgs {
+		ref := p.Source[0]
+		exP, ok := expectedPkgs[ref]
+		if !ok {
+			t.Errorf("missing expected pkg: ref=%+v", ref)
+			continue
+		}
+
+		if p.FoundBy != exP.FoundBy {
+			t.Errorf("bad upstream: %s", p.FoundBy)
+		}
+
+		if exP.Name != p.Name {
+			t.Errorf("bad contents mapping: %+v", p.Source)
+		}
+	}
+}

imgbom/cataloger/controller.go

@@ -17,6 +17,14 @@ func init() {
 	controllerInstance = newController()
 }
 
+func Catalogers() []string {
+	c := make([]string, len(controllerInstance.catalogers))
+	for idx, catalog := range controllerInstance.catalogers {
+		c[idx] = catalog.Name()
+	}
+	return c
+}
+
 func Catalog(s scope.Scope) (*pkg.Catalog, error) {
 	return controllerInstance.catalog(s)
 }
@@ -46,7 +54,7 @@ func (c *controller) catalog(s scope.Scope) (*pkg.Catalog, error) {
 
 	// ask catalogers for files to extract from the image tar
 	for _, a := range c.catalogers {
-		fileSelection = append(fileSelection, a.SelectFiles(s.Trees)...)
+		fileSelection = append(fileSelection, a.SelectFiles(&s)...)
 		log.Debugf("cataloger '%s' selected '%d' files", a.Name(), len(fileSelection))
 	}
 

imgbom/cataloger/dpkg/cataloger.go

@@ -3,8 +3,8 @@ package dpkg
 import (
 	"github.com/anchore/imgbom/imgbom/cataloger/common"
 	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/imgbom/scope"
 	"github.com/anchore/stereoscope/pkg/file"
-	"github.com/anchore/stereoscope/pkg/tree"
 )
 
 type Cataloger struct {
@@ -25,8 +25,8 @@ func (a *Cataloger) Name() string {
 	return "dpkg-cataloger"
 }
 
-func (a *Cataloger) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
+func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
-	return a.cataloger.SelectFiles(trees)
+	return a.cataloger.SelectFiles(resolver)
 }
 
 func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {

imgbom/cataloger/python/cataloger.go

@@ -3,8 +3,8 @@ package python
 import (
 	"github.com/anchore/imgbom/imgbom/cataloger/common"
 	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/imgbom/scope"
 	"github.com/anchore/stereoscope/pkg/file"
-	"github.com/anchore/stereoscope/pkg/tree"
 )
 
 type Cataloger struct {
@@ -26,8 +26,8 @@ func (a *Cataloger) Name() string {
 	return "python-cataloger"
 }
 
-func (a *Cataloger) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
+func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
-	return a.cataloger.SelectFiles(trees)
+	return a.cataloger.SelectFiles(resolver)
 }
 
 func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {

imgbom/distro/identify.go

@@ -20,15 +20,14 @@ func Identify(img *image.Image) *Distro {
 		"/etc/os-release": parseOsRelease,
 		// Debian and Debian-based distros have the same contents linked from this path
 		"/usr/lib/os-release": parseOsRelease,
-		// TODO: change this to /bin/busybox when stereoscope deals with hardlinks
+		"/bin/busybox":        parseBusyBox,
-		"/bin/[": parseBusyBox,
 	}
 
 	for path, fn := range identityFiles {
 		contents, err := img.FileContentsFromSquash(path)
 
 		if err != nil {
-			log.Errorf("unable to get contents from %s: %s", path, err)
+			log.Debugf("unable to get contents from %s: %s", path, err)
 			continue
 		}
 

imgbom/presenter/json/presenter_test.go

@@ -53,7 +53,7 @@ func TestJsonPresenter(t *testing.T) {
 		Name:    "package-1",
 		Version: "1.0.1",
 		Source: []file.Reference{
-			*img.SquashedTree.File("/somefile-1.txt"),
+			*img.SquashedTree().File("/somefile-1.txt"),
 		},
 		Type: pkg.DebPkg,
 	})
@@ -61,7 +61,7 @@ func TestJsonPresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Source: []file.Reference{
-			*img.SquashedTree.File("/somefile-2.txt"),
+			*img.SquashedTree().File("/somefile-2.txt"),
 		},
 		Type: pkg.DebPkg,
 	})

imgbom/scope/file_resolver.go

@@ -0,0 +1,25 @@
+package scope
+
+import (
+	"fmt"
+
+	"github.com/anchore/imgbom/imgbom/scope/resolvers"
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/image"
+)
+
+type FileResolver interface {
+	FilesByPath(paths ...file.Path) ([]file.Reference, error)
+	FilesByGlob(patterns ...string) ([]file.Reference, error)
+}
+
+func getFileResolver(img *image.Image, option Option) (FileResolver, error) {
+	switch option {
+	case SquashedScope:
+		return resolvers.NewImageSquashResolver(img)
+	case AllLayersScope:
+		return resolvers.NewAllLayersResolver(img)
+	default:
+		return nil, fmt.Errorf("bad option provided: %+v", option)
+	}
+}

imgbom/scope/resolvers/all_layers_resolver.go

@@ -0,0 +1,106 @@
+package resolvers
+
+import (
+	"archive/tar"
+	"fmt"
+
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/image"
+)
+
+type AllLayersResolver struct {
+	img    *image.Image
+	layers []int
+}
+
+func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) {
+	if len(img.Layers) == 0 {
+		return nil, fmt.Errorf("the image does not contain any layers")
+	}
+
+	var layers = make([]int, 0)
+	for idx := range img.Layers {
+		layers = append(layers, idx)
+	}
+	return &AllLayersResolver{
+		img:    img,
+		layers: layers,
+	}, nil
+}
+
+func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) {
+	uniqueFiles := make([]file.Reference, 0)
+
+	// since there is potentially considerable work for each symlink/hardlink that needs to be resolved, let's check to see if this is a symlink/hardlink first
+	entry, err := r.img.FileCatalog.Get(ref)
+	if err != nil {
+		return nil, fmt.Errorf("unable to fetch metadata (ref=%+v): %w", ref, err)
+	}
+
+	if entry.Metadata.TypeFlag == tar.TypeLink || entry.Metadata.TypeFlag == tar.TypeSymlink {
+		// a link may resolve in this layer or higher, assuming a squashed tree is used to search
+		// we should search all possible resolutions within the valid scope
+		for _, subLayerIdx := range r.layers[layerIdx:] {
+			resolvedRef, err := r.img.ResolveLinkByLayerSquash(ref, subLayerIdx)
+			if err != nil {
+				return nil, fmt.Errorf("failed to resolve link from layer (layer=%d ref=%+v): %w", subLayerIdx, ref, err)
+			}
+			if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
+				uniqueFileIDs.Add(*resolvedRef)
+				uniqueFiles = append(uniqueFiles, *resolvedRef)
+			}
+		}
+	} else if !uniqueFileIDs.Contains(ref) {
+		uniqueFileIDs.Add(ref)
+		uniqueFiles = append(uniqueFiles, ref)
+	}
+
+	return uniqueFiles, nil
+}
+
+func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, path := range paths {
+		for idx, layerIdx := range r.layers {
+			ref := r.img.Layers[layerIdx].Tree.File(path)
+			if ref == nil {
+				// no file found, keep looking through layers
+				continue
+			}
+
+			results, err := r.fileByRef(*ref, uniqueFileIDs, idx)
+			if err != nil {
+				return nil, err
+			}
+			uniqueFiles = append(uniqueFiles, results...)
+		}
+	}
+
+	return uniqueFiles, nil
+}
+
+func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, pattern := range patterns {
+		for idx, layerIdx := range r.layers {
+			refs, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern)
+			if err != nil {
+				return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
+			}
+
+			for _, ref := range refs {
+				results, err := r.fileByRef(ref, uniqueFileIDs, idx)
+				if err != nil {
+					return nil, err
+				}
+				uniqueFiles = append(uniqueFiles, results...)
+			}
+		}
+	}
+
+	return uniqueFiles, nil
+}

imgbom/scope/resolvers/all_layers_resolver_test.go

@@ -0,0 +1,229 @@
+package resolvers
+
+import (
+	"testing"
+
+	"github.com/anchore/go-testutils"
+	"github.com/anchore/stereoscope/pkg/file"
+)
+
+type resolution struct {
+	layer uint
+	path  string
+}
+
+func TestAllLayersResolver_FilesByPath(t *testing.T) {
+	cases := []struct {
+		name        string
+		linkPath    string
+		resolutions []resolution
+	}{
+		{
+			name:     "link with previous data",
+			linkPath: "/link-1",
+			resolutions: []resolution{
+				{
+					layer: 1,
+					path:  "/file-1.txt",
+				},
+			},
+		},
+		{
+			name:     "link with in layer data",
+			linkPath: "/link-within",
+			resolutions: []resolution{
+				{
+					layer: 5,
+					path:  "/file-3.txt",
+				},
+			},
+		},
+		{
+			name:     "link with overridden data",
+			linkPath: "/link-2",
+			resolutions: []resolution{
+				{
+					layer: 3,
+					path:  "/link-2",
+				},
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name:     "indirect link (with overridden data)",
+			linkPath: "/link-indirect",
+			resolutions: []resolution{
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name:     "dead link",
+			linkPath: "/link-dead",
+			resolutions: []resolution{
+				{
+					layer: 8,
+					path:  "/link-dead",
+				},
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewAllLayersResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByPath(file.Path(c.linkPath))
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != len(c.resolutions) {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			for idx, actual := range refs {
+				expected := c.resolutions[idx]
+
+				if actual.Path != file.Path(expected.path) {
+					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				}
+
+				entry, err := img.FileCatalog.Get(actual)
+				if err != nil {
+					t.Fatalf("failed to get metadata: %+v", err)
+				}
+
+				if entry.Source.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				}
+			}
+		})
+	}
+}
+
+func TestAllLayersResolver_FilesByGlob(t *testing.T) {
+	cases := []struct {
+		name        string
+		glob        string
+		resolutions []resolution
+	}{
+		{
+			name: "link with previous data",
+			glob: "**ink-1",
+			resolutions: []resolution{
+				{
+					layer: 1,
+					path:  "/file-1.txt",
+				},
+			},
+		},
+		{
+			name: "link with in layer data",
+			glob: "**nk-within",
+			resolutions: []resolution{
+				{
+					layer: 5,
+					path:  "/file-3.txt",
+				},
+			},
+		},
+		{
+			name: "link with overridden data",
+			glob: "**ink-2",
+			resolutions: []resolution{
+				{
+					layer: 3,
+					path:  "/link-2",
+				},
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name: "indirect link (with overridden data)",
+			glob: "**nk-indirect",
+			resolutions: []resolution{
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name: "dead link",
+			glob: "**k-dead",
+			resolutions: []resolution{
+				{
+					layer: 8,
+					path:  "/link-dead",
+				},
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewAllLayersResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByGlob(c.glob)
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != len(c.resolutions) {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			for idx, actual := range refs {
+				expected := c.resolutions[idx]
+
+				if actual.Path != file.Path(expected.path) {
+					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				}
+
+				entry, err := img.FileCatalog.Get(actual)
+				if err != nil {
+					t.Fatalf("failed to get metadata: %+v", err)
+				}
+
+				if entry.Source.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				}
+			}
+		})
+	}
+}

imgbom/scope/resolvers/image_squash_resolver.go

@@ -0,0 +1,70 @@
+package resolvers
+
+import (
+	"fmt"
+
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/image"
+)
+
+type ImageSquashResolver struct {
+	img *image.Image
+}
+
+func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) {
+	if img.SquashedTree() == nil {
+		return nil, fmt.Errorf("the image does not have have a squashed tree")
+	}
+	return &ImageSquashResolver{img: img}, nil
+}
+
+func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, path := range paths {
+		ref := r.img.SquashedTree().File(path)
+		if ref == nil {
+			// no file found, keep looking through layers
+			continue
+		}
+
+		resolvedRef, err := r.img.ResolveLinkByImageSquash(*ref)
+		if err != nil {
+			return nil, fmt.Errorf("failed to resolve link from img (ref=%+v): %w", ref, err)
+		}
+		if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
+			uniqueFileIDs.Add(*resolvedRef)
+			uniqueFiles = append(uniqueFiles, *resolvedRef)
+		}
+	}
+
+	return uniqueFiles, nil
+}
+
+func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, pattern := range patterns {
+		refs, err := r.img.SquashedTree().FilesByGlob(pattern)
+		if err != nil {
+			return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
+		}
+
+		for _, ref := range refs {
+			resolvedRefs, err := r.FilesByPath(ref.Path)
+			if err != nil {
+				return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err)
+			}
+			for _, resolvedRef := range resolvedRefs {
+				if !uniqueFileIDs.Contains(resolvedRef) {
+					uniqueFileIDs.Add(resolvedRef)
+					uniqueFiles = append(uniqueFiles, resolvedRef)
+				}
+			}
+		}
+	}
+
+	return uniqueFiles, nil
+}

imgbom/scope/resolvers/image_squash_resolver_test.go

@@ -0,0 +1,158 @@
+package resolvers
+
+import (
+	"testing"
+
+	"github.com/anchore/go-testutils"
+	"github.com/anchore/stereoscope/pkg/file"
+)
+
+func TestImageSquashResolver_FilesByPath(t *testing.T) {
+	cases := []struct {
+		name         string
+		linkPath     string
+		resolveLayer uint
+		resolvePath  string
+	}{
+		{
+			name:         "link with previous data",
+			linkPath:     "/link-1",
+			resolveLayer: 1,
+			resolvePath:  "/file-1.txt",
+		},
+		{
+			name:         "link with in layer data",
+			linkPath:     "/link-within",
+			resolveLayer: 5,
+			resolvePath:  "/file-3.txt",
+		},
+		{
+			name:         "link with overridden data",
+			linkPath:     "/link-2",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "indirect link (with overridden data)",
+			linkPath:     "/link-indirect",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "dead link",
+			linkPath:     "/link-dead",
+			resolveLayer: 8,
+			resolvePath:  "/link-dead",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewImageSquashResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByPath(file.Path(c.linkPath))
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != 1 {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			actual := refs[0]
+
+			if actual.Path != file.Path(c.resolvePath) {
+				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			}
+
+			entry, err := img.FileCatalog.Get(actual)
+			if err != nil {
+				t.Fatalf("failed to get metadata: %+v", err)
+			}
+
+			if entry.Source.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			}
+		})
+	}
+}
+
+func TestImageSquashResolver_FilesByGlob(t *testing.T) {
+	cases := []struct {
+		name         string
+		glob         string
+		resolveLayer uint
+		resolvePath  string
+	}{
+		{
+			name:         "link with previous data",
+			glob:         "**link-1",
+			resolveLayer: 1,
+			resolvePath:  "/file-1.txt",
+		},
+		{
+			name:         "link with in layer data",
+			glob:         "**link-within",
+			resolveLayer: 5,
+			resolvePath:  "/file-3.txt",
+		},
+		{
+			name:         "link with overridden data",
+			glob:         "**link-2",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "indirect link (with overridden data)",
+			glob:         "**link-indirect",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "dead link",
+			glob:         "**link-dead",
+			resolveLayer: 8,
+			resolvePath:  "/link-dead",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewImageSquashResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByGlob(c.glob)
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != 1 {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			actual := refs[0]
+
+			if actual.Path != file.Path(c.resolvePath) {
+				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			}
+
+			entry, err := img.FileCatalog.Get(actual)
+			if err != nil {
+				t.Fatalf("failed to get metadata: %+v", err)
+			}
+
+			if entry.Source.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			}
+		})
+	}
+}

imgbom/scope/resolvers/test-fixtures/image-symlinks/Dockerfile

@@ -0,0 +1,24 @@
+# LAYER 0:
+FROM busybox:latest
+
+# LAYER 1:
+ADD file-1.txt .
+# LAYER 2: link with previous data
+RUN ln -s ./file-1.txt link-1
+
+# LAYER 3: link with future data
+RUN ln -s ./file-2.txt link-2
+# LAYER 4:
+ADD file-2.txt .
+
+# LAYER 5: link with current data
+RUN echo "file 3" > file-3.txt && ln -s ./file-3.txt link-within
+
+# LAYER 6: multiple links (link-indirect > link-2 > file-2.txt)
+RUN ln -s ./link-2 link-indirect
+
+# LAYER 7: override contents / resolution
+ADD new-file-2.txt file-2.txt
+
+# LAYER 8: dead link
+RUN ln -s ./i-dont-exist.txt link-dead

imgbom/scope/resolvers/test-fixtures/image-symlinks/file-1.txt

@@ -0,0 +1 @@
+file 1!

imgbom/scope/resolvers/test-fixtures/image-symlinks/file-2.txt

@@ -0,0 +1 @@
+file 2!

imgbom/scope/resolvers/test-fixtures/image-symlinks/new-file-2.txt

@@ -0,0 +1 @@
+NEW file override!

imgbom/scope/scope.go

@@ -3,44 +3,37 @@ package scope
 import (
 	"fmt"
 
+	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/image"
-	"github.com/anchore/stereoscope/pkg/tree"
 )
 
 type Scope struct {
-	Option Option
+	Option   Option
-	Trees  []tree.FileTreeReader
+	resolver FileResolver
-	Image  *image.Image
+	Image    *image.Image
 }
 
 func NewScope(img *image.Image, option Option) (Scope, error) {
-	var trees = make([]tree.FileTreeReader, 0)
-
 	if img == nil {
 		return Scope{}, fmt.Errorf("no image given")
 	}
 
-	switch option {
+	resolver, err := getFileResolver(img, option)
-	case SquashedScope:
+	if err != nil {
-		if img.SquashedTree == nil {
+		return Scope{}, fmt.Errorf("could not determine file resolver: %w", err)
-			return Scope{}, fmt.Errorf("the image does not have have a squashed tree")
-		}
-		trees = append(trees, img.SquashedTree)
-
-	case AllLayersScope:
-		if len(img.Layers) == 0 {
-			return Scope{}, fmt.Errorf("the image does not contain any layers")
-		}
-		for _, layer := range img.Layers {
-			trees = append(trees, layer.Tree)
-		}
-	default:
-		return Scope{}, fmt.Errorf("bad option provided: %+v", option)
 	}
 
 	return Scope{
-		Option: option,
+		Option:   option,
-		Trees:  trees,
+		resolver: resolver,
-		Image:  img,
+		Image:    img,
 	}, nil
 }
+
+func (s Scope) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+	return s.resolver.FilesByPath(paths...)
+}
+
+func (s Scope) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+	return s.resolver.FilesByGlob(patterns...)
+}

imgbom/scope/scope_test.go

@@ -1,95 +0,0 @@
-package scope
-
-import (
-	"testing"
-
-	"github.com/anchore/stereoscope/pkg/image"
-	"github.com/anchore/stereoscope/pkg/tree"
-)
-
-func testScopeImage(t *testing.T) *image.Image {
-	t.Helper()
-
-	one := image.NewLayer(nil)
-	one.Tree = tree.NewFileTree()
-	one.Tree.AddPath("/tree/first/path.txt")
-
-	two := image.NewLayer(nil)
-	two.Tree = tree.NewFileTree()
-	two.Tree.AddPath("/tree/second/path.txt")
-
-	i := image.NewImage(nil)
-	i.Layers = []image.Layer{one, two}
-	err := i.Squash()
-	if err != nil {
-		t.Fatal("could not squash test image trees")
-	}
-
-	return i
-}
-
-func TestScope(t *testing.T) {
-	refImg := testScopeImage(t)
-
-	cases := []struct {
-		name          string
-		img           *image.Image
-		option        Option
-		expectedTrees []*tree.FileTree
-		err           bool
-	}{
-		{
-			name:          "AllLayersGoCase",
-			option:        AllLayersScope,
-			img:           testScopeImage(t),
-			expectedTrees: []*tree.FileTree{refImg.Layers[0].Tree, refImg.Layers[1].Tree},
-		},
-		{
-			name:          "SquashedGoCase",
-			option:        SquashedScope,
-			img:           testScopeImage(t),
-			expectedTrees: []*tree.FileTree{refImg.SquashedTree},
-		},
-		{
-			name:   "MissingImage",
-			option: SquashedScope,
-			err:    true,
-		},
-		{
-			name:   "MissingSquashedTree",
-			option: SquashedScope,
-			img:    image.NewImage(nil),
-			err:    true,
-		},
-		{
-			name:   "NoLayers",
-			option: AllLayersScope,
-			img:    image.NewImage(nil),
-			err:    true,
-		},
-	}
-
-	for _, c := range cases {
-		actual, err := NewScope(c.img, c.option)
-		if err == nil && c.err {
-			t.Fatal("expected an error but did not find one")
-		} else if err != nil && !c.err {
-			t.Fatal("expected no error but found one:", err)
-		}
-
-		if len(actual.Trees) != len(c.expectedTrees) {
-			t.Fatalf("mismatched tree lengths: %d!=%d", len(actual.Trees), len(c.expectedTrees))
-		}
-
-		for idx, atr := range actual.Trees {
-			at, ok := atr.(*tree.FileTree)
-			if !ok {
-				t.Fatalf("could not extract tree from reader")
-			}
-			if !at.Equal(c.expectedTrees[idx]) {
-				t.Error("mismatched tree @ idx", idx)
-			}
-		}
-	}
-
-}

integration/fixture_image_distro_test.go

@@ -0,0 +1,35 @@
+// +build integration
+
+package integration
+
+import (
+	"testing"
+
+	"github.com/anchore/go-testutils"
+	"github.com/anchore/imgbom/imgbom"
+	"github.com/anchore/imgbom/imgbom/distro"
+	"github.com/go-test/deep"
+)
+
+func TestDistroImage(t *testing.T) {
+	img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-distro-id")
+	defer cleanup()
+
+	actual := imgbom.IdentifyDistro(img)
+	if actual == nil {
+		t.Fatalf("could not find distro")
+	}
+
+	expected, err := distro.NewDistro(distro.Busybox, "1.31.1")
+	if err != nil {
+		t.Fatalf("could not create distro: %+v", err)
+	}
+
+	diffs := deep.Equal(*actual, expected)
+	if len(diffs) != 0 {
+		for _, d := range diffs {
+			t.Errorf("found distro difference: %+v", d)
+		}
+	}
+
+}

integration/fixture_image_language_pkgs_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/anchore/go-testutils"
 	"github.com/anchore/imgbom/imgbom"
+	"github.com/anchore/imgbom/imgbom/cataloger"
 	"github.com/anchore/imgbom/imgbom/pkg"
 	"github.com/anchore/imgbom/imgbom/scope"
 )
@@ -137,4 +138,9 @@ func TestLanguageImage(t *testing.T) {
 		})
 	}
 
+	// ensure that integration test cases stay in sync with the available catalogers
+	if len(cataloger.Catalogers()) < len(cases) {
+		t.Fatalf("probably missed a cataloger during testing, double check that all catalogers are included in testing")
+	}
+
 }

integration/test-fixtures/image-distro-id/Dockerfile

@@ -0,0 +1,3 @@
+FROM busybox:1.31.1
+
+