Fix symlink resolutions for constituent paths (#304)

* bump stereoscope to pull in content API refactors

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* incorporate symlink fixes

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* with filetree.File() adjustments

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* regress all-layers scope to not include dead-links + default tests to squashed scope

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* restore all layers resolver glob behavior (custom + lazy link resolution)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* incorporate filetree link resolution options and restore no-follow dead link option for resolvers

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* removed path from lower-level FileTree.File() calls

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* bump stereoscope to pull in latest link resolution fixes

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* bump doublestar to v2 for directory resolver

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

syft/source/all_layers_resolver.go

@@ -5,6 +5,10 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
+	"github.com/anchore/syft/internal/log"
+
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/image"
 )
@@ -71,19 +75,22 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
 	for _, path := range paths {
 		for idx, layerIdx := range r.layers {
 			tree := r.img.Layers[layerIdx].Tree
-			ref := tree.File(file.Path(path))
+			_, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks, filetree.DoNotFollowDeadBasenameLinks)
+			if err != nil {
+				return nil, err
+			}
 			if ref == nil {
 				// no file found, keep looking through layers
 				continue
 			}
 
 			// don't consider directories (special case: there is no path information for /)
-			if ref.Path == "/" {
+			if ref.RealPath == "/" {
 				continue
 			} else if r.img.FileCatalog.Exists(*ref) {
 				metadata, err := r.img.FileCatalog.Get(*ref)
 				if err != nil {
-					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
 				}
 				if metadata.Metadata.IsDir {
 					continue
@@ -110,31 +117,31 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error)
 
 	for _, pattern := range patterns {
 		for idx, layerIdx := range r.layers {
-			refs, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern)
+			results, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern, filetree.DoNotFollowDeadBasenameLinks)
 			if err != nil {
 				return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
 			}
 
-			for _, ref := range refs {
+			for _, result := range results {
 				// don't consider directories (special case: there is no path information for /)
-				if ref.Path == "/" {
+				if result.RealPath == "/" {
 					continue
-				} else if r.img.FileCatalog.Exists(ref) {
-					metadata, err := r.img.FileCatalog.Get(ref)
+				} else if r.img.FileCatalog.Exists(result.Reference) {
+					metadata, err := r.img.FileCatalog.Get(result.Reference)
 					if err != nil {
-						return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+						return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err)
 					}
 					if metadata.Metadata.IsDir {
 						continue
 					}
 				}
 
-				results, err := r.fileByRef(ref, uniqueFileIDs, idx)
+				refResults, err := r.fileByRef(result.Reference, uniqueFileIDs, idx)
 				if err != nil {
 					return nil, err
 				}
-				for _, result := range results {
-					uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
+				for _, refResult := range refResults {
+					uniqueLocations = append(uniqueLocations, NewLocationFromImage(refResult, r.img))
 				}
 			}
 		}
@@ -151,8 +158,12 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *
 		return nil
 	}
 
-	relativeRef := entry.Layer.SquashedTree.File(file.Path(path))
-	if relativeRef == nil {
+	exists, relativeRef, err := entry.Layer.SquashedTree.File(file.Path(path), filetree.FollowBasenameLinks)
+	if err != nil {
+		log.Errorf("failed to find path=%q in squash: %+w", path, err)
+		return nil
+	}
+	if !exists && relativeRef == nil {
 		return nil
 	}
 

syft/source/all_layers_resolver_test.go

@@ -41,10 +41,6 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			name:     "link with overridden data",
 			linkPath: "/link-2",
 			resolutions: []resolution{
-				{
-					layer: 3,
-					path:  "/link-2",
-				},
 				{
 					layer: 4,
 					path:  "/file-2.txt",
@@ -70,14 +66,9 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			},
 		},
 		{
-			name:     "dead link",
-			linkPath: "/link-dead",
-			resolutions: []resolution{
-				{
-					layer: 8,
-					path:  "/link-dead",
-				},
-			},
+			name:        "dead link",
+			linkPath:    "/link-dead",
+			resolutions: []resolution{},
 		},
 		{
 			name:        "ignore directories",
@@ -132,7 +123,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 	}{
 		{
 			name: "link with previous data",
-			glob: "**ink-1",
+			glob: "**/*ink-1",
 			resolutions: []resolution{
 				{
 					layer: 1,
@@ -142,7 +133,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "link with in layer data",
-			glob: "**nk-within",
+			glob: "**/*nk-within",
 			resolutions: []resolution{
 				{
 					layer: 5,
@@ -152,12 +143,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "link with overridden data",
-			glob: "**ink-2",
+			glob: "**/*ink-2",
 			resolutions: []resolution{
-				{
-					layer: 3,
-					path:  "/link-2",
-				},
 				{
 					layer: 4,
 					path:  "/file-2.txt",
@@ -170,7 +157,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "indirect link (with overridden data)",
-			glob: "**nk-indirect",
+			glob: "**/*nk-indirect",
 			resolutions: []resolution{
 				{
 					layer: 4,
@@ -183,14 +170,9 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 			},
 		},
 		{
-			name: "dead link",
-			glob: "**k-dead",
-			resolutions: []resolution{
-				{
-					layer: 8,
-					path:  "/link-dead",
-				},
-			},
+			name:        "dead link",
+			glob:        "**/*k-dead",
+			resolutions: []resolution{},
 		},
 		{
 			name:        "ignore directories",

syft/source/directory_resolver.go

@@ -8,9 +8,8 @@ import (
 	"path/filepath"
 
 	"github.com/anchore/stereoscope/pkg/file"
-
 	"github.com/anchore/syft/internal/log"
-	"github.com/bmatcuk/doublestar"
+	"github.com/bmatcuk/doublestar/v2"
 )
 
 var _ Resolver = (*DirectoryResolver)(nil)

syft/source/directory_resolver_test.go

@@ -171,8 +171,9 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 2 {
-			t.Errorf("unexpected number of refs: %d != 2", len(refs))
+		expected := 2
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})
@@ -187,8 +188,9 @@ func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 4 {
-			t.Errorf("unexpected number of refs: %d != 4", len(refs))
+		expected := 6
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})
@@ -202,8 +204,9 @@ func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 1 {
-			t.Errorf("unexpected number of refs: %d != 1", len(refs))
+		expected := 1
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})

syft/source/image_squash_resolver.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/image"
 )
@@ -30,19 +32,22 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 
 	for _, path := range paths {
 		tree := r.img.SquashedTree()
-		ref := tree.File(file.Path(path))
+		_, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks)
+		if err != nil {
+			return nil, err
+		}
 		if ref == nil {
 			// no file found, keep looking through layers
 			continue
 		}
 
 		// don't consider directories (special case: there is no path information for /)
-		if ref.Path == "/" {
+		if ref.RealPath == "/" {
 			continue
 		} else if r.img.FileCatalog.Exists(*ref) {
 			metadata, err := r.img.FileCatalog.Get(*ref)
 			if err != nil {
-				return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+				return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
 			}
 			if metadata.Metadata.IsDir {
 				continue
@@ -70,28 +75,28 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error
 	uniqueLocations := make([]Location, 0)
 
 	for _, pattern := range patterns {
-		refs, err := r.img.SquashedTree().FilesByGlob(pattern)
+		results, err := r.img.SquashedTree().FilesByGlob(pattern)
 		if err != nil {
 			return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
 		}
 
-		for _, ref := range refs {
+		for _, result := range results {
 			// don't consider directories (special case: there is no path information for /)
-			if ref.Path == "/" {
+			if result.MatchPath == "/" {
 				continue
-			} else if r.img.FileCatalog.Exists(ref) {
-				metadata, err := r.img.FileCatalog.Get(ref)
+			} else if r.img.FileCatalog.Exists(result.Reference) {
+				metadata, err := r.img.FileCatalog.Get(result.Reference)
 				if err != nil {
-					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err)
 				}
 				if metadata.Metadata.IsDir {
 					continue
 				}
 			}
 
-			resolvedLocations, err := r.FilesByPath(string(ref.Path))
+			resolvedLocations, err := r.FilesByPath(string(result.MatchPath))
 			if err != nil {
-				return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err)
+				return nil, fmt.Errorf("failed to find files by path (result=%+v): %w", result, err)
 			}
 			for _, resolvedLocation := range resolvedLocations {
 				if !uniqueFileIDs.Contains(resolvedLocation.ref) {

syft/source/image_squash_resolver_test.go

@@ -41,13 +41,19 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 			name:         "dead link",
 			linkPath:     "/link-dead",
 			resolveLayer: 8,
-			resolvePath:  "/link-dead",
+			resolvePath:  "",
 		},
 		{
 			name:        "ignore directories",
 			linkPath:    "/bin",
 			resolvePath: "",
 		},
+		{
+			name:         "parent is a link (with overridden data)",
+			linkPath:     "/parent-link/file-4.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
@@ -105,39 +111,51 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 	}{
 		{
 			name:         "link with previous data",
-			glob:         "**link-1",
+			glob:         "**/link-1",
 			resolveLayer: 1,
 			resolvePath:  "/file-1.txt",
 		},
 		{
 			name:         "link with in layer data",
-			glob:         "**link-within",
+			glob:         "**/link-within",
 			resolveLayer: 5,
 			resolvePath:  "/file-3.txt",
 		},
 		{
 			name:         "link with overridden data",
-			glob:         "**link-2",
+			glob:         "**/link-2",
 			resolveLayer: 7,
 			resolvePath:  "/file-2.txt",
 		},
 		{
 			name:         "indirect link (with overridden data)",
-			glob:         "**link-indirect",
+			glob:         "**/link-indirect",
 			resolveLayer: 7,
 			resolvePath:  "/file-2.txt",
 		},
 		{
-			name:         "dead link",
-			glob:         "**link-dead",
-			resolveLayer: 8,
-			resolvePath:  "/link-dead",
+			name: "dead link",
+			glob: "**/link-dead",
+			// dead links are dead! they shouldn't match on globs
+			resolvePath: "",
 		},
 		{
 			name:        "ignore directories",
 			glob:        "**/bin",
 			resolvePath: "",
 		},
+		{
+			name:         "parent without link",
+			glob:         "**/parent/*.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
+		},
+		{
+			name:         "parent is a link (override)",
+			glob:         "**/parent-link/file-4.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {

syft/source/location.go

@@ -27,13 +27,13 @@ func NewLocationFromImage(ref file.Reference, img *image.Image) Location {
 	if err != nil {
 		log.Warnf("unable to find file catalog entry for ref=%+v", ref)
 		return Location{
-			Path: string(ref.Path),
+			Path: string(ref.RealPath),
 			ref:  ref,
 		}
 	}
 
 	return Location{
-		Path:         string(ref.Path),
+		Path:         string(ref.RealPath),
 		FileSystemID: entry.Layer.Metadata.Digest,
 		ref:          ref,
 	}

syft/source/test-fixtures/image-symlinks/Dockerfile

@@ -21,4 +21,12 @@ RUN ln -s ./link-2 link-indirect
 ADD new-file-2.txt file-2.txt
 
 # LAYER 8: dead link
-RUN ln -s ./i-dont-exist.txt link-dead
+RUN ln -s ./i-dont-exist.txt link-dead
+
+# LAYER 9: add the parent dir
+ADD parent /parent
+# LAYER 10: parent is a symlink
+RUN ln -s /parent parent-link
+
+# LAYER 11: parent is a symlink and the child target is overridden
+COPY new-file-4.txt /parent-link/file-4.txt

syft/source/test-fixtures/image-symlinks/new-file-4.txt

@@ -0,0 +1 @@
+override file 4!

syft/source/test-fixtures/image-symlinks/parent/file-4.txt

@@ -0,0 +1 @@
+the 4th file!