add symlink content fetching support

imgbom/scope/resolvers/all_layers_resolver.go

@@ -0,0 +1,106 @@
+package resolvers
+
+import (
+	"archive/tar"
+	"fmt"
+
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/image"
+)
+
+type AllLayersResolver struct {
+	img    *image.Image
+	layers []int
+}
+
+func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) {
+	if len(img.Layers) == 0 {
+		return nil, fmt.Errorf("the image does not contain any layers")
+	}
+
+	var layers = make([]int, 0)
+	for idx := range img.Layers {
+		layers = append(layers, idx)
+	}
+	return &AllLayersResolver{
+		img:    img,
+		layers: layers,
+	}, nil
+}
+
+func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) {
+	uniqueFiles := make([]file.Reference, 0)
+
+	// since there is potentially considerable work for each symlink/hardlink that needs to be resolved, let's check to see if this is a symlink/hardlink first
+	entry, err := r.img.FileCatalog.Get(ref)
+	if err != nil {
+		return nil, fmt.Errorf("unable to fetch metadata (ref=%+v): %w", ref, err)
+	}
+
+	if entry.Metadata.TypeFlag == tar.TypeLink || entry.Metadata.TypeFlag == tar.TypeSymlink {
+		// a link may resolve in this layer or higher, assuming a squashed tree is used to search
+		// we should search all possible resolutions within the valid scope
+		for _, subLayerIdx := range r.layers[layerIdx:] {
+			resolvedRef, err := r.img.ResolveLinkByLayerSquash(ref, subLayerIdx)
+			if err != nil {
+				return nil, fmt.Errorf("failed to resolve link from layer (layer=%d ref=%+v): %w", subLayerIdx, ref, err)
+			}
+			if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
+				uniqueFileIDs.Add(*resolvedRef)
+				uniqueFiles = append(uniqueFiles, *resolvedRef)
+			}
+		}
+	} else if !uniqueFileIDs.Contains(ref) {
+		uniqueFileIDs.Add(ref)
+		uniqueFiles = append(uniqueFiles, ref)
+	}
+
+	return uniqueFiles, nil
+}
+
+func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, path := range paths {
+		for idx, layerIdx := range r.layers {
+			ref := r.img.Layers[layerIdx].Tree.File(path)
+			if ref == nil {
+				// no file found, keep looking through layers
+				continue
+			}
+
+			results, err := r.fileByRef(*ref, uniqueFileIDs, idx)
+			if err != nil {
+				return nil, err
+			}
+			uniqueFiles = append(uniqueFiles, results...)
+		}
+	}
+
+	return uniqueFiles, nil
+}
+
+func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, pattern := range patterns {
+		for idx, layerIdx := range r.layers {
+			refs, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern)
+			if err != nil {
+				return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
+			}
+
+			for _, ref := range refs {
+				results, err := r.fileByRef(ref, uniqueFileIDs, idx)
+				if err != nil {
+					return nil, err
+				}
+				uniqueFiles = append(uniqueFiles, results...)
+			}
+		}
+	}
+
+	return uniqueFiles, nil
+}

imgbom/scope/resolvers/all_layers_resolver_test.go

@@ -0,0 +1,229 @@
+package resolvers
+
+import (
+	"testing"
+
+	"github.com/anchore/go-testutils"
+	"github.com/anchore/stereoscope/pkg/file"
+)
+
+type resolution struct {
+	layer uint
+	path  string
+}
+
+func TestAllLayersResolver_FilesByPath(t *testing.T) {
+	cases := []struct {
+		name        string
+		linkPath    string
+		resolutions []resolution
+	}{
+		{
+			name:     "link with previous data",
+			linkPath: "/link-1",
+			resolutions: []resolution{
+				{
+					layer: 1,
+					path:  "/file-1.txt",
+				},
+			},
+		},
+		{
+			name:     "link with in layer data",
+			linkPath: "/link-within",
+			resolutions: []resolution{
+				{
+					layer: 5,
+					path:  "/file-3.txt",
+				},
+			},
+		},
+		{
+			name:     "link with overridden data",
+			linkPath: "/link-2",
+			resolutions: []resolution{
+				{
+					layer: 3,
+					path:  "/link-2",
+				},
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name:     "indirect link (with overridden data)",
+			linkPath: "/link-indirect",
+			resolutions: []resolution{
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name:     "dead link",
+			linkPath: "/link-dead",
+			resolutions: []resolution{
+				{
+					layer: 8,
+					path:  "/link-dead",
+				},
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewAllLayersResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByPath(file.Path(c.linkPath))
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != len(c.resolutions) {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			for idx, actual := range refs {
+				expected := c.resolutions[idx]
+
+				if actual.Path != file.Path(expected.path) {
+					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				}
+
+				entry, err := img.FileCatalog.Get(actual)
+				if err != nil {
+					t.Fatalf("failed to get metadata: %+v", err)
+				}
+
+				if entry.Source.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				}
+			}
+		})
+	}
+}
+
+func TestAllLayersResolver_FilesByGlob(t *testing.T) {
+	cases := []struct {
+		name        string
+		glob        string
+		resolutions []resolution
+	}{
+		{
+			name: "link with previous data",
+			glob: "**ink-1",
+			resolutions: []resolution{
+				{
+					layer: 1,
+					path:  "/file-1.txt",
+				},
+			},
+		},
+		{
+			name: "link with in layer data",
+			glob: "**nk-within",
+			resolutions: []resolution{
+				{
+					layer: 5,
+					path:  "/file-3.txt",
+				},
+			},
+		},
+		{
+			name: "link with overridden data",
+			glob: "**ink-2",
+			resolutions: []resolution{
+				{
+					layer: 3,
+					path:  "/link-2",
+				},
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name: "indirect link (with overridden data)",
+			glob: "**nk-indirect",
+			resolutions: []resolution{
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name: "dead link",
+			glob: "**k-dead",
+			resolutions: []resolution{
+				{
+					layer: 8,
+					path:  "/link-dead",
+				},
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewAllLayersResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByGlob(c.glob)
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != len(c.resolutions) {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			for idx, actual := range refs {
+				expected := c.resolutions[idx]
+
+				if actual.Path != file.Path(expected.path) {
+					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				}
+
+				entry, err := img.FileCatalog.Get(actual)
+				if err != nil {
+					t.Fatalf("failed to get metadata: %+v", err)
+				}
+
+				if entry.Source.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				}
+			}
+		})
+	}
+}

imgbom/scope/resolvers/image_squash_resolver.go

@@ -0,0 +1,70 @@
+package resolvers
+
+import (
+	"fmt"
+
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/image"
+)
+
+type ImageSquashResolver struct {
+	img *image.Image
+}
+
+func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) {
+	if img.SquashedTree() == nil {
+		return nil, fmt.Errorf("the image does not have have a squashed tree")
+	}
+	return &ImageSquashResolver{img: img}, nil
+}
+
+func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, path := range paths {
+		ref := r.img.SquashedTree().File(path)
+		if ref == nil {
+			// no file found, keep looking through layers
+			continue
+		}
+
+		resolvedRef, err := r.img.ResolveLinkByImageSquash(*ref)
+		if err != nil {
+			return nil, fmt.Errorf("failed to resolve link from img (ref=%+v): %w", ref, err)
+		}
+		if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
+			uniqueFileIDs.Add(*resolvedRef)
+			uniqueFiles = append(uniqueFiles, *resolvedRef)
+		}
+	}
+
+	return uniqueFiles, nil
+}
+
+func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, pattern := range patterns {
+		refs, err := r.img.SquashedTree().FilesByGlob(pattern)
+		if err != nil {
+			return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
+		}
+
+		for _, ref := range refs {
+			resolvedRefs, err := r.FilesByPath(ref.Path)
+			if err != nil {
+				return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err)
+			}
+			for _, resolvedRef := range resolvedRefs {
+				if !uniqueFileIDs.Contains(resolvedRef) {
+					uniqueFileIDs.Add(resolvedRef)
+					uniqueFiles = append(uniqueFiles, resolvedRef)
+				}
+			}
+		}
+	}
+
+	return uniqueFiles, nil
+}

imgbom/scope/resolvers/image_squash_resolver_test.go

@@ -0,0 +1,158 @@
+package resolvers
+
+import (
+	"testing"
+
+	"github.com/anchore/go-testutils"
+	"github.com/anchore/stereoscope/pkg/file"
+)
+
+func TestImageSquashResolver_FilesByPath(t *testing.T) {
+	cases := []struct {
+		name         string
+		linkPath     string
+		resolveLayer uint
+		resolvePath  string
+	}{
+		{
+			name:         "link with previous data",
+			linkPath:     "/link-1",
+			resolveLayer: 1,
+			resolvePath:  "/file-1.txt",
+		},
+		{
+			name:         "link with in layer data",
+			linkPath:     "/link-within",
+			resolveLayer: 5,
+			resolvePath:  "/file-3.txt",
+		},
+		{
+			name:         "link with overridden data",
+			linkPath:     "/link-2",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "indirect link (with overridden data)",
+			linkPath:     "/link-indirect",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "dead link",
+			linkPath:     "/link-dead",
+			resolveLayer: 8,
+			resolvePath:  "/link-dead",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewImageSquashResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByPath(file.Path(c.linkPath))
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != 1 {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			actual := refs[0]
+
+			if actual.Path != file.Path(c.resolvePath) {
+				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			}
+
+			entry, err := img.FileCatalog.Get(actual)
+			if err != nil {
+				t.Fatalf("failed to get metadata: %+v", err)
+			}
+
+			if entry.Source.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			}
+		})
+	}
+}
+
+func TestImageSquashResolver_FilesByGlob(t *testing.T) {
+	cases := []struct {
+		name         string
+		glob         string
+		resolveLayer uint
+		resolvePath  string
+	}{
+		{
+			name:         "link with previous data",
+			glob:         "**link-1",
+			resolveLayer: 1,
+			resolvePath:  "/file-1.txt",
+		},
+		{
+			name:         "link with in layer data",
+			glob:         "**link-within",
+			resolveLayer: 5,
+			resolvePath:  "/file-3.txt",
+		},
+		{
+			name:         "link with overridden data",
+			glob:         "**link-2",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "indirect link (with overridden data)",
+			glob:         "**link-indirect",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "dead link",
+			glob:         "**link-dead",
+			resolveLayer: 8,
+			resolvePath:  "/link-dead",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewImageSquashResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByGlob(c.glob)
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != 1 {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			actual := refs[0]
+
+			if actual.Path != file.Path(c.resolvePath) {
+				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			}
+
+			entry, err := img.FileCatalog.Get(actual)
+			if err != nil {
+				t.Fatalf("failed to get metadata: %+v", err)
+			}
+
+			if entry.Source.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			}
+		})
+	}
+}

imgbom/scope/resolvers/test-fixtures/image-symlinks/Dockerfile

@@ -0,0 +1,24 @@
+# LAYER 0:
+FROM busybox:latest
+
+# LAYER 1:
+ADD file-1.txt .
+# LAYER 2: link with previous data
+RUN ln -s ./file-1.txt link-1
+
+# LAYER 3: link with future data
+RUN ln -s ./file-2.txt link-2
+# LAYER 4:
+ADD file-2.txt .
+
+# LAYER 5: link with current data
+RUN echo "file 3" > file-3.txt && ln -s ./file-3.txt link-within
+
+# LAYER 6: multiple links (link-indirect > link-2 > file-2.txt)
+RUN ln -s ./link-2 link-indirect
+
+# LAYER 7: override contents / resolution
+ADD new-file-2.txt file-2.txt
+
+# LAYER 8: dead link
+RUN ln -s ./i-dont-exist.txt link-dead

imgbom/scope/resolvers/test-fixtures/image-symlinks/file-1.txt

@@ -0,0 +1 @@
+file 1!

imgbom/scope/resolvers/test-fixtures/image-symlinks/file-2.txt

@@ -0,0 +1 @@
+file 2!

imgbom/scope/resolvers/test-fixtures/image-symlinks/new-file-2.txt

@@ -0,0 +1 @@
+NEW file override!