add symlink content fetching support

imgbom/scope/file_resolver.go

@@ -0,0 +1,25 @@
+package scope
+
+import (
+	"fmt"
+
+	"github.com/anchore/imgbom/imgbom/scope/resolvers"
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/image"
+)
+
+type FileResolver interface {
+	FilesByPath(paths ...file.Path) ([]file.Reference, error)
+	FilesByGlob(patterns ...string) ([]file.Reference, error)
+}
+
+func getFileResolver(img *image.Image, option Option) (FileResolver, error) {
+	switch option {
+	case SquashedScope:
+		return resolvers.NewImageSquashResolver(img)
+	case AllLayersScope:
+		return resolvers.NewAllLayersResolver(img)
+	default:
+		return nil, fmt.Errorf("bad option provided: %+v", option)
+	}
+}

imgbom/scope/resolvers/all_layers_resolver.go

@@ -0,0 +1,106 @@
+package resolvers
+
+import (
+	"archive/tar"
+	"fmt"
+
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/image"
+)
+
+type AllLayersResolver struct {
+	img    *image.Image
+	layers []int
+}
+
+func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) {
+	if len(img.Layers) == 0 {
+		return nil, fmt.Errorf("the image does not contain any layers")
+	}
+
+	var layers = make([]int, 0)
+	for idx := range img.Layers {
+		layers = append(layers, idx)
+	}
+	return &AllLayersResolver{
+		img:    img,
+		layers: layers,
+	}, nil
+}
+
+func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) {
+	uniqueFiles := make([]file.Reference, 0)
+
+	// since there is potentially considerable work for each symlink/hardlink that needs to be resolved, let's check to see if this is a symlink/hardlink first
+	entry, err := r.img.FileCatalog.Get(ref)
+	if err != nil {
+		return nil, fmt.Errorf("unable to fetch metadata (ref=%+v): %w", ref, err)
+	}
+
+	if entry.Metadata.TypeFlag == tar.TypeLink || entry.Metadata.TypeFlag == tar.TypeSymlink {
+		// a link may resolve in this layer or higher, assuming a squashed tree is used to search
+		// we should search all possible resolutions within the valid scope
+		for _, subLayerIdx := range r.layers[layerIdx:] {
+			resolvedRef, err := r.img.ResolveLinkByLayerSquash(ref, subLayerIdx)
+			if err != nil {
+				return nil, fmt.Errorf("failed to resolve link from layer (layer=%d ref=%+v): %w", subLayerIdx, ref, err)
+			}
+			if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
+				uniqueFileIDs.Add(*resolvedRef)
+				uniqueFiles = append(uniqueFiles, *resolvedRef)
+			}
+		}
+	} else if !uniqueFileIDs.Contains(ref) {
+		uniqueFileIDs.Add(ref)
+		uniqueFiles = append(uniqueFiles, ref)
+	}
+
+	return uniqueFiles, nil
+}
+
+func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, path := range paths {
+		for idx, layerIdx := range r.layers {
+			ref := r.img.Layers[layerIdx].Tree.File(path)
+			if ref == nil {
+				// no file found, keep looking through layers
+				continue
+			}
+
+			results, err := r.fileByRef(*ref, uniqueFileIDs, idx)
+			if err != nil {
+				return nil, err
+			}
+			uniqueFiles = append(uniqueFiles, results...)
+		}
+	}
+
+	return uniqueFiles, nil
+}
+
+func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, pattern := range patterns {
+		for idx, layerIdx := range r.layers {
+			refs, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern)
+			if err != nil {
+				return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
+			}
+
+			for _, ref := range refs {
+				results, err := r.fileByRef(ref, uniqueFileIDs, idx)
+				if err != nil {
+					return nil, err
+				}
+				uniqueFiles = append(uniqueFiles, results...)
+			}
+		}
+	}
+
+	return uniqueFiles, nil
+}

imgbom/scope/resolvers/all_layers_resolver_test.go

@@ -0,0 +1,229 @@
+package resolvers
+
+import (
+	"testing"
+
+	"github.com/anchore/go-testutils"
+	"github.com/anchore/stereoscope/pkg/file"
+)
+
+type resolution struct {
+	layer uint
+	path  string
+}
+
+func TestAllLayersResolver_FilesByPath(t *testing.T) {
+	cases := []struct {
+		name        string
+		linkPath    string
+		resolutions []resolution
+	}{
+		{
+			name:     "link with previous data",
+			linkPath: "/link-1",
+			resolutions: []resolution{
+				{
+					layer: 1,
+					path:  "/file-1.txt",
+				},
+			},
+		},
+		{
+			name:     "link with in layer data",
+			linkPath: "/link-within",
+			resolutions: []resolution{
+				{
+					layer: 5,
+					path:  "/file-3.txt",
+				},
+			},
+		},
+		{
+			name:     "link with overridden data",
+			linkPath: "/link-2",
+			resolutions: []resolution{
+				{
+					layer: 3,
+					path:  "/link-2",
+				},
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name:     "indirect link (with overridden data)",
+			linkPath: "/link-indirect",
+			resolutions: []resolution{
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name:     "dead link",
+			linkPath: "/link-dead",
+			resolutions: []resolution{
+				{
+					layer: 8,
+					path:  "/link-dead",
+				},
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewAllLayersResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByPath(file.Path(c.linkPath))
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != len(c.resolutions) {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			for idx, actual := range refs {
+				expected := c.resolutions[idx]
+
+				if actual.Path != file.Path(expected.path) {
+					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				}
+
+				entry, err := img.FileCatalog.Get(actual)
+				if err != nil {
+					t.Fatalf("failed to get metadata: %+v", err)
+				}
+
+				if entry.Source.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				}
+			}
+		})
+	}
+}
+
+func TestAllLayersResolver_FilesByGlob(t *testing.T) {
+	cases := []struct {
+		name        string
+		glob        string
+		resolutions []resolution
+	}{
+		{
+			name: "link with previous data",
+			glob: "**ink-1",
+			resolutions: []resolution{
+				{
+					layer: 1,
+					path:  "/file-1.txt",
+				},
+			},
+		},
+		{
+			name: "link with in layer data",
+			glob: "**nk-within",
+			resolutions: []resolution{
+				{
+					layer: 5,
+					path:  "/file-3.txt",
+				},
+			},
+		},
+		{
+			name: "link with overridden data",
+			glob: "**ink-2",
+			resolutions: []resolution{
+				{
+					layer: 3,
+					path:  "/link-2",
+				},
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name: "indirect link (with overridden data)",
+			glob: "**nk-indirect",
+			resolutions: []resolution{
+				{
+					layer: 4,
+					path:  "/file-2.txt",
+				},
+				{
+					layer: 7,
+					path:  "/file-2.txt",
+				},
+			},
+		},
+		{
+			name: "dead link",
+			glob: "**k-dead",
+			resolutions: []resolution{
+				{
+					layer: 8,
+					path:  "/link-dead",
+				},
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewAllLayersResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByGlob(c.glob)
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != len(c.resolutions) {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			for idx, actual := range refs {
+				expected := c.resolutions[idx]
+
+				if actual.Path != file.Path(expected.path) {
+					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				}
+
+				entry, err := img.FileCatalog.Get(actual)
+				if err != nil {
+					t.Fatalf("failed to get metadata: %+v", err)
+				}
+
+				if entry.Source.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				}
+			}
+		})
+	}
+}

imgbom/scope/resolvers/image_squash_resolver.go

@@ -0,0 +1,70 @@
+package resolvers
+
+import (
+	"fmt"
+
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/image"
+)
+
+type ImageSquashResolver struct {
+	img *image.Image
+}
+
+func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) {
+	if img.SquashedTree() == nil {
+		return nil, fmt.Errorf("the image does not have have a squashed tree")
+	}
+	return &ImageSquashResolver{img: img}, nil
+}
+
+func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, path := range paths {
+		ref := r.img.SquashedTree().File(path)
+		if ref == nil {
+			// no file found, keep looking through layers
+			continue
+		}
+
+		resolvedRef, err := r.img.ResolveLinkByImageSquash(*ref)
+		if err != nil {
+			return nil, fmt.Errorf("failed to resolve link from img (ref=%+v): %w", ref, err)
+		}
+		if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
+			uniqueFileIDs.Add(*resolvedRef)
+			uniqueFiles = append(uniqueFiles, *resolvedRef)
+		}
+	}
+
+	return uniqueFiles, nil
+}
+
+func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+	uniqueFileIDs := file.NewFileReferenceSet()
+	uniqueFiles := make([]file.Reference, 0)
+
+	for _, pattern := range patterns {
+		refs, err := r.img.SquashedTree().FilesByGlob(pattern)
+		if err != nil {
+			return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
+		}
+
+		for _, ref := range refs {
+			resolvedRefs, err := r.FilesByPath(ref.Path)
+			if err != nil {
+				return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err)
+			}
+			for _, resolvedRef := range resolvedRefs {
+				if !uniqueFileIDs.Contains(resolvedRef) {
+					uniqueFileIDs.Add(resolvedRef)
+					uniqueFiles = append(uniqueFiles, resolvedRef)
+				}
+			}
+		}
+	}
+
+	return uniqueFiles, nil
+}

imgbom/scope/resolvers/image_squash_resolver_test.go

@@ -0,0 +1,158 @@
+package resolvers
+
+import (
+	"testing"
+
+	"github.com/anchore/go-testutils"
+	"github.com/anchore/stereoscope/pkg/file"
+)
+
+func TestImageSquashResolver_FilesByPath(t *testing.T) {
+	cases := []struct {
+		name         string
+		linkPath     string
+		resolveLayer uint
+		resolvePath  string
+	}{
+		{
+			name:         "link with previous data",
+			linkPath:     "/link-1",
+			resolveLayer: 1,
+			resolvePath:  "/file-1.txt",
+		},
+		{
+			name:         "link with in layer data",
+			linkPath:     "/link-within",
+			resolveLayer: 5,
+			resolvePath:  "/file-3.txt",
+		},
+		{
+			name:         "link with overridden data",
+			linkPath:     "/link-2",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "indirect link (with overridden data)",
+			linkPath:     "/link-indirect",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "dead link",
+			linkPath:     "/link-dead",
+			resolveLayer: 8,
+			resolvePath:  "/link-dead",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewImageSquashResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByPath(file.Path(c.linkPath))
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != 1 {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			actual := refs[0]
+
+			if actual.Path != file.Path(c.resolvePath) {
+				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			}
+
+			entry, err := img.FileCatalog.Get(actual)
+			if err != nil {
+				t.Fatalf("failed to get metadata: %+v", err)
+			}
+
+			if entry.Source.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			}
+		})
+	}
+}
+
+func TestImageSquashResolver_FilesByGlob(t *testing.T) {
+	cases := []struct {
+		name         string
+		glob         string
+		resolveLayer uint
+		resolvePath  string
+	}{
+		{
+			name:         "link with previous data",
+			glob:         "**link-1",
+			resolveLayer: 1,
+			resolvePath:  "/file-1.txt",
+		},
+		{
+			name:         "link with in layer data",
+			glob:         "**link-within",
+			resolveLayer: 5,
+			resolvePath:  "/file-3.txt",
+		},
+		{
+			name:         "link with overridden data",
+			glob:         "**link-2",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "indirect link (with overridden data)",
+			glob:         "**link-indirect",
+			resolveLayer: 7,
+			resolvePath:  "/file-2.txt",
+		},
+		{
+			name:         "dead link",
+			glob:         "**link-dead",
+			resolveLayer: 8,
+			resolvePath:  "/link-dead",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			img, cleanup := testutils.GetFixtureImage(t, "docker-archive", "image-symlinks")
+			defer cleanup()
+
+			resolver, err := NewImageSquashResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			refs, err := resolver.FilesByGlob(c.glob)
+			if err != nil {
+				t.Fatalf("could not use resolver: %+v", err)
+			}
+
+			if len(refs) != 1 {
+				t.Fatalf("unexpected number of resolutions: %d", len(refs))
+			}
+
+			actual := refs[0]
+
+			if actual.Path != file.Path(c.resolvePath) {
+				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			}
+
+			entry, err := img.FileCatalog.Get(actual)
+			if err != nil {
+				t.Fatalf("failed to get metadata: %+v", err)
+			}
+
+			if entry.Source.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			}
+		})
+	}
+}

imgbom/scope/resolvers/test-fixtures/image-symlinks/Dockerfile

@@ -0,0 +1,24 @@
+# LAYER 0:
+FROM busybox:latest
+
+# LAYER 1:
+ADD file-1.txt .
+# LAYER 2: link with previous data
+RUN ln -s ./file-1.txt link-1
+
+# LAYER 3: link with future data
+RUN ln -s ./file-2.txt link-2
+# LAYER 4:
+ADD file-2.txt .
+
+# LAYER 5: link with current data
+RUN echo "file 3" > file-3.txt && ln -s ./file-3.txt link-within
+
+# LAYER 6: multiple links (link-indirect > link-2 > file-2.txt)
+RUN ln -s ./link-2 link-indirect
+
+# LAYER 7: override contents / resolution
+ADD new-file-2.txt file-2.txt
+
+# LAYER 8: dead link
+RUN ln -s ./i-dont-exist.txt link-dead

imgbom/scope/resolvers/test-fixtures/image-symlinks/file-1.txt

@@ -0,0 +1 @@
+file 1!

imgbom/scope/resolvers/test-fixtures/image-symlinks/file-2.txt

@@ -0,0 +1 @@
+file 2!

imgbom/scope/resolvers/test-fixtures/image-symlinks/new-file-2.txt

@@ -0,0 +1 @@
+NEW file override!

imgbom/scope/scope.go

@@ -3,44 +3,37 @@ package scope
 import (
 	"fmt"
 
+	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/image"
-	"github.com/anchore/stereoscope/pkg/tree"
 )
 
 type Scope struct {
-	Option Option
-	Trees  []tree.FileTreeReader
-	Image  *image.Image
+	Option   Option
+	resolver FileResolver
+	Image    *image.Image
 }
 
 func NewScope(img *image.Image, option Option) (Scope, error) {
-	var trees = make([]tree.FileTreeReader, 0)
-
 	if img == nil {
 		return Scope{}, fmt.Errorf("no image given")
 	}
 
-	switch option {
-	case SquashedScope:
-		if img.SquashedTree == nil {
-			return Scope{}, fmt.Errorf("the image does not have have a squashed tree")
-		}
-		trees = append(trees, img.SquashedTree)
-
-	case AllLayersScope:
-		if len(img.Layers) == 0 {
-			return Scope{}, fmt.Errorf("the image does not contain any layers")
-		}
-		for _, layer := range img.Layers {
-			trees = append(trees, layer.Tree)
-		}
-	default:
-		return Scope{}, fmt.Errorf("bad option provided: %+v", option)
+	resolver, err := getFileResolver(img, option)
+	if err != nil {
+		return Scope{}, fmt.Errorf("could not determine file resolver: %w", err)
 	}
 
 	return Scope{
-		Option: option,
-		Trees:  trees,
-		Image:  img,
+		Option:   option,
+		resolver: resolver,
+		Image:    img,
 	}, nil
 }
+
+func (s Scope) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+	return s.resolver.FilesByPath(paths...)
+}
+
+func (s Scope) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+	return s.resolver.FilesByGlob(patterns...)
+}

imgbom/scope/scope_test.go

@@ -1,95 +0,0 @@
-package scope
-
-import (
-	"testing"
-
-	"github.com/anchore/stereoscope/pkg/image"
-	"github.com/anchore/stereoscope/pkg/tree"
-)
-
-func testScopeImage(t *testing.T) *image.Image {
-	t.Helper()
-
-	one := image.NewLayer(nil)
-	one.Tree = tree.NewFileTree()
-	one.Tree.AddPath("/tree/first/path.txt")
-
-	two := image.NewLayer(nil)
-	two.Tree = tree.NewFileTree()
-	two.Tree.AddPath("/tree/second/path.txt")
-
-	i := image.NewImage(nil)
-	i.Layers = []image.Layer{one, two}
-	err := i.Squash()
-	if err != nil {
-		t.Fatal("could not squash test image trees")
-	}
-
-	return i
-}
-
-func TestScope(t *testing.T) {
-	refImg := testScopeImage(t)
-
-	cases := []struct {
-		name          string
-		img           *image.Image
-		option        Option
-		expectedTrees []*tree.FileTree
-		err           bool
-	}{
-		{
-			name:          "AllLayersGoCase",
-			option:        AllLayersScope,
-			img:           testScopeImage(t),
-			expectedTrees: []*tree.FileTree{refImg.Layers[0].Tree, refImg.Layers[1].Tree},
-		},
-		{
-			name:          "SquashedGoCase",
-			option:        SquashedScope,
-			img:           testScopeImage(t),
-			expectedTrees: []*tree.FileTree{refImg.SquashedTree},
-		},
-		{
-			name:   "MissingImage",
-			option: SquashedScope,
-			err:    true,
-		},
-		{
-			name:   "MissingSquashedTree",
-			option: SquashedScope,
-			img:    image.NewImage(nil),
-			err:    true,
-		},
-		{
-			name:   "NoLayers",
-			option: AllLayersScope,
-			img:    image.NewImage(nil),
-			err:    true,
-		},
-	}
-
-	for _, c := range cases {
-		actual, err := NewScope(c.img, c.option)
-		if err == nil && c.err {
-			t.Fatal("expected an error but did not find one")
-		} else if err != nil && !c.err {
-			t.Fatal("expected no error but found one:", err)
-		}
-
-		if len(actual.Trees) != len(c.expectedTrees) {
-			t.Fatalf("mismatched tree lengths: %d!=%d", len(actual.Trees), len(c.expectedTrees))
-		}
-
-		for idx, atr := range actual.Trees {
-			at, ok := atr.(*tree.FileTree)
-			if !ok {
-				t.Fatalf("could not extract tree from reader")
-			}
-			if !at.Equal(c.expectedTrees[idx]) {
-				t.Error("mismatched tree @ idx", idx)
-			}
-		}
-	}
-
-}