rename scope to source

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2020-11-17 12:37:11 -05:00
parent 495fb0a45f
commit 9668341a14
55 changed files with 340 additions and 334 deletions
+158
View File
@@ -0,0 +1,158 @@
package source
import (
"archive/tar"
"fmt"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
)
// AllLayersResolver implements path and content access for the AllLayers source option for container image data sources.
type AllLayersResolver struct {
img *image.Image
layers []int
}
// NewAllLayersResolver returns a new resolver from the perspective of all image layers for the given image.
func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) {
if len(img.Layers) == 0 {
return nil, fmt.Errorf("the image does not contain any layers")
}
var layers = make([]int, 0)
for idx := range img.Layers {
layers = append(layers, idx)
}
return &AllLayersResolver{
img: img,
layers: layers,
}, nil
}
func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) {
uniqueFiles := make([]file.Reference, 0)
// since there is potentially considerable work for each symlink/hardlink that needs to be resolved, let's check to see if this is a symlink/hardlink first
entry, err := r.img.FileCatalog.Get(ref)
if err != nil {
return nil, fmt.Errorf("unable to fetch metadata (ref=%+v): %w", ref, err)
}
if entry.Metadata.TypeFlag == tar.TypeLink || entry.Metadata.TypeFlag == tar.TypeSymlink {
// a link may resolve in this layer or higher, assuming a squashed tree is used to search
// we should search all possible resolutions within the valid source
for _, subLayerIdx := range r.layers[layerIdx:] {
resolvedRef, err := r.img.ResolveLinkByLayerSquash(ref, subLayerIdx)
if err != nil {
return nil, fmt.Errorf("failed to resolve link from layer (layer=%d ref=%+v): %w", subLayerIdx, ref, err)
}
if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
uniqueFileIDs.Add(*resolvedRef)
uniqueFiles = append(uniqueFiles, *resolvedRef)
}
}
} else if !uniqueFileIDs.Contains(ref) {
uniqueFileIDs.Add(ref)
uniqueFiles = append(uniqueFiles, ref)
}
return uniqueFiles, nil
}
// FilesByPath returns all file.References that match the given paths from any layer in the image.
func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueFiles := make([]file.Reference, 0)
for _, path := range paths {
for idx, layerIdx := range r.layers {
tree := r.img.Layers[layerIdx].Tree
ref := tree.File(path)
if ref == nil {
// no file found, keep looking through layers
continue
}
// don't consider directories (special case: there is no path information for /)
if ref.Path == "/" {
continue
} else if r.img.FileCatalog.Exists(*ref) {
metadata, err := r.img.FileCatalog.Get(*ref)
if err != nil {
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
}
if metadata.Metadata.IsDir {
continue
}
}
results, err := r.fileByRef(*ref, uniqueFileIDs, idx)
if err != nil {
return nil, err
}
uniqueFiles = append(uniqueFiles, results...)
}
}
return uniqueFiles, nil
}
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueFiles := make([]file.Reference, 0)
for _, pattern := range patterns {
for idx, layerIdx := range r.layers {
refs, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern)
if err != nil {
return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
}
for _, ref := range refs {
// don't consider directories (special case: there is no path information for /)
if ref.Path == "/" {
continue
} else if r.img.FileCatalog.Exists(ref) {
metadata, err := r.img.FileCatalog.Get(ref)
if err != nil {
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
}
if metadata.Metadata.IsDir {
continue
}
}
results, err := r.fileByRef(ref, uniqueFileIDs, idx)
if err != nil {
return nil, err
}
uniqueFiles = append(uniqueFiles, results...)
}
}
}
return uniqueFiles, nil
}
func (r *AllLayersResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) {
entry, err := r.img.FileCatalog.Get(reference)
if err != nil {
return nil, err
}
return entry.Source.SquashedTree.File(file.Path(path)), nil
}
// MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a
// file.Reference is a path relative to a particular layer.
func (r *AllLayersResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) {
return r.img.MultipleFileContentsByRef(f...)
}
// FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer.
// If the path does not exist an error is returned.
func (r *AllLayersResolver) FileContentsByRef(ref file.Reference) (string, error) {
return r.img.FileContentsByRef(ref)
}
+240
View File
@@ -0,0 +1,240 @@
package source
import (
"testing"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/stereoscope/pkg/file"
)
type resolution struct {
layer uint
path string
}
func TestAllLayersResolver_FilesByPath(t *testing.T) {
cases := []struct {
name string
linkPath string
resolutions []resolution
}{
{
name: "link with previous data",
linkPath: "/link-1",
resolutions: []resolution{
{
layer: 1,
path: "/file-1.txt",
},
},
},
{
name: "link with in layer data",
linkPath: "/link-within",
resolutions: []resolution{
{
layer: 5,
path: "/file-3.txt",
},
},
},
{
name: "link with overridden data",
linkPath: "/link-2",
resolutions: []resolution{
{
layer: 3,
path: "/link-2",
},
{
layer: 4,
path: "/file-2.txt",
},
{
layer: 7,
path: "/file-2.txt",
},
},
},
{
name: "indirect link (with overridden data)",
linkPath: "/link-indirect",
resolutions: []resolution{
{
layer: 4,
path: "/file-2.txt",
},
{
layer: 7,
path: "/file-2.txt",
},
},
},
{
name: "dead link",
linkPath: "/link-dead",
resolutions: []resolution{
{
layer: 8,
path: "/link-dead",
},
},
},
{
name: "ignore directories",
linkPath: "/bin",
resolutions: []resolution{},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
defer cleanup()
resolver, err := NewAllLayersResolver(img)
if err != nil {
t.Fatalf("could not create resolver: %+v", err)
}
refs, err := resolver.FilesByPath(file.Path(c.linkPath))
if err != nil {
t.Fatalf("could not use resolver: %+v", err)
}
if len(refs) != len(c.resolutions) {
t.Fatalf("unexpected number of resolutions: %d", len(refs))
}
for idx, actual := range refs {
expected := c.resolutions[idx]
if actual.Path != file.Path(expected.path) {
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
}
entry, err := img.FileCatalog.Get(actual)
if err != nil {
t.Fatalf("failed to get metadata: %+v", err)
}
if entry.Source.Metadata.Index != expected.layer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
}
}
})
}
}
func TestAllLayersResolver_FilesByGlob(t *testing.T) {
cases := []struct {
name string
glob string
resolutions []resolution
}{
{
name: "link with previous data",
glob: "**ink-1",
resolutions: []resolution{
{
layer: 1,
path: "/file-1.txt",
},
},
},
{
name: "link with in layer data",
glob: "**nk-within",
resolutions: []resolution{
{
layer: 5,
path: "/file-3.txt",
},
},
},
{
name: "link with overridden data",
glob: "**ink-2",
resolutions: []resolution{
{
layer: 3,
path: "/link-2",
},
{
layer: 4,
path: "/file-2.txt",
},
{
layer: 7,
path: "/file-2.txt",
},
},
},
{
name: "indirect link (with overridden data)",
glob: "**nk-indirect",
resolutions: []resolution{
{
layer: 4,
path: "/file-2.txt",
},
{
layer: 7,
path: "/file-2.txt",
},
},
},
{
name: "dead link",
glob: "**k-dead",
resolutions: []resolution{
{
layer: 8,
path: "/link-dead",
},
},
},
{
name: "ignore directories",
glob: "**/bin",
resolutions: []resolution{},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
defer cleanup()
resolver, err := NewAllLayersResolver(img)
if err != nil {
t.Fatalf("could not create resolver: %+v", err)
}
refs, err := resolver.FilesByGlob(c.glob)
if err != nil {
t.Fatalf("could not use resolver: %+v", err)
}
if len(refs) != len(c.resolutions) {
t.Fatalf("unexpected number of resolutions: %d", len(refs))
}
for idx, actual := range refs {
expected := c.resolutions[idx]
if actual.Path != file.Path(expected.path) {
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
}
entry, err := img.FileCatalog.Get(actual)
if err != nil {
t.Fatalf("failed to get metadata: %+v", err)
}
if entry.Source.Metadata.Index != expected.layer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
}
}
})
}
}
+127
View File
@@ -0,0 +1,127 @@
package source
import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal/log"
"github.com/bmatcuk/doublestar"
)
// DirectoryResolver implements path and content access for the directory data source.
type DirectoryResolver struct {
Path string
}
// Stringer to represent a directory path data source
func (s DirectoryResolver) String() string {
return fmt.Sprintf("dir:%s", s.Path)
}
// FilesByPath returns all file.References that match the given paths from the directory.
func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference, error) {
var references = make([]file.Reference, 0)
for _, userPath := range userPaths {
userStrPath := string(userPath)
if filepath.IsAbs(userStrPath) {
// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
userStrPath = path.Join(s.Path, userStrPath)
}
fileMeta, err := os.Stat(userStrPath)
if os.IsNotExist(err) {
continue
} else if err != nil {
log.Errorf("path (%s) is not valid: %v", userStrPath, err)
}
// don't consider directories
if fileMeta.IsDir() {
continue
}
references = append(references, file.NewFileReference(file.Path(userStrPath)))
}
return references, nil
}
func fileContents(path file.Path) ([]byte, error) {
contents, err := ioutil.ReadFile(string(path))
if err != nil {
return nil, err
}
return contents, nil
}
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
result := make([]file.Reference, 0)
for _, pattern := range patterns {
pathPattern := path.Join(s.Path, pattern)
matches, err := doublestar.Glob(pathPattern)
if err != nil {
return result, err
}
for _, match := range matches {
fileMeta, err := os.Stat(match)
if err != nil {
continue
}
// don't consider directories
if fileMeta.IsDir() {
continue
}
matchedPath := file.Path(match)
result = append(result, file.NewFileReference(matchedPath))
}
}
return result, nil
}
func (s *DirectoryResolver) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) {
paths, err := s.FilesByPath(file.Path(path))
if err != nil {
return nil, err
}
if len(paths) == 0 {
return nil, nil
}
return &paths[0], nil
}
// MultipleFileContentsByRef returns the file contents for all file.References relative a directory.
func (s DirectoryResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) {
refContents := make(map[file.Reference]string)
for _, fileRef := range f {
contents, err := fileContents(fileRef.Path)
if err != nil {
return nil, fmt.Errorf("could not read contents of file: %s", fileRef.Path)
}
refContents[fileRef] = string(contents)
}
return refContents, nil
}
// FileContentsByRef fetches file contents for a single file reference relative to a directory.
// If the path does not exist an error is returned.
func (s DirectoryResolver) FileContentsByRef(reference file.Reference) (string, error) {
contents, err := fileContents(reference.Path)
if err != nil {
return "", fmt.Errorf("could not read contents of file: %s", reference.Path)
}
return string(contents), nil
}
+212
View File
@@ -0,0 +1,212 @@
package source
import (
"testing"
"github.com/anchore/stereoscope/pkg/file"
)
func TestDirectoryResolver_FilesByPath(t *testing.T) {
cases := []struct {
name string
root string
input string
expected string
refCount int
}{
{
name: "finds a file (relative)",
root: "./test-fixtures/",
input: "test-fixtures/image-symlinks/file-1.txt",
expected: "test-fixtures/image-symlinks/file-1.txt",
refCount: 1,
},
{
name: "finds a file with relative indirection",
root: "./test-fixtures/../test-fixtures",
input: "test-fixtures/image-symlinks/file-1.txt",
expected: "test-fixtures/image-symlinks/file-1.txt",
refCount: 1,
},
{
// note: this is asserting the old behavior is not supported
name: "relative lookup with wrong path fails",
root: "./test-fixtures/",
input: "image-symlinks/file-1.txt",
refCount: 0,
},
{
name: "managed non-existing files (relative)",
root: "./test-fixtures/",
input: "test-fixtures/image-symlinks/bogus.txt",
refCount: 0,
},
{
name: "finds a file (absolute)",
root: "./test-fixtures/",
input: "/image-symlinks/file-1.txt",
expected: "test-fixtures/image-symlinks/file-1.txt",
refCount: 1,
},
{
name: "directories ignored",
root: "./test-fixtures/",
input: "/image-symlinks",
refCount: 0,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver := DirectoryResolver{c.root}
refs, err := resolver.FilesByPath(file.Path(c.input))
if err != nil {
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
}
if len(refs) != c.refCount {
t.Errorf("unexpected number of refs: %d != %d", len(refs), c.refCount)
}
for _, actual := range refs {
if actual.Path != file.Path(c.expected) {
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.expected)
}
}
})
}
}
func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
cases := []struct {
name string
input []file.Path
refCount int
}{
{
name: "finds multiple files",
input: []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")},
refCount: 2,
},
{
name: "skips non-existing files",
input: []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")},
refCount: 1,
},
{
name: "does not return anything for non-existing directories",
input: []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")},
refCount: 0,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver := DirectoryResolver{"test-fixtures"}
refs, err := resolver.FilesByPath(c.input...)
if err != nil {
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
}
if len(refs) != c.refCount {
t.Errorf("unexpected number of refs: %d != %d", len(refs), c.refCount)
}
})
}
}
func TestDirectoryResolver_MultipleFileContentsByRef(t *testing.T) {
cases := []struct {
name string
input []file.Path
refCount int
contents []string
}{
{
name: "gets multiple file contents",
input: []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")},
refCount: 2,
},
{
name: "skips non-existing files",
input: []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")},
refCount: 1,
},
{
name: "does not return anything for non-existing directories",
input: []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")},
refCount: 0,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
refs := make([]file.Reference, 0)
resolver := DirectoryResolver{"test-fixtures"}
for _, p := range c.input {
newRefs, err := resolver.FilesByPath(p)
if err != nil {
t.Errorf("could not generate refs: %+v", err)
}
for _, ref := range newRefs {
refs = append(refs, ref)
}
}
contents, err := resolver.MultipleFileContentsByRef(refs...)
if err != nil {
t.Fatalf("unable to generate file contents by ref: %+v", err)
}
if len(contents) != c.refCount {
t.Errorf("unexpected number of refs produced: %d != %d", len(contents), c.refCount)
}
})
}
}
func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
t.Run("finds multiple matching files", func(t *testing.T) {
resolver := DirectoryResolver{"test-fixtures"}
refs, err := resolver.FilesByGlob("image-symlinks/file*")
if err != nil {
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
}
if len(refs) != 2 {
t.Errorf("unexpected number of refs: %d != 2", len(refs))
}
})
}
func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
t.Run("finds multiple matching files", func(t *testing.T) {
resolver := DirectoryResolver{"test-fixtures"}
refs, err := resolver.FilesByGlob("**/*.txt")
if err != nil {
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
}
if len(refs) != 4 {
t.Errorf("unexpected number of refs: %d != 4", len(refs))
}
})
}
func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
t.Run("finds multiple matching files", func(t *testing.T) {
resolver := DirectoryResolver{"test-fixtures"}
refs, err := resolver.FilesByGlob("image-symlinks/*1.txt")
if err != nil {
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
}
if len(refs) != 1 {
t.Errorf("unexpected number of refs: %d != 1", len(refs))
}
})
}
+127
View File
@@ -0,0 +1,127 @@
package source
import (
"fmt"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
)
// ImageSquashResolver implements path and content access for the Squashed source option for container image data sources.
type ImageSquashResolver struct {
img *image.Image
}
// NewImageSquashResolver returns a new resolver from the perspective of the squashed representation for the given image.
func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) {
if img.SquashedTree() == nil {
return nil, fmt.Errorf("the image does not have have a squashed tree")
}
return &ImageSquashResolver{img: img}, nil
}
// FilesByPath returns all file.References that match the given paths within the squashed representation of the image.
func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueFiles := make([]file.Reference, 0)
for _, path := range paths {
tree := r.img.SquashedTree()
ref := tree.File(path)
if ref == nil {
// no file found, keep looking through layers
continue
}
// don't consider directories (special case: there is no path information for /)
if ref.Path == "/" {
continue
} else if r.img.FileCatalog.Exists(*ref) {
metadata, err := r.img.FileCatalog.Get(*ref)
if err != nil {
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
}
if metadata.Metadata.IsDir {
continue
}
}
// a file may be a symlink, process it as such and resolve it
resolvedRef, err := r.img.ResolveLinkByImageSquash(*ref)
if err != nil {
return nil, fmt.Errorf("failed to resolve link from img (ref=%+v): %w", ref, err)
}
if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
uniqueFileIDs.Add(*resolvedRef)
uniqueFiles = append(uniqueFiles, *resolvedRef)
}
}
return uniqueFiles, nil
}
// FilesByGlob returns all file.References that match the given path glob pattern within the squashed representation of the image.
func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueFiles := make([]file.Reference, 0)
for _, pattern := range patterns {
refs, err := r.img.SquashedTree().FilesByGlob(pattern)
if err != nil {
return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
}
for _, ref := range refs {
// don't consider directories (special case: there is no path information for /)
if ref.Path == "/" {
continue
} else if r.img.FileCatalog.Exists(ref) {
metadata, err := r.img.FileCatalog.Get(ref)
if err != nil {
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
}
if metadata.Metadata.IsDir {
continue
}
}
resolvedRefs, err := r.FilesByPath(ref.Path)
if err != nil {
return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err)
}
for _, resolvedRef := range resolvedRefs {
if !uniqueFileIDs.Contains(resolvedRef) {
uniqueFileIDs.Add(resolvedRef)
uniqueFiles = append(uniqueFiles, resolvedRef)
}
}
}
}
return uniqueFiles, nil
}
func (r *ImageSquashResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) {
paths, err := r.FilesByPath(file.Path(path))
if err != nil {
return nil, err
}
if len(paths) == 0 {
return nil, nil
}
return &paths[0], nil
}
// MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a
// file.Reference is a path relative to a particular layer, in this case only from the squashed representation.
func (r *ImageSquashResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) {
return r.img.MultipleFileContentsByRef(f...)
}
// FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer.
// If the path does not exist an error is returned.
func (r *ImageSquashResolver) FileContentsByRef(ref file.Reference) (string, error) {
return r.img.FileContentsByRef(ref)
}
+189
View File
@@ -0,0 +1,189 @@
package source
import (
"testing"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/stereoscope/pkg/file"
)
func TestImageSquashResolver_FilesByPath(t *testing.T) {
cases := []struct {
name string
linkPath string
resolveLayer uint
resolvePath string
}{
{
name: "link with previous data",
linkPath: "/link-1",
resolveLayer: 1,
resolvePath: "/file-1.txt",
},
{
name: "link with in layer data",
linkPath: "/link-within",
resolveLayer: 5,
resolvePath: "/file-3.txt",
},
{
name: "link with overridden data",
linkPath: "/link-2",
resolveLayer: 7,
resolvePath: "/file-2.txt",
},
{
name: "indirect link (with overridden data)",
linkPath: "/link-indirect",
resolveLayer: 7,
resolvePath: "/file-2.txt",
},
{
name: "dead link",
linkPath: "/link-dead",
resolveLayer: 8,
resolvePath: "/link-dead",
},
{
name: "ignore directories",
linkPath: "/bin",
resolvePath: "",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
defer cleanup()
resolver, err := NewImageSquashResolver(img)
if err != nil {
t.Fatalf("could not create resolver: %+v", err)
}
refs, err := resolver.FilesByPath(file.Path(c.linkPath))
if err != nil {
t.Fatalf("could not use resolver: %+v", err)
}
expectedRefs := 1
if c.resolvePath == "" {
expectedRefs = 0
}
if len(refs) != expectedRefs {
t.Fatalf("unexpected number of resolutions: %d", len(refs))
}
if expectedRefs == 0 {
// nothing else to assert
return
}
actual := refs[0]
if actual.Path != file.Path(c.resolvePath) {
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
}
entry, err := img.FileCatalog.Get(actual)
if err != nil {
t.Fatalf("failed to get metadata: %+v", err)
}
if entry.Source.Metadata.Index != c.resolveLayer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
}
})
}
}
func TestImageSquashResolver_FilesByGlob(t *testing.T) {
cases := []struct {
name string
glob string
resolveLayer uint
resolvePath string
}{
{
name: "link with previous data",
glob: "**link-1",
resolveLayer: 1,
resolvePath: "/file-1.txt",
},
{
name: "link with in layer data",
glob: "**link-within",
resolveLayer: 5,
resolvePath: "/file-3.txt",
},
{
name: "link with overridden data",
glob: "**link-2",
resolveLayer: 7,
resolvePath: "/file-2.txt",
},
{
name: "indirect link (with overridden data)",
glob: "**link-indirect",
resolveLayer: 7,
resolvePath: "/file-2.txt",
},
{
name: "dead link",
glob: "**link-dead",
resolveLayer: 8,
resolvePath: "/link-dead",
},
{
name: "ignore directories",
glob: "**/bin",
resolvePath: "",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
defer cleanup()
resolver, err := NewImageSquashResolver(img)
if err != nil {
t.Fatalf("could not create resolver: %+v", err)
}
refs, err := resolver.FilesByGlob(c.glob)
if err != nil {
t.Fatalf("could not use resolver: %+v", err)
}
expectedRefs := 1
if c.resolvePath == "" {
expectedRefs = 0
}
if len(refs) != expectedRefs {
t.Fatalf("unexpected number of resolutions: %d", len(refs))
}
if expectedRefs == 0 {
// nothing else to assert
return
}
actual := refs[0]
if actual.Path != file.Path(c.resolvePath) {
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
}
entry, err := img.FileCatalog.Get(actual)
if err != nil {
t.Fatalf("failed to get metadata: %+v", err)
}
if entry.Source.Metadata.Index != c.resolveLayer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
}
})
}
}
+44
View File
@@ -0,0 +1,44 @@
package source
import (
"fmt"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
)
// Resolver is an interface that encompasses how to get specific file references and file contents for a generic data source.
type Resolver interface {
ContentResolver
FileResolver
}
// ContentResolver knows how to get file content for given file.References
type ContentResolver interface {
FileContentsByRef(ref file.Reference) (string, error)
MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error)
// TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering).
}
// FileResolver knows how to get file.References for given string paths and globs
type FileResolver interface {
// FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches)
FilesByPath(paths ...file.Path) ([]file.Reference, error)
// FilesByGlob fetches a set of file references which the given glob matches
FilesByGlob(patterns ...string) ([]file.Reference, error)
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
// This is helpful when attempting to find a file that is in the same layer or lower as another file.
RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error)
}
// getImageResolver returns the appropriate resolve for a container image given the source option
func getImageResolver(img *image.Image, option Scope) (Resolver, error) {
switch option {
case SquashedScope:
return NewImageSquashResolver(img)
case AllLayersScope:
return NewAllLayersResolver(img)
default:
return nil, fmt.Errorf("bad option provided: %+v", option)
}
}
+55
View File
@@ -0,0 +1,55 @@
package source
import (
"fmt"
"strings"
"github.com/anchore/stereoscope/pkg/image"
"github.com/mitchellh/go-homedir"
"github.com/spf13/afero"
)
type Scheme string
const (
UnknownScheme Scheme = "unknown-scheme"
DirectoryScheme Scheme = "directory-scheme"
ImageScheme Scheme = "image-scheme"
)
func detectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, string, error) {
if strings.HasPrefix(userInput, "dir:") {
// blindly trust the user's scheme
dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:"))
if err != nil {
return UnknownScheme, "", fmt.Errorf("unable to expand directory path: %w", err)
}
return DirectoryScheme, dirLocation, nil
}
// we should attempt to let stereoscope determine what the source is first --just because the source is a valid directory
// doesn't mean we yet know if it is an OCI layout directory (to be treated as an image) or if it is a generic filesystem directory.
source, imageSpec, err := imageDetector(userInput)
if err != nil {
return UnknownScheme, "", fmt.Errorf("unable to detect the scheme from %q: %w", userInput, err)
}
if source == image.UnknownSource {
dirLocation, err := homedir.Expand(userInput)
if err != nil {
return UnknownScheme, "", fmt.Errorf("unable to expand potential directory path: %w", err)
}
fileMeta, err := fs.Stat(dirLocation)
if err != nil {
return UnknownScheme, "", nil
}
if fileMeta.IsDir() {
return DirectoryScheme, dirLocation, nil
}
return UnknownScheme, "", nil
}
return ImageScheme, imageSpec, nil
}
+40
View File
@@ -0,0 +1,40 @@
package source
import "strings"
const (
UnknownScope Scope = iota
SquashedScope
AllLayersScope
)
type Scope int
var optionStr = []string{
"UnknownScope",
"Squashed",
"AllLayers",
}
var Options = []Scope{
SquashedScope,
AllLayersScope,
}
func ParseOption(userStr string) Scope {
switch strings.ToLower(userStr) {
case strings.ToLower(SquashedScope.String()):
return SquashedScope
case "all-layers", strings.ToLower(AllLayersScope.String()):
return AllLayersScope
}
return UnknownScope
}
func (o Scope) String() string {
if int(o) >= len(optionStr) || o < 0 {
return optionStr[0]
}
return optionStr[o]
}
+17
View File
@@ -0,0 +1,17 @@
package source
import (
"fmt"
"testing"
)
func TestOptionStringerBoundary(t *testing.T) {
var _ fmt.Stringer = Scope(0)
for _, c := range []int{-1, 0, 3} {
option := Scope(c)
if option.String() != UnknownScope.String() {
t.Errorf("expected Scope(%d) to be unknown, found '%+v'", c, option)
}
}
}
+117
View File
@@ -0,0 +1,117 @@
/*
Package source provides an abstraction to allow a user to loosely define a data source to catalog and expose a common interface that
catalogers and use explore and analyze data from the data source. All valid (cataloggable) data sources are defined
within this package.
*/
package source
import (
"fmt"
"github.com/spf13/afero"
"github.com/anchore/stereoscope"
"github.com/anchore/stereoscope/pkg/image"
)
// ImageSource represents a data source that is a container image
type ImageSource struct {
Img *image.Image // the image object to be cataloged
}
// DirSource represents a data source that is a filesystem directory tree
type DirSource struct {
Path string // the root path to be cataloged
}
// Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used
// in cataloging (based on the data source and configuration)
type Source struct {
Scope Scope // specific perspective to catalog
Resolver Resolver // a Resolver object to use in file path/glob resolution and file contents resolution
Target interface{} // the specific source object to be cataloged
Scheme Scheme // the source data scheme type (directory or image)
}
type sourceDetector func(string) (image.Source, string, error)
// NewSource produces a Source based on userInput like dir: or image:tag
func NewSource(userInput string, o Scope) (Source, func(), error) {
fs := afero.NewOsFs()
parsedScheme, location, err := detectScheme(fs, image.DetectSource, userInput)
if err != nil {
return Source{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err)
}
switch parsedScheme {
case DirectoryScheme:
fileMeta, err := fs.Stat(location)
if err != nil {
return Source{}, func() {}, fmt.Errorf("unable to stat dir=%q: %w", location, err)
}
if !fileMeta.IsDir() {
return Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err)
}
s, err := NewFromDirectory(location)
if err != nil {
return Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err)
}
return s, func() {}, nil
case ImageScheme:
img, err := stereoscope.GetImage(location)
cleanup := func() {
stereoscope.Cleanup()
}
if err != nil || img == nil {
return Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err)
}
s, err := NewFromImage(img, o)
if err != nil {
return Source{}, cleanup, fmt.Errorf("could not populate source with image: %w", err)
}
return s, cleanup, nil
}
return Source{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
}
// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
func NewFromDirectory(path string) (Source, error) {
return Source{
Resolver: &DirectoryResolver{
Path: path,
},
Target: DirSource{
Path: path,
},
Scheme: DirectoryScheme,
}, nil
}
// NewFromImage creates a new source object tailored to catalog a given container image, relative to the
// option given (e.g. all-layers, squashed, etc)
func NewFromImage(img *image.Image, option Scope) (Source, error) {
if img == nil {
return Source{}, fmt.Errorf("no image given")
}
resolver, err := getImageResolver(img, option)
if err != nil {
return Source{}, fmt.Errorf("could not determine file resolver: %w", err)
}
return Source{
Scope: option,
Resolver: resolver,
Target: ImageSource{
Img: img,
},
Scheme: ImageScheme,
}, nil
}
+469
View File
@@ -0,0 +1,469 @@
package source
import (
"os"
"testing"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
"github.com/mitchellh/go-homedir"
"github.com/spf13/afero"
)
func TestNewScopeFromImageFails(t *testing.T) {
t.Run("no image given", func(t *testing.T) {
_, err := NewFromImage(nil, AllLayersScope)
if err == nil {
t.Errorf("expected an error condition but none was given")
}
})
}
func TestNewScopeFromImageUnknownOption(t *testing.T) {
img := image.Image{}
t.Run("unknown option is an error", func(t *testing.T) {
_, err := NewFromImage(&img, UnknownScope)
if err == nil {
t.Errorf("expected an error condition but none was given")
}
})
}
func TestNewScopeFromImage(t *testing.T) {
layer := image.NewLayer(nil)
img := image.Image{
Layers: []*image.Layer{layer},
}
t.Run("create a new Source object from image", func(t *testing.T) {
_, err := NewFromImage(&img, AllLayersScope)
if err != nil {
t.Errorf("unexpected error when creating a new Source from img: %w", err)
}
})
}
func TestDirectoryScope(t *testing.T) {
testCases := []struct {
desc string
input string
expString string
inputPaths []file.Path
expRefs int
}{
{
desc: "no paths exist",
input: "foobar/",
inputPaths: []file.Path{file.Path("/opt/"), file.Path("/other")},
expRefs: 0,
},
{
desc: "path detected",
input: "test-fixtures",
inputPaths: []file.Path{file.Path("test-fixtures/path-detected/.vimrc")},
expRefs: 1,
},
{
desc: "directory ignored",
input: "test-fixtures",
inputPaths: []file.Path{file.Path("test-fixtures/path-detected")},
expRefs: 0,
},
{
desc: "no files-by-path detected",
input: "test-fixtures",
inputPaths: []file.Path{file.Path("test-fixtures/no-path-detected")},
expRefs: 0,
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
p, err := NewFromDirectory(test.input)
if err != nil {
t.Errorf("could not create NewDirScope: %w", err)
}
if p.Target.(DirSource).Path != test.input {
t.Errorf("mismatched stringer: '%s' != '%s'", p.Target.(DirSource).Path, test.input)
}
refs, err := p.Resolver.FilesByPath(test.inputPaths...)
if err != nil {
t.Errorf("FilesByPath call produced an error: %w", err)
}
if len(refs) != test.expRefs {
t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expRefs)
}
})
}
}
func TestMultipleFileContentsByRefContents(t *testing.T) {
testCases := []struct {
desc string
input string
path string
expected string
}{
{
input: "test-fixtures/path-detected",
desc: "empty file",
path: "test-fixtures/path-detected/empty",
expected: "",
},
{
input: "test-fixtures/path-detected",
desc: "file has contents",
path: "test-fixtures/path-detected/.vimrc",
expected: "\" A .vimrc file\n",
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
p, err := NewFromDirectory(test.input)
if err != nil {
t.Errorf("could not create NewDirScope: %w", err)
}
refs, err := p.Resolver.FilesByPath(file.Path(test.path))
if err != nil {
t.Errorf("could not get file references from path: %s, %v", test.path, err)
}
if len(refs) != 1 {
t.Fatalf("expected a single ref to be generated but got: %d", len(refs))
}
ref := refs[0]
contents, err := p.Resolver.MultipleFileContentsByRef(ref)
content := contents[ref]
if content != test.expected {
t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected)
}
})
}
}
func TestMultipleFileContentsByRefNoContents(t *testing.T) {
testCases := []struct {
desc string
input string
path string
expected string
}{
{
input: "test-fixtures/path-detected",
desc: "path does not exist",
path: "foo",
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
p, err := NewFromDirectory(test.input)
if err != nil {
t.Errorf("could not create NewDirScope: %w", err)
}
refs, err := p.Resolver.FilesByPath(file.Path(test.path))
if err != nil {
t.Errorf("could not get file references from path: %s, %v", test.path, err)
}
if len(refs) != 0 {
t.Errorf("didnt' expect a ref, but got: %d", len(refs))
}
})
}
}
func TestFilesByGlob(t *testing.T) {
testCases := []struct {
desc string
input string
glob string
expected int
}{
{
input: "test-fixtures",
desc: "no matches",
glob: "bar/foo",
expected: 0,
},
{
input: "test-fixtures/path-detected",
desc: "a single match",
glob: "*vimrc",
expected: 1,
},
{
input: "test-fixtures/path-detected",
desc: "multiple matches",
glob: "*",
expected: 2,
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
p, err := NewFromDirectory(test.input)
if err != nil {
t.Errorf("could not create NewDirScope: %w", err)
}
contents, err := p.Resolver.FilesByGlob(test.glob)
if len(contents) != test.expected {
t.Errorf("unexpected number of files found by glob (%s): %d != %d", test.glob, len(contents), test.expected)
}
})
}
}
func TestDetectScheme(t *testing.T) {
type detectorResult struct {
src image.Source
ref string
err error
}
testCases := []struct {
name string
userInput string
dirs []string
detection detectorResult
expectedScheme Scheme
expectedLocation string
}{
{
name: "docker-image-ref",
userInput: "wagoodman/dive:latest",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "wagoodman/dive:latest",
},
expectedScheme: ImageScheme,
expectedLocation: "wagoodman/dive:latest",
},
{
name: "docker-image-ref-no-tag",
userInput: "wagoodman/dive",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "wagoodman/dive",
},
expectedScheme: ImageScheme,
expectedLocation: "wagoodman/dive",
},
{
name: "docker-image-explicit-scheme",
userInput: "docker:wagoodman/dive:latest",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "wagoodman/dive:latest",
},
expectedScheme: ImageScheme,
expectedLocation: "wagoodman/dive:latest",
},
{
name: "docker-image-explicit-scheme-no-tag",
userInput: "docker:wagoodman/dive",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "wagoodman/dive",
},
expectedScheme: ImageScheme,
expectedLocation: "wagoodman/dive",
},
{
name: "docker-image-edge-case",
userInput: "docker:latest",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "latest",
},
expectedScheme: ImageScheme,
// we want to be able to handle this case better, however, I don't see a way to do this
// the user will need to provide more explicit input (docker:docker:latest)
expectedLocation: "latest",
},
{
name: "docker-image-edge-case-explicit",
userInput: "docker:docker:latest",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "docker:latest",
},
expectedScheme: ImageScheme,
// we want to be able to handle this case better, however, I don't see a way to do this
// the user will need to provide more explicit input (docker:docker:latest)
expectedLocation: "docker:latest",
},
{
name: "oci-tar",
userInput: "some/path-to-file",
detection: detectorResult{
src: image.OciTarballSource,
ref: "some/path-to-file",
},
expectedScheme: ImageScheme,
expectedLocation: "some/path-to-file",
},
{
name: "oci-dir",
userInput: "some/path-to-dir",
detection: detectorResult{
src: image.OciDirectorySource,
ref: "some/path-to-dir",
},
dirs: []string{"some/path-to-dir"},
expectedScheme: ImageScheme,
expectedLocation: "some/path-to-dir",
},
{
name: "guess-dir",
userInput: "some/path-to-dir",
detection: detectorResult{
src: image.UnknownSource,
ref: "",
},
dirs: []string{"some/path-to-dir"},
expectedScheme: DirectoryScheme,
expectedLocation: "some/path-to-dir",
},
{
name: "generic-dir-does-not-exist",
userInput: "some/path-to-dir",
detection: detectorResult{
src: image.DockerDaemonSource,
ref: "some/path-to-dir",
},
expectedScheme: ImageScheme,
expectedLocation: "some/path-to-dir",
},
{
name: "explicit-dir",
userInput: "dir:some/path-to-dir",
detection: detectorResult{
src: image.UnknownSource,
ref: "",
},
dirs: []string{"some/path-to-dir"},
expectedScheme: DirectoryScheme,
expectedLocation: "some/path-to-dir",
},
{
name: "explicit-current-dir",
userInput: "dir:.",
detection: detectorResult{
src: image.UnknownSource,
ref: "",
},
expectedScheme: DirectoryScheme,
expectedLocation: ".",
},
{
name: "current-dir",
userInput: ".",
detection: detectorResult{
src: image.UnknownSource,
ref: "",
},
expectedScheme: DirectoryScheme,
expectedLocation: ".",
},
// we should support tilde expansion
{
name: "tilde-expansion-image-implicit",
userInput: "~/some-path",
detection: detectorResult{
src: image.OciDirectorySource,
ref: "~/some-path",
},
expectedScheme: ImageScheme,
expectedLocation: "~/some-path",
},
{
name: "tilde-expansion-dir-implicit",
userInput: "~/some-path",
detection: detectorResult{
src: image.UnknownSource,
ref: "",
},
dirs: []string{"~/some-path"},
expectedScheme: DirectoryScheme,
expectedLocation: "~/some-path",
},
{
name: "tilde-expansion-dir-explicit-exists",
userInput: "dir:~/some-path",
dirs: []string{"~/some-path"},
expectedScheme: DirectoryScheme,
expectedLocation: "~/some-path",
},
{
name: "tilde-expansion-dir-explicit-dne",
userInput: "dir:~/some-path",
expectedScheme: DirectoryScheme,
expectedLocation: "~/some-path",
},
{
name: "tilde-expansion-dir-implicit-dne",
userInput: "~/some-path",
expectedScheme: UnknownScheme,
expectedLocation: "",
},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
fs := afero.NewMemMapFs()
for _, p := range test.dirs {
expandedExpectedLocation, err := homedir.Expand(p)
if err != nil {
t.Fatalf("unable to expand path=%q: %+v", p, err)
}
err = fs.Mkdir(expandedExpectedLocation, os.ModePerm)
if err != nil {
t.Fatalf("failed to create dummy tar: %+v", err)
}
}
imageDetector := func(string) (image.Source, string, error) {
// lean on the users real home directory value
switch test.detection.src {
case image.OciDirectorySource, image.DockerTarballSource, image.OciTarballSource:
expandedExpectedLocation, err := homedir.Expand(test.expectedLocation)
if err != nil {
t.Fatalf("unable to expand path=%q: %+v", test.expectedLocation, err)
}
return test.detection.src, expandedExpectedLocation, test.detection.err
default:
return test.detection.src, test.detection.ref, test.detection.err
}
}
actualScheme, actualLocation, err := detectScheme(fs, imageDetector, test.userInput)
if err != nil {
t.Fatalf("unexpected err : %+v", err)
}
if actualScheme != test.expectedScheme {
t.Errorf("expected scheme %q , got %q", test.expectedScheme, actualScheme)
}
// lean on the users real home directory value
expandedExpectedLocation, err := homedir.Expand(test.expectedLocation)
if err != nil {
t.Fatalf("unable to expand path=%q: %+v", test.expectedLocation, err)
}
if actualLocation != expandedExpectedLocation {
t.Errorf("expected location %q , got %q", expandedExpectedLocation, actualLocation)
}
})
}
}
@@ -0,0 +1,24 @@
# LAYER 0:
FROM busybox:latest
# LAYER 1:
ADD file-1.txt .
# LAYER 2: link with previous data
RUN ln -s ./file-1.txt link-1
# LAYER 3: link with future data
RUN ln -s ./file-2.txt link-2
# LAYER 4:
ADD file-2.txt .
# LAYER 5: link with current data
RUN echo "file 3" > file-3.txt && ln -s ./file-3.txt link-within
# LAYER 6: multiple links (link-indirect > link-2 > file-2.txt)
RUN ln -s ./link-2 link-indirect
# LAYER 7: override contents / resolution
ADD new-file-2.txt file-2.txt
# LAYER 8: dead link
RUN ln -s ./i-dont-exist.txt link-dead
@@ -0,0 +1 @@
file 1!
@@ -0,0 +1 @@
file 2!
@@ -0,0 +1 @@
file 3
@@ -0,0 +1 @@
NEW file override!
@@ -0,0 +1 @@
" A .vimrc file