Add query by MIME type to source.FileResolver (#529)

* add query by MIME type to source.FileResolver

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* pull in stereoscope MIME type feature

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

syft/source/all_layers_resolver.go

@@ -188,6 +188,24 @@ func (r *allLayersResolver) FileContentsByLocation(location Location) (io.ReadCl
 	return r.img.FileContentsByRef(location.ref)
 }
 
+func (r *allLayersResolver) FilesByMIMEType(types ...string) ([]Location, error) {
+	var locations []Location
+	for _, layerIdx := range r.layers {
+		layer := r.img.Layers[layerIdx]
+
+		refs, err := layer.FilesByMIMEType(types...)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, ref := range refs {
+			locations = append(locations, NewLocationFromReference(ref))
+		}
+	}
+
+	return locations, nil
+}
+
 func (r *allLayersResolver) AllLocations() <-chan Location {
 	results := make(chan Location)
 	go func() {

syft/source/all_layers_resolver_test.go

@@ -3,6 +3,8 @@ package source
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/anchore/stereoscope/pkg/imagetest"
 )
 
@@ -239,3 +241,34 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		})
 	}
 }
+
+func Test_imageAllLayersResolver_FilesByMIMEType(t *testing.T) {
+
+	tests := []struct {
+		fixtureName   string
+		mimeType      string
+		expectedPaths []string
+	}{
+		{
+			fixtureName:   "image-duplicate-path",
+			mimeType:      "text/plain",
+			expectedPaths: []string{"/somefile-1.txt", "/somefile-1.txt"},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.fixtureName, func(t *testing.T) {
+			img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureName)
+
+			resolver, err := newAllLayersResolver(img)
+			assert.NoError(t, err)
+
+			locations, err := resolver.FilesByMIMEType(test.mimeType)
+			assert.NoError(t, err)
+
+			assert.Len(t, test.expectedPaths, len(locations))
+			for idx, l := range locations {
+				assert.Equal(t, test.expectedPaths[idx], l.RealPath, "does not have path %q", l.RealPath)
+			}
+		})
+	}
+}

syft/source/directory_resolver.go

@@ -8,7 +8,6 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"syscall"
 
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/filetree"
@@ -35,10 +34,11 @@ type directoryResolver struct {
 	path     string
 	cwd      string
 	fileTree *filetree.FileTree
-	infos    map[file.ID]os.FileInfo
+	metadata map[file.ID]FileMetadata
 	// TODO: wire up to report these paths in the json report
-	pathFilterFns []pathFilterFn
-	errPaths      map[string]error
+	pathFilterFns  []pathFilterFn
+	refsByMIMEType map[string][]file.Reference
+	errPaths       map[string]error
 }
 
 func newDirectoryResolver(root string, pathFilters ...pathFilterFn) (*directoryResolver, error) {
@@ -52,12 +52,13 @@ func newDirectoryResolver(root string, pathFilters ...pathFilterFn) (*directoryR
 	}
 
 	resolver := directoryResolver{
-		path:          root,
-		cwd:           cwd,
-		fileTree:      filetree.NewFileTree(),
-		infos:         make(map[file.ID]os.FileInfo),
-		pathFilterFns: pathFilters,
-		errPaths:      make(map[string]error),
+		path:           root,
+		cwd:            cwd,
+		fileTree:       filetree.NewFileTree(),
+		metadata:       make(map[file.ID]FileMetadata),
+		pathFilterFns:  pathFilters,
+		refsByMIMEType: make(map[string][]file.Reference),
+		errPaths:       make(map[string]error),
 	}
 
 	return &resolver, indexAllRoots(root, resolver.indexTree)
@@ -158,7 +159,11 @@ func (r directoryResolver) addPathToIndex(p string, info os.FileInfo) (string, e
 		}
 	}
 
-	r.infos[ref.ID()] = info
+	metadata := fileMetadataFromPath(p, info)
+	if ref != nil {
+		r.refsByMIMEType[metadata.MIMEType] = append(r.refsByMIMEType[metadata.MIMEType], *ref)
+	}
+	r.metadata[ref.ID()] = metadata
 	return newRoot, nil
 }
 
@@ -207,6 +212,7 @@ func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
 			log.Warnf("unable to get file by path=%q : %+v", userPath, err)
 			continue
 		}
+		// TODO: why not use stored metadata?
 		fileMeta, err := os.Stat(userStrPath)
 		if os.IsNotExist(err) {
 			continue
@@ -280,25 +286,24 @@ func (r *directoryResolver) AllLocations() <-chan Location {
 }
 
 func (r *directoryResolver) FileMetadataByLocation(location Location) (FileMetadata, error) {
-	info, exists := r.infos[location.ref.ID()]
+	metadata, exists := r.metadata[location.ref.ID()]
 	if !exists {
 		return FileMetadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
 	}
 
-	uid := -1
-	gid := -1
-	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
-		uid = int(stat.Uid)
-		gid = int(stat.Gid)
-	}
+	return metadata, nil
+}
 
-	return FileMetadata{
-		Mode: info.Mode(),
-		Type: newFileTypeFromMode(info.Mode()),
-		// unsupported across platforms
-		UserID:  uid,
-		GroupID: gid,
-	}, nil
+func (r *directoryResolver) FilesByMIMEType(types ...string) ([]Location, error) {
+	var locations []Location
+	for _, ty := range types {
+		if refs, ok := r.refsByMIMEType[ty]; ok {
+			for _, ref := range refs {
+				locations = append(locations, NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), ref))
+			}
+		}
+	}
+	return locations, nil
 }
 
 func isUnixSystemRuntimePath(path string) bool {

syft/source/directory_resolver_test.go

@@ -7,6 +7,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/scylladb/go-set/strset"
+
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/stretchr/testify/assert"
 	"github.com/wagoodman/go-progress"
@@ -303,7 +305,7 @@ func Test_directoryResolver_index(t *testing.T) {
 			if assert.NoError(t, err) {
 				return
 			}
-			assert.Equal(t, info, r.infos[ref.ID()])
+			assert.Equal(t, info, r.metadata[ref.ID()])
 		})
 	}
 }
@@ -429,3 +431,31 @@ func Test_indexAllRoots(t *testing.T) {
 		})
 	}
 }
+
+func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
+
+	tests := []struct {
+		fixturePath   string
+		mimeType      string
+		expectedPaths *strset.Set
+	}{
+		{
+			fixturePath:   "./test-fixtures/image-simple",
+			mimeType:      "text/plain",
+			expectedPaths: strset.New("test-fixtures/image-simple/file-1.txt", "test-fixtures/image-simple/file-2.txt", "test-fixtures/image-simple/target/really/nested/file-3.txt", "test-fixtures/image-simple/Dockerfile"),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.fixturePath, func(t *testing.T) {
+
+			resolver, err := newDirectoryResolver(test.fixturePath)
+			assert.NoError(t, err)
+			locations, err := resolver.FilesByMIMEType(test.mimeType)
+			assert.NoError(t, err)
+			assert.Equal(t, test.expectedPaths.Size(), len(locations))
+			for _, l := range locations {
+				assert.True(t, test.expectedPaths.Has(l.RealPath), "does not have path %q", l.RealPath)
+			}
+		})
+	}
+}

syft/source/file_metadata.go

@@ -2,6 +2,11 @@ package source
 
 import (
 	"os"
+	"syscall"
+
+	"github.com/anchore/syft/internal/log"
+
+	"github.com/anchore/stereoscope/pkg/file"
 
 	"github.com/anchore/stereoscope/pkg/image"
 )
@@ -13,6 +18,7 @@ type FileMetadata struct {
 	GroupID         int
 	LinkDestination string
 	Size            int64
+	MIMEType        string
 }
 
 func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata, error) {
@@ -28,5 +34,36 @@ func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata,
 		GroupID:         entry.Metadata.GroupID,
 		LinkDestination: entry.Metadata.Linkname,
 		Size:            entry.Metadata.Size,
+		MIMEType:        entry.Metadata.MIMEType,
 	}, nil
 }
+
+func fileMetadataFromPath(path string, info os.FileInfo) FileMetadata {
+	uid := -1
+	gid := -1
+	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+		uid = int(stat.Uid)
+		gid = int(stat.Gid)
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		// TODO: it may be that the file is inaccessible, however, this is not an error or a warning. In the future we need to track these as known-unknowns
+		f = nil
+	} else {
+		defer func() {
+			if err := f.Close(); err != nil {
+				log.Warnf("unable to close file while obtaining metadata: %s", path)
+			}
+		}()
+	}
+
+	return FileMetadata{
+		Mode: info.Mode(),
+		Type: newFileTypeFromMode(info.Mode()),
+		// unsupported across platforms
+		UserID:   uid,
+		GroupID:  gid,
+		MIMEType: file.MIMEType(f),
+	}
+}

syft/source/file_resolver.go

@@ -29,6 +29,8 @@ type FilePathResolver interface {
 	FilesByPath(paths ...string) ([]Location, error)
 	// FilesByGlob fetches a set of file references which the given glob matches
 	FilesByGlob(patterns ...string) ([]Location, error)
+	// FilesByGlob fetches a set of file references which the contents have been classified as one of the given MIME Types
+	FilesByMIMEType(types ...string) ([]Location, error)
 	// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
 	// This is helpful when attempting to find a file that is in the same layer or lower as another file.
 	RelativeFileByPath(_ Location, path string) *Location

syft/source/image_squash_resolver.go

@@ -151,6 +151,20 @@ func (r *imageSquashResolver) AllLocations() <-chan Location {
 	return results
 }
 
+func (r *imageSquashResolver) FilesByMIMEType(types ...string) ([]Location, error) {
+	refs, err := r.img.FilesByMIMETypeFromSquash(types...)
+	if err != nil {
+		return nil, err
+	}
+
+	var locations []Location
+	for _, ref := range refs {
+		locations = append(locations, NewLocationFromReference(ref))
+	}
+
+	return locations, nil
+}
+
 func (r *imageSquashResolver) FileMetadataByLocation(location Location) (FileMetadata, error) {
 	return fileMetadataByLocation(r.img, location)
 }

syft/source/image_squash_resolver_test.go

@@ -3,6 +3,9 @@ package source
 import (
 	"testing"
 
+	"github.com/scylladb/go-set/strset"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/anchore/stereoscope/pkg/imagetest"
 )
 
@@ -225,3 +228,34 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 		})
 	}
 }
+
+func Test_imageSquashResolver_FilesByMIMEType(t *testing.T) {
+
+	tests := []struct {
+		fixtureName   string
+		mimeType      string
+		expectedPaths *strset.Set
+	}{
+		{
+			fixtureName:   "image-simple",
+			mimeType:      "text/plain",
+			expectedPaths: strset.New("/somefile-1.txt", "/somefile-2.txt", "/really/nested/file-3.txt"),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.fixtureName, func(t *testing.T) {
+			img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureName)
+
+			resolver, err := newImageSquashResolver(img)
+			assert.NoError(t, err)
+
+			locations, err := resolver.FilesByMIMEType(test.mimeType)
+			assert.NoError(t, err)
+
+			assert.Equal(t, test.expectedPaths.Size(), len(locations))
+			for _, l := range locations {
+				assert.True(t, test.expectedPaths.Has(l.RealPath), "does not have path %q", l.RealPath)
+			}
+		})
+	}
+}

syft/source/mock_resolver.go

@@ -14,8 +14,9 @@ var _ FileResolver = (*MockResolver)(nil)
 // It provides an implementation that can resolve local filesystem paths using only a provided discrete list of file
 // paths, which are typically paths to test fixtures.
 type MockResolver struct {
-	Locations []Location
-	Metadata  map[Location]FileMetadata
+	locations     []Location
+	metadata      map[Location]FileMetadata
+	mimeTypeIndex map[string][]Location
 }
 
 // NewMockResolverForPaths creates a new MockResolver, where the only resolvable
@@ -26,21 +27,30 @@ func NewMockResolverForPaths(paths ...string) *MockResolver {
 		locations = append(locations, NewLocation(p))
 	}
 
-	return &MockResolver{Locations: locations}
+	return &MockResolver{
+		locations: locations,
+		metadata:  make(map[Location]FileMetadata),
+	}
 }
 
 func NewMockResolverForPathsWithMetadata(metadata map[Location]FileMetadata) *MockResolver {
 	var locations []Location
-	for p := range metadata {
-		locations = append(locations, p)
+	var mimeTypeIndex = make(map[string][]Location)
+	for l, m := range metadata {
+		locations = append(locations, l)
+		mimeTypeIndex[m.MIMEType] = append(mimeTypeIndex[m.MIMEType], l)
 	}
 
-	return &MockResolver{Locations: locations, Metadata: metadata}
+	return &MockResolver{
+		locations:     locations,
+		metadata:      metadata,
+		mimeTypeIndex: mimeTypeIndex,
+	}
 }
 
 // HasPath indicates if the given path exists in the underlying source.
 func (r MockResolver) HasPath(path string) bool {
-	for _, l := range r.Locations {
+	for _, l := range r.locations {
 		if l.RealPath == path {
 			return true
 		}
@@ -50,13 +60,13 @@ func (r MockResolver) HasPath(path string) bool {
 
 // String returns the string representation of the MockResolver.
 func (r MockResolver) String() string {
-	return fmt.Sprintf("mock:(%s,...)", r.Locations[0].RealPath)
+	return fmt.Sprintf("mock:(%s,...)", r.locations[0].RealPath)
 }
 
 // FileContentsByLocation fetches file contents for a single location. If the
 // path does not exist, an error is returned.
 func (r MockResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
-	for _, l := range r.Locations {
+	for _, l := range r.locations {
 		if l == location {
 			return os.Open(location.RealPath)
 		}
@@ -69,7 +79,7 @@ func (r MockResolver) FileContentsByLocation(location Location) (io.ReadCloser,
 func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) {
 	var results []Location
 	for _, p := range paths {
-		for _, location := range r.Locations {
+		for _, location := range r.locations {
 			if p == location.RealPath {
 				results = append(results, NewLocation(p))
 			}
@@ -83,7 +93,7 @@ func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) {
 func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 	var results []Location
 	for _, pattern := range patterns {
-		for _, location := range r.Locations {
+		for _, location := range r.locations {
 			matches, err := doublestar.Match(pattern, location.RealPath)
 			if err != nil {
 				return nil, err
@@ -115,7 +125,7 @@ func (r MockResolver) AllLocations() <-chan Location {
 	results := make(chan Location)
 	go func() {
 		defer close(results)
-		for _, l := range r.Locations {
+		for _, l := range r.locations {
 			results <- l
 		}
 	}()
@@ -142,3 +152,11 @@ func (r MockResolver) FileMetadataByLocation(l Location) (FileMetadata, error) {
 		Size:    info.Size(),
 	}, nil
 }
+
+func (r MockResolver) FilesByMIMEType(types ...string) ([]Location, error) {
+	var locations []Location
+	for _, ty := range types {
+		locations = append(r.mimeTypeIndex[ty], locations...)
+	}
+	return locations, nil
+}

syft/source/test-fixtures/image-duplicate-path/Dockerfile

@@ -0,0 +1,4 @@
+# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one.
+FROM scratch
+ADD file-1.txt /somefile-1.txt
+ADD file-2.txt /somefile-1.txt

syft/source/test-fixtures/image-duplicate-path/file-1.txt

@@ -0,0 +1 @@
+this file has contents

syft/source/test-fixtures/image-duplicate-path/file-2.txt

@@ -0,0 +1 @@
+file-2 contents!