Power user command support for directory scans (#467)

* Power-user directory source support
Signed-off-by: Mikey Strauss <mikey@scribe-security.com>

Signed-off-by: houdini91 <mdstrauss91@gmail.com>

* Remove newline

Signed-off-by: houdini91 <mdstrauss91@gmail.com>

* Shared filetree (#1)

* Shared directory resolver filetree

Signed-off-by: houdini91 <mdstrauss91@gmail.com>

* PR - change error ErrObserve to ErrPath

Signed-off-by: houdini91 <mdstrauss91@gmail.com>

* PR - share directory resolver
* Use pointer to source struct

Signed-off-by: houdini91 <mdstrauss91@gmail.com>

* Fix Lint

Signed-off-by: houdini91 <mdstrauss91@gmail.com>

syft/source/directory_resolver.go

@@ -8,6 +8,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/filetree"
@@ -218,7 +219,12 @@ func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
 			continue
 		}
 
-		references = append(references, NewLocation(r.responsePath(userStrPath)))
+		exists, ref, err := r.fileTree.File(file.Path(userStrPath))
+		if err == nil && exists {
+			references = append(references, NewLocationFromDirectory(r.responsePath(userStrPath), *ref))
+		} else {
+			log.Warnf("path (%s) not found in file tree: Exists: %t Err:%+v", userStrPath, exists, err)
+		}
 	}
 
 	return references, nil
@@ -234,7 +240,7 @@ func (r directoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 			return nil, err
 		}
 		for _, globResult := range globResults {
-			result = append(result, NewLocation(r.responsePath(string(globResult.MatchPath))))
+			result = append(result, NewLocationFromDirectory(r.responsePath(string(globResult.MatchPath)), globResult.Reference))
 		}
 	}
 
@@ -267,7 +273,7 @@ func (r *directoryResolver) AllLocations() <-chan Location {
 	go func() {
 		defer close(results)
 		for _, ref := range r.fileTree.AllFiles() {
-			results <- NewLocation(r.responsePath(string(ref.RealPath)))
+			results <- NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), ref)
 		}
 	}()
 	return results
@@ -276,15 +282,22 @@ func (r *directoryResolver) AllLocations() <-chan Location {
 func (r *directoryResolver) FileMetadataByLocation(location Location) (FileMetadata, error) {
 	info, exists := r.infos[location.ref.ID()]
 	if !exists {
-		return FileMetadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrExist)
+		return FileMetadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
+	}
+
+	uid := -1
+	gid := -1
+	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+		uid = int(stat.Uid)
+		gid = int(stat.Gid)
 	}
 
 	return FileMetadata{
 		Mode: info.Mode(),
 		Type: newFileTypeFromMode(info.Mode()),
 		// unsupported across platforms
-		UserID:  -1,
-		GroupID: -1,
+		UserID:  uid,
+		GroupID: gid,
 	}, nil
 }
 
@@ -297,6 +310,8 @@ func indexAllRoots(root string, indexer func(string, *progress.Stage) ([]string,
 	// in which case we need to additionally index where the link resolves to. it's for this reason why the filetree
 	// must be relative to the root of the filesystem (and not just relative to the given path).
 	pathsToIndex := []string{root}
+	fullPathsMap := map[string]struct{}{}
+
 	stager, prog := indexingProgress(root)
 	defer prog.SetCompleted()
 loop:
@@ -315,7 +330,13 @@ loop:
 		if err != nil {
 			return fmt.Errorf("unable to index filesystem path=%q: %w", currentPath, err)
 		}
-		pathsToIndex = append(pathsToIndex, additionalRoots...)
+
+		for _, newRoot := range additionalRoots {
+			if _, ok := fullPathsMap[newRoot]; !ok {
+				fullPathsMap[newRoot] = struct{}{}
+				pathsToIndex = append(pathsToIndex, newRoot)
+			}
+		}
 	}
 
 	return nil

syft/source/directory_resolver_test.go

@@ -178,9 +178,17 @@ func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
 	assert.Len(t, refs, 6)
 
 	// ensure that symlink indexing outside of root worked
-	assert.Contains(t, refs, Location{
-		RealPath: "test-fixtures/system_paths/outside_root/link_target/place",
-	})
+	ok := false
+	test_location := "test-fixtures/system_paths/outside_root/link_target/place"
+	for _, actual_loc := range refs {
+		if test_location == actual_loc.RealPath {
+			ok = true
+		}
+	}
+
+	if !ok {
+		t.Fatalf("could not find test location=%q", test_location)
+	}
 }
 
 func TestDirectoryResolverUsesPathFilterFunction(t *testing.T) {

syft/source/location.go

@@ -45,6 +45,14 @@ func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Ima
 	}
 }
 
+// NewLocationFromDirectory creates a new Location representing the given path (extracted from the ref) relative to the given directory.
+func NewLocationFromDirectory(responsePath string, ref file.Reference) Location {
+	return Location{
+		RealPath: responsePath,
+		ref:      ref,
+	}
+}
+
 func NewLocationFromReference(ref file.Reference) Location {
 	return Location{
 		VirtualPath: string(ref.RealPath),

syft/source/source.go

@@ -7,6 +7,7 @@ package source
 
 import (
 	"fmt"
+	"sync"
 
 	"github.com/anchore/stereoscope"
 	"github.com/anchore/stereoscope/pkg/image"
@@ -16,58 +17,61 @@ import (
 // Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used
 // in cataloging (based on the data source and configuration)
 type Source struct {
-	Image    *image.Image // the image object to be cataloged (image only)
-	Metadata Metadata
+	Image             *image.Image // the image object to be cataloged (image only)
+	DirectoryResolver *directoryResolver
+	Metadata          Metadata
+	Mutex             *sync.Mutex
 }
 
 type sourceDetector func(string) (image.Source, string, error)
 
 // New produces a Source based on userInput like dir: or image:tag
-func New(userInput string, registryOptions *image.RegistryOptions) (Source, func(), error) {
+func New(userInput string, registryOptions *image.RegistryOptions) (*Source, func(), error) {
 	fs := afero.NewOsFs()
 	parsedScheme, imageSource, location, err := detectScheme(fs, image.DetectSource, userInput)
 	if err != nil {
-		return Source{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err)
+		return &Source{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err)
 	}
 
 	switch parsedScheme {
 	case DirectoryScheme:
 		fileMeta, err := fs.Stat(location)
 		if err != nil {
-			return Source{}, func() {}, fmt.Errorf("unable to stat dir=%q: %w", location, err)
+			return &Source{}, func() {}, fmt.Errorf("unable to stat dir=%q: %w", location, err)
 		}
 
 		if !fileMeta.IsDir() {
-			return Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err)
+			return &Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err)
 		}
 
 		s, err := NewFromDirectory(location)
 		if err != nil {
-			return Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err)
+			return &Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err)
 		}
-		return s, func() {}, nil
+		return &s, func() {}, nil
 
 	case ImageScheme:
 		img, err := stereoscope.GetImageFromSource(location, imageSource, registryOptions)
 		cleanup := stereoscope.Cleanup
 
 		if err != nil || img == nil {
-			return Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err)
+			return &Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err)
 		}
 
 		s, err := NewFromImage(img, location)
 		if err != nil {
-			return Source{}, cleanup, fmt.Errorf("could not populate source with image: %w", err)
+			return &Source{}, cleanup, fmt.Errorf("could not populate source with image: %w", err)
 		}
-		return s, cleanup, nil
+		return &s, cleanup, nil
 	}
 
-	return Source{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
+	return &Source{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
 }
 
 // NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
 func NewFromDirectory(path string) (Source, error) {
 	return Source{
+		Mutex: &sync.Mutex{},
 		Metadata: Metadata{
 			Scheme: DirectoryScheme,
 			Path:   path,
@@ -91,10 +95,19 @@ func NewFromImage(img *image.Image, userImageStr string) (Source, error) {
 	}, nil
 }
 
-func (s Source) FileResolver(scope Scope) (FileResolver, error) {
+func (s *Source) FileResolver(scope Scope) (FileResolver, error) {
 	switch s.Metadata.Scheme {
 	case DirectoryScheme:
-		return newDirectoryResolver(s.Metadata.Path)
+		s.Mutex.Lock()
+		defer s.Mutex.Unlock()
+		if s.DirectoryResolver == nil {
+			directoryResolver, err := newDirectoryResolver(s.Metadata.Path)
+			if err != nil {
+				return nil, err
+			}
+			s.DirectoryResolver = directoryResolver
+		}
+		return s.DirectoryResolver, nil
 	case ImageScheme:
 		switch scope {
 		case SquashedScope:

syft/source/source_test.go

@@ -89,6 +89,68 @@ func TestNewFromDirectory(t *testing.T) {
 	}
 }
 
+func TestNewFromDirectoryShared(t *testing.T) {
+	testCases := []struct {
+		desc       string
+		input      string
+		expString  string
+		notExist   string
+		inputPaths []string
+		expRefs    int
+	}{
+		{
+			desc:       "path detected",
+			input:      "test-fixtures",
+			notExist:   "foobar/",
+			inputPaths: []string{"test-fixtures/path-detected/.vimrc"},
+			expRefs:    1,
+		},
+		{
+			desc:       "directory ignored",
+			input:      "test-fixtures",
+			notExist:   "foobar/",
+			inputPaths: []string{"test-fixtures/path-detected"},
+			expRefs:    0,
+		},
+		{
+			desc:       "no files-by-path detected",
+			input:      "test-fixtures",
+			notExist:   "foobar/",
+			inputPaths: []string{"test-fixtures/no-path-detected"},
+			expRefs:    0,
+		},
+	}
+	for _, test := range testCases {
+		t.Run(test.desc, func(t *testing.T) {
+			src, err := NewFromDirectory(test.input)
+
+			if err != nil {
+				t.Errorf("could not create NewDirScope: %+v", err)
+			}
+			if src.Metadata.Path != test.input {
+				t.Errorf("mismatched stringer: '%s' != '%s'", src.Metadata.Path, test.input)
+			}
+
+			_, err = src.FileResolver(SquashedScope)
+			assert.NoError(t, err)
+
+			src.Metadata.Path = test.notExist
+			resolver2, err := src.FileResolver(SquashedScope)
+			assert.NoError(t, err)
+
+			refs, err := resolver2.FilesByPath(test.inputPaths...)
+			if err != nil {
+				t.Errorf("FilesByPath call produced an error: %+v", err)
+			}
+			if len(refs) != test.expRefs {
+				t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expRefs)
+
+			}
+
+		})
+	}
+}
+
 func TestFilesByPathDoesNotExist(t *testing.T) {
 	testCases := []struct {
 		desc     string