From 40df2d0d913dd6126584e0f0dfe33b1bfcc48527 Mon Sep 17 00:00:00 2001 From: Kudryavcev Nikolay Date: Sun, 29 Jun 2025 17:06:59 +0300 Subject: [PATCH] recursive indexing archives Signed-off-by: Kudryavcev Nikolay --- cmd/syft/internal/commands/scan.go | 1 + cmd/syft/internal/options/source.go | 50 +++-- syft/get_source_config.go | 5 + syft/internal/fileresolver/directory.go | 123 +++++++++++- .../fileresolver/directory_indexer.go | 17 ++ .../fileresolver/filetree_resolver.go | 123 ++++++++++-- .../fileresolver/filetree_resolver_test.go | 178 +++++++++++++----- .../java/internal/maven/resolver_test.go | 6 +- .../directorysource/directory_source.go | 18 +- .../directory_source_provider.go | 29 +-- syft/source/filesource/file_source.go | 23 ++- .../source/filesource/file_source_provider.go | 29 +-- .../sourceproviders/source_provider_config.go | 18 +- .../sourceproviders/source_providers.go | 4 +- 14 files changed, 501 insertions(+), 123 deletions(-) diff --git a/cmd/syft/internal/commands/scan.go b/cmd/syft/internal/commands/scan.go index 0d730fbdc..63831834a 100644 --- a/cmd/syft/internal/commands/scan.go +++ b/cmd/syft/internal/commands/scan.go @@ -225,6 +225,7 @@ func getSource(ctx context.Context, opts *options.Catalog, userInput string, sou Paths: opts.Exclusions, }). WithBasePath(opts.Source.BasePath). + WithMaxArchiveRecursiveIndexDepth(opts.Source.Directory.MaxArchiveRecursiveIndexDepth). WithSources(sources...). WithDefaultImagePullSource(opts.Source.Image.DefaultPullSource) diff --git a/cmd/syft/internal/options/source.go b/cmd/syft/internal/options/source.go index 97e81b945..4e0a7c456 100644 --- a/cmd/syft/internal/options/source.go +++ b/cmd/syft/internal/options/source.go @@ -14,34 +14,23 @@ import ( ) type sourceConfig struct { - Name string `json:"name" yaml:"name" mapstructure:"name"` - Version string `json:"version" yaml:"version" mapstructure:"version"` - BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths - File fileSource `json:"file" yaml:"file" mapstructure:"file"` - Image imageSource `json:"image" yaml:"image" mapstructure:"image"` + Name string `json:"name" yaml:"name" mapstructure:"name"` + Version string `json:"version" yaml:"version" mapstructure:"version"` + BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths + File fileSource `json:"file" yaml:"file" mapstructure:"file"` + Directory directorySource `json:"directory" yaml:"directory" mapstructure:"directory"` + Image imageSource `json:"image" yaml:"image" mapstructure:"image"` } -type fileSource struct { - Digests []string `json:"digests" yaml:"digests" mapstructure:"digests"` -} - -var _ interface { - clio.FieldDescriber -} = (*sourceConfig)(nil) - -var _ clio.PostLoader = (*imageSource)(nil) +var _ clio.FieldDescriber = (*sourceConfig)(nil) func (o *sourceConfig) DescribeFields(descriptions clio.FieldDescriptionSet) { descriptions.Add(&o.File.Digests, `the file digest algorithms to use on the scanned file (options: "md5", "sha1", "sha224", "sha256", "sha384", "sha512")`) + descriptions.Add(&o.Directory.MaxArchiveRecursiveIndexDepth, `depth to recursively resolve archive parents, no limit if == -1`) descriptions.Add(&o.Image.DefaultPullSource, `allows users to specify which image source should be used to generate the sbom valid values are: registry, docker, podman`) } -type imageSource struct { - DefaultPullSource string `json:"default-pull-source" yaml:"default-pull-source" mapstructure:"default-pull-source"` - MaxLayerSize string `json:"max-layer-size" yaml:"max-layer-size" mapstructure:"max-layer-size"` -} - func defaultSourceConfig() sourceConfig { var digests []string for _, alg := range sourceproviders.DefaultConfig().DigestAlgorithms { @@ -54,6 +43,12 @@ func defaultSourceConfig() sourceConfig { } } +var _ clio.PostLoader = (*fileSource)(nil) + +type fileSource struct { + Digests []string `json:"digests" yaml:"digests" mapstructure:"digests"` +} + func (c *fileSource) PostLoad() error { digests := strset.New(c.Digests...).List() sort.Strings(digests) @@ -61,6 +56,23 @@ func (c *fileSource) PostLoad() error { return nil } +var _ clio.PostLoader = (*directorySource)(nil) + +type directorySource struct { + MaxArchiveRecursiveIndexDepth int `json:"max-archive-recursive-index-depth" yaml:"max-archive-recursive-index-depth" mapstructure:"max-archive-recursive-index-depth"` +} + +func (c *directorySource) PostLoad() error { + return nil +} + +var _ clio.PostLoader = (*imageSource)(nil) + +type imageSource struct { + DefaultPullSource string `json:"default-pull-source" yaml:"default-pull-source" mapstructure:"default-pull-source"` + MaxLayerSize string `json:"max-layer-size" yaml:"max-layer-size" mapstructure:"max-layer-size"` +} + func (c *imageSource) PostLoad() error { if c.MaxLayerSize != "" { perFileReadLimit, err := humanize.ParseBytes(c.MaxLayerSize) diff --git a/syft/get_source_config.go b/syft/get_source_config.go index 7a163f1d1..0e7aff5e5 100644 --- a/syft/get_source_config.go +++ b/syft/get_source_config.go @@ -51,6 +51,11 @@ func (c *GetSourceConfig) WithBasePath(basePath string) *GetSourceConfig { return c } +func (c *GetSourceConfig) WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth int) *GetSourceConfig { + c.SourceProviderConfig = c.SourceProviderConfig.WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth) + return c +} + func (c *GetSourceConfig) WithSources(sources ...string) *GetSourceConfig { c.Sources = sources return c diff --git a/syft/internal/fileresolver/directory.go b/syft/internal/fileresolver/directory.go index 0f2df32b3..c517c00f2 100644 --- a/syft/internal/fileresolver/directory.go +++ b/syft/internal/fileresolver/directory.go @@ -3,11 +3,28 @@ package fileresolver import ( "errors" "fmt" + "os" + "path/filepath" + "strings" + "github.com/anchore/archiver/v3" "github.com/anchore/stereoscope/pkg/filetree" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/file" ) +type archiveAccessPath struct { + realPath string + accessPath string + archiveDepth int + archiveRealPath string +} + +const ( + archiveTempPathPattern = "syft-archivePaths-" + archiveContentTempPathPattern = "syft-archive-contents-" +) + var ErrSkipPath = errors.New("skip path") var _ file.Resolver = (*Directory)(nil) @@ -19,13 +36,29 @@ type Directory struct { indexer *directoryIndexer } -func NewFromDirectory(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) { - r, err := newFromDirectoryWithoutIndex(root, base, pathFilters...) +func NewFromDirectory(root, base string, maxArchiveRecursiveIndexDepth int, pathFilters ...PathIndexVisitor) (*Directory, func() error, error) { + var cleanupFn = func() error { return nil } + directory, err := newFromDirectoryWithoutIndex(root, base, pathFilters...) if err != nil { - return nil, err + return nil, cleanupFn, err } - return r, r.buildIndex() + if err = directory.buildIndex(); err != nil { + return nil, cleanupFn, fmt.Errorf("unable to build index: %w", err) + } + + if maxArchiveRecursiveIndexDepth != 0 { + archiveTempDir, err := os.MkdirTemp("", archiveTempPathPattern) + if err != nil { + return nil, cleanupFn, fmt.Errorf("unable to create tempdir for archive processing: %w", err) + } + + return directory, func() error { + return os.RemoveAll(archiveTempDir) + }, directory.buildArchiveIndex(archiveTempDir, directory.indexer.archivePaths, maxArchiveRecursiveIndexDepth) + } + + return directory, cleanupFn, nil } func newFromDirectoryWithoutIndex(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) { @@ -64,6 +97,88 @@ func (r *Directory) buildIndex() error { return nil } +func (r *Directory) buildArchiveIndex(archiveTempDir string, archives []string, maxArchiveIndexDepth int) error { + archivesToIndex := make([]archiveAccessPath, len(archives)) + for i, archive := range archives { + archivesToIndex[i] = archiveAccessPath{realPath: archive, accessPath: archive, archiveRealPath: archive} + } + +loop: + for { + var currentArchivePath archiveAccessPath + switch len(archivesToIndex) { + case 0: + break loop + case 1: + currentArchivePath, archivesToIndex = archivesToIndex[0], nil + default: + currentArchivePath, archivesToIndex = archivesToIndex[0], archivesToIndex[1:] + } + + if maxArchiveIndexDepth != -1 && currentArchivePath.archiveDepth >= maxArchiveIndexDepth { + continue + } + + archivePath, err := os.MkdirTemp(archiveTempDir, archiveContentTempPathPattern) + if err != nil { + return fmt.Errorf("unable to create tempdir for archive processing: %w", err) + } + + archiveRealPath, err := filepath.EvalSymlinks(archivePath) + if err != nil { + var pathErr *os.PathError + if errors.As(err, &pathErr) { + // we can't index the path, but we shouldn't consider this to be fatal + // TODO: known-unknowns + log.WithFields("archivePath", archivePath, "error", err).Trace("unable to evaluate symlink while indexing branch") + return nil + } + return err + } + + envelopedUnarchiver, err := archiver.ByExtension(currentArchivePath.archiveRealPath) + if err != nil { + return err + } + + unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver) + if !ok { + return ErrSkipPath + } + + if err = unarchiver.Unarchive(currentArchivePath.archiveRealPath, archiveRealPath); err != nil { + return err + } + + d, err := newFromDirectoryWithoutIndex(archiveRealPath, "") + if err != nil { + return err + } + + if err = d.buildIndex(); err != nil { + return err + } + + for _, archive := range d.indexer.archivePaths { + archivesToIndex = append(archivesToIndex, archiveAccessPath{ + realPath: currentArchivePath.realPath, + accessPath: strings.Replace(archive, archiveRealPath, currentArchivePath.accessPath, 1), + archiveDepth: currentArchivePath.archiveDepth + 1, + archiveRealPath: archive, + }) + } + + d.chroot = r.chroot + d.realPath = currentArchivePath.realPath + d.accessPath = currentArchivePath.accessPath + d.tempDir = archiveRealPath + + r.archives = append(r.archives, &d.filetreeResolver) + } + + return nil +} + // Stringer to represent a directory path data source func (r Directory) String() string { return fmt.Sprintf("dir:%s", r.path) diff --git a/syft/internal/fileresolver/directory_indexer.go b/syft/internal/fileresolver/directory_indexer.go index 0e3509655..79ac96fb0 100644 --- a/syft/internal/fileresolver/directory_indexer.go +++ b/syft/internal/fileresolver/directory_indexer.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" + "github.com/anchore/archiver/v3" "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" @@ -29,6 +30,7 @@ type directoryIndexer struct { errPaths map[string]error tree filetree.ReadWriter index filetree.Index + archivePaths []string } func newDirectoryIndexer(path, base string, visitors ...PathIndexVisitor) *directoryIndexer { @@ -146,6 +148,10 @@ func (r *directoryIndexer) indexTree(root string, stager *progress.Stage) ([]str func(path string, info os.FileInfo, err error) error { stager.Current = path + if isArchive(path, info) { + r.archivePaths = append(r.archivePaths, path) + } + newRoot, err := r.indexPath(path, info, err) if err != nil { @@ -166,6 +172,17 @@ func (r *directoryIndexer) indexTree(root string, stager *progress.Stage) ([]str return roots, nil } +// isArchive returns true if the path appears to be an archive +func isArchive(path string, info os.FileInfo) bool { + if info == nil || !info.Mode().IsRegular() { + return false + } + + envelopedUnarchiver, err := archiver.ByExtension(path) + _, ok := envelopedUnarchiver.(archiver.Unarchiver) + return err == nil && ok +} + func isRealPath(root string) (bool, error) { rootParent := filepath.Clean(filepath.Dir(root)) diff --git a/syft/internal/fileresolver/filetree_resolver.go b/syft/internal/fileresolver/filetree_resolver.go index 15d051166..b8126bfd4 100644 --- a/syft/internal/fileresolver/filetree_resolver.go +++ b/syft/internal/fileresolver/filetree_resolver.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "strings" stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree" @@ -19,24 +20,57 @@ type filetreeResolver struct { tree filetree.Reader index filetree.IndexReader searchContext filetree.Searcher + + realPath string + accessPath string + tempDir string + archives []*filetreeResolver } func (r *filetreeResolver) requestPath(userPath string) (string, error) { - return r.chroot.ToNativePath(userPath) + requestPath, err := r.chroot.ToNativePath(userPath) + if err != nil { + return "", err + } + + if r.accessPath != "" && r.tempDir != "" { + return strings.Replace(requestPath, r.accessPath, r.tempDir, 1), nil + } + + return requestPath, nil } // responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the file resolver. func (r filetreeResolver) responsePath(path string) string { + if r.tempDir != "" && strings.HasPrefix(path, r.tempDir) { + path = r.realPath + } return r.chroot.ToChrootPath(path) } +func (r filetreeResolver) responseAccessPath(path string) string { + responsePath := strings.Replace(path, r.tempDir, r.accessPath, 1) + return r.chroot.ToChrootPath(responsePath) +} + // HasPath indicates if the given path exists in the underlying source. func (r *filetreeResolver) HasPath(userPath string) bool { requestPath, err := r.requestPath(userPath) if err != nil { return false } - return r.tree.HasPath(stereoscopeFile.Path(requestPath)) + + if r.tree.HasPath(stereoscopeFile.Path(requestPath)) { + return true + } + + for _, archive := range r.archives { + if archive.HasPath(userPath) { + return true + } + } + + return false } // FilesByPath returns all file.References that match the given paths from the file index. @@ -80,18 +114,31 @@ func (r filetreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err references = append(references, file.NewVirtualLocationFromDirectory( r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root - r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root + r.responseAccessPath(userStrPath), // the path used to access this file, relative to the resolver root *ref.Reference, ), ) } } + for _, archive := range r.archives { + locations, err := archive.FilesByPath(userPaths...) + if err != nil { + return nil, err + } + references = append(references, locations...) + } + return references, nil } func (r filetreeResolver) requestGlob(pattern string) (string, error) { - return r.chroot.ToNativeGlob(pattern) + nativeGlob, err := r.chroot.ToNativeGlob(pattern) + if err != nil { + return "", err + } + + return strings.Replace(nativeGlob, r.accessPath, r.tempDir, 1), nil } // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. @@ -123,8 +170,8 @@ func (r filetreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro } loc := file.NewVirtualLocationFromDirectory( - r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root - r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root + r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root + r.responseAccessPath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root *refVia.Reference, ) uniqueFileIDs.Add(*refVia.Reference) @@ -132,6 +179,20 @@ func (r filetreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro } } + for _, archive := range r.archives { + archiveUniqueLocations, err := archive.FilesByGlob(patterns...) + if err != nil { + return nil, err + } + + for _, archiveUniqueLocation := range archiveUniqueLocations { + if uniqueFileIDs.Contains(archiveUniqueLocation.Reference()) { + continue + } + uniqueLocations = append(uniqueLocations, archiveUniqueLocation) + } + } + return uniqueLocations, nil } @@ -157,6 +218,14 @@ func (r filetreeResolver) FileContentsByLocation(location file.Location) (io.Rea } entry, err := r.index.Get(location.Reference()) + if err != nil { + for _, archive := range r.archives { + entry, err = archive.index.Get(location.Reference()) + if err == nil { + break + } + } + } if err != nil { return nil, err } @@ -184,21 +253,39 @@ func (r *filetreeResolver) AllLocations(ctx context.Context) <-chan file.Locatio select { case <-ctx.Done(): return - case results <- file.NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), ref): + case results <- file.NewVirtualLocationFromDirectory(r.responsePath(string(ref.RealPath)), r.responseAccessPath(string(ref.RealPath)), ref): continue } } + + for _, archive := range r.archives { + for location := range archive.AllLocations(ctx) { + select { + case <-ctx.Done(): + return + case results <- location: + continue + } + } + } }() return results } func (r *filetreeResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) { entry, err := r.index.Get(location.Reference()) - if err != nil { - return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist) + if err == nil { + return entry.Metadata, nil } - return entry.Metadata, nil + for _, archive := range r.archives { + entry, err = archive.index.Get(location.Reference()) + if err == nil { + return entry.Metadata, nil + } + } + + return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist) } func (r *filetreeResolver) FilesByMIMEType(types ...string) ([]file.Location, error) { @@ -218,12 +305,26 @@ func (r *filetreeResolver) FilesByMIMEType(types ...string) ([]file.Location, er } location := file.NewVirtualLocationFromDirectory( r.responsePath(string(refVia.RealPath)), - r.responsePath(string(refVia.RequestPath)), + r.responseAccessPath(string(refVia.RequestPath)), *refVia.Reference, ) uniqueFileIDs.Add(*refVia.Reference) uniqueLocations = append(uniqueLocations, location) } + for _, archive := range r.archives { + archiveUniqueLocations, err := archive.FilesByMIMEType(types...) + if err != nil { + return nil, err + } + + for _, archiveUniqueLocation := range archiveUniqueLocations { + if uniqueFileIDs.Contains(archiveUniqueLocation.Reference()) { + continue + } + uniqueLocations = append(uniqueLocations, archiveUniqueLocation) + } + } + return uniqueLocations, nil } diff --git a/syft/internal/fileresolver/filetree_resolver_test.go b/syft/internal/fileresolver/filetree_resolver_test.go index adbc77502..97de6d2b5 100644 --- a/syft/internal/fileresolver/filetree_resolver_test.go +++ b/syft/internal/fileresolver/filetree_resolver_test.go @@ -506,9 +506,12 @@ func TestDirectoryResolver_FilesByPath_request_response(t *testing.T) { require.NoError(t, os.Chdir(testDir)) }) - resolver, err := NewFromDirectory(c.root, c.base) + resolver, cleanupFn, err := NewFromDirectory(c.root, c.base, 0) require.NoError(t, err) require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) refs, err := resolver.FilesByPath(c.input) require.NoError(t, err) @@ -558,8 +561,12 @@ func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - resolver, err := NewFromDirectory(c.relativeRoot, "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory(c.relativeRoot, "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) refs, err := resolver.FilesByPath(c.input) require.NoError(t, err) @@ -613,8 +620,12 @@ func TestDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) { absRoot, err := filepath.Abs(c.relativeRoot) require.NoError(t, err) - resolver, err := NewFromDirectory(absRoot, "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory(absRoot, "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) refs, err := resolver.FilesByPath(c.input) require.NoError(t, err) @@ -674,8 +685,12 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - resolver, err := NewFromDirectory(c.root, "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory(c.root, "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) hasPath := resolver.HasPath(c.input) if !c.forcePositiveHasPath { @@ -722,8 +737,13 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures", "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures", "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) + refs, err := resolver.FilesByPath(c.input...) assert.NoError(t, err) @@ -735,8 +755,13 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) { } func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures", "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures", "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) + refs, err := resolver.FilesByGlob("**/image-symlinks/file*") assert.NoError(t, err) @@ -744,16 +769,26 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) { } func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures/image-symlinks", "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/image-symlinks", "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) + refs, err := resolver.FilesByGlob("**/*.txt") assert.NoError(t, err) assert.Len(t, refs, 6) } func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures", "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures", "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) + refs, err := resolver.FilesByGlob("**/image-symlinks/*1.txt") assert.NoError(t, err) @@ -779,8 +814,12 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-simple", "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) refs, err := resolver.FilesByPath(test.fixture) require.NoError(t, err) @@ -802,8 +841,12 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) { func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) { // let's make certain that "dev/place" is not ignored, since it is not "/dev/place" - resolver, err := NewFromDirectory("test-fixtures/system_paths/target", "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory("test-fixtures/system_paths/target", "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) // all paths should be found (non filtering matches a path) locations, err := resolver.FilesByGlob("**/place") @@ -841,8 +884,13 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) { } for _, test := range tests { t.Run(test.fixturePath, func(t *testing.T) { - resolver, err := NewFromDirectory(test.fixturePath, "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory(test.fixturePath, "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) + locations, err := resolver.FilesByMIMEType(test.mimeType) assert.NoError(t, err) assert.Equal(t, test.expectedPaths.Size(), len(locations)) @@ -854,8 +902,12 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) { } func Test_IndexingNestedSymLinks(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "") + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-simple", "", 0) require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) // check that we can get the real path locations, err := resolver.FilesByPath("./readme") @@ -910,8 +962,12 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) { return nil } - resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "", filterFn) + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-simple", "", 0, filterFn) require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) // the path to the real file is PRUNED from the index, so we should NOT expect a location returned locations, err := resolver.FilesByPath("./readme") @@ -930,8 +986,12 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) { } func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures/symlinks-multiple-roots/root", "") + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-multiple-roots/root", "", 0) require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) // check that we can get the real path locations, err := resolver.FilesByPath("./readme") @@ -948,8 +1008,12 @@ func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) { } func Test_RootViaSymlink(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures/symlinked-root/nested/link-root", "") + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinked-root/nested/link-root", "", 0) require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) locations, err := resolver.FilesByPath("./file1.txt") require.NoError(t, err) @@ -968,10 +1032,14 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) { cwd, err := os.Getwd() require.NoError(t, err) - r, err := NewFromDirectory(".", "") + resolver, cleanupFn, err := NewFromDirectory(".", "", 0) require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) - exists, existingPath, err := r.tree.File(stereoscopeFile.Path(filepath.Join(cwd, "test-fixtures/image-simple/file-1.txt"))) + exists, existingPath, err := resolver.tree.File(stereoscopeFile.Path(filepath.Join(cwd, "test-fixtures/image-simple/file-1.txt"))) require.True(t, exists) require.NoError(t, err) require.True(t, existingPath.HasReference()) @@ -995,8 +1063,7 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - - actual, err := r.FileContentsByLocation(test.location) + actual, err := resolver.FileContentsByLocation(test.location) if test.err { require.Error(t, err) return @@ -1014,8 +1081,12 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) { func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) { test := func(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures/symlinks-loop", "") + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-loop", "", 0) require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) locations, err := resolver.FilesByGlob("**/file.target") require.NoError(t, err) @@ -1085,8 +1156,12 @@ func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - resolver, err := NewFromDirectory(c.root, c.root) - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory(c.root, c.root, 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) refs, err := resolver.FilesByPath(c.input) require.NoError(t, err) @@ -1229,9 +1304,12 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "") + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "", 0) require.NoError(t, err) - assert.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) actual := test.runner(resolver) @@ -1241,8 +1319,12 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) { } func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures/symlinks-prune-indexing", "") + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-prune-indexing", "", 0) require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) var allRealPaths []stereoscopeFile.Path for l := range resolver.AllLocations(context.Background()) { @@ -1264,8 +1346,12 @@ func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) { func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) { defer goleak.VerifyNone(t) - resolver, err := NewFromDirectory("./test-fixtures/system_paths", "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/system_paths", "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) var dirLoc *file.Location ctx, cancel := context.WithCancel(context.Background()) @@ -1287,8 +1373,12 @@ func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) { } func TestDirectoryResolver_AllLocations(t *testing.T) { - resolver, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "") - assert.NoError(t, err) + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "", 0) + require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) paths := strset.New() for loc := range resolver.AllLocations(context.Background()) { @@ -1320,8 +1410,13 @@ func TestDirectoryResolver_AllLocations(t *testing.T) { func TestAllLocationsDoesNotLeakGoRoutine(t *testing.T) { defer goleak.VerifyNone(t) - resolver, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "") + resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "", 0) require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) + ctx, cancel := context.WithCancel(context.Background()) for range resolver.AllLocations(ctx) { break @@ -1542,6 +1637,7 @@ func Test_fileResolver_FileContentsByLocation(t *testing.T) { } func TestFileResolver_AllLocations_errorOnDirRequest(t *testing.T) { + defer goleak.VerifyNone(t) filePath := "./test-fixtures/system_paths/target/home/place" parentPath, err := absoluteSymlinkFreePathToParent(filePath) require.NoError(t, err) @@ -1565,8 +1661,6 @@ func TestFileResolver_AllLocations_errorOnDirRequest(t *testing.T) { reader, err := resolver.FileContentsByLocation(*dirLoc) require.Error(t, err) require.Nil(t, reader) - - goleak.VerifyNone(t) } func TestFileResolver_AllLocations(t *testing.T) { diff --git a/syft/pkg/cataloger/java/internal/maven/resolver_test.go b/syft/pkg/cataloger/java/internal/maven/resolver_test.go index 34042593a..7f1a37f87 100644 --- a/syft/pkg/cataloger/java/internal/maven/resolver_test.go +++ b/syft/pkg/cataloger/java/internal/maven/resolver_test.go @@ -273,8 +273,12 @@ func Test_mavenResolverRemote(t *testing.T) { } func Test_relativePathParent(t *testing.T) { - resolver, err := fileresolver.NewFromDirectory("test-fixtures/local", "") + resolver, cleanupFn, err := fileresolver.NewFromDirectory("test-fixtures/local", "", 0) require.NoError(t, err) + require.NotNil(t, resolver) + t.Cleanup(func() { + require.NoError(t, cleanupFn()) + }) ctx := context.Background() diff --git a/syft/source/directorysource/directory_source.go b/syft/source/directorysource/directory_source.go index 0ab06980f..f249d4106 100644 --- a/syft/source/directorysource/directory_source.go +++ b/syft/source/directorysource/directory_source.go @@ -21,10 +21,11 @@ import ( var _ source.Source = (*directorySource)(nil) type Config struct { - Path string - Base string - Exclude source.ExcludeConfig - Alias source.Alias + Path string + Base string + Exclude source.ExcludeConfig + Alias source.Alias + MaxArchiveRecursiveIndexDepth int } type directorySource struct { @@ -32,6 +33,7 @@ type directorySource struct { config Config resolver *fileresolver.Directory mutex *sync.Mutex + closer func() error } func NewFromPath(path string) (source.Source, error) { @@ -145,11 +147,12 @@ func (s *directorySource) FileResolver(_ source.Scope) (file.Resolver, error) { // this should be the only file resolver that might have overlap with where files are cached exclusionFunctions = append(exclusionFunctions, excludeCachePathVisitors()...) - res, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, exclusionFunctions...) + res, cleanupFn, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, s.config.MaxArchiveRecursiveIndexDepth, exclusionFunctions...) if err != nil { return nil, fmt.Errorf("unable to create directory resolver: %w", err) } + s.closer = cleanupFn s.resolver = res } @@ -159,8 +162,11 @@ func (s *directorySource) FileResolver(_ source.Scope) (file.Resolver, error) { func (s *directorySource) Close() error { s.mutex.Lock() defer s.mutex.Unlock() + if s.closer == nil { + return nil + } s.resolver = nil - return nil + return s.closer() } func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileresolver.PathIndexVisitor, error) { diff --git a/syft/source/directorysource/directory_source_provider.go b/syft/source/directorysource/directory_source_provider.go index 59acbc3a2..14fa3542c 100644 --- a/syft/source/directorysource/directory_source_provider.go +++ b/syft/source/directorysource/directory_source_provider.go @@ -10,20 +10,22 @@ import ( "github.com/anchore/syft/syft/source" ) -func NewSourceProvider(path string, exclude source.ExcludeConfig, alias source.Alias, basePath string) source.Provider { +func NewSourceProvider(path string, exclude source.ExcludeConfig, alias source.Alias, basePath string, maxArchiveRecursiveIndexDepth int) source.Provider { return &directorySourceProvider{ - path: path, - basePath: basePath, - exclude: exclude, - alias: alias, + path: path, + basePath: basePath, + exclude: exclude, + alias: alias, + maxArchiveRecursiveIndexDepth: maxArchiveRecursiveIndexDepth, } } type directorySourceProvider struct { - path string - basePath string - exclude source.ExcludeConfig - alias source.Alias + path string + basePath string + exclude source.ExcludeConfig + alias source.Alias + maxArchiveRecursiveIndexDepth int } func (l directorySourceProvider) Name() string { @@ -48,10 +50,11 @@ func (l directorySourceProvider) Provide(_ context.Context) (source.Source, erro return New( Config{ - Path: location, - Base: basePath(l.basePath, location), - Exclude: l.exclude, - Alias: l.alias, + Path: location, + Base: basePath(l.basePath, location), + Exclude: l.exclude, + Alias: l.alias, + MaxArchiveRecursiveIndexDepth: l.maxArchiveRecursiveIndexDepth, }, ) } diff --git a/syft/source/filesource/file_source.go b/syft/source/filesource/file_source.go index 09a4422ce..62a520e05 100644 --- a/syft/source/filesource/file_source.go +++ b/syft/source/filesource/file_source.go @@ -26,11 +26,12 @@ import ( var _ source.Source = (*fileSource)(nil) type Config struct { - Path string - Exclude source.ExcludeConfig - DigestAlgorithms []crypto.Hash - Alias source.Alias - SkipExtractArchive bool + Path string + Exclude source.ExcludeConfig + DigestAlgorithms []crypto.Hash + Alias source.Alias + SkipExtractArchive bool + MaxArchiveRecursiveIndexDepth int } type fileSource struct { @@ -171,11 +172,18 @@ func (s fileSource) FileResolver(_ source.Scope) (file.Resolver, error) { if isArchiveAnalysis { // this is an analysis of an archive file... we should scan the directory where the archive contents - res, err := fileresolver.NewFromDirectory(s.analysisPath, "", exclusionFunctions...) + res, cleanupFn, err := fileresolver.NewFromDirectory(s.analysisPath, "", s.config.MaxArchiveRecursiveIndexDepth, exclusionFunctions...) if err != nil { return nil, fmt.Errorf("unable to create directory resolver: %w", err) } s.resolver = res + s.closer = func() error { + if err := cleanupFn(); err != nil { + return err + } + + return s.closer() + } return s.resolver, nil } @@ -201,6 +209,9 @@ func absoluteSymlinkFreePathToParent(path string) (string, error) { } func (s *fileSource) Close() error { + s.mutex.Lock() + defer s.mutex.Unlock() + if s.closer == nil { return nil } diff --git a/syft/source/filesource/file_source_provider.go b/syft/source/filesource/file_source_provider.go index adf80ec88..895a85cab 100644 --- a/syft/source/filesource/file_source_provider.go +++ b/syft/source/filesource/file_source_provider.go @@ -11,20 +11,22 @@ import ( "github.com/anchore/syft/syft/source" ) -func NewSourceProvider(path string, exclude source.ExcludeConfig, digestAlgorithms []crypto.Hash, alias source.Alias) source.Provider { +func NewSourceProvider(path string, exclude source.ExcludeConfig, digestAlgorithms []crypto.Hash, alias source.Alias, maxArchiveRecursiveIndexDepth int) source.Provider { return &fileSourceProvider{ - path: path, - exclude: exclude, - digestAlgorithms: digestAlgorithms, - alias: alias, + path: path, + exclude: exclude, + digestAlgorithms: digestAlgorithms, + alias: alias, + maxArchiveRecursiveIndexDepth: maxArchiveRecursiveIndexDepth, } } type fileSourceProvider struct { - path string - exclude source.ExcludeConfig - digestAlgorithms []crypto.Hash - alias source.Alias + path string + exclude source.ExcludeConfig + digestAlgorithms []crypto.Hash + alias source.Alias + maxArchiveRecursiveIndexDepth int } func (p fileSourceProvider) Name() string { @@ -49,10 +51,11 @@ func (p fileSourceProvider) Provide(_ context.Context) (source.Source, error) { return New( Config{ - Path: location, - Exclude: p.exclude, - DigestAlgorithms: p.digestAlgorithms, - Alias: p.alias, + Path: location, + Exclude: p.exclude, + DigestAlgorithms: p.digestAlgorithms, + Alias: p.alias, + MaxArchiveRecursiveIndexDepth: p.maxArchiveRecursiveIndexDepth, }, ) } diff --git a/syft/source/sourceproviders/source_provider_config.go b/syft/source/sourceproviders/source_provider_config.go index dbc48bc0b..662c48f01 100644 --- a/syft/source/sourceproviders/source_provider_config.go +++ b/syft/source/sourceproviders/source_provider_config.go @@ -9,12 +9,13 @@ import ( // Config is the uber-configuration for all Syft source providers type Config struct { - Platform *image.Platform - Alias source.Alias - RegistryOptions *image.RegistryOptions - Exclude source.ExcludeConfig - DigestAlgorithms []crypto.Hash - BasePath string + Platform *image.Platform + Alias source.Alias + RegistryOptions *image.RegistryOptions + Exclude source.ExcludeConfig + DigestAlgorithms []crypto.Hash + BasePath string + MaxArchiveRecursiveIndexDepth int } func (c *Config) WithAlias(alias source.Alias) *Config { @@ -47,6 +48,11 @@ func (c *Config) WithBasePath(basePath string) *Config { return c } +func (c *Config) WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth int) *Config { + c.MaxArchiveRecursiveIndexDepth = maxArchiveRecursiveIndexDepth + return c +} + func DefaultConfig() *Config { return &Config{ DigestAlgorithms: []crypto.Hash{ diff --git a/syft/source/sourceproviders/source_providers.go b/syft/source/sourceproviders/source_providers.go index 107f3ad48..4363f575c 100644 --- a/syft/source/sourceproviders/source_providers.go +++ b/syft/source/sourceproviders/source_providers.go @@ -26,8 +26,8 @@ func All(userInput string, cfg *Config) []collections.TaggedValue[source.Provide return collections.TaggedValueSet[source.Provider]{}. // --from file, dir, oci-archive, etc. Join(stereoscopeProviders.Select(FileTag, DirTag)...). - Join(tagProvider(filesource.NewSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias), FileTag)). - Join(tagProvider(directorysource.NewSourceProvider(userInput, cfg.Exclude, cfg.Alias, cfg.BasePath), DirTag)). + Join(tagProvider(filesource.NewSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias, cfg.MaxArchiveRecursiveIndexDepth), FileTag)). + Join(tagProvider(directorysource.NewSourceProvider(userInput, cfg.Exclude, cfg.Alias, cfg.BasePath, cfg.MaxArchiveRecursiveIndexDepth), DirTag)). // --from docker, registry, etc. Join(stereoscopeProviders.Select(PullTag)...)