oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-10-30 14:02:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

recursive indexing archives

Browse Source 
Signed-off-by: Kudryavcev Nikolay <kydry.nikolau@gmail.com>
This commit is contained in:
Kudryavcev Nikolay
2025-06-29 17:06:59 +03:00
parent 4eb8ba4575
commit 40df2d0d91
14 changed files with 501 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/syft/internal/commands/scan.go
+1
View File
@@ -225,6 +225,7 @@ func getSource(ctx context.Context, opts *options.Catalog, userInput string, sou
Paths: opts.Exclusions,
}).
WithBasePath(opts.Source.BasePath).
WithMaxArchiveRecursiveIndexDepth(opts.Source.Directory.MaxArchiveRecursiveIndexDepth).
WithSources(sources...).
WithDefaultImagePullSource(opts.Source.Image.DefaultPullSource)
cmd/syft/internal/options/source.go
+31 -19
View File
@@ -14,34 +14,23 @@ import (
)
type sourceConfig struct {
Name string `json:"name" yaml:"name" mapstructure:"name"`
Version string `json:"version" yaml:"version" mapstructure:"version"`
BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths
File fileSource `json:"file" yaml:"file" mapstructure:"file"`
Image imageSource `json:"image" yaml:"image" mapstructure:"image"`
Name string `json:"name" yaml:"name" mapstructure:"name"`
Version string `json:"version" yaml:"version" mapstructure:"version"`
BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths
File fileSource `json:"file" yaml:"file" mapstructure:"file"`
Directory directorySource `json:"directory" yaml:"directory" mapstructure:"directory"`
Image imageSource `json:"image" yaml:"image" mapstructure:"image"`
}
type fileSource struct {
Digests []string `json:"digests" yaml:"digests" mapstructure:"digests"`
}
var _ interface {
clio.FieldDescriber
} = (*sourceConfig)(nil)
var _ clio.PostLoader = (*imageSource)(nil)
var _ clio.FieldDescriber = (*sourceConfig)(nil)
func (o *sourceConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
descriptions.Add(&o.File.Digests, `the file digest algorithms to use on the scanned file (options: "md5", "sha1", "sha224", "sha256", "sha384", "sha512")`)
descriptions.Add(&o.Directory.MaxArchiveRecursiveIndexDepth, `depth to recursively resolve archive parents, no limit if == -1`)
descriptions.Add(&o.Image.DefaultPullSource, `allows users to specify which image source should be used to generate the sbom
valid values are: registry, docker, podman`)
}
type imageSource struct {
DefaultPullSource string `json:"default-pull-source" yaml:"default-pull-source" mapstructure:"default-pull-source"`
MaxLayerSize string `json:"max-layer-size" yaml:"max-layer-size" mapstructure:"max-layer-size"`
}
func defaultSourceConfig() sourceConfig {
var digests []string
for _, alg := range sourceproviders.DefaultConfig().DigestAlgorithms {
@@ -54,6 +43,12 @@ func defaultSourceConfig() sourceConfig {
}
}
var _ clio.PostLoader = (*fileSource)(nil)
type fileSource struct {
Digests []string `json:"digests" yaml:"digests" mapstructure:"digests"`
}
func (c *fileSource) PostLoad() error {
digests := strset.New(c.Digests...).List()
sort.Strings(digests)
@@ -61,6 +56,23 @@ func (c *fileSource) PostLoad() error {
return nil
}
var _ clio.PostLoader = (*directorySource)(nil)
type directorySource struct {
MaxArchiveRecursiveIndexDepth int `json:"max-archive-recursive-index-depth" yaml:"max-archive-recursive-index-depth" mapstructure:"max-archive-recursive-index-depth"`
}
func (c *directorySource) PostLoad() error {
return nil
}
var _ clio.PostLoader = (*imageSource)(nil)
type imageSource struct {
DefaultPullSource string `json:"default-pull-source" yaml:"default-pull-source" mapstructure:"default-pull-source"`
MaxLayerSize string `json:"max-layer-size" yaml:"max-layer-size" mapstructure:"max-layer-size"`
}
func (c *imageSource) PostLoad() error {
if c.MaxLayerSize != "" {
perFileReadLimit, err := humanize.ParseBytes(c.MaxLayerSize)
syft/get_source_config.go
+5
View File
@@ -51,6 +51,11 @@ func (c *GetSourceConfig) WithBasePath(basePath string) *GetSourceConfig {
return c
}
func (c *GetSourceConfig) WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth int) *GetSourceConfig {
c.SourceProviderConfig = c.SourceProviderConfig.WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth)
return c
}
func (c *GetSourceConfig) WithSources(sources ...string) *GetSourceConfig {
c.Sources = sources
return c
syft/internal/fileresolver/directory.go
+119 -4
View File
@@ -3,11 +3,28 @@ package fileresolver
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/anchore/archiver/v3"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
)
type archiveAccessPath struct {
realPath string
accessPath string
archiveDepth int
archiveRealPath string
}
const (
archiveTempPathPattern = "syft-archivePaths-"
archiveContentTempPathPattern = "syft-archive-contents-"
)
var ErrSkipPath = errors.New("skip path")
var _ file.Resolver = (*Directory)(nil)
@@ -19,13 +36,29 @@ type Directory struct {
indexer *directoryIndexer
}
func NewFromDirectory(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
r, err := newFromDirectoryWithoutIndex(root, base, pathFilters...)
func NewFromDirectory(root, base string, maxArchiveRecursiveIndexDepth int, pathFilters ...PathIndexVisitor) (*Directory, func() error, error) {
var cleanupFn = func() error { return nil }
directory, err := newFromDirectoryWithoutIndex(root, base, pathFilters...)
if err != nil {
return nil, err
return nil, cleanupFn, err
}
return r, r.buildIndex()
if err = directory.buildIndex(); err != nil {
return nil, cleanupFn, fmt.Errorf("unable to build index: %w", err)
}
if maxArchiveRecursiveIndexDepth != 0 {
archiveTempDir, err := os.MkdirTemp("", archiveTempPathPattern)
if err != nil {
return nil, cleanupFn, fmt.Errorf("unable to create tempdir for archive processing: %w", err)
}
return directory, func() error {
return os.RemoveAll(archiveTempDir)
}, directory.buildArchiveIndex(archiveTempDir, directory.indexer.archivePaths, maxArchiveRecursiveIndexDepth)
}
return directory, cleanupFn, nil
}
func newFromDirectoryWithoutIndex(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
@@ -64,6 +97,88 @@ func (r *Directory) buildIndex() error {
return nil
}
func (r *Directory) buildArchiveIndex(archiveTempDir string, archives []string, maxArchiveIndexDepth int) error {
archivesToIndex := make([]archiveAccessPath, len(archives))
for i, archive := range archives {
archivesToIndex[i] = archiveAccessPath{realPath: archive, accessPath: archive, archiveRealPath: archive}
}
loop:
for {
var currentArchivePath archiveAccessPath
switch len(archivesToIndex) {
case 0:
break loop
case 1:
currentArchivePath, archivesToIndex = archivesToIndex[0], nil
default:
currentArchivePath, archivesToIndex = archivesToIndex[0], archivesToIndex[1:]
}
if maxArchiveIndexDepth != -1 && currentArchivePath.archiveDepth >= maxArchiveIndexDepth {
continue
}
archivePath, err := os.MkdirTemp(archiveTempDir, archiveContentTempPathPattern)
if err != nil {
return fmt.Errorf("unable to create tempdir for archive processing: %w", err)
}
archiveRealPath, err := filepath.EvalSymlinks(archivePath)
if err != nil {
var pathErr *os.PathError
if errors.As(err, &pathErr) {
// we can't index the path, but we shouldn't consider this to be fatal
// TODO: known-unknowns
log.WithFields("archivePath", archivePath, "error", err).Trace("unable to evaluate symlink while indexing branch")
return nil
}
return err
}
envelopedUnarchiver, err := archiver.ByExtension(currentArchivePath.archiveRealPath)
if err != nil {
return err
}
unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver)
if !ok {
return ErrSkipPath
}
if err = unarchiver.Unarchive(currentArchivePath.archiveRealPath, archiveRealPath); err != nil {
return err
}
d, err := newFromDirectoryWithoutIndex(archiveRealPath, "")
if err != nil {
return err
}
if err = d.buildIndex(); err != nil {
return err
}
for _, archive := range d.indexer.archivePaths {
archivesToIndex = append(archivesToIndex, archiveAccessPath{
realPath: currentArchivePath.realPath,
accessPath: strings.Replace(archive, archiveRealPath, currentArchivePath.accessPath, 1),
archiveDepth: currentArchivePath.archiveDepth + 1,
archiveRealPath: archive,
})
}
d.chroot = r.chroot
d.realPath = currentArchivePath.realPath
d.accessPath = currentArchivePath.accessPath
d.tempDir = archiveRealPath
r.archives = append(r.archives, &d.filetreeResolver)
}
return nil
}
// Stringer to represent a directory path data source
func (r Directory) String() string {
return fmt.Sprintf("dir:%s", r.path)
syft/internal/fileresolver/directory_indexer.go
+17
View File
@@ -9,6 +9,7 @@ import (
"path/filepath"
"strings"
"github.com/anchore/archiver/v3"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
@@ -29,6 +30,7 @@ type directoryIndexer struct {
errPaths map[string]error
tree filetree.ReadWriter
index filetree.Index
archivePaths []string
}
func newDirectoryIndexer(path, base string, visitors ...PathIndexVisitor) *directoryIndexer {
@@ -146,6 +148,10 @@ func (r *directoryIndexer) indexTree(root string, stager *progress.Stage) ([]str
func(path string, info os.FileInfo, err error) error {
stager.Current = path
if isArchive(path, info) {
r.archivePaths = append(r.archivePaths, path)
}
newRoot, err := r.indexPath(path, info, err)
if err != nil {
@@ -166,6 +172,17 @@ func (r *directoryIndexer) indexTree(root string, stager *progress.Stage) ([]str
return roots, nil
}
// isArchive returns true if the path appears to be an archive
func isArchive(path string, info os.FileInfo) bool {
if info == nil || !info.Mode().IsRegular() {
return false
}
envelopedUnarchiver, err := archiver.ByExtension(path)
_, ok := envelopedUnarchiver.(archiver.Unarchiver)
return err == nil && ok
}
func isRealPath(root string) (bool, error) {
rootParent := filepath.Clean(filepath.Dir(root))
syft/internal/fileresolver/filetree_resolver.go
+112 -11
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"os"
"strings"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
@@ -19,24 +20,57 @@ type filetreeResolver struct {
tree filetree.Reader
index filetree.IndexReader
searchContext filetree.Searcher
realPath string
accessPath string
tempDir string
archives []*filetreeResolver
}
func (r *filetreeResolver) requestPath(userPath string) (string, error) {
return r.chroot.ToNativePath(userPath)
requestPath, err := r.chroot.ToNativePath(userPath)
if err != nil {
return "", err
}
if r.accessPath != "" && r.tempDir != "" {
return strings.Replace(requestPath, r.accessPath, r.tempDir, 1), nil
}
return requestPath, nil
}
// responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the file resolver.
func (r filetreeResolver) responsePath(path string) string {
if r.tempDir != "" && strings.HasPrefix(path, r.tempDir) {
path = r.realPath
}
return r.chroot.ToChrootPath(path)
}
func (r filetreeResolver) responseAccessPath(path string) string {
responsePath := strings.Replace(path, r.tempDir, r.accessPath, 1)
return r.chroot.ToChrootPath(responsePath)
}
// HasPath indicates if the given path exists in the underlying source.
func (r *filetreeResolver) HasPath(userPath string) bool {
requestPath, err := r.requestPath(userPath)
if err != nil {
return false
}
return r.tree.HasPath(stereoscopeFile.Path(requestPath))
if r.tree.HasPath(stereoscopeFile.Path(requestPath)) {
return true
}
for _, archive := range r.archives {
if archive.HasPath(userPath) {
return true
}
}
return false
}
// FilesByPath returns all file.References that match the given paths from the file index.
@@ -80,18 +114,31 @@ func (r filetreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err
references = append(references,
file.NewVirtualLocationFromDirectory(
r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root
r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root
r.responseAccessPath(userStrPath), // the path used to access this file, relative to the resolver root
*ref.Reference,
),
)
}
}
for _, archive := range r.archives {
locations, err := archive.FilesByPath(userPaths...)
if err != nil {
return nil, err
}
references = append(references, locations...)
}
return references, nil
}
func (r filetreeResolver) requestGlob(pattern string) (string, error) {
return r.chroot.ToNativeGlob(pattern)
nativeGlob, err := r.chroot.ToNativeGlob(pattern)
if err != nil {
return "", err
}
return strings.Replace(nativeGlob, r.accessPath, r.tempDir, 1), nil
}
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
@@ -123,8 +170,8 @@ func (r filetreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro
}
loc := file.NewVirtualLocationFromDirectory(
r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root
r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root
r.responseAccessPath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
*refVia.Reference,
)
uniqueFileIDs.Add(*refVia.Reference)
@@ -132,6 +179,20 @@ func (r filetreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro
}
}
for _, archive := range r.archives {
archiveUniqueLocations, err := archive.FilesByGlob(patterns...)
if err != nil {
return nil, err
}
for _, archiveUniqueLocation := range archiveUniqueLocations {
if uniqueFileIDs.Contains(archiveUniqueLocation.Reference()) {
continue
}
uniqueLocations = append(uniqueLocations, archiveUniqueLocation)
}
}
return uniqueLocations, nil
}
@@ -157,6 +218,14 @@ func (r filetreeResolver) FileContentsByLocation(location file.Location) (io.Rea
}
entry, err := r.index.Get(location.Reference())
if err != nil {
for _, archive := range r.archives {
entry, err = archive.index.Get(location.Reference())
if err == nil {
break
}
}
}
if err != nil {
return nil, err
}
@@ -184,21 +253,39 @@ func (r *filetreeResolver) AllLocations(ctx context.Context) <-chan file.Locatio
select {
case <-ctx.Done():
return
case results <- file.NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), ref):
case results <- file.NewVirtualLocationFromDirectory(r.responsePath(string(ref.RealPath)), r.responseAccessPath(string(ref.RealPath)), ref):
continue
}
}
for _, archive := range r.archives {
for location := range archive.AllLocations(ctx) {
select {
case <-ctx.Done():
return
case results <- location:
continue
}
}
}
}()
return results
}
func (r *filetreeResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
entry, err := r.index.Get(location.Reference())
if err != nil {
return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
if err == nil {
return entry.Metadata, nil
}
return entry.Metadata, nil
for _, archive := range r.archives {
entry, err = archive.index.Get(location.Reference())
if err == nil {
return entry.Metadata, nil
}
}
return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
}
func (r *filetreeResolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
@@ -218,12 +305,26 @@ func (r *filetreeResolver) FilesByMIMEType(types ...string) ([]file.Location, er
}
location := file.NewVirtualLocationFromDirectory(
r.responsePath(string(refVia.RealPath)),
r.responsePath(string(refVia.RequestPath)),
r.responseAccessPath(string(refVia.RequestPath)),
*refVia.Reference,
)
uniqueFileIDs.Add(*refVia.Reference)
uniqueLocations = append(uniqueLocations, location)
}
for _, archive := range r.archives {
archiveUniqueLocations, err := archive.FilesByMIMEType(types...)
if err != nil {
return nil, err
}
for _, archiveUniqueLocation := range archiveUniqueLocations {
if uniqueFileIDs.Contains(archiveUniqueLocation.Reference()) {
continue
}
uniqueLocations = append(uniqueLocations, archiveUniqueLocation)
}
}
return uniqueLocations, nil
}
syft/internal/fileresolver/filetree_resolver_test.go
+136 -42
View File
@@ -506,9 +506,12 @@ func TestDirectoryResolver_FilesByPath_request_response(t *testing.T) {
require.NoError(t, os.Chdir(testDir))
})
resolver, err := NewFromDirectory(c.root, c.base)
resolver, cleanupFn, err := NewFromDirectory(c.root, c.base, 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
refs, err := resolver.FilesByPath(c.input)
require.NoError(t, err)
@@ -558,8 +561,12 @@ func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := NewFromDirectory(c.relativeRoot, "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory(c.relativeRoot, "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
refs, err := resolver.FilesByPath(c.input)
require.NoError(t, err)
@@ -613,8 +620,12 @@ func TestDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) {
absRoot, err := filepath.Abs(c.relativeRoot)
require.NoError(t, err)
resolver, err := NewFromDirectory(absRoot, "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory(absRoot, "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
refs, err := resolver.FilesByPath(c.input)
require.NoError(t, err)
@@ -674,8 +685,12 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := NewFromDirectory(c.root, "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory(c.root, "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
hasPath := resolver.HasPath(c.input)
if !c.forcePositiveHasPath {
@@ -722,8 +737,13 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures", "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
refs, err := resolver.FilesByPath(c.input...)
assert.NoError(t, err)
@@ -735,8 +755,13 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
}
func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures", "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
refs, err := resolver.FilesByGlob("**/image-symlinks/file*")
assert.NoError(t, err)
@@ -744,16 +769,26 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
}
func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures/image-symlinks", "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/image-symlinks", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
refs, err := resolver.FilesByGlob("**/*.txt")
assert.NoError(t, err)
assert.Len(t, refs, 6)
}
func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures", "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
refs, err := resolver.FilesByGlob("**/image-symlinks/*1.txt")
assert.NoError(t, err)
@@ -779,8 +814,12 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-simple", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
refs, err := resolver.FilesByPath(test.fixture)
require.NoError(t, err)
@@ -802,8 +841,12 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
// let's make certain that "dev/place" is not ignored, since it is not "/dev/place"
resolver, err := NewFromDirectory("test-fixtures/system_paths/target", "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory("test-fixtures/system_paths/target", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
// all paths should be found (non filtering matches a path)
locations, err := resolver.FilesByGlob("**/place")
@@ -841,8 +884,13 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
}
for _, test := range tests {
t.Run(test.fixturePath, func(t *testing.T) {
resolver, err := NewFromDirectory(test.fixturePath, "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory(test.fixturePath, "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
locations, err := resolver.FilesByMIMEType(test.mimeType)
assert.NoError(t, err)
assert.Equal(t, test.expectedPaths.Size(), len(locations))
@@ -854,8 +902,12 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
}
func Test_IndexingNestedSymLinks(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "")
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-simple", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
// check that we can get the real path
locations, err := resolver.FilesByPath("./readme")
@@ -910,8 +962,12 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
return nil
}
resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "", filterFn)
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-simple", "", 0, filterFn)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
// the path to the real file is PRUNED from the index, so we should NOT expect a location returned
locations, err := resolver.FilesByPath("./readme")
@@ -930,8 +986,12 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
}
func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures/symlinks-multiple-roots/root", "")
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-multiple-roots/root", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
// check that we can get the real path
locations, err := resolver.FilesByPath("./readme")
@@ -948,8 +1008,12 @@ func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
}
func Test_RootViaSymlink(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures/symlinked-root/nested/link-root", "")
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinked-root/nested/link-root", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
locations, err := resolver.FilesByPath("./file1.txt")
require.NoError(t, err)
@@ -968,10 +1032,14 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
cwd, err := os.Getwd()
require.NoError(t, err)
r, err := NewFromDirectory(".", "")
resolver, cleanupFn, err := NewFromDirectory(".", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
exists, existingPath, err := r.tree.File(stereoscopeFile.Path(filepath.Join(cwd, "test-fixtures/image-simple/file-1.txt")))
exists, existingPath, err := resolver.tree.File(stereoscopeFile.Path(filepath.Join(cwd, "test-fixtures/image-simple/file-1.txt")))
require.True(t, exists)
require.NoError(t, err)
require.True(t, existingPath.HasReference())
@@ -995,8 +1063,7 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual, err := r.FileContentsByLocation(test.location)
actual, err := resolver.FileContentsByLocation(test.location)
if test.err {
require.Error(t, err)
return
@@ -1014,8 +1081,12 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) {
test := func(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures/symlinks-loop", "")
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-loop", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
locations, err := resolver.FilesByGlob("**/file.target")
require.NoError(t, err)
@@ -1085,8 +1156,12 @@ func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := NewFromDirectory(c.root, c.root)
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory(c.root, c.root, 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
refs, err := resolver.FilesByPath(c.input)
require.NoError(t, err)
@@ -1229,9 +1304,12 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "")
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "", 0)
require.NoError(t, err)
assert.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
actual := test.runner(resolver)
@@ -1241,8 +1319,12 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) {
}
func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures/symlinks-prune-indexing", "")
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-prune-indexing", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
var allRealPaths []stereoscopeFile.Path
for l := range resolver.AllLocations(context.Background()) {
@@ -1264,8 +1346,12 @@ func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
defer goleak.VerifyNone(t)
resolver, err := NewFromDirectory("./test-fixtures/system_paths", "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/system_paths", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
var dirLoc *file.Location
ctx, cancel := context.WithCancel(context.Background())
@@ -1287,8 +1373,12 @@ func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
}
func TestDirectoryResolver_AllLocations(t *testing.T) {
resolver, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "")
assert.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
paths := strset.New()
for loc := range resolver.AllLocations(context.Background()) {
@@ -1320,8 +1410,13 @@ func TestDirectoryResolver_AllLocations(t *testing.T) {
func TestAllLocationsDoesNotLeakGoRoutine(t *testing.T) {
defer goleak.VerifyNone(t)
resolver, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "")
resolver, cleanupFn, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
ctx, cancel := context.WithCancel(context.Background())
for range resolver.AllLocations(ctx) {
break
@@ -1542,6 +1637,7 @@ func Test_fileResolver_FileContentsByLocation(t *testing.T) {
}
func TestFileResolver_AllLocations_errorOnDirRequest(t *testing.T) {
defer goleak.VerifyNone(t)
filePath := "./test-fixtures/system_paths/target/home/place"
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
require.NoError(t, err)
@@ -1565,8 +1661,6 @@ func TestFileResolver_AllLocations_errorOnDirRequest(t *testing.T) {
reader, err := resolver.FileContentsByLocation(*dirLoc)
require.Error(t, err)
require.Nil(t, reader)
goleak.VerifyNone(t)
}
func TestFileResolver_AllLocations(t *testing.T) {
syft/pkg/cataloger/java/internal/maven/resolver_test.go
+5 -1
View File
@@ -273,8 +273,12 @@ func Test_mavenResolverRemote(t *testing.T) {
}
func Test_relativePathParent(t *testing.T) {
resolver, err := fileresolver.NewFromDirectory("test-fixtures/local", "")
resolver, cleanupFn, err := fileresolver.NewFromDirectory("test-fixtures/local", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
ctx := context.Background()
syft/source/directorysource/directory_source.go
+12 -6
View File
@@ -21,10 +21,11 @@ import (
var _ source.Source = (*directorySource)(nil)
type Config struct {
Path string
Base string
Exclude source.ExcludeConfig
Alias source.Alias
Path string
Base string
Exclude source.ExcludeConfig
Alias source.Alias
MaxArchiveRecursiveIndexDepth int
}
type directorySource struct {
@@ -32,6 +33,7 @@ type directorySource struct {
config Config
resolver *fileresolver.Directory
mutex *sync.Mutex
closer func() error
}
func NewFromPath(path string) (source.Source, error) {
@@ -145,11 +147,12 @@ func (s *directorySource) FileResolver(_ source.Scope) (file.Resolver, error) {
// this should be the only file resolver that might have overlap with where files are cached
exclusionFunctions = append(exclusionFunctions, excludeCachePathVisitors()...)
res, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, exclusionFunctions...)
res, cleanupFn, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, s.config.MaxArchiveRecursiveIndexDepth, exclusionFunctions...)
if err != nil {
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
}
s.closer = cleanupFn
s.resolver = res
}
@@ -159,8 +162,11 @@ func (s *directorySource) FileResolver(_ source.Scope) (file.Resolver, error) {
func (s *directorySource) Close() error {
s.mutex.Lock()
defer s.mutex.Unlock()
if s.closer == nil {
return nil
}
s.resolver = nil
return nil
return s.closer()
}
func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileresolver.PathIndexVisitor, error) {
syft/source/directorysource/directory_source_provider.go
+16 -13
View File
@@ -10,20 +10,22 @@ import (
"github.com/anchore/syft/syft/source"
)
func NewSourceProvider(path string, exclude source.ExcludeConfig, alias source.Alias, basePath string) source.Provider {
func NewSourceProvider(path string, exclude source.ExcludeConfig, alias source.Alias, basePath string, maxArchiveRecursiveIndexDepth int) source.Provider {
return &directorySourceProvider{
path: path,
basePath: basePath,
exclude: exclude,
alias: alias,
path: path,
basePath: basePath,
exclude: exclude,
alias: alias,
maxArchiveRecursiveIndexDepth: maxArchiveRecursiveIndexDepth,
}
}
type directorySourceProvider struct {
path string
basePath string
exclude source.ExcludeConfig
alias source.Alias
path string
basePath string
exclude source.ExcludeConfig
alias source.Alias
maxArchiveRecursiveIndexDepth int
}
func (l directorySourceProvider) Name() string {
@@ -48,10 +50,11 @@ func (l directorySourceProvider) Provide(_ context.Context) (source.Source, erro
return New(
Config{
Path: location,
Base: basePath(l.basePath, location),
Exclude: l.exclude,
Alias: l.alias,
Path: location,
Base: basePath(l.basePath, location),
Exclude: l.exclude,
Alias: l.alias,
MaxArchiveRecursiveIndexDepth: l.maxArchiveRecursiveIndexDepth,
},
)
}
syft/source/filesource/file_source.go
+17 -6
View File
@@ -26,11 +26,12 @@ import (
var _ source.Source = (*fileSource)(nil)
type Config struct {
Path string
Exclude source.ExcludeConfig
DigestAlgorithms []crypto.Hash
Alias source.Alias
SkipExtractArchive bool
Path string
Exclude source.ExcludeConfig
DigestAlgorithms []crypto.Hash
Alias source.Alias
SkipExtractArchive bool
MaxArchiveRecursiveIndexDepth int
}
type fileSource struct {
@@ -171,11 +172,18 @@ func (s fileSource) FileResolver(_ source.Scope) (file.Resolver, error) {
if isArchiveAnalysis {
// this is an analysis of an archive file... we should scan the directory where the archive contents
res, err := fileresolver.NewFromDirectory(s.analysisPath, "", exclusionFunctions...)
res, cleanupFn, err := fileresolver.NewFromDirectory(s.analysisPath, "", s.config.MaxArchiveRecursiveIndexDepth, exclusionFunctions...)
if err != nil {
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
}
s.resolver = res
s.closer = func() error {
if err := cleanupFn(); err != nil {
return err
}
return s.closer()
}
return s.resolver, nil
}
@@ -201,6 +209,9 @@ func absoluteSymlinkFreePathToParent(path string) (string, error) {
}
func (s *fileSource) Close() error {
s.mutex.Lock()
defer s.mutex.Unlock()
if s.closer == nil {
return nil
}
syft/source/filesource/file_source_provider.go
+16 -13
View File
@@ -11,20 +11,22 @@ import (
"github.com/anchore/syft/syft/source"
)
func NewSourceProvider(path string, exclude source.ExcludeConfig, digestAlgorithms []crypto.Hash, alias source.Alias) source.Provider {
func NewSourceProvider(path string, exclude source.ExcludeConfig, digestAlgorithms []crypto.Hash, alias source.Alias, maxArchiveRecursiveIndexDepth int) source.Provider {
return &fileSourceProvider{
path: path,
exclude: exclude,
digestAlgorithms: digestAlgorithms,
alias: alias,
path: path,
exclude: exclude,
digestAlgorithms: digestAlgorithms,
alias: alias,
maxArchiveRecursiveIndexDepth: maxArchiveRecursiveIndexDepth,
}
}
type fileSourceProvider struct {
path string
exclude source.ExcludeConfig
digestAlgorithms []crypto.Hash
alias source.Alias
path string
exclude source.ExcludeConfig
digestAlgorithms []crypto.Hash
alias source.Alias
maxArchiveRecursiveIndexDepth int
}
func (p fileSourceProvider) Name() string {
@@ -49,10 +51,11 @@ func (p fileSourceProvider) Provide(_ context.Context) (source.Source, error) {
return New(
Config{
Path: location,
Exclude: p.exclude,
DigestAlgorithms: p.digestAlgorithms,
Alias: p.alias,
Path: location,
Exclude: p.exclude,
DigestAlgorithms: p.digestAlgorithms,
Alias: p.alias,
MaxArchiveRecursiveIndexDepth: p.maxArchiveRecursiveIndexDepth,
},
)
}
syft/source/sourceproviders/source_provider_config.go
+12 -6
View File
@@ -9,12 +9,13 @@ import (
// Config is the uber-configuration for all Syft source providers
type Config struct {
Platform *image.Platform
Alias source.Alias
RegistryOptions *image.RegistryOptions
Exclude source.ExcludeConfig
DigestAlgorithms []crypto.Hash
BasePath string
Platform *image.Platform
Alias source.Alias
RegistryOptions *image.RegistryOptions
Exclude source.ExcludeConfig
DigestAlgorithms []crypto.Hash
BasePath string
MaxArchiveRecursiveIndexDepth int
}
func (c *Config) WithAlias(alias source.Alias) *Config {
@@ -47,6 +48,11 @@ func (c *Config) WithBasePath(basePath string) *Config {
return c
}
func (c *Config) WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth int) *Config {
c.MaxArchiveRecursiveIndexDepth = maxArchiveRecursiveIndexDepth
return c
}
func DefaultConfig() *Config {
return &Config{
DigestAlgorithms: []crypto.Hash{
syft/source/sourceproviders/source_providers.go
+2 -2
View File
@@ -26,8 +26,8 @@ func All(userInput string, cfg *Config) []collections.TaggedValue[source.Provide
return collections.TaggedValueSet[source.Provider]{}.
// --from file, dir, oci-archive, etc.
Join(stereoscopeProviders.Select(FileTag, DirTag)...).
Join(tagProvider(filesource.NewSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias), FileTag)).
Join(tagProvider(directorysource.NewSourceProvider(userInput, cfg.Exclude, cfg.Alias, cfg.BasePath), DirTag)).
Join(tagProvider(filesource.NewSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias, cfg.MaxArchiveRecursiveIndexDepth), FileTag)).
Join(tagProvider(directorysource.NewSourceProvider(userInput, cfg.Exclude, cfg.Alias, cfg.BasePath, cfg.MaxArchiveRecursiveIndexDepth), DirTag)).
// --from docker, registry, etc.
Join(stereoscopeProviders.Select(PullTag)...)