chore: revert original recursive-archive implementations from PR #4044 and #4761

Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
Keith Zantow
2026-09-09 17:22:51 -04:00
parent 50100f6531
commit 0a93e60b16
28 changed files with 227 additions and 2964 deletions
-1
View File
@@ -17,7 +17,6 @@ type configurationAuditTrail struct {
Packages pkgcataloging.Config `json:"packages" yaml:"packages" mapstructure:"packages"`
Files filecataloging.Config `json:"files" yaml:"files" mapstructure:"files"`
Licenses cataloging.LicenseConfig `json:"licenses" yaml:"licenses" mapstructure:"licenses"`
Archive cataloging.ArchiveSearchConfig `json:"archive" yaml:"archive" mapstructure:"archive"`
Catalogers catalogerManifest `json:"catalogers" yaml:"catalogers" mapstructure:"catalogers"`
ExtraConfigs any `json:"extra,omitempty" yaml:"extra" mapstructure:"extra"`
}
+1 -72
View File
@@ -10,7 +10,6 @@ import (
"github.com/scylladb/go-set/strset"
"github.com/anchore/go-sync"
intArchive "github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/licenses"
"github.com/anchore/syft/internal/log"
@@ -20,12 +19,9 @@ import (
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/internal/fileresolver"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/source/directorysource"
)
// CreateSBOM creates a software bill-of-materials from the given source. If the CreateSBOMConfig is nil, then
@@ -64,7 +60,6 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
Packages: cfg.Packages,
Files: cfg.Files,
Licenses: cfg.Licenses,
Archive: cfg.Archive,
Catalogers: *audit,
ExtraConfigs: cfg.ToolConfiguration,
},
@@ -97,17 +92,9 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
packageCatalogingProgress := monitorPackageCatalogingTask()
builder := sbomsync.NewBuilder(&s, monitorPackageCount(packageCatalogingProgress))
// determine the effective resolver: if recursive archive extraction is enabled,
// wrap the resolver with the archive orchestrator
effectiveResolver, archiveCleanup, archiveRelationships := setupArchiveOrchestration(ctx, resolver, cfg)
if archiveCleanup != nil {
defer archiveCleanup()
}
for i := range taskGroups {
err = sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(taskGroups[i]), func(t task.Task) (any, error) {
return nil, task.RunTask(ctx, t, effectiveResolver, builder, catalogingProgress)
return nil, task.RunTask(ctx, t, resolver, builder, catalogingProgress)
}, nil)
if err != nil {
// TODO: tie this to the open progress monitors...
@@ -115,70 +102,12 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
}
}
// add archive containment relationships to the SBOM
if len(archiveRelationships) > 0 {
builder.AddRelationships(archiveRelationships...)
}
packageCatalogingProgress.SetCompleted()
catalogingProgress.SetCompleted()
return &s, nil
}
// setupArchiveOrchestration wraps the resolver with an archive orchestrator when recursive
// archive extraction is enabled (MaxDepth > 0). It performs the iterative extract loop
// and returns the effective resolver, a cleanup function, and any archive relationships.
func setupArchiveOrchestration(ctx context.Context, baseResolver file.Resolver, cfg *CreateSBOMConfig) (file.Resolver, func(), []artifact.Relationship) {
if cfg.Archive.MaxDepth <= 0 {
return baseResolver, nil, nil
}
td := tmpdir.FromContext(ctx)
if td == nil {
log.Warn("archive orchestration requested but no temp dir available")
return baseResolver, nil, nil
}
archiveTmpDir, archiveTmpCleanup, err := td.NewChild("archive-extraction") //nolint:gocritic // cleanup is returned to caller, not deferred here
if err != nil {
log.WithFields("error", err).Warn("unable to create temp dir for archive extraction")
return baseResolver, nil, nil
}
// build user-exclusion path filters once and reuse them for every child
// resolver, so that --exclude patterns also apply inside extracted archives
resolverFactory := func(root string) (file.Resolver, error) {
filters, err := directorysource.GetDirectoryExclusionFunctions(root, append([]string(nil), cfg.Exclusions...))
if err != nil {
return nil, fmt.Errorf("invalid exclusion patterns for archive child resolver: %w", err)
}
// depth 0: the archive Orchestrator (PR #4761) drives recursion here, so the
// resolver-level recursive archive indexing (PR #4044) is disabled to avoid
// double extraction. With depth 0 no temp dir is created, so the cleanup is a no-op.
res, _, err := fileresolver.NewFromDirectory(root, "", 0, filters...)
return res, err
}
orch := intArchive.NewOrchestrator(baseResolver, cfg.Archive, archiveTmpDir, resolverFactory)
// iteratively discover and extract archives up to the configured max depth
for depth := 0; depth < cfg.Archive.MaxDepth; depth++ {
newArchives := orch.DiscoverAndExtract(ctx, depth)
if newArchives == 0 {
break
}
log.WithFields("depth", depth+1, "archives", newArchives).Debug("extracted archives for recursive cataloging")
}
cleanup := func() {
orch.Cleanup()
archiveTmpCleanup()
}
return orch.Resolver(), cleanup, orch.Relationships()
}
func setupContext(ctx context.Context, cfg *CreateSBOMConfig) (context.Context, error) {
// configure parallel executors
ctx = setContextExecutors(ctx, cfg)
-22
View File
@@ -28,16 +28,9 @@ type CreateSBOMConfig struct {
Packages pkgcataloging.Config
Licenses cataloging.LicenseConfig
Files filecataloging.Config
Archive cataloging.ArchiveSearchConfig
Parallelism int
CatalogerSelection cataloging.SelectionRequest
// Exclusions are user-provided glob patterns that the source resolver already
// applies; they are also propagated into child resolvers built during recursive
// archive extraction so that a "--exclude=**/foo" pattern keeps working inside
// extracted archive contents.
Exclusions []string
// audit what tool is being used to generate the SBOM
ToolName string
ToolVersion string
@@ -56,7 +49,6 @@ func DefaultCreateSBOMConfig() *CreateSBOMConfig {
Packages: pkgcataloging.DefaultConfig(),
Licenses: cataloging.DefaultLicenseConfig(),
Files: filecataloging.DefaultConfig(),
Archive: cataloging.DefaultArchiveSearchConfig(),
Parallelism: 0, // use default: run in parallel based on number of CPUs
packageTaskFactories: task.DefaultPackageTaskFactories(),
@@ -161,20 +153,6 @@ func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig {
return c
}
// WithArchiveConfig allows for defining archive extraction and recursive cataloging parameters.
func (c *CreateSBOMConfig) WithArchiveConfig(cfg cataloging.ArchiveSearchConfig) *CreateSBOMConfig {
c.Archive = cfg
return c
}
// WithExclusions sets the glob patterns that should be propagated to child resolvers
// constructed during recursive archive extraction. Patterns must follow the same
// "./", "*/", or "**/"-prefix convention as the syft --exclude flag.
func (c *CreateSBOMConfig) WithExclusions(exclusions []string) *CreateSBOMConfig {
c.Exclusions = exclusions
return c
}
// WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag.
func (c *CreateSBOMConfig) WithCatalogerSelection(selection cataloging.SelectionRequest) *CreateSBOMConfig {
c.CatalogerSelection = selection
-5
View File
@@ -51,11 +51,6 @@ func (c *GetSourceConfig) WithBasePath(basePath string) *GetSourceConfig {
return c
}
func (c *GetSourceConfig) WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth int) *GetSourceConfig {
c.SourceProviderConfig = c.SourceProviderConfig.WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth)
return c
}
func (c *GetSourceConfig) WithSources(sources ...string) *GetSourceConfig {
c.Sources = sources
return c
+4 -122
View File
@@ -3,28 +3,11 @@ package fileresolver
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/anchore/archiver/v3"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
)
type archiveAccessPath struct {
realPath string
accessPath string
archiveRealPath string
archiveDepth int
}
const (
archiveTempPathPattern = "syft-archivePaths-"
archiveContentTempPathPattern = "syft-archive-contents-"
)
var ErrSkipPath = errors.New("skip path")
var _ file.Resolver = (*Directory)(nil)
@@ -36,29 +19,13 @@ type Directory struct {
indexer *directoryIndexer
}
func NewFromDirectory(root, base string, maxArchiveRecursiveIndexDepth int, pathFilters ...PathIndexVisitor) (*Directory, func() error, error) {
var cleanupFn = func() error { return nil }
directory, err := newFromDirectoryWithoutIndex(root, base, pathFilters...)
func NewFromDirectory(root, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
resolver, err := newFromDirectoryWithoutIndex(root, base, pathFilters...)
if err != nil {
return nil, cleanupFn, err
return nil, err
}
if err = directory.buildIndex(); err != nil {
return nil, cleanupFn, fmt.Errorf("unable to build index: %w", err)
}
if maxArchiveRecursiveIndexDepth != 0 && len(directory.indexer.archivePaths) > 0 {
archiveTempDir, err := os.MkdirTemp("", archiveTempPathPattern)
if err != nil {
return nil, cleanupFn, fmt.Errorf("unable to create tempdir for archive processing: %w", err)
}
return directory, func() error {
return os.RemoveAll(archiveTempDir)
}, directory.buildArchiveIndex(archiveTempDir, directory.indexer.archivePaths, maxArchiveRecursiveIndexDepth)
}
return directory, cleanupFn, nil
return resolver, resolver.buildIndex()
}
func newFromDirectoryWithoutIndex(root, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
@@ -98,91 +65,6 @@ func (r *Directory) buildIndex() error {
return nil
}
func (r *Directory) buildArchiveIndex(archiveTempDir string, archives []string, maxArchiveIndexDepth int) error {
archivesToIndex := make([]archiveAccessPath, len(archives))
for i, archive := range archives {
archivesToIndex[i] = archiveAccessPath{
realPath: r.Chroot.ToChrootPath(archive),
accessPath: r.Chroot.ToChrootPath(archive),
archiveRealPath: archive,
}
}
loop:
for {
var currentArchivePath archiveAccessPath
switch len(archivesToIndex) {
case 0:
break loop
case 1:
currentArchivePath, archivesToIndex = archivesToIndex[0], nil
default:
currentArchivePath, archivesToIndex = archivesToIndex[0], archivesToIndex[1:]
}
if maxArchiveIndexDepth != -1 && currentArchivePath.archiveDepth >= maxArchiveIndexDepth {
continue
}
envelopedUnarchiver, err := archiver.ByExtension(currentArchivePath.archiveRealPath)
if err != nil {
return fmt.Errorf("could not determine archive envelope type: %w", err)
}
unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver)
if !ok {
return fmt.Errorf("unarchiver %T is not an archiver", envelopedUnarchiver)
}
archiveDestinationTempDir, err := os.MkdirTemp(archiveTempDir, archiveContentTempPathPattern)
if err != nil {
return fmt.Errorf("unable to create tempdir for archive processing: %w", err)
}
archiveDestinationPath, err := filepath.EvalSymlinks(archiveDestinationTempDir)
if err != nil {
var pathErr *os.PathError
if errors.As(err, &pathErr) {
// we can't index the path, but we shouldn't consider this to be fatal
// TODO: known-unknowns
log.WithFields("archivePath", archiveDestinationTempDir, "error", err).Trace("unable to evaluate symlink while indexing branch")
continue
}
return err
}
if err = unarchiver.Unarchive(currentArchivePath.archiveRealPath, archiveDestinationPath); err != nil {
return fmt.Errorf("unable to unarchive nested file: %w", err)
}
d, err := newFromDirectoryWithoutIndex(archiveDestinationPath, "")
if err != nil {
return err
}
if err = d.buildIndex(); err != nil {
return fmt.Errorf("unable to build index of archive: %w", err)
}
for _, archive := range d.indexer.archivePaths {
archivesToIndex = append(archivesToIndex, archiveAccessPath{
realPath: currentArchivePath.realPath,
accessPath: strings.Replace(archive, archiveDestinationPath, currentArchivePath.accessPath, 1),
archiveRealPath: archive,
archiveDepth: currentArchivePath.archiveDepth + 1,
})
}
d.realPath = currentArchivePath.realPath
d.accessPath = currentArchivePath.accessPath
d.archiveRealPath = archiveDestinationPath
r.archives = append(r.archives, &d.FiletreeResolver)
}
return nil
}
// Stringer to represent a directory path data source
func (r *Directory) String() string {
return fmt.Sprintf("dir:%s", r.path)
@@ -9,13 +9,13 @@ import (
"path/filepath"
"strings"
"github.com/anchore/archiver/v3"
"github.com/wagoodman/go-progress"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/internal/windows"
"github.com/wagoodman/go-progress"
)
type PathIndexVisitor func(string, string, os.FileInfo, error) error
@@ -27,7 +27,6 @@ type directoryIndexer struct {
errPaths map[string]error
tree filetree.ReadWriter
index filetree.Index
archivePaths []string
}
func newDirectoryIndexer(path, base string, visitors ...PathIndexVisitor) *directoryIndexer {
@@ -145,10 +144,6 @@ func (r *directoryIndexer) indexTree(root string, stager *progress.AtomicStage)
func(path string, info os.FileInfo, err error) error {
stager.Set(path)
if isArchive(path, info) {
r.archivePaths = append(r.archivePaths, path)
}
newRoot, err := r.indexPath(path, info, err)
if err != nil {
@@ -169,17 +164,6 @@ func (r *directoryIndexer) indexTree(root string, stager *progress.AtomicStage)
return roots, nil
}
// isArchive returns true if the path appears to be an archive
func isArchive(path string, info os.FileInfo) bool {
if info == nil || !info.Mode().IsRegular() {
return false
}
envelopedUnarchiver, err := archiver.ByExtension(path)
_, ok := envelopedUnarchiver.(archiver.Unarchiver)
return err == nil && ok
}
func isRealPath(root string) (bool, error) {
rootParent := filepath.Clean(filepath.Dir(root))
+11 -116
View File
@@ -6,7 +6,6 @@ import (
"fmt"
"io"
"os"
"strings"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
@@ -22,11 +21,6 @@ type FiletreeResolver struct {
Index filetree.IndexReader
SearchContext filetree.Searcher
Opener func(stereoscopeFile.Reference) (io.ReadCloser, error)
realPath string
accessPath string
archiveRealPath string
archives []*FiletreeResolver
}
func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) {
@@ -41,16 +35,7 @@ func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) {
}
func (r *FiletreeResolver) requestPath(userPath string) (string, error) {
requestPath, err := r.Chroot.ToNativePath(userPath)
if err != nil {
return "", err
}
if r.accessPath != "" && r.archiveRealPath != "" {
return strings.Replace(requestPath, r.accessPath, r.archiveRealPath, 1), nil
}
return requestPath, nil
return r.Chroot.ToNativePath(userPath)
}
// responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the file resolver.
@@ -58,32 +43,13 @@ func (r FiletreeResolver) responsePath(path string) string {
return r.Chroot.ToChrootPath(path)
}
func (r FiletreeResolver) responseAccessPath(path string) string {
if r.accessPath != "" && r.archiveRealPath != "" {
path = strings.Replace(path, r.archiveRealPath, r.accessPath, 1)
}
return r.Chroot.ToChrootPath(path)
}
// HasPath indicates if the given path exists in the underlying source.
func (r *FiletreeResolver) HasPath(userPath string) bool {
requestPath, err := r.requestPath(userPath)
if err != nil {
return false
}
if r.Tree.HasPath(stereoscopeFile.Path(requestPath)) {
return true
}
for _, archive := range r.archives {
if archive.HasPath(userPath) {
return true
}
}
return false
return r.Tree.HasPath(stereoscopeFile.Path(requestPath))
}
// FilesByPath returns all file.References that match the given paths from the file index.
@@ -127,35 +93,18 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err
references = append(references,
file.NewVirtualLocationFromDirectory(
r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root
r.responseAccessPath(userStrPath), // the path used to access this file, relative to the resolver root
r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root
*ref.Reference,
),
)
}
}
for _, archive := range r.archives {
locations, err := archive.FilesByPath(userPaths...)
if err != nil {
return nil, err
}
references = append(references, locations...)
}
return references, nil
}
func (r FiletreeResolver) requestGlob(pattern string) (string, error) {
nativeGlob, err := r.Chroot.ToNativeGlob(pattern)
if err != nil {
return "", err
}
if r.accessPath != "" && r.archiveRealPath != "" {
return strings.Replace(nativeGlob, r.accessPath, r.archiveRealPath, 1), nil
}
return nativeGlob, nil
return r.Chroot.ToNativeGlob(pattern)
}
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
@@ -187,8 +136,8 @@ func (r FiletreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro
}
loc := file.NewVirtualLocationFromDirectory(
r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root
r.responseAccessPath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root
r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
*refVia.Reference,
)
uniqueFileIDs.Add(*refVia.Reference)
@@ -196,20 +145,6 @@ func (r FiletreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro
}
}
for _, archive := range r.archives {
archiveUniqueLocations, err := archive.FilesByGlob(patterns...)
if err != nil {
return nil, err
}
for _, archiveUniqueLocation := range archiveUniqueLocations {
if uniqueFileIDs.Contains(archiveUniqueLocation.Reference()) {
continue
}
uniqueLocations = append(uniqueLocations, archiveUniqueLocation)
}
}
return uniqueLocations, nil
}
@@ -235,14 +170,6 @@ func (r FiletreeResolver) FileContentsByLocation(location file.Location) (io.Rea
}
entry, err := r.Index.Get(location.Reference())
if err != nil {
for _, archive := range r.archives {
entry, err = archive.Index.Get(location.Reference())
if err == nil {
break
}
}
}
if err != nil {
return nil, err
}
@@ -263,39 +190,21 @@ func (r *FiletreeResolver) AllLocations(ctx context.Context) <-chan file.Locatio
select {
case <-ctx.Done():
return
case results <- file.NewVirtualLocationFromDirectory(r.responsePath(string(ref.RealPath)), r.responseAccessPath(string(ref.RealPath)), ref):
case results <- file.NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), "", ref):
continue
}
}
for _, archive := range r.archives {
for location := range archive.AllLocations(ctx) {
select {
case <-ctx.Done():
return
case results <- location:
continue
}
}
}
}()
return results
}
func (r *FiletreeResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
entry, err := r.Index.Get(location.Reference())
if err == nil {
return entry.Metadata, nil
if err != nil {
return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
}
for _, archive := range r.archives {
entry, err = archive.Index.Get(location.Reference())
if err == nil {
return entry.Metadata, nil
}
}
return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
return entry.Metadata, nil
}
func (r *FiletreeResolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
@@ -315,26 +224,12 @@ func (r *FiletreeResolver) FilesByMIMEType(types ...string) ([]file.Location, er
}
location := file.NewVirtualLocationFromDirectory(
r.responsePath(string(refVia.RealPath)),
r.responseAccessPath(string(refVia.RequestPath)),
r.responsePath(string(refVia.RequestPath)),
*refVia.Reference,
)
uniqueFileIDs.Add(*refVia.Reference)
uniqueLocations = append(uniqueLocations, location)
}
for _, archive := range r.archives {
archiveUniqueLocations, err := archive.FilesByMIMEType(types...)
if err != nil {
return nil, err
}
for _, archiveUniqueLocation := range archiveUniqueLocations {
if uniqueFileIDs.Contains(archiveUniqueLocation.Reference()) {
continue
}
uniqueLocations = append(uniqueLocations, archiveUniqueLocation)
}
}
return uniqueLocations, nil
}
@@ -505,12 +505,9 @@ func TestDirectoryResolver_FilesByPath_request_response(t *testing.T) {
require.NoError(t, os.Chdir(testDir))
})
resolver, cleanupFn, err := NewFromDirectory(c.root, c.base, 0)
resolver, err := NewFromDirectory(c.root, c.base)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
refs, err := resolver.FilesByPath(c.input)
require.NoError(t, err)
@@ -560,12 +557,8 @@ func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory(c.relativeRoot, "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory(c.relativeRoot, "")
assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input)
require.NoError(t, err)
@@ -619,12 +612,8 @@ func TestDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) {
absRoot, err := filepath.Abs(c.relativeRoot)
require.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory(absRoot, "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory(absRoot, "")
assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input)
require.NoError(t, err)
@@ -684,12 +673,8 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory(c.root, "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory(c.root, "")
assert.NoError(t, err)
hasPath := resolver.HasPath(c.input)
if !c.forcePositiveHasPath {
@@ -736,13 +721,8 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory("./testdata", "")
assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input...)
assert.NoError(t, err)
@@ -754,13 +734,8 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
}
func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory("./testdata", "")
assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/image-symlinks/file*")
assert.NoError(t, err)
@@ -768,26 +743,16 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
}
func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata/image-symlinks", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory("./testdata/image-symlinks", "")
assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/*.txt")
assert.NoError(t, err)
assert.Len(t, refs, 6)
}
func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory("./testdata", "")
assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/image-symlinks/*1.txt")
assert.NoError(t, err)
@@ -813,12 +778,8 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-simple", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory("./testdata/symlinks-simple", "")
assert.NoError(t, err)
refs, err := resolver.FilesByPath(test.fixture)
require.NoError(t, err)
@@ -840,12 +801,8 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
// let's make certain that "dev/place" is not ignored, since it is not "/dev/place"
resolver, cleanupFn, err := NewFromDirectory("testdata/system_paths/target", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory("testdata/system_paths/target", "")
assert.NoError(t, err)
// all paths should be found (non filtering matches a path)
locations, err := resolver.FilesByGlob("**/place")
@@ -883,13 +840,8 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
}
for _, test := range tests {
t.Run(test.fixturePath, func(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory(test.fixturePath, "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory(test.fixturePath, "")
assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType(test.mimeType)
assert.NoError(t, err)
assert.Equal(t, test.expectedPaths.Size(), len(locations))
@@ -901,12 +853,8 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
}
func Test_IndexingNestedSymLinks(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-simple", "", 0)
resolver, err := NewFromDirectory("./testdata/symlinks-simple", "")
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
// check that we can get the real path
locations, err := resolver.FilesByPath("./readme")
@@ -961,12 +909,8 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
return nil
}
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-simple", "", 0, filterFn)
resolver, err := NewFromDirectory("./testdata/symlinks-simple", "", filterFn)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
// the path to the real file is PRUNED from the index, so we should NOT expect a location returned
locations, err := resolver.FilesByPath("./readme")
@@ -985,12 +929,8 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
}
func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-multiple-roots/root", "", 0)
resolver, err := NewFromDirectory("./testdata/symlinks-multiple-roots/root", "")
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
// check that we can get the real path
locations, err := resolver.FilesByPath("./readme")
@@ -1007,12 +947,8 @@ func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
}
func Test_RootViaSymlink(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinked-root/nested/link-root", "", 0)
resolver, err := NewFromDirectory("./testdata/symlinked-root/nested/link-root", "")
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
locations, err := resolver.FilesByPath("./file1.txt")
require.NoError(t, err)
@@ -1031,14 +967,10 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
cwd, err := os.Getwd()
require.NoError(t, err)
resolver, cleanupFn, err := NewFromDirectory(".", "", 0)
r, err := NewFromDirectory(".", "")
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
exists, existingPath, err := resolver.Tree.File(stereoscopeFile.Path(filepath.Join(cwd, "testdata/image-simple/file-1.txt")))
exists, existingPath, err := r.Tree.File(stereoscopeFile.Path(filepath.Join(cwd, "testdata/image-simple/file-1.txt")))
require.True(t, exists)
require.NoError(t, err)
require.True(t, existingPath.HasReference())
@@ -1062,7 +994,8 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual, err := resolver.FileContentsByLocation(test.location)
actual, err := r.FileContentsByLocation(test.location)
if test.err {
require.Error(t, err)
return
@@ -1080,12 +1013,8 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) {
test := func(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-loop", "", 0)
resolver, err := NewFromDirectory("./testdata/symlinks-loop", "")
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
locations, err := resolver.FilesByGlob("**/file.target")
require.NoError(t, err)
@@ -1155,12 +1084,8 @@ func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory(c.root, c.root, 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory(c.root, c.root)
assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input)
require.NoError(t, err)
@@ -1303,12 +1228,9 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "", 0)
resolver, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "")
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
assert.NoError(t, err)
actual := test.runner(resolver)
@@ -1318,12 +1240,8 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) {
}
func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-prune-indexing", "", 0)
resolver, err := NewFromDirectory("./testdata/symlinks-prune-indexing", "")
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
var allRealPaths []stereoscopeFile.Path
for l := range resolver.AllLocations(context.Background()) {
@@ -1345,12 +1263,8 @@ func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
defer goleak.VerifyNone(t)
resolver, cleanupFn, err := NewFromDirectory("./testdata/system_paths", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory("./testdata/system_paths", "")
assert.NoError(t, err)
var dirLoc *file.Location
ctx, cancel := context.WithCancel(context.Background())
@@ -1372,12 +1286,8 @@ func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
}
func TestDirectoryResolver_AllLocations(t *testing.T) {
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "", 0)
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
resolver, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "")
assert.NoError(t, err)
paths := strset.New()
for loc := range resolver.AllLocations(context.Background()) {
@@ -1409,13 +1319,8 @@ func TestDirectoryResolver_AllLocations(t *testing.T) {
func TestAllLocationsDoesNotLeakGoRoutine(t *testing.T) {
defer goleak.VerifyNone(t)
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "", 0)
resolver, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "")
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
ctx, cancel := context.WithCancel(context.Background())
for range resolver.AllLocations(ctx) {
break
@@ -1648,7 +1553,6 @@ func Test_fileResolver_FileContentsByLocation(t *testing.T) {
}
func TestFileResolver_AllLocations_errorOnDirRequest(t *testing.T) {
defer goleak.VerifyNone(t)
filePath := "./testdata/system_paths/target/home/place"
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
require.NoError(t, err)
@@ -1675,10 +1579,11 @@ func TestFileResolver_AllLocations_errorOnDirRequest(t *testing.T) {
reader, err := resolver.FileContentsByLocation(*dirLoc)
require.Error(t, err)
require.Nil(t, reader)
goleak.VerifyNone(t)
}
func TestFileResolver_AllLocations(t *testing.T) {
defer goleak.VerifyNone(t)
// Verify both the parent and the file itself are indexed
filePath := "./testdata/system_paths/target/home/place"
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
@@ -1703,10 +1608,11 @@ func TestFileResolver_AllLocations(t *testing.T) {
sort.Strings(pathsList)
assert.ElementsMatchf(t, expected, pathsList, "expected all paths to be indexed, but found different paths: \n%s", cmp.Diff(expected, paths.List()))
goleak.VerifyNone(t)
}
func Test_FileResolver_AllLocationsDoesNotLeakGoRoutine(t *testing.T) {
defer goleak.VerifyNone(t)
filePath := "./testdata/system_paths/target/home/place"
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
require.NoError(t, err)
@@ -1723,4 +1629,6 @@ func Test_FileResolver_AllLocationsDoesNotLeakGoRoutine(t *testing.T) {
break
}
cancel()
goleak.VerifyNone(t)
}
@@ -273,12 +273,8 @@ func Test_mavenResolverRemote(t *testing.T) {
}
func Test_relativePathParent(t *testing.T) {
resolver, cleanupFn, err := fileresolver.NewFromDirectory("testdata/local", "", 0)
resolver, err := fileresolver.NewFromDirectory("testdata/local", "")
require.NoError(t, err)
require.NotNil(t, resolver)
t.Cleanup(func() {
require.NoError(t, cleanupFn())
})
ctx := context.Background()
@@ -21,11 +21,10 @@ import (
var _ source.Source = (*directorySource)(nil)
type Config struct {
Path string
Base string
Exclude source.ExcludeConfig
Alias source.Alias
MaxArchiveRecursiveIndexDepth int
Path string
Base string
Exclude source.ExcludeConfig
Alias source.Alias
}
type directorySource struct {
@@ -33,7 +32,6 @@ type directorySource struct {
config Config
resolver file.Resolver
mutex *sync.Mutex
closer func() error
}
func NewFromPath(path string) (source.Source, error) {
@@ -107,12 +105,11 @@ func (s *directorySource) FileResolver(_ source.Scope) (file.Resolver, error) {
// this should be the only file resolver that might have overlap with where files are cached
exclusionFunctions = append(exclusionFunctions, excludeCachePathVisitors()...)
res, cleanupFn, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, s.config.MaxArchiveRecursiveIndexDepth, exclusionFunctions...)
res, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, exclusionFunctions...)
if err != nil {
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
}
s.closer = cleanupFn
s.resolver = res
return s.resolver, nil
}
@@ -122,10 +119,7 @@ func (s *directorySource) Close() error {
defer s.mutex.Unlock()
s.resolver = nil
if s.closer == nil {
return nil
}
return s.closer()
return nil
}
func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileresolver.PathIndexVisitor, error) {
@@ -10,22 +10,20 @@ import (
"github.com/anchore/syft/syft/source"
)
func NewSourceProvider(path string, exclude source.ExcludeConfig, alias source.Alias, basePath string, maxArchiveRecursiveIndexDepth int) source.Provider {
func NewSourceProvider(path string, exclude source.ExcludeConfig, alias source.Alias, basePath string) source.Provider {
return &directorySourceProvider{
path: path,
basePath: basePath,
exclude: exclude,
alias: alias,
maxArchiveRecursiveIndexDepth: maxArchiveRecursiveIndexDepth,
path: path,
basePath: basePath,
exclude: exclude,
alias: alias,
}
}
type directorySourceProvider struct {
path string
basePath string
exclude source.ExcludeConfig
alias source.Alias
maxArchiveRecursiveIndexDepth int
path string
basePath string
exclude source.ExcludeConfig
alias source.Alias
}
func (l directorySourceProvider) Name() string {
@@ -50,11 +48,10 @@ func (l directorySourceProvider) Provide(_ context.Context) (source.Source, erro
return New(
Config{
Path: location,
Base: basePath(l.basePath, location),
Exclude: l.exclude,
Alias: l.alias,
MaxArchiveRecursiveIndexDepth: l.maxArchiveRecursiveIndexDepth,
Path: location,
Base: basePath(l.basePath, location),
Exclude: l.exclude,
Alias: l.alias,
},
)
}
+6 -14
View File
@@ -27,12 +27,11 @@ import (
var _ source.Source = (*fileSource)(nil)
type Config struct {
Path string
Exclude source.ExcludeConfig
DigestAlgorithms []crypto.Hash
Alias source.Alias
SkipExtractArchive bool
MaxArchiveRecursiveIndexDepth int
Path string
Exclude source.ExcludeConfig
DigestAlgorithms []crypto.Hash
Alias source.Alias
SkipExtractArchive bool
}
type fileSource struct {
@@ -154,19 +153,12 @@ func (s fileSource) FileResolver(_ source.Scope) (file.Resolver, error) {
if isArchiveAnalysis := fi.IsDir(); isArchiveAnalysis {
// this is an analysis of an archive file... we should scan the directory where the archive contents
res, cleanupFn, err := fileresolver.NewFromDirectory(s.analysisPath, "", s.config.MaxArchiveRecursiveIndexDepth, exclusionFunctions...)
res, err := fileresolver.NewFromDirectory(s.analysisPath, "", exclusionFunctions...)
if err != nil {
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
}
s.resolver = res
s.closer = func() error {
if err := cleanupFn(); err != nil {
return err
}
return s.closer()
}
return s.resolver, nil
}
+13 -16
View File
@@ -11,22 +11,20 @@ import (
"github.com/anchore/syft/syft/source"
)
func NewSourceProvider(path string, exclude source.ExcludeConfig, digestAlgorithms []crypto.Hash, alias source.Alias, maxArchiveRecursiveIndexDepth int) source.Provider {
func NewSourceProvider(path string, exclude source.ExcludeConfig, digestAlgorithms []crypto.Hash, alias source.Alias) source.Provider {
return &fileSourceProvider{
path: path,
exclude: exclude,
digestAlgorithms: digestAlgorithms,
alias: alias,
maxArchiveRecursiveIndexDepth: maxArchiveRecursiveIndexDepth,
path: path,
exclude: exclude,
digestAlgorithms: digestAlgorithms,
alias: alias,
}
}
type fileSourceProvider struct {
path string
exclude source.ExcludeConfig
digestAlgorithms []crypto.Hash
alias source.Alias
maxArchiveRecursiveIndexDepth int
path string
exclude source.ExcludeConfig
digestAlgorithms []crypto.Hash
alias source.Alias
}
func (p fileSourceProvider) Name() string {
@@ -51,11 +49,10 @@ func (p fileSourceProvider) Provide(_ context.Context) (source.Source, error) {
return New(
Config{
Path: location,
Exclude: p.exclude,
DigestAlgorithms: p.digestAlgorithms,
Alias: p.alias,
MaxArchiveRecursiveIndexDepth: p.maxArchiveRecursiveIndexDepth,
Path: location,
Exclude: p.exclude,
DigestAlgorithms: p.digestAlgorithms,
Alias: p.alias,
},
)
}
@@ -9,13 +9,12 @@ import (
// Config is the uber-configuration for all Syft source providers
type Config struct {
Platform *image.Platform
Alias source.Alias
RegistryOptions *image.RegistryOptions
Exclude source.ExcludeConfig
DigestAlgorithms []crypto.Hash
BasePath string
MaxArchiveRecursiveIndexDepth int
Platform *image.Platform
Alias source.Alias
RegistryOptions *image.RegistryOptions
Exclude source.ExcludeConfig
DigestAlgorithms []crypto.Hash
BasePath string
}
func (c *Config) WithAlias(alias source.Alias) *Config {
@@ -48,11 +47,6 @@ func (c *Config) WithBasePath(basePath string) *Config {
return c
}
func (c *Config) WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth int) *Config {
c.MaxArchiveRecursiveIndexDepth = maxArchiveRecursiveIndexDepth
return c
}
func DefaultConfig() *Config {
return &Config{
DigestAlgorithms: []crypto.Hash{
@@ -36,8 +36,8 @@ func All(userInput string, cfg *Config) []collections.TaggedValue[source.Provide
Join(tagProvider(snapsource.NewLocalSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias), SnapTag)).
// 2. try unspecific, local sources after other local sources last...
Join(tagProvider(filesource.NewSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias, cfg.MaxArchiveRecursiveIndexDepth), FileTag)).
Join(tagProvider(directorysource.NewSourceProvider(userInput, cfg.Exclude, cfg.Alias, cfg.BasePath, cfg.MaxArchiveRecursiveIndexDepth), DirTag)).
Join(tagProvider(filesource.NewSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias), FileTag)).
Join(tagProvider(directorysource.NewSourceProvider(userInput, cfg.Exclude, cfg.Alias, cfg.BasePath), DirTag)).
// 3. try remote sources after everything else...