oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-10-30 14:02:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat(sbom): wire archive orchestrator into CreateSBOM pipeline

Browse Source 
Integrate the archive extraction orchestrator into the SBOM creation
pipeline, enabling recursive archive cataloging when configured:

- Add Archive field to CreateSBOMConfig with WithArchiveConfig builder
- Add Exclusions field to CreateSBOMConfig with WithExclusions builder
  so user --exclude patterns also filter content inside extracted
  archives, not only the source filesystem
- Add setupArchiveOrchestration() that creates the orchestrator when
  MaxDepth > 0, runs the iterative extract loop, and returns the
  composite resolver with archive contents visible to all catalogers
- Include archive config in configuration audit trail
- Add CLI archive config section (max-depth, size/file limits,
  exclude-extensions) wired through Catalog.ToArchiveConfig()
- CLI Catalog.ToSBOMConfig threads cfg.Exclusions through
  WithExclusions so child resolvers respect them

When MaxDepth is 0 (default), behavior is unchanged. When > 0, archives
discovered during scanning are extracted and their contents made visible
to all catalogers via the CompositeResolver.

Signed-off-by: Thomas Bechtold <thomasbechtold@jpberlin.de>
This commit is contained in:
Thomas Bechtold
2026-04-27 08:49:00 +02:00
parent 5d6ad0b9e8
commit 3fc5194f77
5 changed files with 143 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/syft/internal/options/archive.go
+36
View File
@@ -0,0 +1,36 @@
package options
import (
"github.com/anchore/clio"
"github.com/anchore/syft/syft/cataloging"
)
type archiveConfig struct {
MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"`
MaxExtractionSizeBytes int64 `yaml:"max-extraction-size-bytes" json:"max-extraction-size-bytes" mapstructure:"max-extraction-size-bytes"`
MaxFileCount int `yaml:"max-file-count" json:"max-file-count" mapstructure:"max-file-count"`
MaxTotalExtractionBytes int64 `yaml:"max-total-extraction-bytes" json:"max-total-extraction-bytes" mapstructure:"max-total-extraction-bytes"`
ExcludeExtensions []string `yaml:"exclude-extensions" json:"exclude-extensions" mapstructure:"exclude-extensions"`
}
var _ interface {
clio.FieldDescriber
} = (*archiveConfig)(nil)
func defaultArchiveConfig() archiveConfig {
return archiveConfig{
MaxDepth: cataloging.DefaultArchiveMaxDepth,
MaxExtractionSizeBytes: cataloging.DefaultArchiveMaxExtractionSizeBytes,
MaxFileCount: cataloging.DefaultArchiveMaxFileCount,
MaxTotalExtractionBytes: cataloging.DefaultArchiveMaxTotalExtractionBytes,
}
}
func (o *archiveConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
descriptions.Add(&o.MaxDepth, `maximum depth of recursive archive extraction (0 = disabled, no recursive extraction)
enabling this allows syft to discover packages inside nested archives (e.g., a tar.gz containing Python packages)`)
descriptions.Add(&o.MaxExtractionSizeBytes, `maximum total bytes to extract from a single archive before stopping`)
descriptions.Add(&o.MaxFileCount, `maximum number of files to extract from a single archive before stopping`)
descriptions.Add(&o.MaxTotalExtractionBytes, `maximum total bytes to extract across all archives before stopping`)
descriptions.Add(&o.ExcludeExtensions, `archive file extensions to skip during recursive extraction (e.g., [".rpm", ".deb"])`)
}
cmd/syft/internal/options/catalog.go
+16
View File
@@ -37,6 +37,7 @@ type Catalog struct {
Package packageConfig `yaml:"package" json:"package" mapstructure:"package"`
License licenseConfig `yaml:"license" json:"license" mapstructure:"license"`
File fileConfig `yaml:"file" json:"file" mapstructure:"file"`
Archive archiveConfig `yaml:"archive" json:"archive" mapstructure:"archive"`
Scope string `yaml:"scope" json:"scope" mapstructure:"scope"`
Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel
Relationships relationshipsConfig `yaml:"relationships" json:"relationships" mapstructure:"relationships"`
@@ -76,6 +77,7 @@ func DefaultCatalog() Catalog {
Scope: source.SquashedScope.String(),
Package: defaultPackageConfig(),
License: defaultLicenseConfig(),
Archive: defaultArchiveConfig(),
LinuxKernel: defaultLinuxKernelConfig(),
JavaScript: defaultJavaScriptConfig(),
Python: defaultPythonConfig(),
@@ -102,6 +104,8 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
WithPackagesConfig(cfg.ToPackagesConfig()).
WithLicenseConfig(cfg.ToLicenseConfig()).
WithFilesConfig(cfg.ToFilesConfig()).
WithArchiveConfig(cfg.ToArchiveConfig()).
WithExclusions(cfg.Exclusions).
WithCatalogerSelection(
cataloging.NewSelectionRequest().
WithDefaults(cfg.DefaultCatalogers...).
@@ -109,6 +113,18 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
)
}
func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig {
return cataloging.ArchiveSearchConfig{
IncludeIndexedArchives: cfg.Package.SearchIndexedArchives,
IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
MaxDepth: cfg.Archive.MaxDepth,
MaxExtractionSizeBytes: cfg.Archive.MaxExtractionSizeBytes,
MaxFileCount: cfg.Archive.MaxFileCount,
MaxTotalExtractionBytes: cfg.Archive.MaxTotalExtractionBytes,
ExcludeExtensions: cfg.Archive.ExcludeExtensions,
}
}
func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig {
return cataloging.SearchConfig{
Scope: source.ParseScope(cfg.Scope),
syft/configuration_audit_trail.go
+1
View File
@@ -17,6 +17,7 @@ type configurationAuditTrail struct {
Packages pkgcataloging.Config `json:"packages" yaml:"packages" mapstructure:"packages"`
Files filecataloging.Config `json:"files" yaml:"files" mapstructure:"files"`
Licenses cataloging.LicenseConfig `json:"licenses" yaml:"licenses" mapstructure:"licenses"`
Archive cataloging.ArchiveSearchConfig `json:"archive" yaml:"archive" mapstructure:"archive"`
Catalogers catalogerManifest `json:"catalogers" yaml:"catalogers" mapstructure:"catalogers"`
ExtraConfigs any `json:"extra,omitempty" yaml:"extra" mapstructure:"extra"`
}
syft/create_sbom.go
+68 -1
View File
@@ -10,6 +10,7 @@ import (
"github.com/scylladb/go-set/strset"
"github.com/anchore/go-sync"
intArchive "github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/licenses"
"github.com/anchore/syft/internal/log"
@@ -19,9 +20,12 @@ import (
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/internal/fileresolver"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/source/directorysource"
)
// CreateSBOM creates a software bill-of-materials from the given source. If the CreateSBOMConfig is nil, then
@@ -60,6 +64,7 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
Packages: cfg.Packages,
Files: cfg.Files,
Licenses: cfg.Licenses,
Archive: cfg.Archive,
Catalogers: *audit,
ExtraConfigs: cfg.ToolConfiguration,
},
@@ -92,9 +97,17 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
packageCatalogingProgress := monitorPackageCatalogingTask()
builder := sbomsync.NewBuilder(&s, monitorPackageCount(packageCatalogingProgress))
// determine the effective resolver: if recursive archive extraction is enabled,
// wrap the resolver with the archive orchestrator
effectiveResolver, archiveCleanup, archiveRelationships := setupArchiveOrchestration(ctx, resolver, cfg)
if archiveCleanup != nil {
defer archiveCleanup()
}
for i := range taskGroups {
err = sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(taskGroups[i]), func(t task.Task) (any, error) {
return nil, task.RunTask(ctx, t, resolver, builder, catalogingProgress)
return nil, task.RunTask(ctx, t, effectiveResolver, builder, catalogingProgress)
}, nil)
if err != nil {
// TODO: tie this to the open progress monitors...
@@ -102,12 +115,66 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
}
}
// add archive containment relationships to the SBOM
if len(archiveRelationships) > 0 {
builder.AddRelationships(archiveRelationships...)
}
packageCatalogingProgress.SetCompleted()
catalogingProgress.SetCompleted()
return &s, nil
}
// setupArchiveOrchestration wraps the resolver with an archive orchestrator when recursive
// archive extraction is enabled (MaxDepth > 0). It performs the iterative extract loop
// and returns the effective resolver, a cleanup function, and any archive relationships.
func setupArchiveOrchestration(ctx context.Context, baseResolver file.Resolver, cfg *CreateSBOMConfig) (file.Resolver, func(), []artifact.Relationship) {
if cfg.Archive.MaxDepth <= 0 {
return baseResolver, nil, nil
}
td := tmpdir.FromContext(ctx)
if td == nil {
log.Warn("archive orchestration requested but no temp dir available")
return baseResolver, nil, nil
}
archiveTmpDir, archiveTmpCleanup, err := td.NewChild("archive-extraction") //nolint:gocritic // cleanup is returned to caller, not deferred here
if err != nil {
log.WithFields("error", err).Warn("unable to create temp dir for archive extraction")
return baseResolver, nil, nil
}
// build user-exclusion path filters once and reuse them for every child
// resolver, so that --exclude patterns also apply inside extracted archives
resolverFactory := func(root string) (file.Resolver, error) {
filters, err := directorysource.GetDirectoryExclusionFunctions(root, append([]string(nil), cfg.Exclusions...))
if err != nil {
return nil, fmt.Errorf("invalid exclusion patterns for archive child resolver: %w", err)
}
return fileresolver.NewFromDirectory(root, "", filters...)
}
orch := intArchive.NewOrchestrator(baseResolver, cfg.Archive, archiveTmpDir, resolverFactory)
// iteratively discover and extract archives up to the configured max depth
for depth := 0; depth < cfg.Archive.MaxDepth; depth++ {
newArchives := orch.DiscoverAndExtract(ctx, depth)
if newArchives == 0 {
break
}
log.WithFields("depth", depth+1, "archives", newArchives).Debug("extracted archives for recursive cataloging")
}
cleanup := func() {
orch.Cleanup()
archiveTmpCleanup()
}
return orch.Resolver(), cleanup, orch.Relationships()
}
func setupContext(ctx context.Context, cfg *CreateSBOMConfig) (context.Context, error) {
// configure parallel executors
ctx = setContextExecutors(ctx, cfg)
syft/create_sbom_config.go
+22
View File
@@ -28,9 +28,16 @@ type CreateSBOMConfig struct {
Packages pkgcataloging.Config
Licenses cataloging.LicenseConfig
Files filecataloging.Config
Archive cataloging.ArchiveSearchConfig
Parallelism int
CatalogerSelection cataloging.SelectionRequest
// Exclusions are user-provided glob patterns that the source resolver already
// applies; they are also propagated into child resolvers built during recursive
// archive extraction so that a "--exclude=**/foo" pattern keeps working inside
// extracted archive contents.
Exclusions []string
// audit what tool is being used to generate the SBOM
ToolName string
ToolVersion string
@@ -49,6 +56,7 @@ func DefaultCreateSBOMConfig() *CreateSBOMConfig {
Packages: pkgcataloging.DefaultConfig(),
Licenses: cataloging.DefaultLicenseConfig(),
Files: filecataloging.DefaultConfig(),
Archive: cataloging.DefaultArchiveSearchConfig(),
Parallelism: 0, // use default: run in parallel based on number of CPUs
packageTaskFactories: task.DefaultPackageTaskFactories(),
@@ -153,6 +161,20 @@ func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig {
return c
}
// WithArchiveConfig allows for defining archive extraction and recursive cataloging parameters.
func (c *CreateSBOMConfig) WithArchiveConfig(cfg cataloging.ArchiveSearchConfig) *CreateSBOMConfig {
c.Archive = cfg
return c
}
// WithExclusions sets the glob patterns that should be propagated to child resolvers
// constructed during recursive archive extraction. Patterns must follow the same
// "./", "*/", or "**/"-prefix convention as the syft --exclude flag.
func (c *CreateSBOMConfig) WithExclusions(exclusions []string) *CreateSBOMConfig {
c.Exclusions = exclusions
return c
}
// WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag.
func (c *CreateSBOMConfig) WithCatalogerSelection(selection cataloging.SelectionRequest) *CreateSBOMConfig {
c.CatalogerSelection = selection