From 3fc5194f77fb065ce9331441376680804b86859f Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Wed, 25 Mar 2026 20:29:11 +0100 Subject: [PATCH] feat(sbom): wire archive orchestrator into CreateSBOM pipeline Integrate the archive extraction orchestrator into the SBOM creation pipeline, enabling recursive archive cataloging when configured: - Add Archive field to CreateSBOMConfig with WithArchiveConfig builder - Add Exclusions field to CreateSBOMConfig with WithExclusions builder so user --exclude patterns also filter content inside extracted archives, not only the source filesystem - Add setupArchiveOrchestration() that creates the orchestrator when MaxDepth > 0, runs the iterative extract loop, and returns the composite resolver with archive contents visible to all catalogers - Include archive config in configuration audit trail - Add CLI archive config section (max-depth, size/file limits, exclude-extensions) wired through Catalog.ToArchiveConfig() - CLI Catalog.ToSBOMConfig threads cfg.Exclusions through WithExclusions so child resolvers respect them When MaxDepth is 0 (default), behavior is unchanged. When > 0, archives discovered during scanning are extracted and their contents made visible to all catalogers via the CompositeResolver. Signed-off-by: Thomas Bechtold --- cmd/syft/internal/options/archive.go | 36 +++++++++++++++ cmd/syft/internal/options/catalog.go | 16 +++++++ syft/configuration_audit_trail.go | 1 + syft/create_sbom.go | 69 +++++++++++++++++++++++++++- syft/create_sbom_config.go | 22 +++++++++ 5 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 cmd/syft/internal/options/archive.go diff --git a/cmd/syft/internal/options/archive.go b/cmd/syft/internal/options/archive.go new file mode 100644 index 000000000..d2a71e1b0 --- /dev/null +++ b/cmd/syft/internal/options/archive.go @@ -0,0 +1,36 @@ +package options + +import ( + "github.com/anchore/clio" + "github.com/anchore/syft/syft/cataloging" +) + +type archiveConfig struct { + MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"` + MaxExtractionSizeBytes int64 `yaml:"max-extraction-size-bytes" json:"max-extraction-size-bytes" mapstructure:"max-extraction-size-bytes"` + MaxFileCount int `yaml:"max-file-count" json:"max-file-count" mapstructure:"max-file-count"` + MaxTotalExtractionBytes int64 `yaml:"max-total-extraction-bytes" json:"max-total-extraction-bytes" mapstructure:"max-total-extraction-bytes"` + ExcludeExtensions []string `yaml:"exclude-extensions" json:"exclude-extensions" mapstructure:"exclude-extensions"` +} + +var _ interface { + clio.FieldDescriber +} = (*archiveConfig)(nil) + +func defaultArchiveConfig() archiveConfig { + return archiveConfig{ + MaxDepth: cataloging.DefaultArchiveMaxDepth, + MaxExtractionSizeBytes: cataloging.DefaultArchiveMaxExtractionSizeBytes, + MaxFileCount: cataloging.DefaultArchiveMaxFileCount, + MaxTotalExtractionBytes: cataloging.DefaultArchiveMaxTotalExtractionBytes, + } +} + +func (o *archiveConfig) DescribeFields(descriptions clio.FieldDescriptionSet) { + descriptions.Add(&o.MaxDepth, `maximum depth of recursive archive extraction (0 = disabled, no recursive extraction) +enabling this allows syft to discover packages inside nested archives (e.g., a tar.gz containing Python packages)`) + descriptions.Add(&o.MaxExtractionSizeBytes, `maximum total bytes to extract from a single archive before stopping`) + descriptions.Add(&o.MaxFileCount, `maximum number of files to extract from a single archive before stopping`) + descriptions.Add(&o.MaxTotalExtractionBytes, `maximum total bytes to extract across all archives before stopping`) + descriptions.Add(&o.ExcludeExtensions, `archive file extensions to skip during recursive extraction (e.g., [".rpm", ".deb"])`) +} diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index bcb8ab66f..7483983d2 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -37,6 +37,7 @@ type Catalog struct { Package packageConfig `yaml:"package" json:"package" mapstructure:"package"` License licenseConfig `yaml:"license" json:"license" mapstructure:"license"` File fileConfig `yaml:"file" json:"file" mapstructure:"file"` + Archive archiveConfig `yaml:"archive" json:"archive" mapstructure:"archive"` Scope string `yaml:"scope" json:"scope" mapstructure:"scope"` Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel Relationships relationshipsConfig `yaml:"relationships" json:"relationships" mapstructure:"relationships"` @@ -76,6 +77,7 @@ func DefaultCatalog() Catalog { Scope: source.SquashedScope.String(), Package: defaultPackageConfig(), License: defaultLicenseConfig(), + Archive: defaultArchiveConfig(), LinuxKernel: defaultLinuxKernelConfig(), JavaScript: defaultJavaScriptConfig(), Python: defaultPythonConfig(), @@ -102,6 +104,8 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { WithPackagesConfig(cfg.ToPackagesConfig()). WithLicenseConfig(cfg.ToLicenseConfig()). WithFilesConfig(cfg.ToFilesConfig()). + WithArchiveConfig(cfg.ToArchiveConfig()). + WithExclusions(cfg.Exclusions). WithCatalogerSelection( cataloging.NewSelectionRequest(). WithDefaults(cfg.DefaultCatalogers...). @@ -109,6 +113,18 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { ) } +func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig { + return cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: cfg.Package.SearchIndexedArchives, + IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives, + MaxDepth: cfg.Archive.MaxDepth, + MaxExtractionSizeBytes: cfg.Archive.MaxExtractionSizeBytes, + MaxFileCount: cfg.Archive.MaxFileCount, + MaxTotalExtractionBytes: cfg.Archive.MaxTotalExtractionBytes, + ExcludeExtensions: cfg.Archive.ExcludeExtensions, + } +} + func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig { return cataloging.SearchConfig{ Scope: source.ParseScope(cfg.Scope), diff --git a/syft/configuration_audit_trail.go b/syft/configuration_audit_trail.go index aa7a51f4d..b63eb9be4 100644 --- a/syft/configuration_audit_trail.go +++ b/syft/configuration_audit_trail.go @@ -17,6 +17,7 @@ type configurationAuditTrail struct { Packages pkgcataloging.Config `json:"packages" yaml:"packages" mapstructure:"packages"` Files filecataloging.Config `json:"files" yaml:"files" mapstructure:"files"` Licenses cataloging.LicenseConfig `json:"licenses" yaml:"licenses" mapstructure:"licenses"` + Archive cataloging.ArchiveSearchConfig `json:"archive" yaml:"archive" mapstructure:"archive"` Catalogers catalogerManifest `json:"catalogers" yaml:"catalogers" mapstructure:"catalogers"` ExtraConfigs any `json:"extra,omitempty" yaml:"extra" mapstructure:"extra"` } diff --git a/syft/create_sbom.go b/syft/create_sbom.go index 817e698c6..ff4e1dc4b 100644 --- a/syft/create_sbom.go +++ b/syft/create_sbom.go @@ -10,6 +10,7 @@ import ( "github.com/scylladb/go-set/strset" "github.com/anchore/go-sync" + intArchive "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/log" @@ -19,9 +20,12 @@ import ( "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/event/monitor" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/fileresolver" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/source/directorysource" ) // CreateSBOM creates a software bill-of-materials from the given source. If the CreateSBOMConfig is nil, then @@ -60,6 +64,7 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( Packages: cfg.Packages, Files: cfg.Files, Licenses: cfg.Licenses, + Archive: cfg.Archive, Catalogers: *audit, ExtraConfigs: cfg.ToolConfiguration, }, @@ -92,9 +97,17 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( packageCatalogingProgress := monitorPackageCatalogingTask() builder := sbomsync.NewBuilder(&s, monitorPackageCount(packageCatalogingProgress)) + + // determine the effective resolver: if recursive archive extraction is enabled, + // wrap the resolver with the archive orchestrator + effectiveResolver, archiveCleanup, archiveRelationships := setupArchiveOrchestration(ctx, resolver, cfg) + if archiveCleanup != nil { + defer archiveCleanup() + } + for i := range taskGroups { err = sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(taskGroups[i]), func(t task.Task) (any, error) { - return nil, task.RunTask(ctx, t, resolver, builder, catalogingProgress) + return nil, task.RunTask(ctx, t, effectiveResolver, builder, catalogingProgress) }, nil) if err != nil { // TODO: tie this to the open progress monitors... @@ -102,12 +115,66 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( } } + // add archive containment relationships to the SBOM + if len(archiveRelationships) > 0 { + builder.AddRelationships(archiveRelationships...) + } + packageCatalogingProgress.SetCompleted() catalogingProgress.SetCompleted() return &s, nil } +// setupArchiveOrchestration wraps the resolver with an archive orchestrator when recursive +// archive extraction is enabled (MaxDepth > 0). It performs the iterative extract loop +// and returns the effective resolver, a cleanup function, and any archive relationships. +func setupArchiveOrchestration(ctx context.Context, baseResolver file.Resolver, cfg *CreateSBOMConfig) (file.Resolver, func(), []artifact.Relationship) { + if cfg.Archive.MaxDepth <= 0 { + return baseResolver, nil, nil + } + + td := tmpdir.FromContext(ctx) + if td == nil { + log.Warn("archive orchestration requested but no temp dir available") + return baseResolver, nil, nil + } + + archiveTmpDir, archiveTmpCleanup, err := td.NewChild("archive-extraction") //nolint:gocritic // cleanup is returned to caller, not deferred here + if err != nil { + log.WithFields("error", err).Warn("unable to create temp dir for archive extraction") + return baseResolver, nil, nil + } + + // build user-exclusion path filters once and reuse them for every child + // resolver, so that --exclude patterns also apply inside extracted archives + resolverFactory := func(root string) (file.Resolver, error) { + filters, err := directorysource.GetDirectoryExclusionFunctions(root, append([]string(nil), cfg.Exclusions...)) + if err != nil { + return nil, fmt.Errorf("invalid exclusion patterns for archive child resolver: %w", err) + } + return fileresolver.NewFromDirectory(root, "", filters...) + } + + orch := intArchive.NewOrchestrator(baseResolver, cfg.Archive, archiveTmpDir, resolverFactory) + + // iteratively discover and extract archives up to the configured max depth + for depth := 0; depth < cfg.Archive.MaxDepth; depth++ { + newArchives := orch.DiscoverAndExtract(ctx, depth) + if newArchives == 0 { + break + } + log.WithFields("depth", depth+1, "archives", newArchives).Debug("extracted archives for recursive cataloging") + } + + cleanup := func() { + orch.Cleanup() + archiveTmpCleanup() + } + + return orch.Resolver(), cleanup, orch.Relationships() +} + func setupContext(ctx context.Context, cfg *CreateSBOMConfig) (context.Context, error) { // configure parallel executors ctx = setContextExecutors(ctx, cfg) diff --git a/syft/create_sbom_config.go b/syft/create_sbom_config.go index 1a84c14b6..f08c914de 100644 --- a/syft/create_sbom_config.go +++ b/syft/create_sbom_config.go @@ -28,9 +28,16 @@ type CreateSBOMConfig struct { Packages pkgcataloging.Config Licenses cataloging.LicenseConfig Files filecataloging.Config + Archive cataloging.ArchiveSearchConfig Parallelism int CatalogerSelection cataloging.SelectionRequest + // Exclusions are user-provided glob patterns that the source resolver already + // applies; they are also propagated into child resolvers built during recursive + // archive extraction so that a "--exclude=**/foo" pattern keeps working inside + // extracted archive contents. + Exclusions []string + // audit what tool is being used to generate the SBOM ToolName string ToolVersion string @@ -49,6 +56,7 @@ func DefaultCreateSBOMConfig() *CreateSBOMConfig { Packages: pkgcataloging.DefaultConfig(), Licenses: cataloging.DefaultLicenseConfig(), Files: filecataloging.DefaultConfig(), + Archive: cataloging.DefaultArchiveSearchConfig(), Parallelism: 0, // use default: run in parallel based on number of CPUs packageTaskFactories: task.DefaultPackageTaskFactories(), @@ -153,6 +161,20 @@ func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig { return c } +// WithArchiveConfig allows for defining archive extraction and recursive cataloging parameters. +func (c *CreateSBOMConfig) WithArchiveConfig(cfg cataloging.ArchiveSearchConfig) *CreateSBOMConfig { + c.Archive = cfg + return c +} + +// WithExclusions sets the glob patterns that should be propagated to child resolvers +// constructed during recursive archive extraction. Patterns must follow the same +// "./", "*/", or "**/"-prefix convention as the syft --exclude flag. +func (c *CreateSBOMConfig) WithExclusions(exclusions []string) *CreateSBOMConfig { + c.Exclusions = exclusions + return c +} + // WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag. func (c *CreateSBOMConfig) WithCatalogerSelection(selection cataloging.SelectionRequest) *CreateSBOMConfig { c.CatalogerSelection = selection