From f7ea0729007174deebe56d7ee7c1b9c5ae97aafd Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Wed, 25 Mar 2026 20:23:33 +0100 Subject: [PATCH] feat(archive): add Orchestrator for iterative archive extraction Implement the Orchestrator that manages the iterative extract-catalog loop for recursive archive discovery: - Scans the composite resolver for candidate archive locations, detecting format by peeking 512 bytes via FileContentsByLocation rather than copying every non-archive file to a probe-temp file - Extracts identified archives to per-archive temp directories - Builds child resolvers via an injectable ResolverFactory (kept pluggable so tests can substitute a mock resolver without importing syft/internal/fileresolver) - Tracks extraction state to avoid reprocessing - Enforces safety limits (depth, per-archive size, file count, total extraction budget) - Generates ContainsRelationship entries linking each archive to its extracted-contents filesystem - Manages temp directory lifecycle with Cleanup() Signed-off-by: Thomas Bechtold --- internal/archive/orchestrator.go | 362 ++++++++++++++++++++++++++ internal/archive/orchestrator_test.go | 315 ++++++++++++++++++++++ 2 files changed, 677 insertions(+) create mode 100644 internal/archive/orchestrator.go create mode 100644 internal/archive/orchestrator_test.go diff --git a/internal/archive/orchestrator.go b/internal/archive/orchestrator.go new file mode 100644 index 000000000..0a8e1a934 --- /dev/null +++ b/internal/archive/orchestrator.go @@ -0,0 +1,362 @@ +package archive + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "sync" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/file" +) + +// ResolverFactory creates a file.Resolver for a directory path. +// This is injectable to avoid direct dependency on syft/internal/fileresolver. +type ResolverFactory func(root string) (file.Resolver, error) + +// Orchestrator manages iterative archive extraction and cataloging. It discovers +// archives in the file resolver, extracts them, builds child resolvers for their +// contents, and tracks the relationships between archives and their contents. +type Orchestrator struct { + composite *CompositeResolver + extractors []Extractor + config cataloging.ArchiveSearchConfig + tmpDir string + resolverFactory ResolverFactory + + // tracking state + processed map[processedKey]bool // archives we've already considered for extraction + totalBytesWritten int64 + archiveNodes []*archiveNode // for relationship tracking + cleanups []func() + mu sync.Mutex +} + +// archiveNode tracks an extracted archive for relationship generation. +type archiveNode struct { + location file.Location // where the archive is in the resolver + fsID string // the filesystem ID assigned to this archive's contents + depth int +} + +// processedKey identifies an archive location uniquely. We key the processed +// map on a struct (not a ":" string) because both fields can +// legally contain ':' — fsIDs are emitted as "archive:" and POSIX paths +// permit colons — so a string key would have an ambiguous boundary. +type processedKey struct { + fsID string + realPath string +} + +// NewOrchestrator creates an Orchestrator that manages archive extraction. +// The resolverFactory is used to build file.Resolvers for extracted archive directories. +func NewOrchestrator(baseResolver file.Resolver, cfg cataloging.ArchiveSearchConfig, tmpDir string, resolverFactory ResolverFactory) *Orchestrator { + return &Orchestrator{ + composite: NewCompositeResolver(baseResolver), + extractors: DefaultExtractors(), + config: cfg, + tmpDir: tmpDir, + resolverFactory: resolverFactory, + processed: make(map[processedKey]bool), + } +} + +// Resolver returns the composite resolver that includes all extracted archive contents. +func (o *Orchestrator) Resolver() *CompositeResolver { + return o.composite +} + +// DiscoverAndExtract finds archives in the current resolver view, extracts them, +// and adds their contents as child resolvers. Returns the number of new archives extracted. +// The depth parameter indicates the current nesting depth (0 = scanning base filesystem). +func (o *Orchestrator) DiscoverAndExtract(ctx context.Context, depth int) int { + if depth >= o.config.MaxDepth { + return 0 + } + + o.mu.Lock() + defer o.mu.Unlock() + + if o.config.MaxTotalExtractionBytes > 0 && o.totalBytesWritten >= o.config.MaxTotalExtractionBytes { + log.Debug("archive orchestrator: total extraction size limit reached, stopping") + return 0 + } + + // build the list of extractors actually permitted by config — every + // non-archive location in the resolver is going to pay a peek read against + // these, so excluding disabled types up-front avoids reading files we + // would only end up rejecting after detection. + enabled := o.enabledExtractors() + if len(enabled) == 0 { + return 0 + } + + newArchives := 0 + + // scan all locations in the current resolver for archives + for loc := range o.composite.AllLocations(ctx) { + if ctx.Err() != nil { + break + } + + // skip already-processed archives + key := processedKey{fsID: loc.FileSystemID, realPath: loc.RealPath} + if o.processed[key] { + continue + } + + // skip excluded extensions + if IsExcludedExtension(loc.RealPath, o.config.ExcludeExtensions) { + continue + } + + // check if this location is an archive we can extract — peek a small + // buffer rather than materializing the full file content, which would + // be a full disk-to-disk copy of every non-archive file in the resolver. + ext := o.detectFormat(ctx, loc, enabled) + if ext == nil { + continue + } + + // mark as processed before extraction to avoid re-processing + o.processed[key] = true + + // extract the archive + extracted, err := o.extractArchive(ctx, ext, loc, depth+1) + if err != nil { + log.WithFields("archive", loc.Path(), "error", err).Debug("failed to extract archive") + continue + } + if extracted { + newArchives++ + } + } + + return newArchives +} + +// enabledExtractors returns the subset of o.extractors whose corresponding +// config gate is on (IncludeIndexedArchives for ZipExtractor, +// IncludeUnindexedArchives for TarExtractor). +func (o *Orchestrator) enabledExtractors() []Extractor { + var out []Extractor + for _, ext := range o.extractors { + switch ext.(type) { + case *ZipExtractor: + if o.config.IncludeIndexedArchives { + out = append(out, ext) + } + case *TarExtractor: + if o.config.IncludeUnindexedArchives { + out = append(out, ext) + } + default: + // unknown extractor types: include by default so plugin extractors + // added in the future aren't silently disabled + out = append(out, ext) + } + } + return out +} + +// extractArchive extracts a single archive and registers its contents as a child resolver. +func (o *Orchestrator) extractArchive(ctx context.Context, ext Extractor, loc file.Location, depth int) (bool, error) { + // create a temp dir for this archive's contents + extractDir, err := os.MkdirTemp(o.tmpDir, fmt.Sprintf("archive-depth%d-*", depth)) + if err != nil { + return false, fmt.Errorf("unable to create temp dir: %w", err) + } + o.cleanups = append(o.cleanups, func() { + os.RemoveAll(extractDir) + }) + + // extractors need a file path, so materialize the archive contents to a temp file + tmpPath, err := o.copyArchiveToTempFile(loc) + if err != nil { + return false, err + } + defer os.Remove(tmpPath) + + // calculate remaining budget + limits := ExtractionLimits{ + MaxExtractionSizeBytes: o.config.MaxExtractionSizeBytes, + MaxFileCount: o.config.MaxFileCount, + } + if o.config.MaxTotalExtractionBytes > 0 { + remaining := o.config.MaxTotalExtractionBytes - o.totalBytesWritten + if remaining <= 0 { + return false, fmt.Errorf("total extraction budget exhausted") + } + if limits.MaxExtractionSizeBytes == 0 || remaining < limits.MaxExtractionSizeBytes { + limits.MaxExtractionSizeBytes = remaining + } + } + + result, err := ext.Extract(ctx, tmpPath, extractDir, limits) + if err != nil { + log.WithFields("archive", loc.Path(), "error", err).Debug("archive extraction stopped") + // partial extraction is still useful, continue if we got some files + if result.FilesExtracted == 0 { + return false, err + } + } + + o.totalBytesWritten += result.BytesWritten + + if result.FilesExtracted == 0 { + return false, nil + } + + // build a directory resolver for the extracted contents + childResolver, resolverErr := o.resolverFactory(extractDir) + if resolverErr != nil { + return false, fmt.Errorf("unable to build resolver for extracted archive: %w", resolverErr) + } + + fsID := o.composite.AddChild(childResolver, loc, depth) + + // track for relationship generation + o.archiveNodes = append(o.archiveNodes, &archiveNode{ + location: loc, + fsID: fsID, + depth: depth, + }) + + log.WithFields( + "archive", loc.Path(), + "depth", depth, + "files", result.FilesExtracted, + "bytes", result.BytesWritten, + ).Debug("extracted archive contents") + + return true, nil +} + +// formatProbeBytes is the size of the buffer peeked from each candidate file +// to decide whether it's an extractable archive. +// +// Single-step formats only need their magic bytes (ZIP: 4, gzip/bzip2/xz/zstd: +// 2-6, tar: a 512-byte first-block header). The size that matters is for +// compressed-tar without a recognized extension: there mholt/archives must +// gzip-decode our peek buffer and feed the result to tar.NewReader.Next(), +// which needs a full 512-byte tar header out the back. 4 KiB of compressed +// input gives gzip a comfortable margin and is still a single filesystem +// block read. +const formatProbeBytes = 4096 + +// detectFormat returns the first enabled extractor that can handle loc by +// peeking a small buffer of its contents (no full materialization). Returns +// nil when no extractor matches or when the file can't be opened. The caller +// is responsible for narrowing extractors to those permitted by config so we +// don't pay peek I/O for archive types that would only be rejected later. +func (o *Orchestrator) detectFormat(ctx context.Context, loc file.Location, extractors []Extractor) Extractor { + reader, err := o.composite.FileContentsByLocation(loc) + if err != nil { + return nil + } + defer internal.CloseAndLogError(reader, loc.RealPath) + + var peek [formatProbeBytes]byte + n, _ := io.ReadFull(reader, peek[:]) + if n == 0 { + return nil + } + for _, ext := range extractors { + if ext.CanExtract(ctx, loc.RealPath, bytes.NewReader(peek[:n])) { + return ext + } + } + return nil +} + +// copyArchiveToTempFile reads the archive contents from the resolver and writes them +// to a freshly-created temp file under o.tmpDir. The returned path must be removed +// by the caller (typically via defer). +func (o *Orchestrator) copyArchiveToTempFile(loc file.Location) (string, error) { + reader, err := o.composite.FileContentsByLocation(loc) + if err != nil { + return "", fmt.Errorf("unable to get archive contents: %w", err) + } + defer internal.CloseAndLogError(reader, loc.RealPath) + + tmpFile, err := os.CreateTemp(o.tmpDir, "archive-src-*") + if err != nil { + return "", fmt.Errorf("unable to create temp file: %w", err) + } + _, copyErr := safeIOCopy(tmpFile, reader) + tmpFile.Close() + if copyErr != nil { + os.Remove(tmpFile.Name()) + return "", fmt.Errorf("unable to copy archive to temp: %w", copyErr) + } + return tmpFile.Name(), nil +} + +// Relationships returns the archive containment relationships discovered during extraction. +func (o *Orchestrator) Relationships() []artifact.Relationship { + o.mu.Lock() + defer o.mu.Unlock() + + var rels []artifact.Relationship + for _, node := range o.archiveNodes { + // Create a "contains" relationship: the archive contains its extracted content + // The archive file (identified by its coordinates) contains the child filesystem + rels = append(rels, artifact.Relationship{ + From: node.location.Coordinates, + To: file.Coordinates{RealPath: "/", FileSystemID: node.fsID}, + Type: artifact.ContainsRelationship, + }) + } + return rels +} + +// Cleanup removes all temporary directories created during extraction. +func (o *Orchestrator) Cleanup() { + o.mu.Lock() + defer o.mu.Unlock() + for _, cleanup := range o.cleanups { + cleanup() + } + o.cleanups = nil +} + +// safeIOCopy copies with a size limit to prevent decompression bombs. +func safeIOCopy(dst *os.File, src interface{ Read([]byte) (int, error) }) (int64, error) { + const maxCopySize = 2 * 1024 * 1024 * 1024 // 2GB + return limitedCopy(dst, src, maxCopySize) +} + +func limitedCopy(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) { + n, err := copyN(dst, src, limit) + if n >= limit { + return n, fmt.Errorf("copy size limit reached (%d bytes)", limit) + } + return n, err +} + +func copyN(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) { + buf := make([]byte, 32*1024) + var written int64 + for written < limit { + nr, readErr := src.Read(buf) + if nr > 0 { + nw, writeErr := dst.Write(buf[:nr]) + written += int64(nw) + if writeErr != nil { + return written, writeErr + } + } + if readErr != nil { + if readErr == io.EOF { + return written, nil + } + return written, readErr + } + } + return written, nil +} diff --git a/internal/archive/orchestrator_test.go b/internal/archive/orchestrator_test.go new file mode 100644 index 000000000..72b060fbf --- /dev/null +++ b/internal/archive/orchestrator_test.go @@ -0,0 +1,315 @@ +package archive + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/file" +) + +// testResolverFactory creates a simple mock resolver factory for testing. +func testResolverFactory(root string) (file.Resolver, error) { + // Walk the directory and create a mock resolver from its contents + files := make(map[string]string) + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + relPath, _ := filepath.Rel(root, path) + content, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + files["/"+relPath] = string(content) + return nil + }) + if err != nil { + return nil, err + } + return newMockResolver(files), nil +} + +func TestOrchestrator_DiscoverAndExtract_ZipArchive(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create a zip file in the "source" directory + zipFiles := map[string]string{ + "inner.txt": "hello from inside the zip", + "lib/a.so": "library content", + } + zipPath := createTestZip(t, dir, zipFiles) + + // Create a parent resolver that sees the zip file + parentFiles := map[string]string{ + "/test.zip": "placeholder", // the resolver sees the file + } + parent := newMockResolver(parentFiles) + + // Override FileContentsByLocation to return actual zip content + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + parent.contents["/test.zip"] = string(zipContent) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, newCount) + + // the composite resolver should now have the child files + resolver := orch.Resolver() + assert.Equal(t, 1, resolver.ChildCount()) +} + +func TestOrchestrator_DiscoverAndExtract_TarGzArchive(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + tarFiles := map[string]string{ + "inner.txt": "hello from tar", + } + tarPath := createTestTarGz(t, dir, tarFiles) + + tarContent, err := os.ReadFile(tarPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.tar.gz": string(tarContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeUnindexedArchives = true // tar needs this enabled + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, newCount) + assert.Equal(t, 1, orch.Resolver().ChildCount()) +} + +func TestOrchestrator_DiscoverAndExtract_DepthLimit(t *testing.T) { + tmpDir := t.TempDir() + ctx := context.Background() + + parent := newMockResolver(map[string]string{ + "/file.txt": "just a file", + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // at depth 1 (== MaxDepth), should extract nothing + newCount := orch.DiscoverAndExtract(ctx, 1) + assert.Equal(t, 0, newCount) +} + +func TestOrchestrator_DiscoverAndExtract_NoArchives(t *testing.T) { + tmpDir := t.TempDir() + ctx := context.Background() + + parent := newMockResolver(map[string]string{ + "/file.txt": "plain text", + "/data.csv": "a,b,c", + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 3 + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) + assert.Equal(t, 0, orch.Resolver().ChildCount()) +} + +func TestOrchestrator_DiscoverAndExtract_ExcludeExtension(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.ExcludeExtensions = []string{".zip"} + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) +} + +func TestOrchestrator_DiscoverAndExtract_TotalSizeLimit(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.MaxTotalExtractionBytes = 1 // 1 byte total limit - will be exceeded + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // should still extract but hit the limit + _ = orch.DiscoverAndExtract(ctx, 0) + // second call should be blocked by total limit + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) +} + +func TestOrchestrator_Relationships(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + orch.DiscoverAndExtract(ctx, 0) + + rels := orch.Relationships() + assert.NotEmpty(t, rels) + + // should have a contains relationship for the archive + for _, rel := range rels { + assert.Equal(t, "contains", string(rel.Type)) + } +} + +func TestOrchestrator_Cleanup(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + + orch.DiscoverAndExtract(ctx, 0) + + // verify temp dirs exist + entries, err := os.ReadDir(tmpDir) + require.NoError(t, err) + initialCount := len(entries) + assert.True(t, initialCount > 0, "expected temp dirs to be created") + + orch.Cleanup() + + // verify temp dirs are cleaned up + entries, err = os.ReadDir(tmpDir) + require.NoError(t, err) + assert.True(t, len(entries) < initialCount, "expected temp dirs to be cleaned up") +} + +func TestOrchestrator_DoesNotReprocess(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 2 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // first call extracts the archive + first := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, first) + + // second call should not re-extract the same archive + second := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, second) + assert.Equal(t, 1, orch.Resolver().ChildCount()) +} + +func TestOrchestrator_ZipDisabledByConfig(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = false // disable zip extraction + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) +}