mirror of
https://github.com/anchore/syft.git
synced 2026-11-01 06:56:40 +01:00
feat(archive): add Orchestrator for iterative archive extraction
Implement the Orchestrator that manages the iterative extract-catalog loop for recursive archive discovery: - Scans the composite resolver for candidate archive locations, detecting format by peeking 512 bytes via FileContentsByLocation rather than copying every non-archive file to a probe-temp file - Extracts identified archives to per-archive temp directories - Builds child resolvers via an injectable ResolverFactory (kept pluggable so tests can substitute a mock resolver without importing syft/internal/fileresolver) - Tracks extraction state to avoid reprocessing - Enforces safety limits (depth, per-archive size, file count, total extraction budget) - Generates ContainsRelationship entries linking each archive to its extracted-contents filesystem - Manages temp directory lifecycle with Cleanup() Signed-off-by: Thomas Bechtold <thomasbechtold@jpberlin.de>
This commit is contained in:
@@ -0,0 +1,362 @@
|
|||||||
|
package archive
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/anchore/syft/internal"
|
||||||
|
"github.com/anchore/syft/internal/log"
|
||||||
|
"github.com/anchore/syft/syft/artifact"
|
||||||
|
"github.com/anchore/syft/syft/cataloging"
|
||||||
|
"github.com/anchore/syft/syft/file"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ResolverFactory creates a file.Resolver for a directory path.
|
||||||
|
// This is injectable to avoid direct dependency on syft/internal/fileresolver.
|
||||||
|
type ResolverFactory func(root string) (file.Resolver, error)
|
||||||
|
|
||||||
|
// Orchestrator manages iterative archive extraction and cataloging. It discovers
|
||||||
|
// archives in the file resolver, extracts them, builds child resolvers for their
|
||||||
|
// contents, and tracks the relationships between archives and their contents.
|
||||||
|
type Orchestrator struct {
|
||||||
|
composite *CompositeResolver
|
||||||
|
extractors []Extractor
|
||||||
|
config cataloging.ArchiveSearchConfig
|
||||||
|
tmpDir string
|
||||||
|
resolverFactory ResolverFactory
|
||||||
|
|
||||||
|
// tracking state
|
||||||
|
processed map[processedKey]bool // archives we've already considered for extraction
|
||||||
|
totalBytesWritten int64
|
||||||
|
archiveNodes []*archiveNode // for relationship tracking
|
||||||
|
cleanups []func()
|
||||||
|
mu sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// archiveNode tracks an extracted archive for relationship generation.
|
||||||
|
type archiveNode struct {
|
||||||
|
location file.Location // where the archive is in the resolver
|
||||||
|
fsID string // the filesystem ID assigned to this archive's contents
|
||||||
|
depth int
|
||||||
|
}
|
||||||
|
|
||||||
|
// processedKey identifies an archive location uniquely. We key the processed
|
||||||
|
// map on a struct (not a "<fsID>:<realPath>" string) because both fields can
|
||||||
|
// legally contain ':' — fsIDs are emitted as "archive:<hex>" and POSIX paths
|
||||||
|
// permit colons — so a string key would have an ambiguous boundary.
|
||||||
|
type processedKey struct {
|
||||||
|
fsID string
|
||||||
|
realPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewOrchestrator creates an Orchestrator that manages archive extraction.
|
||||||
|
// The resolverFactory is used to build file.Resolvers for extracted archive directories.
|
||||||
|
func NewOrchestrator(baseResolver file.Resolver, cfg cataloging.ArchiveSearchConfig, tmpDir string, resolverFactory ResolverFactory) *Orchestrator {
|
||||||
|
return &Orchestrator{
|
||||||
|
composite: NewCompositeResolver(baseResolver),
|
||||||
|
extractors: DefaultExtractors(),
|
||||||
|
config: cfg,
|
||||||
|
tmpDir: tmpDir,
|
||||||
|
resolverFactory: resolverFactory,
|
||||||
|
processed: make(map[processedKey]bool),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolver returns the composite resolver that includes all extracted archive contents.
|
||||||
|
func (o *Orchestrator) Resolver() *CompositeResolver {
|
||||||
|
return o.composite
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiscoverAndExtract finds archives in the current resolver view, extracts them,
|
||||||
|
// and adds their contents as child resolvers. Returns the number of new archives extracted.
|
||||||
|
// The depth parameter indicates the current nesting depth (0 = scanning base filesystem).
|
||||||
|
func (o *Orchestrator) DiscoverAndExtract(ctx context.Context, depth int) int {
|
||||||
|
if depth >= o.config.MaxDepth {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
o.mu.Lock()
|
||||||
|
defer o.mu.Unlock()
|
||||||
|
|
||||||
|
if o.config.MaxTotalExtractionBytes > 0 && o.totalBytesWritten >= o.config.MaxTotalExtractionBytes {
|
||||||
|
log.Debug("archive orchestrator: total extraction size limit reached, stopping")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// build the list of extractors actually permitted by config — every
|
||||||
|
// non-archive location in the resolver is going to pay a peek read against
|
||||||
|
// these, so excluding disabled types up-front avoids reading files we
|
||||||
|
// would only end up rejecting after detection.
|
||||||
|
enabled := o.enabledExtractors()
|
||||||
|
if len(enabled) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
newArchives := 0
|
||||||
|
|
||||||
|
// scan all locations in the current resolver for archives
|
||||||
|
for loc := range o.composite.AllLocations(ctx) {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// skip already-processed archives
|
||||||
|
key := processedKey{fsID: loc.FileSystemID, realPath: loc.RealPath}
|
||||||
|
if o.processed[key] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// skip excluded extensions
|
||||||
|
if IsExcludedExtension(loc.RealPath, o.config.ExcludeExtensions) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// check if this location is an archive we can extract — peek a small
|
||||||
|
// buffer rather than materializing the full file content, which would
|
||||||
|
// be a full disk-to-disk copy of every non-archive file in the resolver.
|
||||||
|
ext := o.detectFormat(ctx, loc, enabled)
|
||||||
|
if ext == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// mark as processed before extraction to avoid re-processing
|
||||||
|
o.processed[key] = true
|
||||||
|
|
||||||
|
// extract the archive
|
||||||
|
extracted, err := o.extractArchive(ctx, ext, loc, depth+1)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields("archive", loc.Path(), "error", err).Debug("failed to extract archive")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if extracted {
|
||||||
|
newArchives++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return newArchives
|
||||||
|
}
|
||||||
|
|
||||||
|
// enabledExtractors returns the subset of o.extractors whose corresponding
|
||||||
|
// config gate is on (IncludeIndexedArchives for ZipExtractor,
|
||||||
|
// IncludeUnindexedArchives for TarExtractor).
|
||||||
|
func (o *Orchestrator) enabledExtractors() []Extractor {
|
||||||
|
var out []Extractor
|
||||||
|
for _, ext := range o.extractors {
|
||||||
|
switch ext.(type) {
|
||||||
|
case *ZipExtractor:
|
||||||
|
if o.config.IncludeIndexedArchives {
|
||||||
|
out = append(out, ext)
|
||||||
|
}
|
||||||
|
case *TarExtractor:
|
||||||
|
if o.config.IncludeUnindexedArchives {
|
||||||
|
out = append(out, ext)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
// unknown extractor types: include by default so plugin extractors
|
||||||
|
// added in the future aren't silently disabled
|
||||||
|
out = append(out, ext)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractArchive extracts a single archive and registers its contents as a child resolver.
|
||||||
|
func (o *Orchestrator) extractArchive(ctx context.Context, ext Extractor, loc file.Location, depth int) (bool, error) {
|
||||||
|
// create a temp dir for this archive's contents
|
||||||
|
extractDir, err := os.MkdirTemp(o.tmpDir, fmt.Sprintf("archive-depth%d-*", depth))
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("unable to create temp dir: %w", err)
|
||||||
|
}
|
||||||
|
o.cleanups = append(o.cleanups, func() {
|
||||||
|
os.RemoveAll(extractDir)
|
||||||
|
})
|
||||||
|
|
||||||
|
// extractors need a file path, so materialize the archive contents to a temp file
|
||||||
|
tmpPath, err := o.copyArchiveToTempFile(loc)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
defer os.Remove(tmpPath)
|
||||||
|
|
||||||
|
// calculate remaining budget
|
||||||
|
limits := ExtractionLimits{
|
||||||
|
MaxExtractionSizeBytes: o.config.MaxExtractionSizeBytes,
|
||||||
|
MaxFileCount: o.config.MaxFileCount,
|
||||||
|
}
|
||||||
|
if o.config.MaxTotalExtractionBytes > 0 {
|
||||||
|
remaining := o.config.MaxTotalExtractionBytes - o.totalBytesWritten
|
||||||
|
if remaining <= 0 {
|
||||||
|
return false, fmt.Errorf("total extraction budget exhausted")
|
||||||
|
}
|
||||||
|
if limits.MaxExtractionSizeBytes == 0 || remaining < limits.MaxExtractionSizeBytes {
|
||||||
|
limits.MaxExtractionSizeBytes = remaining
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := ext.Extract(ctx, tmpPath, extractDir, limits)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields("archive", loc.Path(), "error", err).Debug("archive extraction stopped")
|
||||||
|
// partial extraction is still useful, continue if we got some files
|
||||||
|
if result.FilesExtracted == 0 {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
o.totalBytesWritten += result.BytesWritten
|
||||||
|
|
||||||
|
if result.FilesExtracted == 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// build a directory resolver for the extracted contents
|
||||||
|
childResolver, resolverErr := o.resolverFactory(extractDir)
|
||||||
|
if resolverErr != nil {
|
||||||
|
return false, fmt.Errorf("unable to build resolver for extracted archive: %w", resolverErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
fsID := o.composite.AddChild(childResolver, loc, depth)
|
||||||
|
|
||||||
|
// track for relationship generation
|
||||||
|
o.archiveNodes = append(o.archiveNodes, &archiveNode{
|
||||||
|
location: loc,
|
||||||
|
fsID: fsID,
|
||||||
|
depth: depth,
|
||||||
|
})
|
||||||
|
|
||||||
|
log.WithFields(
|
||||||
|
"archive", loc.Path(),
|
||||||
|
"depth", depth,
|
||||||
|
"files", result.FilesExtracted,
|
||||||
|
"bytes", result.BytesWritten,
|
||||||
|
).Debug("extracted archive contents")
|
||||||
|
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatProbeBytes is the size of the buffer peeked from each candidate file
|
||||||
|
// to decide whether it's an extractable archive.
|
||||||
|
//
|
||||||
|
// Single-step formats only need their magic bytes (ZIP: 4, gzip/bzip2/xz/zstd:
|
||||||
|
// 2-6, tar: a 512-byte first-block header). The size that matters is for
|
||||||
|
// compressed-tar without a recognized extension: there mholt/archives must
|
||||||
|
// gzip-decode our peek buffer and feed the result to tar.NewReader.Next(),
|
||||||
|
// which needs a full 512-byte tar header out the back. 4 KiB of compressed
|
||||||
|
// input gives gzip a comfortable margin and is still a single filesystem
|
||||||
|
// block read.
|
||||||
|
const formatProbeBytes = 4096
|
||||||
|
|
||||||
|
// detectFormat returns the first enabled extractor that can handle loc by
|
||||||
|
// peeking a small buffer of its contents (no full materialization). Returns
|
||||||
|
// nil when no extractor matches or when the file can't be opened. The caller
|
||||||
|
// is responsible for narrowing extractors to those permitted by config so we
|
||||||
|
// don't pay peek I/O for archive types that would only be rejected later.
|
||||||
|
func (o *Orchestrator) detectFormat(ctx context.Context, loc file.Location, extractors []Extractor) Extractor {
|
||||||
|
reader, err := o.composite.FileContentsByLocation(loc)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer internal.CloseAndLogError(reader, loc.RealPath)
|
||||||
|
|
||||||
|
var peek [formatProbeBytes]byte
|
||||||
|
n, _ := io.ReadFull(reader, peek[:])
|
||||||
|
if n == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, ext := range extractors {
|
||||||
|
if ext.CanExtract(ctx, loc.RealPath, bytes.NewReader(peek[:n])) {
|
||||||
|
return ext
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// copyArchiveToTempFile reads the archive contents from the resolver and writes them
|
||||||
|
// to a freshly-created temp file under o.tmpDir. The returned path must be removed
|
||||||
|
// by the caller (typically via defer).
|
||||||
|
func (o *Orchestrator) copyArchiveToTempFile(loc file.Location) (string, error) {
|
||||||
|
reader, err := o.composite.FileContentsByLocation(loc)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("unable to get archive contents: %w", err)
|
||||||
|
}
|
||||||
|
defer internal.CloseAndLogError(reader, loc.RealPath)
|
||||||
|
|
||||||
|
tmpFile, err := os.CreateTemp(o.tmpDir, "archive-src-*")
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("unable to create temp file: %w", err)
|
||||||
|
}
|
||||||
|
_, copyErr := safeIOCopy(tmpFile, reader)
|
||||||
|
tmpFile.Close()
|
||||||
|
if copyErr != nil {
|
||||||
|
os.Remove(tmpFile.Name())
|
||||||
|
return "", fmt.Errorf("unable to copy archive to temp: %w", copyErr)
|
||||||
|
}
|
||||||
|
return tmpFile.Name(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Relationships returns the archive containment relationships discovered during extraction.
|
||||||
|
func (o *Orchestrator) Relationships() []artifact.Relationship {
|
||||||
|
o.mu.Lock()
|
||||||
|
defer o.mu.Unlock()
|
||||||
|
|
||||||
|
var rels []artifact.Relationship
|
||||||
|
for _, node := range o.archiveNodes {
|
||||||
|
// Create a "contains" relationship: the archive contains its extracted content
|
||||||
|
// The archive file (identified by its coordinates) contains the child filesystem
|
||||||
|
rels = append(rels, artifact.Relationship{
|
||||||
|
From: node.location.Coordinates,
|
||||||
|
To: file.Coordinates{RealPath: "/", FileSystemID: node.fsID},
|
||||||
|
Type: artifact.ContainsRelationship,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return rels
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cleanup removes all temporary directories created during extraction.
|
||||||
|
func (o *Orchestrator) Cleanup() {
|
||||||
|
o.mu.Lock()
|
||||||
|
defer o.mu.Unlock()
|
||||||
|
for _, cleanup := range o.cleanups {
|
||||||
|
cleanup()
|
||||||
|
}
|
||||||
|
o.cleanups = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// safeIOCopy copies with a size limit to prevent decompression bombs.
|
||||||
|
func safeIOCopy(dst *os.File, src interface{ Read([]byte) (int, error) }) (int64, error) {
|
||||||
|
const maxCopySize = 2 * 1024 * 1024 * 1024 // 2GB
|
||||||
|
return limitedCopy(dst, src, maxCopySize)
|
||||||
|
}
|
||||||
|
|
||||||
|
func limitedCopy(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) {
|
||||||
|
n, err := copyN(dst, src, limit)
|
||||||
|
if n >= limit {
|
||||||
|
return n, fmt.Errorf("copy size limit reached (%d bytes)", limit)
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyN(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) {
|
||||||
|
buf := make([]byte, 32*1024)
|
||||||
|
var written int64
|
||||||
|
for written < limit {
|
||||||
|
nr, readErr := src.Read(buf)
|
||||||
|
if nr > 0 {
|
||||||
|
nw, writeErr := dst.Write(buf[:nr])
|
||||||
|
written += int64(nw)
|
||||||
|
if writeErr != nil {
|
||||||
|
return written, writeErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if readErr != nil {
|
||||||
|
if readErr == io.EOF {
|
||||||
|
return written, nil
|
||||||
|
}
|
||||||
|
return written, readErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return written, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,315 @@
|
|||||||
|
package archive
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"github.com/anchore/syft/syft/cataloging"
|
||||||
|
"github.com/anchore/syft/syft/file"
|
||||||
|
)
|
||||||
|
|
||||||
|
// testResolverFactory creates a simple mock resolver factory for testing.
|
||||||
|
func testResolverFactory(root string) (file.Resolver, error) {
|
||||||
|
// Walk the directory and create a mock resolver from its contents
|
||||||
|
files := make(map[string]string)
|
||||||
|
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil || info.IsDir() {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
relPath, _ := filepath.Rel(root, path)
|
||||||
|
content, readErr := os.ReadFile(path)
|
||||||
|
if readErr != nil {
|
||||||
|
return readErr
|
||||||
|
}
|
||||||
|
files["/"+relPath] = string(content)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return newMockResolver(files), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrchestrator_DiscoverAndExtract_ZipArchive(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Create a zip file in the "source" directory
|
||||||
|
zipFiles := map[string]string{
|
||||||
|
"inner.txt": "hello from inside the zip",
|
||||||
|
"lib/a.so": "library content",
|
||||||
|
}
|
||||||
|
zipPath := createTestZip(t, dir, zipFiles)
|
||||||
|
|
||||||
|
// Create a parent resolver that sees the zip file
|
||||||
|
parentFiles := map[string]string{
|
||||||
|
"/test.zip": "placeholder", // the resolver sees the file
|
||||||
|
}
|
||||||
|
parent := newMockResolver(parentFiles)
|
||||||
|
|
||||||
|
// Override FileContentsByLocation to return actual zip content
|
||||||
|
zipContent, err := os.ReadFile(zipPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
parent.contents["/test.zip"] = string(zipContent)
|
||||||
|
|
||||||
|
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||||
|
cfg.MaxDepth = 1
|
||||||
|
cfg.IncludeIndexedArchives = true
|
||||||
|
|
||||||
|
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||||
|
defer orch.Cleanup()
|
||||||
|
|
||||||
|
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||||
|
assert.Equal(t, 1, newCount)
|
||||||
|
|
||||||
|
// the composite resolver should now have the child files
|
||||||
|
resolver := orch.Resolver()
|
||||||
|
assert.Equal(t, 1, resolver.ChildCount())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrchestrator_DiscoverAndExtract_TarGzArchive(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
tarFiles := map[string]string{
|
||||||
|
"inner.txt": "hello from tar",
|
||||||
|
}
|
||||||
|
tarPath := createTestTarGz(t, dir, tarFiles)
|
||||||
|
|
||||||
|
tarContent, err := os.ReadFile(tarPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
parent := newMockResolver(map[string]string{
|
||||||
|
"/test.tar.gz": string(tarContent),
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||||
|
cfg.MaxDepth = 1
|
||||||
|
cfg.IncludeUnindexedArchives = true // tar needs this enabled
|
||||||
|
|
||||||
|
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||||
|
defer orch.Cleanup()
|
||||||
|
|
||||||
|
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||||
|
assert.Equal(t, 1, newCount)
|
||||||
|
assert.Equal(t, 1, orch.Resolver().ChildCount())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrchestrator_DiscoverAndExtract_DepthLimit(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
parent := newMockResolver(map[string]string{
|
||||||
|
"/file.txt": "just a file",
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||||
|
cfg.MaxDepth = 1
|
||||||
|
|
||||||
|
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||||
|
defer orch.Cleanup()
|
||||||
|
|
||||||
|
// at depth 1 (== MaxDepth), should extract nothing
|
||||||
|
newCount := orch.DiscoverAndExtract(ctx, 1)
|
||||||
|
assert.Equal(t, 0, newCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrchestrator_DiscoverAndExtract_NoArchives(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
parent := newMockResolver(map[string]string{
|
||||||
|
"/file.txt": "plain text",
|
||||||
|
"/data.csv": "a,b,c",
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||||
|
cfg.MaxDepth = 3
|
||||||
|
|
||||||
|
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||||
|
defer orch.Cleanup()
|
||||||
|
|
||||||
|
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||||
|
assert.Equal(t, 0, newCount)
|
||||||
|
assert.Equal(t, 0, orch.Resolver().ChildCount())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrchestrator_DiscoverAndExtract_ExcludeExtension(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||||
|
zipContent, err := os.ReadFile(zipPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
parent := newMockResolver(map[string]string{
|
||||||
|
"/test.zip": string(zipContent),
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||||
|
cfg.MaxDepth = 1
|
||||||
|
cfg.IncludeIndexedArchives = true
|
||||||
|
cfg.ExcludeExtensions = []string{".zip"}
|
||||||
|
|
||||||
|
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||||
|
defer orch.Cleanup()
|
||||||
|
|
||||||
|
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||||
|
assert.Equal(t, 0, newCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrchestrator_DiscoverAndExtract_TotalSizeLimit(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||||
|
zipContent, err := os.ReadFile(zipPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
parent := newMockResolver(map[string]string{
|
||||||
|
"/test.zip": string(zipContent),
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||||
|
cfg.MaxDepth = 1
|
||||||
|
cfg.IncludeIndexedArchives = true
|
||||||
|
cfg.MaxTotalExtractionBytes = 1 // 1 byte total limit - will be exceeded
|
||||||
|
|
||||||
|
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||||
|
defer orch.Cleanup()
|
||||||
|
|
||||||
|
// should still extract but hit the limit
|
||||||
|
_ = orch.DiscoverAndExtract(ctx, 0)
|
||||||
|
// second call should be blocked by total limit
|
||||||
|
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||||
|
assert.Equal(t, 0, newCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrchestrator_Relationships(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||||
|
zipContent, err := os.ReadFile(zipPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
parent := newMockResolver(map[string]string{
|
||||||
|
"/test.zip": string(zipContent),
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||||
|
cfg.MaxDepth = 1
|
||||||
|
cfg.IncludeIndexedArchives = true
|
||||||
|
|
||||||
|
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||||
|
defer orch.Cleanup()
|
||||||
|
|
||||||
|
orch.DiscoverAndExtract(ctx, 0)
|
||||||
|
|
||||||
|
rels := orch.Relationships()
|
||||||
|
assert.NotEmpty(t, rels)
|
||||||
|
|
||||||
|
// should have a contains relationship for the archive
|
||||||
|
for _, rel := range rels {
|
||||||
|
assert.Equal(t, "contains", string(rel.Type))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrchestrator_Cleanup(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||||
|
zipContent, err := os.ReadFile(zipPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
parent := newMockResolver(map[string]string{
|
||||||
|
"/test.zip": string(zipContent),
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||||
|
cfg.MaxDepth = 1
|
||||||
|
cfg.IncludeIndexedArchives = true
|
||||||
|
|
||||||
|
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||||
|
|
||||||
|
orch.DiscoverAndExtract(ctx, 0)
|
||||||
|
|
||||||
|
// verify temp dirs exist
|
||||||
|
entries, err := os.ReadDir(tmpDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
initialCount := len(entries)
|
||||||
|
assert.True(t, initialCount > 0, "expected temp dirs to be created")
|
||||||
|
|
||||||
|
orch.Cleanup()
|
||||||
|
|
||||||
|
// verify temp dirs are cleaned up
|
||||||
|
entries, err = os.ReadDir(tmpDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, len(entries) < initialCount, "expected temp dirs to be cleaned up")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrchestrator_DoesNotReprocess(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||||
|
zipContent, err := os.ReadFile(zipPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
parent := newMockResolver(map[string]string{
|
||||||
|
"/test.zip": string(zipContent),
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||||
|
cfg.MaxDepth = 2
|
||||||
|
cfg.IncludeIndexedArchives = true
|
||||||
|
|
||||||
|
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||||
|
defer orch.Cleanup()
|
||||||
|
|
||||||
|
// first call extracts the archive
|
||||||
|
first := orch.DiscoverAndExtract(ctx, 0)
|
||||||
|
assert.Equal(t, 1, first)
|
||||||
|
|
||||||
|
// second call should not re-extract the same archive
|
||||||
|
second := orch.DiscoverAndExtract(ctx, 0)
|
||||||
|
assert.Equal(t, 0, second)
|
||||||
|
assert.Equal(t, 1, orch.Resolver().ChildCount())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrchestrator_ZipDisabledByConfig(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||||
|
zipContent, err := os.ReadFile(zipPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
parent := newMockResolver(map[string]string{
|
||||||
|
"/test.zip": string(zipContent),
|
||||||
|
})
|
||||||
|
|
||||||
|
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||||
|
cfg.MaxDepth = 1
|
||||||
|
cfg.IncludeIndexedArchives = false // disable zip extraction
|
||||||
|
|
||||||
|
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||||
|
defer orch.Cleanup()
|
||||||
|
|
||||||
|
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||||
|
assert.Equal(t, 0, newCount)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user