From 05b4cf1c8433c881bd20a208bc9a1a527d142c5f Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Wed, 25 Mar 2026 20:14:59 +0100 Subject: [PATCH 01/10] feat(cataloging): extend ArchiveSearchConfig with recursive extraction settings Add configuration fields for generic recursive archive extraction: - MaxDepth: maximum nesting depth for archive extraction (default 0 = disabled) - MaxExtractionSizeBytes: per-archive extraction size limit (default 500MB) - MaxFileCount: per-archive file count limit (default 10,000) - MaxTotalExtractionBytes: total extraction size limit (default 2GB) - ExcludeExtensions: archive extensions to skip during extraction These fields lay the groundwork for cross-cataloger recursive archive cataloging, where archives discovered during scanning can be extracted and their contents made visible to all catalogers. Signed-off-by: Thomas Bechtold --- syft/cataloging/archive_search.go | 51 ++++++++++++++++++++++++++ syft/cataloging/archive_search_test.go | 44 ++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 syft/cataloging/archive_search_test.go diff --git a/syft/cataloging/archive_search.go b/syft/cataloging/archive_search.go index 56dff6cc6..464913909 100644 --- a/syft/cataloging/archive_search.go +++ b/syft/cataloging/archive_search.go @@ -1,17 +1,43 @@ package cataloging +const ( + DefaultArchiveMaxDepth = 0 // disabled by default for backwards compatibility + DefaultArchiveMaxExtractionSizeBytes = 500 * 1024 * 1024 // 500 MB per archive + DefaultArchiveMaxFileCount = 10000 + DefaultArchiveMaxTotalExtractionBytes = 2 * 1024 * 1024 * 1024 // 2 GB total +) + type ArchiveSearchConfig struct { // IncludeIndexedArchives indicates whether to search within indexed archive files (e.g., .zip). IncludeIndexedArchives bool `yaml:"include-indexed-archives" json:"include-indexed-archives" mapstructure:"include-indexed-archives"` // IncludeUnindexedArchives indicates whether to search within unindexed archive files (e.g., .tar*). IncludeUnindexedArchives bool `yaml:"include-unindexed-archives" json:"include-unindexed-archives" mapstructure:"include-unindexed-archives"` + + // MaxDepth is the maximum depth of recursive archive extraction (0 = disabled, no recursive extraction). + MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"` + + // MaxExtractionSizeBytes is the maximum total bytes to extract from a single archive before stopping. + MaxExtractionSizeBytes int64 `yaml:"max-extraction-size-bytes" json:"max-extraction-size-bytes" mapstructure:"max-extraction-size-bytes"` + + // MaxFileCount is the maximum number of files to extract from a single archive before stopping. + MaxFileCount int `yaml:"max-file-count" json:"max-file-count" mapstructure:"max-file-count"` + + // MaxTotalExtractionBytes is the maximum total bytes to extract across all archives before stopping. + MaxTotalExtractionBytes int64 `yaml:"max-total-extraction-bytes" json:"max-total-extraction-bytes" mapstructure:"max-total-extraction-bytes"` + + // ExcludeExtensions is a list of archive extensions to skip during recursive extraction (e.g., [".rpm"]). + ExcludeExtensions []string `yaml:"exclude-extensions" json:"exclude-extensions" mapstructure:"exclude-extensions"` } func DefaultArchiveSearchConfig() ArchiveSearchConfig { return ArchiveSearchConfig{ IncludeIndexedArchives: true, IncludeUnindexedArchives: false, + MaxDepth: DefaultArchiveMaxDepth, + MaxExtractionSizeBytes: DefaultArchiveMaxExtractionSizeBytes, + MaxFileCount: DefaultArchiveMaxFileCount, + MaxTotalExtractionBytes: DefaultArchiveMaxTotalExtractionBytes, } } @@ -24,3 +50,28 @@ func (c ArchiveSearchConfig) WithIncludeUnindexedArchives(include bool) ArchiveS c.IncludeUnindexedArchives = include return c } + +func (c ArchiveSearchConfig) WithMaxDepth(depth int) ArchiveSearchConfig { + c.MaxDepth = depth + return c +} + +func (c ArchiveSearchConfig) WithMaxExtractionSizeBytes(size int64) ArchiveSearchConfig { + c.MaxExtractionSizeBytes = size + return c +} + +func (c ArchiveSearchConfig) WithMaxFileCount(count int) ArchiveSearchConfig { + c.MaxFileCount = count + return c +} + +func (c ArchiveSearchConfig) WithMaxTotalExtractionBytes(size int64) ArchiveSearchConfig { + c.MaxTotalExtractionBytes = size + return c +} + +func (c ArchiveSearchConfig) WithExcludeExtensions(extensions []string) ArchiveSearchConfig { + c.ExcludeExtensions = extensions + return c +} diff --git a/syft/cataloging/archive_search_test.go b/syft/cataloging/archive_search_test.go new file mode 100644 index 000000000..f15e76b34 --- /dev/null +++ b/syft/cataloging/archive_search_test.go @@ -0,0 +1,44 @@ +package cataloging + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDefaultArchiveSearchConfig(t *testing.T) { + cfg := DefaultArchiveSearchConfig() + + assert.True(t, cfg.IncludeIndexedArchives) + assert.False(t, cfg.IncludeUnindexedArchives) + assert.Equal(t, DefaultArchiveMaxDepth, cfg.MaxDepth) + assert.Equal(t, int64(DefaultArchiveMaxExtractionSizeBytes), cfg.MaxExtractionSizeBytes) + assert.Equal(t, DefaultArchiveMaxFileCount, cfg.MaxFileCount) + assert.Equal(t, int64(DefaultArchiveMaxTotalExtractionBytes), cfg.MaxTotalExtractionBytes) + assert.Nil(t, cfg.ExcludeExtensions) +} + +func TestArchiveSearchConfig_WithMethods(t *testing.T) { + cfg := DefaultArchiveSearchConfig() + + cfg = cfg.WithMaxDepth(5) + assert.Equal(t, 5, cfg.MaxDepth) + + cfg = cfg.WithMaxExtractionSizeBytes(1024) + assert.Equal(t, int64(1024), cfg.MaxExtractionSizeBytes) + + cfg = cfg.WithMaxFileCount(100) + assert.Equal(t, 100, cfg.MaxFileCount) + + cfg = cfg.WithMaxTotalExtractionBytes(2048) + assert.Equal(t, int64(2048), cfg.MaxTotalExtractionBytes) + + cfg = cfg.WithExcludeExtensions([]string{".rpm", ".deb"}) + assert.Equal(t, []string{".rpm", ".deb"}, cfg.ExcludeExtensions) + + cfg = cfg.WithIncludeIndexedArchives(false) + assert.False(t, cfg.IncludeIndexedArchives) + + cfg = cfg.WithIncludeUnindexedArchives(true) + assert.True(t, cfg.IncludeUnindexedArchives) +} From 4df9a0ce101c188d88edb30159bcfb274b55700a Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Wed, 25 Mar 2026 20:16:55 +0100 Subject: [PATCH 02/10] feat(archive): add extractor interface and zip/tar implementations Introduce the internal/archive package with a generic Extractor interface for extracting archive contents to a destination directory. Includes two implementations: - ZipExtractor: handles zip-based archives (.zip, .jar, .war, etc.) - TarExtractor: handles tar-based archives (.tar, .tar.gz, .tar.bz2, etc.) Both extractors enforce configurable safety limits and protect against common malicious-archive vectors: - Per-archive extraction size limit - Per-archive file count limit - Zip-slip path traversal prevention via existing SafeJoin - Symlink entries are written as real symlinks only when their target resolves inside destDir; symlinks pointing outside are dropped to avoid escape via symlink-then-write - Non-regular non-symlink entries (devices, fifos, sockets) are skipped because their contents can't safely round-trip to disk Also provides FindExtractor for auto-detecting archive format, and IsExcludedExtension for filtering by extension. Signed-off-by: Thomas Bechtold --- internal/archive/extractor.go | 285 +++++++++++++++++++ internal/archive/extractor_test.go | 432 +++++++++++++++++++++++++++++ 2 files changed, 717 insertions(+) create mode 100644 internal/archive/extractor.go create mode 100644 internal/archive/extractor_test.go diff --git a/internal/archive/extractor.go b/internal/archive/extractor.go new file mode 100644 index 000000000..96525570d --- /dev/null +++ b/internal/archive/extractor.go @@ -0,0 +1,285 @@ +package archive + +import ( + "context" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/mholt/archives" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/cataloging" +) + +// Extractor extracts archive contents to a destination directory. +type Extractor interface { + // CanExtract returns true if this extractor can handle the given file. + CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool + + // Extract extracts the archive contents to destDir, respecting the given limits. + // Returns the number of files extracted and total bytes written. + Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) +} + +// ExtractionLimits defines safety limits for archive extraction. +type ExtractionLimits struct { + MaxExtractionSizeBytes int64 + MaxFileCount int +} + +// ExtractionResult holds the result of an extraction operation. +type ExtractionResult struct { + FilesExtracted int + BytesWritten int64 +} + +// DefaultExtractionLimits returns limits from the given config. +func DefaultExtractionLimits(cfg cataloging.ArchiveSearchConfig) ExtractionLimits { + return ExtractionLimits{ + MaxExtractionSizeBytes: cfg.MaxExtractionSizeBytes, + MaxFileCount: cfg.MaxFileCount, + } +} + +// DefaultExtractors returns the set of built-in archive extractors. +func DefaultExtractors() []Extractor { + return []Extractor{ + &ZipExtractor{}, + &TarExtractor{}, + } +} + +// FindExtractor finds an extractor that can handle the given file, or nil if none can. +func FindExtractor(ctx context.Context, extractors []Extractor, path string) Extractor { + f, err := os.Open(path) + if err != nil { + log.Tracef("unable to open file for archive detection: %v", err) + return nil + } + defer f.Close() + + for _, ext := range extractors { + if ext.CanExtract(ctx, path, f) { + return ext + } + // reset for next extractor + if _, err := f.Seek(0, io.SeekStart); err != nil { + log.Tracef("unable to seek file for archive detection: %v", err) + return nil + } + } + return nil +} + +// IsExcludedExtension checks if a file path has an excluded extension. +func IsExcludedExtension(path string, excludeExtensions []string) bool { + for _, ext := range excludeExtensions { + if strings.HasSuffix(strings.ToLower(path), strings.ToLower(ext)) { + return true + } + } + return false +} + +// ZipExtractor extracts zip-based archives (zip, jar, war, ear, etc.). +type ZipExtractor struct{} + +func (z *ZipExtractor) CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool { + format, _, err := intFile.IdentifyArchive(ctx, path, reader) + if err != nil { + return false + } + _, ok := format.(archives.Zip) + return ok +} + +func (z *ZipExtractor) Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) { + var result ExtractionResult + + f, err := os.Open(path) + if err != nil { + return result, fmt.Errorf("unable to open zip archive %q: %w", path, err) + } + defer f.Close() + + return result, archives.Zip{}.Extract(ctx, f, func(_ context.Context, file archives.FileInfo) error { + if file.IsDir() { + joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive) + if err != nil { + return err + } + return os.MkdirAll(joinedPath, file.Mode()) + } + + if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount { + return fmt.Errorf("archive file count limit reached (%d files)", limits.MaxFileCount) + } + + joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive) + if err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(joinedPath), 0o755); err != nil { + return fmt.Errorf("unable to create parent dir: %w", err) + } + + return extractFileWithLimits(file, joinedPath, destDir, &result, limits) + }) +} + +// TarExtractor extracts tar-based archives (tar, tar.gz, tar.bz2, tar.xz, tar.zst). +type TarExtractor struct{} + +func (t *TarExtractor) CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool { + format, _, err := intFile.IdentifyArchive(ctx, path, reader) + if err != nil { + return false + } + // mholt/archives returns a compound type for tar+compression, check if it's an extractor but not a zip + if _, isZip := format.(archives.Zip); isZip { + return false + } + _, ok := format.(archives.Extractor) + return ok +} + +func (t *TarExtractor) Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) { + var result ExtractionResult + + f, err := os.Open(path) + if err != nil { + return result, fmt.Errorf("unable to open tar archive %q: %w", path, err) + } + defer f.Close() + + format, readerAfterIdentify, err := intFile.IdentifyArchive(ctx, path, f) + if err != nil { + return result, fmt.Errorf("unable to identify archive format for %q: %w", path, err) + } + + extractor, ok := format.(archives.Extractor) + if !ok { + return result, fmt.Errorf("file format does not support extraction: %s", path) + } + + // use the reader returned by IdentifyArchive since it may have consumed some bytes + extractReader := readerAfterIdentify + if extractReader == nil { + extractReader = f + } + + return result, extractor.Extract(ctx, extractReader, func(_ context.Context, file archives.FileInfo) error { + if file.IsDir() { + joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive) + if err != nil { + return err + } + return os.MkdirAll(joinedPath, file.Mode()) + } + + if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount { + return fmt.Errorf("archive file count limit reached (%d files)", limits.MaxFileCount) + } + + joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive) + if err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(joinedPath), 0o755); err != nil { + return fmt.Errorf("unable to create parent dir: %w", err) + } + + return extractFileWithLimits(file, joinedPath, destDir, &result, limits) + }) +} + +// extractFileWithLimits extracts a single file from an archive, respecting size limits. +// Symlinks are written as real symlinks (only when the target stays inside destDir); +// non-regular file types other than symlinks are skipped to avoid writing bogus content. +func extractFileWithLimits(file archives.FileInfo, destPath, destDir string, result *ExtractionResult, limits ExtractionLimits) error { + mode := file.Mode() + if mode.Type()&fs.ModeSymlink != 0 { + if err := writeSafeSymlink(file.LinkTarget, destPath, destDir); err != nil { + log.WithFields("entry", file.NameInArchive, "target", file.LinkTarget, "error", err).Debug("skipping unsafe symlink in archive") + return nil + } + result.FilesExtracted++ + return nil + } + if !mode.IsRegular() { + // skip device nodes, fifos, sockets, etc. — they can't safely round-trip + log.WithFields("entry", file.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry") + return nil + } + + src, err := file.Open() + if err != nil { + return fmt.Errorf("unable to open archive entry %q: %w", file.NameInArchive, err) + } + defer src.Close() + + dst, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("unable to create dest file %q: %w", destPath, err) + } + defer dst.Close() + + var reader io.Reader = src + + // enforce per-archive total size limit + if limits.MaxExtractionSizeBytes > 0 { + remaining := limits.MaxExtractionSizeBytes - result.BytesWritten + if remaining <= 0 { + return fmt.Errorf("archive extraction size limit reached (%d bytes)", limits.MaxExtractionSizeBytes) + } + reader = io.LimitReader(src, remaining+1) // +1 to detect exceeding the limit + } + + n, err := io.Copy(dst, reader) + result.BytesWritten += n + result.FilesExtracted++ + + if limits.MaxExtractionSizeBytes > 0 && result.BytesWritten > limits.MaxExtractionSizeBytes { + return fmt.Errorf("archive extraction size limit reached (%d bytes)", limits.MaxExtractionSizeBytes) + } + + return err +} + +// writeSafeSymlink creates a symlink at destPath only if target is relative and +// resolves — relative to destPath's directory — to a location inside destDir. +// +// Absolute targets are rejected outright: os.Symlink writes the literal target +// string, so an absolute target like "/etc/passwd" would resolve on the host +// filesystem when read, regardless of any safety check we performed against +// destDir at extraction time. +// +// NOTE: this guards against the single-symlink-escape pattern, but a malicious +// archive can still attempt a multi-step chain where two cooperating link +// entries together resolve outside destDir (e.g., one entry creates a benign +// link, a later entry writes through it). Fully preventing that requires +// either openat/O_NOFOLLOW per path component during extraction or +// transitively resolving every newly-created symlink against all prior ones +// — neither is implemented here. +func writeSafeSymlink(target, destPath, destDir string) error { + if target == "" { + return fmt.Errorf("empty link target") + } + if filepath.IsAbs(target) { + return fmt.Errorf("absolute symlink target not allowed") + } + resolved := filepath.Join(filepath.Dir(destPath), target) + cleanRoot := filepath.Clean(destDir) + cleanResolved := filepath.Clean(resolved) + if cleanResolved != cleanRoot && !strings.HasPrefix(cleanResolved, cleanRoot+string(filepath.Separator)) { + return fmt.Errorf("symlink target escapes extraction root") + } + return os.Symlink(target, destPath) +} diff --git a/internal/archive/extractor_test.go b/internal/archive/extractor_test.go new file mode 100644 index 000000000..b244f3dcd --- /dev/null +++ b/internal/archive/extractor_test.go @@ -0,0 +1,432 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cataloging" +) + +func createTestZip(t *testing.T, dir string, files map[string]string) string { + t.Helper() + zipPath := filepath.Join(dir, "test.zip") + f, err := os.Create(zipPath) + require.NoError(t, err) + defer f.Close() + + w := zip.NewWriter(f) + for name, content := range files { + fw, err := w.Create(name) + require.NoError(t, err) + _, err = fw.Write([]byte(content)) + require.NoError(t, err) + } + require.NoError(t, w.Close()) + return zipPath +} + +func createTestTarGz(t *testing.T, dir string, files map[string]string) string { + t.Helper() + tarPath := filepath.Join(dir, "test.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + defer f.Close() + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Mode: 0o644, + Size: int64(len(content)), + } + require.NoError(t, tw.WriteHeader(hdr)) + _, err = tw.Write([]byte(content)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + return tarPath +} + +func TestZipExtractor_CanExtract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) + + ext := &ZipExtractor{} + + f, err := os.Open(zipPath) + require.NoError(t, err) + defer f.Close() + + assert.True(t, ext.CanExtract(ctx, zipPath, f)) + + // non-zip file + nonZipPath := filepath.Join(dir, "notazip.txt") + require.NoError(t, os.WriteFile(nonZipPath, []byte("just text"), 0o644)) + + f2, err := os.Open(nonZipPath) + require.NoError(t, err) + defer f2.Close() + + assert.False(t, ext.CanExtract(ctx, nonZipPath, f2)) +} + +func TestZipExtractor_Extract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": "content1", + "dir/file2.txt": "content2", + } + zipPath := createTestZip(t, dir, files) + + ext := &ZipExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + result, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + assert.Equal(t, 2, result.FilesExtracted) + + // verify extracted files + content, err := os.ReadFile(filepath.Join(destDir, "file1.txt")) + require.NoError(t, err) + assert.Equal(t, "content1", string(content)) + + content, err = os.ReadFile(filepath.Join(destDir, "dir", "file2.txt")) + require.NoError(t, err) + assert.Equal(t, "content2", string(content)) +} + +func TestZipExtractor_Extract_FileLimitReached(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": "a", + "file2.txt": "b", + "file3.txt": "c", + } + zipPath := createTestZip(t, dir, files) + + ext := &ZipExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{MaxFileCount: 2}) + require.Error(t, err) + assert.Contains(t, err.Error(), "file count limit") +} + +func TestZipExtractor_Extract_SizeLimitReached(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": strings.Repeat("x", 100), + } + zipPath := createTestZip(t, dir, files) + + ext := &ZipExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{MaxExtractionSizeBytes: 50}) + require.Error(t, err) + assert.Contains(t, err.Error(), "size limit") +} + +func TestTarExtractor_CanExtract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) + + ext := &TarExtractor{} + + f, err := os.Open(tarPath) + require.NoError(t, err) + defer f.Close() + + assert.True(t, ext.CanExtract(ctx, tarPath, f)) +} + +func TestTarExtractor_Extract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": "content1", + "dir/file2.txt": "content2", + } + tarPath := createTestTarGz(t, dir, files) + + ext := &TarExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + result, err := ext.Extract(ctx, tarPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + assert.Equal(t, 2, result.FilesExtracted) + + // verify extracted files + content, err := os.ReadFile(filepath.Join(destDir, "file1.txt")) + require.NoError(t, err) + assert.Equal(t, "content1", string(content)) + + content, err = os.ReadFile(filepath.Join(destDir, "dir", "file2.txt")) + require.NoError(t, err) + assert.Equal(t, "content2", string(content)) +} + +func TestFindExtractor(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + extractors := DefaultExtractors() + + // zip file should match + zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) + ext := FindExtractor(ctx, extractors, zipPath) + require.NotNil(t, ext) + assert.IsType(t, &ZipExtractor{}, ext) + + // tar.gz should match + tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) + ext = FindExtractor(ctx, extractors, tarPath) + require.NotNil(t, ext) + assert.IsType(t, &TarExtractor{}, ext) + + // non-archive should return nil + textPath := filepath.Join(dir, "plain.txt") + require.NoError(t, os.WriteFile(textPath, []byte("just text"), 0o644)) + ext = FindExtractor(ctx, extractors, textPath) + assert.Nil(t, ext) +} + +func TestIsExcludedExtension(t *testing.T) { + assert.True(t, IsExcludedExtension("/path/to/file.rpm", []string{".rpm", ".deb"})) + assert.True(t, IsExcludedExtension("/path/to/file.deb", []string{".rpm", ".deb"})) + assert.False(t, IsExcludedExtension("/path/to/file.zip", []string{".rpm", ".deb"})) + assert.False(t, IsExcludedExtension("/path/to/file.zip", nil)) +} + +func TestFindExtractor_NoReaders(t *testing.T) { + ctx := context.Background() + ext := FindExtractor(ctx, DefaultExtractors(), "/nonexistent/path") + assert.Nil(t, ext) +} + +func TestTarExtractor_CanExtract_NonTarFile(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + // create a plain text file + path := filepath.Join(dir, "plain.txt") + require.NoError(t, os.WriteFile(path, []byte("not a tar"), 0o644)) + + ext := &TarExtractor{} + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + + assert.False(t, ext.CanExtract(ctx, path, f)) +} + +func TestTarExtractor_CanExtract_ZipFileReturnsFalse(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) + + ext := &TarExtractor{} + f, err := os.Open(zipPath) + require.NoError(t, err) + defer f.Close() + + // tar extractor should NOT claim to handle zip files + assert.False(t, ext.CanExtract(ctx, zipPath, f)) +} + +func TestDefaultExtractionLimits(t *testing.T) { + cfg := cataloging.ArchiveSearchConfig{ + MaxExtractionSizeBytes: 1024, + MaxFileCount: 50, + } + limits := DefaultExtractionLimits(cfg) + assert.Equal(t, int64(1024), limits.MaxExtractionSizeBytes) + assert.Equal(t, 50, limits.MaxFileCount) +} + +func TestZipExtractor_Extract_ZipSlipPrevented(t *testing.T) { + // Create a zip with a path traversal entry + dir := t.TempDir() + ctx := context.Background() + + zipPath := filepath.Join(dir, "evil.zip") + f, err := os.Create(zipPath) + require.NoError(t, err) + + w := zip.NewWriter(f) + fw, err := w.Create("../../etc/passwd") + require.NoError(t, err) + _, err = fw.Write([]byte("evil")) + require.NoError(t, err) + require.NoError(t, w.Close()) + require.NoError(t, f.Close()) + + ext := &ZipExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err = ext.Extract(ctx, zipPath, destDir, ExtractionLimits{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "path traversal") + + // verify evil file was not created + _, statErr := os.Stat(filepath.Join(dir, "etc", "passwd")) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestTarExtractor_Extract_SymlinkInsideRootIsCreated(t *testing.T) { + // A tar archive with a symlink whose target stays inside the extraction + // directory should be written as a real symlink, not as a regular file + // containing the link target as bytes. + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "with-link.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + target := []byte("real content") + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "real.txt", + Mode: 0o644, + Size: int64(len(target)), + })) + _, err = tw.Write(target) + require.NoError(t, err) + + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "link.txt", + Linkname: "real.txt", + Typeflag: tar.TypeSymlink, + Mode: 0o777, + })) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err = ext.Extract(ctx, tarPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + + // the link should be a real symlink + info, err := os.Lstat(filepath.Join(destDir, "link.txt")) + require.NoError(t, err) + assert.NotZero(t, info.Mode()&fs.ModeSymlink, "expected link.txt to be a symlink, got mode %v", info.Mode()) + + // reading through the link should return the target's content + readBack, err := os.ReadFile(filepath.Join(destDir, "link.txt")) + require.NoError(t, err) + assert.Equal(t, "real content", string(readBack)) +} + +func TestTarExtractor_Extract_SymlinkEscapingRootIsSkipped(t *testing.T) { + // A symlink whose target resolves outside the extraction directory must not + // be created — neither as a symlink nor as a regular file. Extraction itself + // should succeed (the unsafe entry is skipped, not fatal). + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "evil-link.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "passwd", + Linkname: "../../../../etc/passwd", + Typeflag: tar.TypeSymlink, + Mode: 0o777, + })) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err = ext.Extract(ctx, tarPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + + // nothing should have been created at the destination + _, statErr := os.Lstat(filepath.Join(destDir, "passwd")) + assert.True(t, os.IsNotExist(statErr), "expected no entry to be created for escaping symlink") +} + +func TestTarExtractor_Extract_AbsoluteSymlinkTargetIsSkipped(t *testing.T) { + // Absolute symlink targets must be rejected: os.Symlink writes the literal + // target, so once the symlink is read it resolves on the host filesystem + // regardless of any safety check at extraction time. Verifying both that + // the symlink itself is not created and (importantly) that nothing was + // written under destDir along the absolute path. + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "absolute-link.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "shadow", + Linkname: "/etc/shadow", + Typeflag: tar.TypeSymlink, + Mode: 0o777, + })) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err = ext.Extract(ctx, tarPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + + // the symlink itself must not exist + _, statErr := os.Lstat(filepath.Join(destDir, "shadow")) + assert.True(t, os.IsNotExist(statErr), "expected absolute-target symlink to be skipped") +} From 4240cb964f53a71d809a7309e8ec531516f477a0 Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Wed, 25 Mar 2026 20:19:20 +0100 Subject: [PATCH 03/10] feat(archive): add CompositeResolver for transparent archive content access Implement a file.Resolver decorator that composites a parent resolver with child resolvers created from extracted archive contents. Files from child resolvers appear transparently alongside parent files. Key behaviors: - FilesByPath/FilesByGlob/FilesByMIMEType delegate to parent AND children - FileContentsByLocation routes to the correct resolver via FileSystemID - Locations from children get archive-prefixed AccessPath (using ":" separator matching Java cataloger convention) and unique FileSystemID - Thread-safe via RWMutex for concurrent access Signed-off-by: Thomas Bechtold --- internal/archive/resolver.go | 274 +++++++++++++++++++ internal/archive/resolver_test.go | 429 ++++++++++++++++++++++++++++++ 2 files changed, 703 insertions(+) create mode 100644 internal/archive/resolver.go create mode 100644 internal/archive/resolver_test.go diff --git a/internal/archive/resolver.go b/internal/archive/resolver.go new file mode 100644 index 000000000..a9baddd85 --- /dev/null +++ b/internal/archive/resolver.go @@ -0,0 +1,274 @@ +package archive + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "sync" + + "github.com/anchore/syft/syft/file" +) + +const ( + // ArchivePathSeparator is used to encode nested archive paths in AccessPath, + // matching the convention established by the Java cataloger. + ArchivePathSeparator = ":" +) + +// childResolver holds a resolver for an extracted archive along with metadata about the archive. +type childResolver struct { + resolver file.Resolver + archiveLocation file.Location // where the archive file lives in the parent resolver + fsID string // unique filesystem ID for this archive + depth int // nesting depth (0 = base filesystem) +} + +// CompositeResolver implements file.Resolver by compositing a parent resolver with +// child resolvers created from extracted archive contents. Files from child resolvers +// appear transparently alongside files from the parent. +type CompositeResolver struct { + parent file.Resolver + children []*childResolver + mu sync.RWMutex +} + +var _ file.Resolver = (*CompositeResolver)(nil) + +// NewCompositeResolver creates a new CompositeResolver wrapping the given parent resolver. +func NewCompositeResolver(parent file.Resolver) *CompositeResolver { + return &CompositeResolver{ + parent: parent, + } +} + +// AddChild registers a child resolver for an extracted archive. +func (r *CompositeResolver) AddChild(resolver file.Resolver, archiveLocation file.Location, depth int) string { + fsID := generateArchiveFSID(archiveLocation) + r.mu.Lock() + defer r.mu.Unlock() + r.children = append(r.children, &childResolver{ + resolver: resolver, + archiveLocation: archiveLocation, + fsID: fsID, + depth: depth, + }) + return fsID +} + +// ChildCount returns the number of child resolvers registered. +func (r *CompositeResolver) ChildCount() int { + r.mu.RLock() + defer r.mu.RUnlock() + return len(r.children) +} + +// HasPath checks if the path exists in the parent or any child resolver. +func (r *CompositeResolver) HasPath(path string) bool { + if r.parent.HasPath(path) { + return true + } + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + if child.resolver.HasPath(path) { + return true + } + } + return false +} + +// FilesByPath returns locations from the parent and all child resolvers. +func (r *CompositeResolver) FilesByPath(paths ...string) ([]file.Location, error) { + var results []file.Location + + parentLocs, err := r.parent.FilesByPath(paths...) + if err != nil { + return nil, err + } + results = append(results, parentLocs...) + + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + childLocs, err := child.resolver.FilesByPath(paths...) + if err != nil { + continue + } + for _, loc := range childLocs { + results = append(results, r.transformLocation(loc, child)) + } + } + + return results, nil +} + +// FilesByGlob returns locations from the parent and all child resolvers. +func (r *CompositeResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + var results []file.Location + + parentLocs, err := r.parent.FilesByGlob(patterns...) + if err != nil { + return nil, err + } + results = append(results, parentLocs...) + + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + childLocs, err := child.resolver.FilesByGlob(patterns...) + if err != nil { + continue + } + for _, loc := range childLocs { + results = append(results, r.transformLocation(loc, child)) + } + } + + return results, nil +} + +// FilesByMIMEType returns locations from the parent and all child resolvers. +func (r *CompositeResolver) FilesByMIMEType(types ...string) ([]file.Location, error) { + var results []file.Location + + parentLocs, err := r.parent.FilesByMIMEType(types...) + if err != nil { + return nil, err + } + results = append(results, parentLocs...) + + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + childLocs, err := child.resolver.FilesByMIMEType(types...) + if err != nil { + continue + } + for _, loc := range childLocs { + results = append(results, r.transformLocation(loc, child)) + } + } + + return results, nil +} + +// RelativeFileByPath looks up a file relative to a given location. +func (r *CompositeResolver) RelativeFileByPath(location file.Location, path string) *file.Location { + // try parent first + if loc := r.parent.RelativeFileByPath(location, path); loc != nil { + return loc + } + + // try children - match on the archive's FSID + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + if location.FileSystemID == child.fsID { + if loc := child.resolver.RelativeFileByPath(location, path); loc != nil { + transformed := r.transformLocation(*loc, child) + return &transformed + } + } + } + return nil +} + +// FileContentsByLocation routes to the appropriate resolver based on FileSystemID. +func (r *CompositeResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { + // check if this location belongs to a child resolver + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + if location.FileSystemID == child.fsID { + // look up the file by path in the child resolver to get a location with + // a valid internal reference (needed by directory resolvers that use + // stereoscope file references for content lookup) + childLocs, err := child.resolver.FilesByPath(location.RealPath) + if err != nil || len(childLocs) == 0 { + // fallback: try with a constructed location + childLoc := file.NewLocation(location.RealPath) + return child.resolver.FileContentsByLocation(childLoc) + } + return child.resolver.FileContentsByLocation(childLocs[0]) + } + } + + // fall through to parent + return r.parent.FileContentsByLocation(location) +} + +// AllLocations returns locations from the parent and all child resolvers. +func (r *CompositeResolver) AllLocations(ctx context.Context) <-chan file.Location { + results := make(chan file.Location) + go func() { + defer close(results) + + // parent locations + for loc := range r.parent.AllLocations(ctx) { + select { + case <-ctx.Done(): + return + case results <- loc: + } + } + + // child locations + r.mu.RLock() + children := make([]*childResolver, len(r.children)) + copy(children, r.children) + r.mu.RUnlock() + + for _, child := range children { + for loc := range child.resolver.AllLocations(ctx) { + transformed := r.transformLocation(loc, child) + select { + case <-ctx.Done(): + return + case results <- transformed: + } + } + } + }() + return results +} + +// FileMetadataByLocation routes to the appropriate resolver based on FileSystemID. +func (r *CompositeResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) { + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + if location.FileSystemID == child.fsID { + childLocs, err := child.resolver.FilesByPath(location.RealPath) + if err != nil || len(childLocs) == 0 { + childLoc := file.NewLocation(location.RealPath) + return child.resolver.FileMetadataByLocation(childLoc) + } + return child.resolver.FileMetadataByLocation(childLocs[0]) + } + } + return r.parent.FileMetadataByLocation(location) +} + +// transformLocation transforms a location from a child resolver to include the archive context. +func (r *CompositeResolver) transformLocation(loc file.Location, child *childResolver) file.Location { + archivePath := child.archiveLocation.Path() + return file.Location{ + LocationData: file.LocationData{ + Coordinates: file.Coordinates{ + RealPath: loc.RealPath, + FileSystemID: child.fsID, + }, + AccessPath: archivePath + ArchivePathSeparator + loc.Path(), + }, + LocationMetadata: loc.LocationMetadata, + } +} + +// generateArchiveFSID creates a deterministic filesystem ID for an archive based on its location. +func generateArchiveFSID(archiveLocation file.Location) string { + h := sha256.New() + h.Write([]byte(archiveLocation.RealPath)) + h.Write([]byte(archiveLocation.FileSystemID)) + return fmt.Sprintf("archive:%x", h.Sum(nil)[:12]) +} diff --git a/internal/archive/resolver_test.go b/internal/archive/resolver_test.go new file mode 100644 index 000000000..7eef566ae --- /dev/null +++ b/internal/archive/resolver_test.go @@ -0,0 +1,429 @@ +package archive + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/file" +) + +// mockResolver implements file.Resolver for testing. +type mockResolver struct { + locations map[string]file.Location // path -> location + contents map[string]string // realPath -> content +} + +func newMockResolver(files map[string]string) *mockResolver { + r := &mockResolver{ + locations: make(map[string]file.Location), + contents: files, + } + for path := range files { + r.locations[path] = file.NewLocation(path) + } + return r +} + +func (m *mockResolver) HasPath(path string) bool { + _, ok := m.locations[path] + return ok +} + +func (m *mockResolver) FilesByPath(paths ...string) ([]file.Location, error) { + var results []file.Location + for _, p := range paths { + if loc, ok := m.locations[p]; ok { + results = append(results, loc) + } + } + return results, nil +} + +func (m *mockResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + var results []file.Location + for _, pattern := range patterns { + for path, loc := range m.locations { + matched, _ := filepath.Match(pattern, path) + if matched { + results = append(results, loc) + } + // also try with the basename for simple glob patterns + if !matched { + matched, _ = filepath.Match(pattern, filepath.Base(path)) + if matched { + results = append(results, loc) + } + } + } + } + return results, nil +} + +func (m *mockResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) { + return nil, nil +} + +func (m *mockResolver) RelativeFileByPath(_ file.Location, path string) *file.Location { + if loc, ok := m.locations[path]; ok { + return &loc + } + return nil +} + +func (m *mockResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { + if content, ok := m.contents[location.RealPath]; ok { + return io.NopCloser(strings.NewReader(content)), nil + } + return nil, os.ErrNotExist +} + +func (m *mockResolver) AllLocations(ctx context.Context) <-chan file.Location { + ch := make(chan file.Location) + go func() { + defer close(ch) + for _, loc := range m.locations { + select { + case <-ctx.Done(): + return + case ch <- loc: + } + } + }() + return ch +} + +func (m *mockResolver) FileMetadataByLocation(_ file.Location) (file.Metadata, error) { + return file.Metadata{}, nil +} + +// strictMockResolver mimics real directory resolvers (fileresolver.Directory) which +// require that FileContentsByLocation receives a location object that was originally +// returned by FilesByPath/FilesByGlob — not a freshly constructed file.NewLocation(). +// Real directory resolvers need the internal stereoscope file.Reference for index lookup. +// We simulate this by annotating returned locations and only accepting those back. +type strictMockResolver struct { + mockResolver + resolvedAnnotation string // annotation key that marks locations as "resolved" +} + +func newStrictMockResolver(files map[string]string) *strictMockResolver { + return &strictMockResolver{ + mockResolver: *newMockResolver(files), + resolvedAnnotation: "strict-resolved", + } +} + +func (s *strictMockResolver) annotate(loc file.Location) file.Location { + return loc.WithAnnotation(s.resolvedAnnotation, "true") +} + +func (s *strictMockResolver) isResolved(loc file.Location) bool { + return loc.Annotations[s.resolvedAnnotation] == "true" +} + +func (s *strictMockResolver) FilesByPath(paths ...string) ([]file.Location, error) { + locs, err := s.mockResolver.FilesByPath(paths...) + for i := range locs { + locs[i] = s.annotate(locs[i]) + } + return locs, err +} + +func (s *strictMockResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + locs, err := s.mockResolver.FilesByGlob(patterns...) + for i := range locs { + locs[i] = s.annotate(locs[i]) + } + return locs, err +} + +func (s *strictMockResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { + if !s.isResolved(location) { + return nil, fmt.Errorf("location %q was not resolved via FilesByPath/FilesByGlob first (simulating real directory resolver behavior)", location.RealPath) + } + return s.mockResolver.FileContentsByLocation(location) +} + +func (s *strictMockResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) { + if !s.isResolved(location) { + return file.Metadata{}, fmt.Errorf("location %q was not resolved via FilesByPath/FilesByGlob first", location.RealPath) + } + return s.mockResolver.FileMetadataByLocation(location) +} + +// TestCompositeResolver_FileContentsByLocation_ChildRequiresLookup tests that +// FileContentsByLocation works when the child resolver requires locations to be +// resolved via FilesByPath first (as real directory resolvers do). This catches +// the bug where we constructed a bare file.NewLocation() without a valid internal +// reference, which fails on real fileresolver.Directory implementations. +func TestCompositeResolver_FileContentsByLocation_ChildRequiresLookup(t *testing.T) { + parent := newMockResolver(map[string]string{}) + child := newStrictMockResolver(map[string]string{ + "/lib/data.txt": "important data", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.zip") + fsID := composite.AddChild(child, archiveLoc, 1) + + // Simulate what a cataloger does: find the file, then read its content. + // The location returned has the archive's fsID, not the child's internal ref. + locs, err := composite.FilesByPath("/lib/data.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + assert.Equal(t, fsID, locs[0].FileSystemID) + + // Now read content - this must work even though the location was transformed + reader, err := composite.FileContentsByLocation(locs[0]) + require.NoError(t, err, "FileContentsByLocation should resolve the file in the child resolver via FilesByPath before reading") + defer reader.Close() + + content, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "important data", string(content)) +} + +// TestCompositeResolver_FileMetadataByLocation_ChildRequiresLookup is the same +// test as above but for FileMetadataByLocation. +func TestCompositeResolver_FileMetadataByLocation_ChildRequiresLookup(t *testing.T) { + parent := newMockResolver(map[string]string{}) + child := newStrictMockResolver(map[string]string{ + "/lib/data.txt": "important data", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.zip") + fsID := composite.AddChild(child, archiveLoc, 1) + + locs, err := composite.FilesByPath("/lib/data.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + assert.Equal(t, fsID, locs[0].FileSystemID) + + // FileMetadataByLocation must also resolve via FilesByPath first + _, err = composite.FileMetadataByLocation(locs[0]) + require.NoError(t, err, "FileMetadataByLocation should resolve the file in the child resolver via FilesByPath before reading metadata") +} + +func TestCompositeResolver_FilesByPath_ParentOnly(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/parent.txt": "parent content", + }) + composite := NewCompositeResolver(parent) + + locs, err := composite.FilesByPath("/parent.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + assert.Equal(t, "/parent.txt", locs[0].RealPath) +} + +func TestCompositeResolver_FilesByPath_WithChild(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/parent.txt": "parent content", + }) + child := newMockResolver(map[string]string{ + "/child.txt": "child content", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.tar.gz") + composite.AddChild(child, archiveLoc, 1) + + locs, err := composite.FilesByPath("/child.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + assert.Contains(t, locs[0].AccessPath, "archive.tar.gz:") + assert.Contains(t, locs[0].AccessPath, "child.txt") +} + +func TestCompositeResolver_FileContentsByLocation_Parent(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/parent.txt": "parent content", + }) + composite := NewCompositeResolver(parent) + + locs, err := composite.FilesByPath("/parent.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + + reader, err := composite.FileContentsByLocation(locs[0]) + require.NoError(t, err) + defer reader.Close() + + content, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "parent content", string(content)) +} + +func TestCompositeResolver_FileContentsByLocation_Child(t *testing.T) { + parent := newMockResolver(map[string]string{}) + child := newMockResolver(map[string]string{ + "/child.txt": "child content", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.zip") + fsID := composite.AddChild(child, archiveLoc, 1) + + // construct a location as returned by FilesByPath + loc := file.Location{ + LocationData: file.LocationData{ + Coordinates: file.Coordinates{ + RealPath: "/child.txt", + FileSystemID: fsID, + }, + AccessPath: "/archive.zip:/child.txt", + }, + } + + reader, err := composite.FileContentsByLocation(loc) + require.NoError(t, err) + defer reader.Close() + + content, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "child content", string(content)) +} + +func TestCompositeResolver_HasPath(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/parent.txt": "p", + }) + child := newMockResolver(map[string]string{ + "/child.txt": "c", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.zip") + composite.AddChild(child, archiveLoc, 1) + + assert.True(t, composite.HasPath("/parent.txt")) + assert.True(t, composite.HasPath("/child.txt")) + assert.False(t, composite.HasPath("/nonexistent.txt")) +} + +func TestCompositeResolver_AllLocations(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/parent.txt": "p", + }) + child := newMockResolver(map[string]string{ + "/child.txt": "c", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.zip") + composite.AddChild(child, archiveLoc, 1) + + ctx := context.Background() + var allLocs []file.Location + for loc := range composite.AllLocations(ctx) { + allLocs = append(allLocs, loc) + } + + assert.Len(t, allLocs, 2) + + hasArchivePrefix := false + for _, loc := range allLocs { + if strings.Contains(loc.AccessPath, "archive.zip:") { + hasArchivePrefix = true + break + } + } + assert.True(t, hasArchivePrefix) +} + +func TestCompositeResolver_ChildCount(t *testing.T) { + parent := newMockResolver(map[string]string{}) + composite := NewCompositeResolver(parent) + assert.Equal(t, 0, composite.ChildCount()) + + child := newMockResolver(map[string]string{"/a.txt": "a"}) + archiveLoc := file.NewLocation("/archive.zip") + composite.AddChild(child, archiveLoc, 1) + assert.Equal(t, 1, composite.ChildCount()) +} + +func TestCompositeResolver_TransformLocation(t *testing.T) { + composite := NewCompositeResolver(nil) + + archiveLoc := file.NewLocation("/path/to/archive.tar.gz") + child := &childResolver{ + archiveLocation: archiveLoc, + fsID: "archive:abc123", + depth: 1, + } + + loc := file.NewLocation("/inner/file.txt") + transformed := composite.transformLocation(loc, child) + + assert.Equal(t, "/inner/file.txt", transformed.RealPath) + assert.Equal(t, "archive:abc123", transformed.FileSystemID) + assert.Equal(t, "/path/to/archive.tar.gz:/inner/file.txt", transformed.AccessPath) +} + +func TestGenerateArchiveFSID(t *testing.T) { + loc1 := file.NewLocation("/archive1.zip") + loc2 := file.NewLocation("/archive2.zip") + + fsID1 := generateArchiveFSID(loc1) + fsID2 := generateArchiveFSID(loc2) + + assert.NotEqual(t, fsID1, fsID2) + assert.Equal(t, fsID1, generateArchiveFSID(loc1)) + assert.True(t, strings.HasPrefix(fsID1, "archive:")) +} + +func TestCompositeResolver_MultipleChildren(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/base.txt": "base", + }) + child1 := newMockResolver(map[string]string{ + "/lib.so": "library1", + }) + child2 := newMockResolver(map[string]string{ + "/lib.so": "library2", + }) + + composite := NewCompositeResolver(parent) + composite.AddChild(child1, file.NewLocation("/archive1.zip"), 1) + composite.AddChild(child2, file.NewLocation("/archive2.zip"), 1) + + assert.Equal(t, 2, composite.ChildCount()) + + // searching for /lib.so should find it in both children + locs, err := composite.FilesByPath("/lib.so") + require.NoError(t, err) + assert.Len(t, locs, 2) + + // each should have different archive prefixes + var accessPaths []string + for _, loc := range locs { + accessPaths = append(accessPaths, loc.AccessPath) + } + assert.Contains(t, accessPaths[0], "archive1.zip:") + assert.Contains(t, accessPaths[1], "archive2.zip:") +} + +func TestCompositeResolver_NoChildren_BehavesLikeParent(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/file.txt": "content", + }) + composite := NewCompositeResolver(parent) + + // FilesByPath + parentLocs, _ := parent.FilesByPath("/file.txt") + compositeLocs, _ := composite.FilesByPath("/file.txt") + assert.Equal(t, len(parentLocs), len(compositeLocs)) + + // HasPath + assert.Equal(t, parent.HasPath("/file.txt"), composite.HasPath("/file.txt")) + assert.Equal(t, parent.HasPath("/nope"), composite.HasPath("/nope")) +} From b26337759e513e6737a8a657dcc75c8502ca8ab9 Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Wed, 25 Mar 2026 20:20:08 +0100 Subject: [PATCH 04/10] test(archive): add edge case tests for CompositeResolver Add tests for important edge cases: - Nested children (child-in-child): verifies that archives within archives produce correct multi-level AccessPath chains like "/archive1.tar.gz:/inner-archive.zip:/deep-file.txt" - Empty child resolver: no locations leak from empty archives - Same file path in parent and child: both versions accessible with distinct FileSystemID routing - Context cancellation in AllLocations: no hanging Signed-off-by: Thomas Bechtold --- internal/archive/resolver_test.go | 155 ++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/internal/archive/resolver_test.go b/internal/archive/resolver_test.go index 7eef566ae..05bf40bb3 100644 --- a/internal/archive/resolver_test.go +++ b/internal/archive/resolver_test.go @@ -412,6 +412,161 @@ func TestCompositeResolver_MultipleChildren(t *testing.T) { assert.Contains(t, accessPaths[1], "archive2.zip:") } +func TestCompositeResolver_NestedChildren_ChildInChild(t *testing.T) { + // Simulates: base filesystem -> archive1.tar.gz -> archive2.zip -> inner file + // In practice the orchestrator adds children at different depths, but the resolver + // itself just has a flat list of children. The nesting is encoded in the AccessPath. + parent := newMockResolver(map[string]string{ + "/base.txt": "base", + }) + + // First level: contents of archive1.tar.gz + child1 := newMockResolver(map[string]string{ + "/inner-archive.zip": "zipdata", // this is an archive inside archive1 + "/readme.txt": "readme from archive1", + }) + + // Second level: contents of inner-archive.zip (which was inside archive1.tar.gz) + child2 := newMockResolver(map[string]string{ + "/deep-file.txt": "deeply nested content", + }) + + composite := NewCompositeResolver(parent) + + // Add child1 as extracted from archive1.tar.gz (depth 1) + archive1Loc := file.NewLocation("/archive1.tar.gz") + fsID1 := composite.AddChild(child1, archive1Loc, 1) + + // Add child2 as extracted from inner-archive.zip which was in archive1 (depth 2) + // The archive location for child2 refers to the inner-archive.zip as it appears in child1 + innerArchiveLoc := file.Location{ + LocationData: file.LocationData{ + Coordinates: file.Coordinates{ + RealPath: "/inner-archive.zip", + FileSystemID: fsID1, + }, + AccessPath: "/archive1.tar.gz:/inner-archive.zip", + }, + } + fsID2 := composite.AddChild(child2, innerArchiveLoc, 2) + + assert.Equal(t, 2, composite.ChildCount()) + assert.NotEqual(t, fsID1, fsID2) + + // Should find base file + locs, err := composite.FilesByPath("/base.txt") + require.NoError(t, err) + assert.Len(t, locs, 1) + assert.Equal(t, "/base.txt", locs[0].AccessPath) + + // Should find readme from archive1 + locs, err = composite.FilesByPath("/readme.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + assert.Equal(t, "/archive1.tar.gz:/readme.txt", locs[0].AccessPath) + assert.Equal(t, fsID1, locs[0].FileSystemID) + + // Should find deeply nested file + locs, err = composite.FilesByPath("/deep-file.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + // The access path shows the full chain: outer-archive:inner-archive:file + assert.Equal(t, "/archive1.tar.gz:/inner-archive.zip:/deep-file.txt", locs[0].AccessPath) + assert.Equal(t, fsID2, locs[0].FileSystemID) + + // Should be able to read the deeply nested content + reader, err := composite.FileContentsByLocation(locs[0]) + require.NoError(t, err) + defer reader.Close() + content, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "deeply nested content", string(content)) +} + +func TestCompositeResolver_EmptyChild(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/file.txt": "content", + }) + child := newMockResolver(map[string]string{}) + + composite := NewCompositeResolver(parent) + composite.AddChild(child, file.NewLocation("/empty.zip"), 1) + + // should still find parent files + locs, err := composite.FilesByPath("/file.txt") + require.NoError(t, err) + assert.Len(t, locs, 1) + + // no extra locations from empty child + ctx := context.Background() + var all []file.Location + for loc := range composite.AllLocations(ctx) { + all = append(all, loc) + } + assert.Len(t, all, 1) +} + +func TestCompositeResolver_SameFileInParentAndChild(t *testing.T) { + // File with the same path exists in both parent and child + parent := newMockResolver(map[string]string{ + "/config.yaml": "parent version", + }) + child := newMockResolver(map[string]string{ + "/config.yaml": "child version", + }) + + composite := NewCompositeResolver(parent) + fsID := composite.AddChild(child, file.NewLocation("/archive.zip"), 1) + + locs, err := composite.FilesByPath("/config.yaml") + require.NoError(t, err) + // Should find both versions + assert.Len(t, locs, 2) + + // parent version has no archive prefix + parentLoc := locs[0] + assert.Equal(t, "/config.yaml", parentLoc.AccessPath) + assert.Empty(t, parentLoc.FileSystemID) + + // child version has archive prefix + childLoc := locs[1] + assert.Equal(t, "/archive.zip:/config.yaml", childLoc.AccessPath) + assert.Equal(t, fsID, childLoc.FileSystemID) + + // can read both + r1, err := composite.FileContentsByLocation(parentLoc) + require.NoError(t, err) + c1, _ := io.ReadAll(r1) + r1.Close() + assert.Equal(t, "parent version", string(c1)) + + r2, err := composite.FileContentsByLocation(childLoc) + require.NoError(t, err) + c2, _ := io.ReadAll(r2) + r2.Close() + assert.Equal(t, "child version", string(c2)) +} + +func TestCompositeResolver_AllLocations_ContextCancellation(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/a.txt": "a", + "/b.txt": "b", + "/c.txt": "c", + }) + + composite := NewCompositeResolver(parent) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + var count int + for range composite.AllLocations(ctx) { + count++ + } + // with cancelled context, we may get 0 or some locations, but shouldn't hang + assert.True(t, count <= 3) +} + func TestCompositeResolver_NoChildren_BehavesLikeParent(t *testing.T) { parent := newMockResolver(map[string]string{ "/file.txt": "content", From f7ea0729007174deebe56d7ee7c1b9c5ae97aafd Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Wed, 25 Mar 2026 20:23:33 +0100 Subject: [PATCH 05/10] feat(archive): add Orchestrator for iterative archive extraction Implement the Orchestrator that manages the iterative extract-catalog loop for recursive archive discovery: - Scans the composite resolver for candidate archive locations, detecting format by peeking 512 bytes via FileContentsByLocation rather than copying every non-archive file to a probe-temp file - Extracts identified archives to per-archive temp directories - Builds child resolvers via an injectable ResolverFactory (kept pluggable so tests can substitute a mock resolver without importing syft/internal/fileresolver) - Tracks extraction state to avoid reprocessing - Enforces safety limits (depth, per-archive size, file count, total extraction budget) - Generates ContainsRelationship entries linking each archive to its extracted-contents filesystem - Manages temp directory lifecycle with Cleanup() Signed-off-by: Thomas Bechtold --- internal/archive/orchestrator.go | 362 ++++++++++++++++++++++++++ internal/archive/orchestrator_test.go | 315 ++++++++++++++++++++++ 2 files changed, 677 insertions(+) create mode 100644 internal/archive/orchestrator.go create mode 100644 internal/archive/orchestrator_test.go diff --git a/internal/archive/orchestrator.go b/internal/archive/orchestrator.go new file mode 100644 index 000000000..0a8e1a934 --- /dev/null +++ b/internal/archive/orchestrator.go @@ -0,0 +1,362 @@ +package archive + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "sync" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/file" +) + +// ResolverFactory creates a file.Resolver for a directory path. +// This is injectable to avoid direct dependency on syft/internal/fileresolver. +type ResolverFactory func(root string) (file.Resolver, error) + +// Orchestrator manages iterative archive extraction and cataloging. It discovers +// archives in the file resolver, extracts them, builds child resolvers for their +// contents, and tracks the relationships between archives and their contents. +type Orchestrator struct { + composite *CompositeResolver + extractors []Extractor + config cataloging.ArchiveSearchConfig + tmpDir string + resolverFactory ResolverFactory + + // tracking state + processed map[processedKey]bool // archives we've already considered for extraction + totalBytesWritten int64 + archiveNodes []*archiveNode // for relationship tracking + cleanups []func() + mu sync.Mutex +} + +// archiveNode tracks an extracted archive for relationship generation. +type archiveNode struct { + location file.Location // where the archive is in the resolver + fsID string // the filesystem ID assigned to this archive's contents + depth int +} + +// processedKey identifies an archive location uniquely. We key the processed +// map on a struct (not a ":" string) because both fields can +// legally contain ':' — fsIDs are emitted as "archive:" and POSIX paths +// permit colons — so a string key would have an ambiguous boundary. +type processedKey struct { + fsID string + realPath string +} + +// NewOrchestrator creates an Orchestrator that manages archive extraction. +// The resolverFactory is used to build file.Resolvers for extracted archive directories. +func NewOrchestrator(baseResolver file.Resolver, cfg cataloging.ArchiveSearchConfig, tmpDir string, resolverFactory ResolverFactory) *Orchestrator { + return &Orchestrator{ + composite: NewCompositeResolver(baseResolver), + extractors: DefaultExtractors(), + config: cfg, + tmpDir: tmpDir, + resolverFactory: resolverFactory, + processed: make(map[processedKey]bool), + } +} + +// Resolver returns the composite resolver that includes all extracted archive contents. +func (o *Orchestrator) Resolver() *CompositeResolver { + return o.composite +} + +// DiscoverAndExtract finds archives in the current resolver view, extracts them, +// and adds their contents as child resolvers. Returns the number of new archives extracted. +// The depth parameter indicates the current nesting depth (0 = scanning base filesystem). +func (o *Orchestrator) DiscoverAndExtract(ctx context.Context, depth int) int { + if depth >= o.config.MaxDepth { + return 0 + } + + o.mu.Lock() + defer o.mu.Unlock() + + if o.config.MaxTotalExtractionBytes > 0 && o.totalBytesWritten >= o.config.MaxTotalExtractionBytes { + log.Debug("archive orchestrator: total extraction size limit reached, stopping") + return 0 + } + + // build the list of extractors actually permitted by config — every + // non-archive location in the resolver is going to pay a peek read against + // these, so excluding disabled types up-front avoids reading files we + // would only end up rejecting after detection. + enabled := o.enabledExtractors() + if len(enabled) == 0 { + return 0 + } + + newArchives := 0 + + // scan all locations in the current resolver for archives + for loc := range o.composite.AllLocations(ctx) { + if ctx.Err() != nil { + break + } + + // skip already-processed archives + key := processedKey{fsID: loc.FileSystemID, realPath: loc.RealPath} + if o.processed[key] { + continue + } + + // skip excluded extensions + if IsExcludedExtension(loc.RealPath, o.config.ExcludeExtensions) { + continue + } + + // check if this location is an archive we can extract — peek a small + // buffer rather than materializing the full file content, which would + // be a full disk-to-disk copy of every non-archive file in the resolver. + ext := o.detectFormat(ctx, loc, enabled) + if ext == nil { + continue + } + + // mark as processed before extraction to avoid re-processing + o.processed[key] = true + + // extract the archive + extracted, err := o.extractArchive(ctx, ext, loc, depth+1) + if err != nil { + log.WithFields("archive", loc.Path(), "error", err).Debug("failed to extract archive") + continue + } + if extracted { + newArchives++ + } + } + + return newArchives +} + +// enabledExtractors returns the subset of o.extractors whose corresponding +// config gate is on (IncludeIndexedArchives for ZipExtractor, +// IncludeUnindexedArchives for TarExtractor). +func (o *Orchestrator) enabledExtractors() []Extractor { + var out []Extractor + for _, ext := range o.extractors { + switch ext.(type) { + case *ZipExtractor: + if o.config.IncludeIndexedArchives { + out = append(out, ext) + } + case *TarExtractor: + if o.config.IncludeUnindexedArchives { + out = append(out, ext) + } + default: + // unknown extractor types: include by default so plugin extractors + // added in the future aren't silently disabled + out = append(out, ext) + } + } + return out +} + +// extractArchive extracts a single archive and registers its contents as a child resolver. +func (o *Orchestrator) extractArchive(ctx context.Context, ext Extractor, loc file.Location, depth int) (bool, error) { + // create a temp dir for this archive's contents + extractDir, err := os.MkdirTemp(o.tmpDir, fmt.Sprintf("archive-depth%d-*", depth)) + if err != nil { + return false, fmt.Errorf("unable to create temp dir: %w", err) + } + o.cleanups = append(o.cleanups, func() { + os.RemoveAll(extractDir) + }) + + // extractors need a file path, so materialize the archive contents to a temp file + tmpPath, err := o.copyArchiveToTempFile(loc) + if err != nil { + return false, err + } + defer os.Remove(tmpPath) + + // calculate remaining budget + limits := ExtractionLimits{ + MaxExtractionSizeBytes: o.config.MaxExtractionSizeBytes, + MaxFileCount: o.config.MaxFileCount, + } + if o.config.MaxTotalExtractionBytes > 0 { + remaining := o.config.MaxTotalExtractionBytes - o.totalBytesWritten + if remaining <= 0 { + return false, fmt.Errorf("total extraction budget exhausted") + } + if limits.MaxExtractionSizeBytes == 0 || remaining < limits.MaxExtractionSizeBytes { + limits.MaxExtractionSizeBytes = remaining + } + } + + result, err := ext.Extract(ctx, tmpPath, extractDir, limits) + if err != nil { + log.WithFields("archive", loc.Path(), "error", err).Debug("archive extraction stopped") + // partial extraction is still useful, continue if we got some files + if result.FilesExtracted == 0 { + return false, err + } + } + + o.totalBytesWritten += result.BytesWritten + + if result.FilesExtracted == 0 { + return false, nil + } + + // build a directory resolver for the extracted contents + childResolver, resolverErr := o.resolverFactory(extractDir) + if resolverErr != nil { + return false, fmt.Errorf("unable to build resolver for extracted archive: %w", resolverErr) + } + + fsID := o.composite.AddChild(childResolver, loc, depth) + + // track for relationship generation + o.archiveNodes = append(o.archiveNodes, &archiveNode{ + location: loc, + fsID: fsID, + depth: depth, + }) + + log.WithFields( + "archive", loc.Path(), + "depth", depth, + "files", result.FilesExtracted, + "bytes", result.BytesWritten, + ).Debug("extracted archive contents") + + return true, nil +} + +// formatProbeBytes is the size of the buffer peeked from each candidate file +// to decide whether it's an extractable archive. +// +// Single-step formats only need their magic bytes (ZIP: 4, gzip/bzip2/xz/zstd: +// 2-6, tar: a 512-byte first-block header). The size that matters is for +// compressed-tar without a recognized extension: there mholt/archives must +// gzip-decode our peek buffer and feed the result to tar.NewReader.Next(), +// which needs a full 512-byte tar header out the back. 4 KiB of compressed +// input gives gzip a comfortable margin and is still a single filesystem +// block read. +const formatProbeBytes = 4096 + +// detectFormat returns the first enabled extractor that can handle loc by +// peeking a small buffer of its contents (no full materialization). Returns +// nil when no extractor matches or when the file can't be opened. The caller +// is responsible for narrowing extractors to those permitted by config so we +// don't pay peek I/O for archive types that would only be rejected later. +func (o *Orchestrator) detectFormat(ctx context.Context, loc file.Location, extractors []Extractor) Extractor { + reader, err := o.composite.FileContentsByLocation(loc) + if err != nil { + return nil + } + defer internal.CloseAndLogError(reader, loc.RealPath) + + var peek [formatProbeBytes]byte + n, _ := io.ReadFull(reader, peek[:]) + if n == 0 { + return nil + } + for _, ext := range extractors { + if ext.CanExtract(ctx, loc.RealPath, bytes.NewReader(peek[:n])) { + return ext + } + } + return nil +} + +// copyArchiveToTempFile reads the archive contents from the resolver and writes them +// to a freshly-created temp file under o.tmpDir. The returned path must be removed +// by the caller (typically via defer). +func (o *Orchestrator) copyArchiveToTempFile(loc file.Location) (string, error) { + reader, err := o.composite.FileContentsByLocation(loc) + if err != nil { + return "", fmt.Errorf("unable to get archive contents: %w", err) + } + defer internal.CloseAndLogError(reader, loc.RealPath) + + tmpFile, err := os.CreateTemp(o.tmpDir, "archive-src-*") + if err != nil { + return "", fmt.Errorf("unable to create temp file: %w", err) + } + _, copyErr := safeIOCopy(tmpFile, reader) + tmpFile.Close() + if copyErr != nil { + os.Remove(tmpFile.Name()) + return "", fmt.Errorf("unable to copy archive to temp: %w", copyErr) + } + return tmpFile.Name(), nil +} + +// Relationships returns the archive containment relationships discovered during extraction. +func (o *Orchestrator) Relationships() []artifact.Relationship { + o.mu.Lock() + defer o.mu.Unlock() + + var rels []artifact.Relationship + for _, node := range o.archiveNodes { + // Create a "contains" relationship: the archive contains its extracted content + // The archive file (identified by its coordinates) contains the child filesystem + rels = append(rels, artifact.Relationship{ + From: node.location.Coordinates, + To: file.Coordinates{RealPath: "/", FileSystemID: node.fsID}, + Type: artifact.ContainsRelationship, + }) + } + return rels +} + +// Cleanup removes all temporary directories created during extraction. +func (o *Orchestrator) Cleanup() { + o.mu.Lock() + defer o.mu.Unlock() + for _, cleanup := range o.cleanups { + cleanup() + } + o.cleanups = nil +} + +// safeIOCopy copies with a size limit to prevent decompression bombs. +func safeIOCopy(dst *os.File, src interface{ Read([]byte) (int, error) }) (int64, error) { + const maxCopySize = 2 * 1024 * 1024 * 1024 // 2GB + return limitedCopy(dst, src, maxCopySize) +} + +func limitedCopy(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) { + n, err := copyN(dst, src, limit) + if n >= limit { + return n, fmt.Errorf("copy size limit reached (%d bytes)", limit) + } + return n, err +} + +func copyN(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) { + buf := make([]byte, 32*1024) + var written int64 + for written < limit { + nr, readErr := src.Read(buf) + if nr > 0 { + nw, writeErr := dst.Write(buf[:nr]) + written += int64(nw) + if writeErr != nil { + return written, writeErr + } + } + if readErr != nil { + if readErr == io.EOF { + return written, nil + } + return written, readErr + } + } + return written, nil +} diff --git a/internal/archive/orchestrator_test.go b/internal/archive/orchestrator_test.go new file mode 100644 index 000000000..72b060fbf --- /dev/null +++ b/internal/archive/orchestrator_test.go @@ -0,0 +1,315 @@ +package archive + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/file" +) + +// testResolverFactory creates a simple mock resolver factory for testing. +func testResolverFactory(root string) (file.Resolver, error) { + // Walk the directory and create a mock resolver from its contents + files := make(map[string]string) + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + relPath, _ := filepath.Rel(root, path) + content, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + files["/"+relPath] = string(content) + return nil + }) + if err != nil { + return nil, err + } + return newMockResolver(files), nil +} + +func TestOrchestrator_DiscoverAndExtract_ZipArchive(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create a zip file in the "source" directory + zipFiles := map[string]string{ + "inner.txt": "hello from inside the zip", + "lib/a.so": "library content", + } + zipPath := createTestZip(t, dir, zipFiles) + + // Create a parent resolver that sees the zip file + parentFiles := map[string]string{ + "/test.zip": "placeholder", // the resolver sees the file + } + parent := newMockResolver(parentFiles) + + // Override FileContentsByLocation to return actual zip content + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + parent.contents["/test.zip"] = string(zipContent) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, newCount) + + // the composite resolver should now have the child files + resolver := orch.Resolver() + assert.Equal(t, 1, resolver.ChildCount()) +} + +func TestOrchestrator_DiscoverAndExtract_TarGzArchive(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + tarFiles := map[string]string{ + "inner.txt": "hello from tar", + } + tarPath := createTestTarGz(t, dir, tarFiles) + + tarContent, err := os.ReadFile(tarPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.tar.gz": string(tarContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeUnindexedArchives = true // tar needs this enabled + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, newCount) + assert.Equal(t, 1, orch.Resolver().ChildCount()) +} + +func TestOrchestrator_DiscoverAndExtract_DepthLimit(t *testing.T) { + tmpDir := t.TempDir() + ctx := context.Background() + + parent := newMockResolver(map[string]string{ + "/file.txt": "just a file", + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // at depth 1 (== MaxDepth), should extract nothing + newCount := orch.DiscoverAndExtract(ctx, 1) + assert.Equal(t, 0, newCount) +} + +func TestOrchestrator_DiscoverAndExtract_NoArchives(t *testing.T) { + tmpDir := t.TempDir() + ctx := context.Background() + + parent := newMockResolver(map[string]string{ + "/file.txt": "plain text", + "/data.csv": "a,b,c", + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 3 + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) + assert.Equal(t, 0, orch.Resolver().ChildCount()) +} + +func TestOrchestrator_DiscoverAndExtract_ExcludeExtension(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.ExcludeExtensions = []string{".zip"} + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) +} + +func TestOrchestrator_DiscoverAndExtract_TotalSizeLimit(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.MaxTotalExtractionBytes = 1 // 1 byte total limit - will be exceeded + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // should still extract but hit the limit + _ = orch.DiscoverAndExtract(ctx, 0) + // second call should be blocked by total limit + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) +} + +func TestOrchestrator_Relationships(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + orch.DiscoverAndExtract(ctx, 0) + + rels := orch.Relationships() + assert.NotEmpty(t, rels) + + // should have a contains relationship for the archive + for _, rel := range rels { + assert.Equal(t, "contains", string(rel.Type)) + } +} + +func TestOrchestrator_Cleanup(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + + orch.DiscoverAndExtract(ctx, 0) + + // verify temp dirs exist + entries, err := os.ReadDir(tmpDir) + require.NoError(t, err) + initialCount := len(entries) + assert.True(t, initialCount > 0, "expected temp dirs to be created") + + orch.Cleanup() + + // verify temp dirs are cleaned up + entries, err = os.ReadDir(tmpDir) + require.NoError(t, err) + assert.True(t, len(entries) < initialCount, "expected temp dirs to be cleaned up") +} + +func TestOrchestrator_DoesNotReprocess(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 2 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // first call extracts the archive + first := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, first) + + // second call should not re-extract the same archive + second := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, second) + assert.Equal(t, 1, orch.Resolver().ChildCount()) +} + +func TestOrchestrator_ZipDisabledByConfig(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = false // disable zip extraction + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) +} From 5d6ad0b9e869fb6a3ebae09afe77cb4fca187ec4 Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Mon, 27 Apr 2026 08:15:19 +0200 Subject: [PATCH 06/10] test(archive): add edge case tests for Orchestrator Add tests for important orchestrator edge cases: - Nested archives (zip-inside-zip): verifies multi-round extraction discovers archives within previously-extracted archives - Context cancellation: graceful handling without hanging - Multiple archives at same level: both extracted in one round - Corrupt archive: skipped without error, doesn't block other work - Empty archive: not counted as extracted - Per-archive file limit: partial extraction handled correctly - Relationship structure: contains relationships emit the archive's coordinates as the source side Signed-off-by: Thomas Bechtold --- internal/archive/orchestrator_test.go | 222 ++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) diff --git a/internal/archive/orchestrator_test.go b/internal/archive/orchestrator_test.go index 72b060fbf..bae414c5e 100644 --- a/internal/archive/orchestrator_test.go +++ b/internal/archive/orchestrator_test.go @@ -290,6 +290,228 @@ func TestOrchestrator_DoesNotReprocess(t *testing.T) { assert.Equal(t, 1, orch.Resolver().ChildCount()) } +func TestOrchestrator_NestedArchives_ZipInsideZip(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create inner zip + innerZipPath := createTestZip(t, dir, map[string]string{ + "deep.txt": "deeply nested content", + }) + innerZipContent, err := os.ReadFile(innerZipPath) + require.NoError(t, err) + + // Create outer zip containing the inner zip + outerDir := filepath.Join(dir, "outer") + require.NoError(t, os.MkdirAll(outerDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(outerDir, "inner.zip"), innerZipContent, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(outerDir, "readme.txt"), []byte("readme"), 0o644)) + outerZipPath := createTestZip(t, outerDir, map[string]string{ + "inner.zip": string(innerZipContent), + "readme.txt": "readme", + }) + outerZipContent, err := os.ReadFile(outerZipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/outer.zip": string(outerZipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 3 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Round 1: extract outer.zip + count1 := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, count1, "should extract outer.zip") + assert.Equal(t, 1, orch.Resolver().ChildCount()) + + // Round 2: extract inner.zip found inside outer.zip + count2 := orch.DiscoverAndExtract(ctx, 1) + assert.Equal(t, 1, count2, "should extract inner.zip from within outer.zip") + assert.Equal(t, 2, orch.Resolver().ChildCount()) + + // Round 3: no more archives to extract + count3 := orch.DiscoverAndExtract(ctx, 2) + assert.Equal(t, 0, count3, "no more archives to extract") +} + +func TestOrchestrator_ContextCancellation(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + // should handle gracefully without hanging + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.True(t, newCount >= 0) // may be 0 or 1 depending on timing +} + +func TestOrchestrator_MultipleArchivesSameLevel(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zip1Path := createTestZip(t, dir, map[string]string{"file1.txt": "from zip1"}) + zip1Content, err := os.ReadFile(zip1Path) + require.NoError(t, err) + + // Need a different dir to avoid overwriting test.zip + dir2 := t.TempDir() + zip2Path := createTestZip(t, dir2, map[string]string{"file2.txt": "from zip2"}) + zip2Content, err := os.ReadFile(zip2Path) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/archive1.zip": string(zip1Content), + "/archive2.zip": string(zip2Content), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 2, newCount, "should extract both archives") + assert.Equal(t, 2, orch.Resolver().ChildCount()) +} + +func TestOrchestrator_CorruptArchiveSkipped(t *testing.T) { + tmpDir := t.TempDir() + ctx := context.Background() + + // A file with .zip extension but invalid content + parent := newMockResolver(map[string]string{ + "/corrupt.zip": "this is not a valid zip file", + "/normal.txt": "just text", + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Should not panic or error out; corrupt archives should be skipped + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount, "corrupt archive should be skipped") +} + +func TestOrchestrator_EmptyArchive(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create an empty zip + emptyZipPath := createTestZip(t, dir, map[string]string{}) + emptyZipContent, err := os.ReadFile(emptyZipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/empty.zip": string(emptyZipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount, "empty archive should not count as extracted") +} + +func TestOrchestrator_PerArchiveFileLimitApplied(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create a zip with many files + files := map[string]string{} + for i := 0; i < 10; i++ { + files[filepath.Join("dir", string(rune('a'+i))+".txt")] = "content" + } + zipPath := createTestZip(t, dir, files) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/big.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.MaxFileCount = 3 // only allow 3 files to be extracted + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Should partially extract but still register the archive + newCount := orch.DiscoverAndExtract(ctx, 0) + // Extraction will fail after 3 files, but partial extraction counts + assert.True(t, newCount >= 0) +} + +func TestOrchestrator_RelationshipsHaveArchiveCoordinates(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"file.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/archive.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + orch.DiscoverAndExtract(ctx, 0) + + rels := orch.Relationships() + require.Len(t, rels, 1) + + // verify the relationship structure + rel := rels[0] + assert.Equal(t, "contains", string(rel.Type)) + + // From should be the archive's coordinates + fromCoords, ok := rel.From.(file.Coordinates) + require.True(t, ok) + assert.Equal(t, "/archive.zip", fromCoords.RealPath) +} + func TestOrchestrator_ZipDisabledByConfig(t *testing.T) { dir := t.TempDir() tmpDir := t.TempDir() From 3fc5194f77fb065ce9331441376680804b86859f Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Wed, 25 Mar 2026 20:29:11 +0100 Subject: [PATCH 07/10] feat(sbom): wire archive orchestrator into CreateSBOM pipeline Integrate the archive extraction orchestrator into the SBOM creation pipeline, enabling recursive archive cataloging when configured: - Add Archive field to CreateSBOMConfig with WithArchiveConfig builder - Add Exclusions field to CreateSBOMConfig with WithExclusions builder so user --exclude patterns also filter content inside extracted archives, not only the source filesystem - Add setupArchiveOrchestration() that creates the orchestrator when MaxDepth > 0, runs the iterative extract loop, and returns the composite resolver with archive contents visible to all catalogers - Include archive config in configuration audit trail - Add CLI archive config section (max-depth, size/file limits, exclude-extensions) wired through Catalog.ToArchiveConfig() - CLI Catalog.ToSBOMConfig threads cfg.Exclusions through WithExclusions so child resolvers respect them When MaxDepth is 0 (default), behavior is unchanged. When > 0, archives discovered during scanning are extracted and their contents made visible to all catalogers via the CompositeResolver. Signed-off-by: Thomas Bechtold --- cmd/syft/internal/options/archive.go | 36 +++++++++++++++ cmd/syft/internal/options/catalog.go | 16 +++++++ syft/configuration_audit_trail.go | 1 + syft/create_sbom.go | 69 +++++++++++++++++++++++++++- syft/create_sbom_config.go | 22 +++++++++ 5 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 cmd/syft/internal/options/archive.go diff --git a/cmd/syft/internal/options/archive.go b/cmd/syft/internal/options/archive.go new file mode 100644 index 000000000..d2a71e1b0 --- /dev/null +++ b/cmd/syft/internal/options/archive.go @@ -0,0 +1,36 @@ +package options + +import ( + "github.com/anchore/clio" + "github.com/anchore/syft/syft/cataloging" +) + +type archiveConfig struct { + MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"` + MaxExtractionSizeBytes int64 `yaml:"max-extraction-size-bytes" json:"max-extraction-size-bytes" mapstructure:"max-extraction-size-bytes"` + MaxFileCount int `yaml:"max-file-count" json:"max-file-count" mapstructure:"max-file-count"` + MaxTotalExtractionBytes int64 `yaml:"max-total-extraction-bytes" json:"max-total-extraction-bytes" mapstructure:"max-total-extraction-bytes"` + ExcludeExtensions []string `yaml:"exclude-extensions" json:"exclude-extensions" mapstructure:"exclude-extensions"` +} + +var _ interface { + clio.FieldDescriber +} = (*archiveConfig)(nil) + +func defaultArchiveConfig() archiveConfig { + return archiveConfig{ + MaxDepth: cataloging.DefaultArchiveMaxDepth, + MaxExtractionSizeBytes: cataloging.DefaultArchiveMaxExtractionSizeBytes, + MaxFileCount: cataloging.DefaultArchiveMaxFileCount, + MaxTotalExtractionBytes: cataloging.DefaultArchiveMaxTotalExtractionBytes, + } +} + +func (o *archiveConfig) DescribeFields(descriptions clio.FieldDescriptionSet) { + descriptions.Add(&o.MaxDepth, `maximum depth of recursive archive extraction (0 = disabled, no recursive extraction) +enabling this allows syft to discover packages inside nested archives (e.g., a tar.gz containing Python packages)`) + descriptions.Add(&o.MaxExtractionSizeBytes, `maximum total bytes to extract from a single archive before stopping`) + descriptions.Add(&o.MaxFileCount, `maximum number of files to extract from a single archive before stopping`) + descriptions.Add(&o.MaxTotalExtractionBytes, `maximum total bytes to extract across all archives before stopping`) + descriptions.Add(&o.ExcludeExtensions, `archive file extensions to skip during recursive extraction (e.g., [".rpm", ".deb"])`) +} diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index bcb8ab66f..7483983d2 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -37,6 +37,7 @@ type Catalog struct { Package packageConfig `yaml:"package" json:"package" mapstructure:"package"` License licenseConfig `yaml:"license" json:"license" mapstructure:"license"` File fileConfig `yaml:"file" json:"file" mapstructure:"file"` + Archive archiveConfig `yaml:"archive" json:"archive" mapstructure:"archive"` Scope string `yaml:"scope" json:"scope" mapstructure:"scope"` Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel Relationships relationshipsConfig `yaml:"relationships" json:"relationships" mapstructure:"relationships"` @@ -76,6 +77,7 @@ func DefaultCatalog() Catalog { Scope: source.SquashedScope.String(), Package: defaultPackageConfig(), License: defaultLicenseConfig(), + Archive: defaultArchiveConfig(), LinuxKernel: defaultLinuxKernelConfig(), JavaScript: defaultJavaScriptConfig(), Python: defaultPythonConfig(), @@ -102,6 +104,8 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { WithPackagesConfig(cfg.ToPackagesConfig()). WithLicenseConfig(cfg.ToLicenseConfig()). WithFilesConfig(cfg.ToFilesConfig()). + WithArchiveConfig(cfg.ToArchiveConfig()). + WithExclusions(cfg.Exclusions). WithCatalogerSelection( cataloging.NewSelectionRequest(). WithDefaults(cfg.DefaultCatalogers...). @@ -109,6 +113,18 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { ) } +func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig { + return cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: cfg.Package.SearchIndexedArchives, + IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives, + MaxDepth: cfg.Archive.MaxDepth, + MaxExtractionSizeBytes: cfg.Archive.MaxExtractionSizeBytes, + MaxFileCount: cfg.Archive.MaxFileCount, + MaxTotalExtractionBytes: cfg.Archive.MaxTotalExtractionBytes, + ExcludeExtensions: cfg.Archive.ExcludeExtensions, + } +} + func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig { return cataloging.SearchConfig{ Scope: source.ParseScope(cfg.Scope), diff --git a/syft/configuration_audit_trail.go b/syft/configuration_audit_trail.go index aa7a51f4d..b63eb9be4 100644 --- a/syft/configuration_audit_trail.go +++ b/syft/configuration_audit_trail.go @@ -17,6 +17,7 @@ type configurationAuditTrail struct { Packages pkgcataloging.Config `json:"packages" yaml:"packages" mapstructure:"packages"` Files filecataloging.Config `json:"files" yaml:"files" mapstructure:"files"` Licenses cataloging.LicenseConfig `json:"licenses" yaml:"licenses" mapstructure:"licenses"` + Archive cataloging.ArchiveSearchConfig `json:"archive" yaml:"archive" mapstructure:"archive"` Catalogers catalogerManifest `json:"catalogers" yaml:"catalogers" mapstructure:"catalogers"` ExtraConfigs any `json:"extra,omitempty" yaml:"extra" mapstructure:"extra"` } diff --git a/syft/create_sbom.go b/syft/create_sbom.go index 817e698c6..ff4e1dc4b 100644 --- a/syft/create_sbom.go +++ b/syft/create_sbom.go @@ -10,6 +10,7 @@ import ( "github.com/scylladb/go-set/strset" "github.com/anchore/go-sync" + intArchive "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/log" @@ -19,9 +20,12 @@ import ( "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/event/monitor" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/fileresolver" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/source/directorysource" ) // CreateSBOM creates a software bill-of-materials from the given source. If the CreateSBOMConfig is nil, then @@ -60,6 +64,7 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( Packages: cfg.Packages, Files: cfg.Files, Licenses: cfg.Licenses, + Archive: cfg.Archive, Catalogers: *audit, ExtraConfigs: cfg.ToolConfiguration, }, @@ -92,9 +97,17 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( packageCatalogingProgress := monitorPackageCatalogingTask() builder := sbomsync.NewBuilder(&s, monitorPackageCount(packageCatalogingProgress)) + + // determine the effective resolver: if recursive archive extraction is enabled, + // wrap the resolver with the archive orchestrator + effectiveResolver, archiveCleanup, archiveRelationships := setupArchiveOrchestration(ctx, resolver, cfg) + if archiveCleanup != nil { + defer archiveCleanup() + } + for i := range taskGroups { err = sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(taskGroups[i]), func(t task.Task) (any, error) { - return nil, task.RunTask(ctx, t, resolver, builder, catalogingProgress) + return nil, task.RunTask(ctx, t, effectiveResolver, builder, catalogingProgress) }, nil) if err != nil { // TODO: tie this to the open progress monitors... @@ -102,12 +115,66 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( } } + // add archive containment relationships to the SBOM + if len(archiveRelationships) > 0 { + builder.AddRelationships(archiveRelationships...) + } + packageCatalogingProgress.SetCompleted() catalogingProgress.SetCompleted() return &s, nil } +// setupArchiveOrchestration wraps the resolver with an archive orchestrator when recursive +// archive extraction is enabled (MaxDepth > 0). It performs the iterative extract loop +// and returns the effective resolver, a cleanup function, and any archive relationships. +func setupArchiveOrchestration(ctx context.Context, baseResolver file.Resolver, cfg *CreateSBOMConfig) (file.Resolver, func(), []artifact.Relationship) { + if cfg.Archive.MaxDepth <= 0 { + return baseResolver, nil, nil + } + + td := tmpdir.FromContext(ctx) + if td == nil { + log.Warn("archive orchestration requested but no temp dir available") + return baseResolver, nil, nil + } + + archiveTmpDir, archiveTmpCleanup, err := td.NewChild("archive-extraction") //nolint:gocritic // cleanup is returned to caller, not deferred here + if err != nil { + log.WithFields("error", err).Warn("unable to create temp dir for archive extraction") + return baseResolver, nil, nil + } + + // build user-exclusion path filters once and reuse them for every child + // resolver, so that --exclude patterns also apply inside extracted archives + resolverFactory := func(root string) (file.Resolver, error) { + filters, err := directorysource.GetDirectoryExclusionFunctions(root, append([]string(nil), cfg.Exclusions...)) + if err != nil { + return nil, fmt.Errorf("invalid exclusion patterns for archive child resolver: %w", err) + } + return fileresolver.NewFromDirectory(root, "", filters...) + } + + orch := intArchive.NewOrchestrator(baseResolver, cfg.Archive, archiveTmpDir, resolverFactory) + + // iteratively discover and extract archives up to the configured max depth + for depth := 0; depth < cfg.Archive.MaxDepth; depth++ { + newArchives := orch.DiscoverAndExtract(ctx, depth) + if newArchives == 0 { + break + } + log.WithFields("depth", depth+1, "archives", newArchives).Debug("extracted archives for recursive cataloging") + } + + cleanup := func() { + orch.Cleanup() + archiveTmpCleanup() + } + + return orch.Resolver(), cleanup, orch.Relationships() +} + func setupContext(ctx context.Context, cfg *CreateSBOMConfig) (context.Context, error) { // configure parallel executors ctx = setContextExecutors(ctx, cfg) diff --git a/syft/create_sbom_config.go b/syft/create_sbom_config.go index 1a84c14b6..f08c914de 100644 --- a/syft/create_sbom_config.go +++ b/syft/create_sbom_config.go @@ -28,9 +28,16 @@ type CreateSBOMConfig struct { Packages pkgcataloging.Config Licenses cataloging.LicenseConfig Files filecataloging.Config + Archive cataloging.ArchiveSearchConfig Parallelism int CatalogerSelection cataloging.SelectionRequest + // Exclusions are user-provided glob patterns that the source resolver already + // applies; they are also propagated into child resolvers built during recursive + // archive extraction so that a "--exclude=**/foo" pattern keeps working inside + // extracted archive contents. + Exclusions []string + // audit what tool is being used to generate the SBOM ToolName string ToolVersion string @@ -49,6 +56,7 @@ func DefaultCreateSBOMConfig() *CreateSBOMConfig { Packages: pkgcataloging.DefaultConfig(), Licenses: cataloging.DefaultLicenseConfig(), Files: filecataloging.DefaultConfig(), + Archive: cataloging.DefaultArchiveSearchConfig(), Parallelism: 0, // use default: run in parallel based on number of CPUs packageTaskFactories: task.DefaultPackageTaskFactories(), @@ -153,6 +161,20 @@ func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig { return c } +// WithArchiveConfig allows for defining archive extraction and recursive cataloging parameters. +func (c *CreateSBOMConfig) WithArchiveConfig(cfg cataloging.ArchiveSearchConfig) *CreateSBOMConfig { + c.Archive = cfg + return c +} + +// WithExclusions sets the glob patterns that should be propagated to child resolvers +// constructed during recursive archive extraction. Patterns must follow the same +// "./", "*/", or "**/"-prefix convention as the syft --exclude flag. +func (c *CreateSBOMConfig) WithExclusions(exclusions []string) *CreateSBOMConfig { + c.Exclusions = exclusions + return c +} + // WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag. func (c *CreateSBOMConfig) WithCatalogerSelection(selection cataloging.SelectionRequest) *CreateSBOMConfig { c.CatalogerSelection = selection From bb7dbf2ff83bfd91c6e82ccc78597c92e62957e2 Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Wed, 25 Mar 2026 20:33:28 +0100 Subject: [PATCH 08/10] test(archive): add integration tests for recursive archive cataloging Add integration tests covering the full archive extraction pipeline: - tar.gz containing Python package metadata (single-level) - zip containing a JAR with MANIFEST.MF (two-level, same format) - Mixed archive types (zip + tar.gz at same level) - Depth limit enforcement (3-level zip nesting, limited to 1) - Extension exclusion filtering - 3-level mixed format nesting: tar.gz -> zip -> tar.gz -> files verifying cross-format extraction and full access path chains - Reverse format crossing: tar.gz -> zip -> tar.gz verifying content readability at all levels and correct relationship count Signed-off-by: Thomas Bechtold --- internal/archive/integration_test.go | 524 +++++++++++++++++++++++++++ 1 file changed, 524 insertions(+) create mode 100644 internal/archive/integration_test.go diff --git a/internal/archive/integration_test.go b/internal/archive/integration_test.go new file mode 100644 index 000000000..3566ac518 --- /dev/null +++ b/internal/archive/integration_test.go @@ -0,0 +1,524 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cataloging" +) + +// TestIntegration_TarGzContainingPythonPackage tests the scenario where a tar.gz +// archive contains Python package metadata files that would normally be found by +// the Python cataloger. This verifies the orchestrator makes those files visible. +func TestIntegration_TarGzContainingPythonPackage(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create a tar.gz containing Python dist-info metadata + tarPath := filepath.Join(dir, "python-packages.tar.gz") + createTarGzWithFiles(t, tarPath, map[string]string{ + "lib/python3.9/site-packages/requests-2.28.0.dist-info/METADATA": `Metadata-Version: 2.1 +Name: requests +Version: 2.28.0 +Summary: Python HTTP for Humans. +`, + "lib/python3.9/site-packages/requests-2.28.0.dist-info/RECORD": "requests/__init__.py,sha256=abc,1234", + "lib/python3.9/site-packages/requests/__init__.py": "# requests library", + }) + tarContent, err := os.ReadFile(tarPath) + require.NoError(t, err) + + // Base resolver sees the tar.gz as a single opaque file + parent := newMockResolver(map[string]string{ + "/app/python-packages.tar.gz": string(tarContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeUnindexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Extract the archive + count := orch.DiscoverAndExtract(ctx, 0) + require.Equal(t, 1, count, "should extract the tar.gz") + + // The Python METADATA file should now be visible through the composite resolver + resolver := orch.Resolver() + locs, err := resolver.FilesByPath("/lib/python3.9/site-packages/requests-2.28.0.dist-info/METADATA") + require.NoError(t, err) + require.Len(t, locs, 1, "METADATA file should be visible through the composite resolver") + + // Verify the access path shows it came from the archive + assert.Contains(t, locs[0].AccessPath, "python-packages.tar.gz:") + assert.Contains(t, locs[0].AccessPath, "METADATA") + + // Verify we can read the content + reader, err := resolver.FileContentsByLocation(locs[0]) + require.NoError(t, err) + defer reader.Close() + + buf := make([]byte, 1024) + n, _ := reader.Read(buf) + assert.Contains(t, string(buf[:n]), "Name: requests") +} + +// TestIntegration_ZipContainingJar tests a zip archive containing a JAR file, +// which itself should be extractable in a second round. +func TestIntegration_ZipContainingJar(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create an inner JAR (which is just a zip) with a MANIFEST.MF + innerJarPath := filepath.Join(dir, "inner.jar") + createZipWithFiles(t, innerJarPath, map[string]string{ + "META-INF/MANIFEST.MF": `Manifest-Version: 1.0 +Implementation-Title: example-lib +Implementation-Version: 1.2.3 +`, + }) + innerJarContent, err := os.ReadFile(innerJarPath) + require.NoError(t, err) + + // Create an outer zip containing the JAR and other files + outerZipPath := filepath.Join(dir, "app-bundle.zip") + createZipWithFiles(t, outerZipPath, map[string]string{ + "lib/example-lib-1.2.3.jar": string(innerJarContent), + "README.md": "# Application Bundle", + }) + outerZipContent, err := os.ReadFile(outerZipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/downloads/app-bundle.zip": string(outerZipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 2 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Round 1: extract outer zip + count1 := orch.DiscoverAndExtract(ctx, 0) + require.Equal(t, 1, count1, "should extract outer zip") + + // Round 2: extract inner JAR found inside the zip + count2 := orch.DiscoverAndExtract(ctx, 1) + require.Equal(t, 1, count2, "should extract inner JAR") + + resolver := orch.Resolver() + + // The MANIFEST.MF from inside the JAR should be visible + locs, err := resolver.FilesByPath("/META-INF/MANIFEST.MF") + require.NoError(t, err) + require.Len(t, locs, 1, "MANIFEST.MF should be visible from nested JAR") + + // The access path should show the full nesting chain + accessPath := locs[0].AccessPath + assert.Contains(t, accessPath, "app-bundle.zip:") + assert.Contains(t, accessPath, "example-lib-1.2.3.jar:") + assert.Contains(t, accessPath, "MANIFEST.MF") + + // Verify relationships + rels := orch.Relationships() + assert.Len(t, rels, 2, "should have relationships for both extracted archives") +} + +// TestIntegration_MixedArchiveTypes tests a directory containing both +// zip and tar.gz archives that should be extracted in a single round. +func TestIntegration_MixedArchiveTypes(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create a zip + zipPath := createTestZip(t, dir, map[string]string{ + "from-zip.txt": "zip content", + }) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + // Create a tar.gz + tarPath := createTestTarGz(t, dir, map[string]string{ + "from-tar.txt": "tar content", + }) + tarContent, err := os.ReadFile(tarPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/data/archive.zip": string(zipContent), + "/data/archive.tar.gz": string(tarContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.IncludeUnindexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + count := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 2, count, "should extract both archives") + + resolver := orch.Resolver() + + // Both files should be visible + zipLocs, err := resolver.FilesByPath("/from-zip.txt") + require.NoError(t, err) + assert.Len(t, zipLocs, 1, "file from zip should be visible") + + tarLocs, err := resolver.FilesByPath("/from-tar.txt") + require.NoError(t, err) + assert.Len(t, tarLocs, 1, "file from tar should be visible") +} + +// TestIntegration_DepthLimitPreventsDeepExtraction verifies that the +// depth limit correctly stops extraction at the configured level. +func TestIntegration_DepthLimitPreventsDeepExtraction(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create a deeply nested archive: zip -> zip -> zip -> file + innermostDir := filepath.Join(dir, "inner") + require.NoError(t, os.MkdirAll(innermostDir, 0o755)) + innermost := createTestZip(t, innermostDir, map[string]string{ + "secret.txt": "you found me!", + }) + innermostContent, err := os.ReadFile(innermost) + require.NoError(t, err) + + middleDir := filepath.Join(dir, "middle") + require.NoError(t, os.MkdirAll(middleDir, 0o755)) + middle := createTestZip(t, middleDir, map[string]string{ + "innermost.zip": string(innermostContent), + }) + middleContent, err := os.ReadFile(middle) + require.NoError(t, err) + + outerDir := filepath.Join(dir, "outer") + require.NoError(t, os.MkdirAll(outerDir, 0o755)) + outerZip := createTestZip(t, outerDir, map[string]string{ + "middle.zip": string(middleContent), + }) + outerContent, err := os.ReadFile(outerZip) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/nested.zip": string(outerContent), + }) + + // Only allow depth 1 — should only extract the outer zip + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + count := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, count, "should only extract the outer zip") + + // middle.zip should be visible but secret.txt should NOT be + resolver := orch.Resolver() + + middleLocs, err := resolver.FilesByPath("/middle.zip") + require.NoError(t, err) + assert.Len(t, middleLocs, 1, "middle.zip should be visible at depth 1") + + secretLocs, err := resolver.FilesByPath("/secret.txt") + require.NoError(t, err) + assert.Len(t, secretLocs, 0, "secret.txt should NOT be visible (depth limit)") +} + +// TestIntegration_ExcludeExtensionPreventsExtraction verifies that +// excluded extensions are respected. +func TestIntegration_ExcludeExtensionPreventsExtraction(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{ + "data.txt": "from zip", + }) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/archive.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.ExcludeExtensions = []string{".zip"} // exclude zip files + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + count := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, count, "zip should be excluded by extension") +} + +// TestIntegration_MultiLevelMixedFormats tests a 3-level nesting chain with +// different archive formats: tar.gz -> zip -> tar.gz -> files. +// This is the most realistic cross-format recursive extraction scenario. +func TestIntegration_MultiLevelMixedFormats(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Level 3 (deepest): a tar.gz containing application files + level3Dir := filepath.Join(dir, "level3") + require.NoError(t, os.MkdirAll(level3Dir, 0o755)) + level3Path := filepath.Join(level3Dir, "app-data.tar.gz") + createTarGzWithFiles(t, level3Path, map[string]string{ + "config.yaml": "app: myapp\nversion: 1.0", + "bin/run.sh": "#!/bin/sh\necho hello", + }) + level3Content, err := os.ReadFile(level3Path) + require.NoError(t, err) + + // Level 2: a zip containing the level3 tar.gz and some metadata + level2Dir := filepath.Join(dir, "level2") + require.NoError(t, os.MkdirAll(level2Dir, 0o755)) + level2Path := filepath.Join(level2Dir, "package.zip") + createZipWithFiles(t, level2Path, map[string]string{ + "app-data.tar.gz": string(level3Content), + "package-info.json": `{"name": "mypackage", "version": "2.0"}`, + }) + level2Content, err := os.ReadFile(level2Path) + require.NoError(t, err) + + // Level 1 (outermost): a tar.gz containing the level2 zip + level1Dir := filepath.Join(dir, "level1") + require.NoError(t, os.MkdirAll(level1Dir, 0o755)) + level1Path := filepath.Join(level1Dir, "distribution.tar.gz") + createTarGzWithFiles(t, level1Path, map[string]string{ + "package.zip": string(level2Content), + "LICENSE": "MIT License", + }) + level1Content, err := os.ReadFile(level1Path) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/distribution.tar.gz": string(level1Content), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 3 + cfg.IncludeIndexedArchives = true + cfg.IncludeUnindexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Round 1: extract distribution.tar.gz + count1 := orch.DiscoverAndExtract(ctx, 0) + require.Equal(t, 1, count1, "round 1: should extract distribution.tar.gz") + + // Round 2: extract package.zip from inside distribution.tar.gz + count2 := orch.DiscoverAndExtract(ctx, 1) + require.Equal(t, 1, count2, "round 2: should extract package.zip") + + // Round 3: extract app-data.tar.gz from inside package.zip + count3 := orch.DiscoverAndExtract(ctx, 2) + require.Equal(t, 1, count3, "round 3: should extract app-data.tar.gz") + + resolver := orch.Resolver() + + // Verify files from all levels are accessible: + + // Level 1: LICENSE from distribution.tar.gz + locs, err := resolver.FilesByPath("/LICENSE") + require.NoError(t, err) + require.Len(t, locs, 1, "LICENSE from level 1 should be visible") + assert.Contains(t, locs[0].AccessPath, "distribution.tar.gz:") + + // Level 2: package-info.json from package.zip + locs, err = resolver.FilesByPath("/package-info.json") + require.NoError(t, err) + require.Len(t, locs, 1, "package-info.json from level 2 should be visible") + assert.Contains(t, locs[0].AccessPath, "package.zip:") + + // Level 3: config.yaml from app-data.tar.gz + locs, err = resolver.FilesByPath("/config.yaml") + require.NoError(t, err) + require.Len(t, locs, 1, "config.yaml from level 3 should be visible") + assert.Contains(t, locs[0].AccessPath, "app-data.tar.gz:") + + // Level 3: bin/run.sh from app-data.tar.gz + locs, err = resolver.FilesByPath("/bin/run.sh") + require.NoError(t, err) + require.Len(t, locs, 1, "bin/run.sh from level 3 should be visible") + + // Verify we can read deeply nested content + reader, err := resolver.FileContentsByLocation(locs[0]) + require.NoError(t, err) + defer reader.Close() + buf := make([]byte, 1024) + n, _ := reader.Read(buf) + assert.Contains(t, string(buf[:n]), "echo hello") + + // Verify relationships cover all 3 levels + rels := orch.Relationships() + assert.Len(t, rels, 3, "should have 3 relationships (one per extracted archive)") +} + +// TestIntegration_TarInsideZipInsideTar tests tar -> zip -> tar nesting +// (the reverse of the above test), ensuring bidirectional format crossing. +func TestIntegration_TarInsideZipInsideTar(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Innermost: a tar.gz with a single file + innerTarDir := filepath.Join(dir, "innertar") + require.NoError(t, os.MkdirAll(innerTarDir, 0o755)) + innerTarPath := filepath.Join(innerTarDir, "inner.tar.gz") + createTarGzWithFiles(t, innerTarPath, map[string]string{ + "deeply-nested.txt": "found at the bottom", + }) + innerTarContent, err := os.ReadFile(innerTarPath) + require.NoError(t, err) + + // Middle: a zip containing the inner tar + middleZipDir := filepath.Join(dir, "middlezip") + require.NoError(t, os.MkdirAll(middleZipDir, 0o755)) + middleZipPath := filepath.Join(middleZipDir, "middle.zip") + createZipWithFiles(t, middleZipPath, map[string]string{ + "inner.tar.gz": string(innerTarContent), + "middle.txt": "middle level", + }) + middleZipContent, err := os.ReadFile(middleZipPath) + require.NoError(t, err) + + // Outermost: a tar.gz containing the middle zip + outerTarDir := filepath.Join(dir, "outertar") + require.NoError(t, os.MkdirAll(outerTarDir, 0o755)) + outerTarPath := filepath.Join(outerTarDir, "outer.tar.gz") + createTarGzWithFiles(t, outerTarPath, map[string]string{ + "middle.zip": string(middleZipContent), + "outer.txt": "outer level", + }) + outerTarContent, err := os.ReadFile(outerTarPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/outer.tar.gz": string(outerTarContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 3 + cfg.IncludeIndexedArchives = true + cfg.IncludeUnindexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Extract all 3 levels + count1 := orch.DiscoverAndExtract(ctx, 0) + require.Equal(t, 1, count1) + count2 := orch.DiscoverAndExtract(ctx, 1) + require.Equal(t, 1, count2) + count3 := orch.DiscoverAndExtract(ctx, 2) + require.Equal(t, 1, count3) + + resolver := orch.Resolver() + + // Check all levels are accessible + outerLocs, err := resolver.FilesByPath("/outer.txt") + require.NoError(t, err) + require.Len(t, outerLocs, 1) + + middleLocs, err := resolver.FilesByPath("/middle.txt") + require.NoError(t, err) + require.Len(t, middleLocs, 1) + + deepLocs, err := resolver.FilesByPath("/deeply-nested.txt") + require.NoError(t, err) + require.Len(t, deepLocs, 1) + + // Read the deeply nested content + reader, err := resolver.FileContentsByLocation(deepLocs[0]) + require.NoError(t, err) + defer reader.Close() + buf := make([]byte, 1024) + n, _ := reader.Read(buf) + assert.Equal(t, "found at the bottom", string(buf[:n])) + + // The access path should show the full chain: tar -> zip -> tar -> file + accessPath := deepLocs[0].AccessPath + assert.Contains(t, accessPath, "outer.tar.gz:") + assert.Contains(t, accessPath, "middle.zip:") + assert.Contains(t, accessPath, "inner.tar.gz:") + assert.Contains(t, accessPath, "deeply-nested.txt") +} + +// createZipWithFiles creates a zip at the given path with the given files. +func createZipWithFiles(t *testing.T, zipPath string, files map[string]string) { + t.Helper() + f, err := os.Create(zipPath) + require.NoError(t, err) + defer f.Close() + + w := zip.NewWriter(f) + for name, content := range files { + fw, err := w.Create(name) + require.NoError(t, err) + _, err = fw.Write([]byte(content)) + require.NoError(t, err) + } + require.NoError(t, w.Close()) +} + +// createTarGzWithFiles creates a tar.gz at the given path with the given files. +func createTarGzWithFiles(t *testing.T, tarPath string, files map[string]string) { + t.Helper() + f, err := os.Create(tarPath) + require.NoError(t, err) + defer f.Close() + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + // sort keys for deterministic output + for name, content := range files { + // create parent dirs as needed + dir := filepath.Dir(name) + if dir != "." { + hdr := &tar.Header{ + Typeflag: tar.TypeDir, + Name: dir + "/", + Mode: 0o755, + } + require.NoError(t, tw.WriteHeader(hdr)) + } + + hdr := &tar.Header{ + Name: name, + Mode: 0o644, + Size: int64(len(content)), + } + require.NoError(t, tw.WriteHeader(hdr)) + _, err = tw.Write([]byte(content)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) +} + From 41a1998d45351bea238ea611e79e0f333a985d4e Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Fri, 10 Apr 2026 14:49:31 +0200 Subject: [PATCH 09/10] fix(java): skip META-INF/maven pom.xml in pom cataloger The java-pom-cataloger was processing pom.xml files found inside META-INF/maven/ directories, which are archive metadata embedded by the Maven build process. These are verbatim copies of the original build POM containing unresolved ${property} references and dependency versions inherited from parent POMs that are unavailable inside the archive. This caused phantom packages with UNKNOWN versions to appear when recursive archive cataloging opened JARs and exposed their internal pom.xml files to the pom cataloger. The java-archive-cataloger already uses pom.properties (which always contains resolved values) to identify these archives, so the pom cataloger should skip META-INF/maven/ pom.xml files entirely. Signed-off-by: Thomas Bechtold --- syft/pkg/cataloger/java/parse_pom_xml.go | 18 +++ syft/pkg/cataloger/java/parse_pom_xml_test.go | 117 ++++++++++++++++++ .../META-INF/maven/com.example/my-lib/pom.xml | 24 ++++ .../maven/com.example/embedded-lib/pom.xml | 18 +++ .../pom/mixed-meta-inf-and-project/pom.xml | 18 +++ 5 files changed, 195 insertions(+) create mode 100644 syft/pkg/cataloger/java/testdata/pom/meta-inf-archive/META-INF/maven/com.example/my-lib/pom.xml create mode 100644 syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/META-INF/maven/com.example/embedded-lib/pom.xml create mode 100644 syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/pom.xml diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 572dfed8e..357b426a6 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -3,6 +3,7 @@ package java import ( "context" "errors" + "path/filepath" "strings" "github.com/anchore/syft/internal" @@ -40,6 +41,9 @@ func (p pomXMLCataloger) Catalog(ctx context.Context, fileResolver file.Resolver var poms []*maven.Project pomLocations := map[*maven.Project]file.Location{} for _, pomLocation := range locations { + if isArchiveMetaPom(pomLocation) { + continue + } pom, err := readPomFromLocation(fileResolver, pomLocation) if err != nil || pom == nil { log.WithFields("error", err, "pomLocation", pomLocation).Debug("error while reading pom") @@ -299,6 +303,20 @@ func pomParent(ctx context.Context, r *maven.Resolver, pom *maven.Project) *pkg. } } +// isArchiveMetaPom returns true if the pom.xml location is inside a META-INF/maven directory, +// indicating it is archive metadata embedded by the Maven build process rather than a project +// pom.xml. These embedded pom.xml files are verbatim copies of the original build POM with +// potentially unresolved ${property} references and dependency versions inherited from parent +// POMs that are not available inside the archive. The java-archive-cataloger already uses +// pom.properties (which always contains resolved values) to identify these archives, so the +// pom cataloger should skip them to avoid creating phantom dependency packages. +func isArchiveMetaPom(location file.Location) bool { + p := location.Path() + // normalize to forward slashes for consistent matching + p = filepath.ToSlash(p) + return strings.Contains(p, "META-INF/maven/") +} + func cleanDescription(original string) (cleaned string) { descriptionLines := strings.Split(original, "\n") for _, line := range descriptionLines { diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 44dc5da05..94fde2542 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -690,6 +690,123 @@ func getCommonsTextExpectedPackages(resolved bool) expected { return expected{pkgs, relationships} } +func Test_isArchiveMetaPom(t *testing.T) { + tests := []struct { + name string + path string + expected bool + }{ + { + name: "META-INF maven pom", + path: "META-INF/maven/com.example/my-lib/pom.xml", + expected: true, + }, + { + name: "nested META-INF maven pom", + path: "some/path/META-INF/maven/org.apache/commons/pom.xml", + expected: true, + }, + { + name: "project pom.xml", + path: "pom.xml", + expected: false, + }, + { + name: "module pom.xml", + path: "submodule/pom.xml", + expected: false, + }, + { + name: "META-INF but not maven", + path: "META-INF/pom.xml", + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loc := file.NewLocation(test.path) + assert.Equal(t, test.expected, isArchiveMetaPom(loc)) + }) + } +} + +func Test_pomCatalogerSkipsMetaInfPoms(t *testing.T) { + // A directory that only contains a pom.xml under META-INF/maven/ should produce + // no packages, since these are archive metadata not project files. + cat := NewPomCataloger(ArchiveCatalogerConfig{ + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: true, + }, + }) + + pkgtest.NewCatalogTester(). + FromDirectory(t, "testdata/pom/meta-inf-archive"). + Expects(nil, nil). + TestCataloger(t, cat) +} + +func Test_pomCatalogerSkipsMetaInfButKeepsProjectPom(t *testing.T) { + // When a directory contains both a project pom.xml and a META-INF/maven pom.xml, + // only the project pom.xml should be cataloged. The META-INF pom.xml and its + // dependencies should be ignored. + pomLocation := file.NewLocationSet(file.NewLocation("pom.xml")) + + myApp := pkg.Package{ + Name: "my-app", + Version: "2.0.0", + PURL: "pkg:maven/org.anchore/my-app@2.0.0", + Language: pkg.Java, + Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, + Locations: pomLocation, + Metadata: pkg.JavaArchive{ + PomProject: &pkg.JavaPomProject{ + GroupID: "org.anchore", + ArtifactID: "my-app", + Version: "2.0.0", + }, + }, + } + finalizePackage(&myApp) + + guava := pkg.Package{ + Name: "guava", + Version: "31.1-jre", + PURL: "pkg:maven/com.google.guava/guava@31.1-jre", + Language: pkg.Java, + Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, + Locations: pomLocation, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "com.google.guava", + ArtifactID: "guava", + }, + }, + } + finalizePackage(&guava) + + expectedPkgs := []pkg.Package{myApp, guava} + expectedRelationships := []artifact.Relationship{ + { + From: guava, + To: myApp, + Type: artifact.DependencyOfRelationship, + }, + } + + cat := NewPomCataloger(ArchiveCatalogerConfig{ + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: true, + }, + }) + + pkgtest.TestCataloger(t, "testdata/pom/mixed-meta-inf-and-project", cat, expectedPkgs, expectedRelationships) +} + func expectedTransientPackageData() expected { epl2 := pkg.NewLicenseSet(pkg.License{ Value: "Eclipse Public License v2.0", diff --git a/syft/pkg/cataloger/java/testdata/pom/meta-inf-archive/META-INF/maven/com.example/my-lib/pom.xml b/syft/pkg/cataloger/java/testdata/pom/meta-inf-archive/META-INF/maven/com.example/my-lib/pom.xml new file mode 100644 index 000000000..ff3d28844 --- /dev/null +++ b/syft/pkg/cataloger/java/testdata/pom/meta-inf-archive/META-INF/maven/com.example/my-lib/pom.xml @@ -0,0 +1,24 @@ + + + 4.0.0 + + com.example + my-lib + 1.0.0 + + + + org.phantom + phantom-dep + ${phantom.version} + + + org.testonly + test-dep + 2.0.0 + test + + + diff --git a/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/META-INF/maven/com.example/embedded-lib/pom.xml b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/META-INF/maven/com.example/embedded-lib/pom.xml new file mode 100644 index 000000000..cf2ab1b72 --- /dev/null +++ b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/META-INF/maven/com.example/embedded-lib/pom.xml @@ -0,0 +1,18 @@ + + + 4.0.0 + + com.example + embedded-lib + 3.0.0 + + + + org.phantom + should-not-appear + ${unresolvable.version} + + + diff --git a/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/pom.xml b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/pom.xml new file mode 100644 index 000000000..32b8e2426 --- /dev/null +++ b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/pom.xml @@ -0,0 +1,18 @@ + + + 4.0.0 + + org.anchore + my-app + 2.0.0 + + + + com.google.guava + guava + 31.1-jre + + + From 5e5760e132443263019dcba551d62dbfc4daef5d Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Fri, 10 Apr 2026 14:49:35 +0200 Subject: [PATCH 10/10] fix(archive): use writable permissions for extracted directories Archives may contain directory entries with read-only permissions (e.g., META-INF/ with mode 0555 in JARs). When extracted to temp directories, these restricted permissions prevented writing files into those directories, causing "permission denied" errors for subsequent file entries like META-INF/MANIFEST.MF. Always use 0755 for extracted directories since they are temporary working directories owned by the extraction process. Signed-off-by: Thomas Bechtold --- internal/archive/extractor.go | 4 +- internal/archive/extractor_test.go | 93 ++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/internal/archive/extractor.go b/internal/archive/extractor.go index 96525570d..d11cce5ca 100644 --- a/internal/archive/extractor.go +++ b/internal/archive/extractor.go @@ -113,7 +113,7 @@ func (z *ZipExtractor) Extract(ctx context.Context, path string, destDir string, if err != nil { return err } - return os.MkdirAll(joinedPath, file.Mode()) + return os.MkdirAll(joinedPath, 0o755) } if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount { @@ -180,7 +180,7 @@ func (t *TarExtractor) Extract(ctx context.Context, path string, destDir string, if err != nil { return err } - return os.MkdirAll(joinedPath, file.Mode()) + return os.MkdirAll(joinedPath, 0o755) } if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount { diff --git a/internal/archive/extractor_test.go b/internal/archive/extractor_test.go index b244f3dcd..715f90418 100644 --- a/internal/archive/extractor_test.go +++ b/internal/archive/extractor_test.go @@ -430,3 +430,96 @@ func TestTarExtractor_Extract_AbsoluteSymlinkTargetIsSkipped(t *testing.T) { _, statErr := os.Lstat(filepath.Join(destDir, "shadow")) assert.True(t, os.IsNotExist(statErr), "expected absolute-target symlink to be skipped") } + +func TestZipExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { + // JARs may contain directory entries with read-only permissions (e.g., META-INF/ with mode 0o555). + // The extractor must still be able to write files into those directories. + dir := t.TempDir() + ctx := context.Background() + + zipPath := filepath.Join(dir, "readonly-dirs.zip") + f, err := os.Create(zipPath) + require.NoError(t, err) + + w := zip.NewWriter(f) + + // create a directory entry with read-only permissions (0o555) + dirHeader := &zip.FileHeader{ + Name: "META-INF/", + } + dirHeader.SetMode(0o555) + _, err = w.CreateHeader(dirHeader) + require.NoError(t, err) + + // create a file inside that directory + fileHeader := &zip.FileHeader{ + Name: "META-INF/MANIFEST.MF", + } + fileHeader.SetMode(0o644) + fw, err := w.CreateHeader(fileHeader) + require.NoError(t, err) + _, err = fw.Write([]byte("Manifest-Version: 1.0")) + require.NoError(t, err) + + require.NoError(t, w.Close()) + require.NoError(t, f.Close()) + + ext := &ZipExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + result, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + assert.Equal(t, 1, result.FilesExtracted) + + content, err := os.ReadFile(filepath.Join(destDir, "META-INF", "MANIFEST.MF")) + require.NoError(t, err) + assert.Equal(t, "Manifest-Version: 1.0", string(content)) +} + +func TestTarExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { + // Tar archives may contain directory entries with read-only permissions. + // The extractor must still be able to write files into those directories. + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "readonly-dirs.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + // write a directory entry with read-only permissions + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "META-INF/", + Typeflag: tar.TypeDir, + Mode: 0o555, + })) + + // write a file inside that directory + content := []byte("Manifest-Version: 1.0") + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "META-INF/MANIFEST.MF", + Mode: 0o644, + Size: int64(len(content)), + })) + _, err = tw.Write(content) + require.NoError(t, err) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + result, err := ext.Extract(ctx, tarPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + assert.Equal(t, 1, result.FilesExtracted) + + extracted, err := os.ReadFile(filepath.Join(destDir, "META-INF", "MANIFEST.MF")) + require.NoError(t, err) + assert.Equal(t, "Manifest-Version: 1.0", string(extracted)) +}