diff --git a/cmd/syft/internal/options/archive.go b/cmd/syft/internal/options/archive.go new file mode 100644 index 000000000..d2a71e1b0 --- /dev/null +++ b/cmd/syft/internal/options/archive.go @@ -0,0 +1,36 @@ +package options + +import ( + "github.com/anchore/clio" + "github.com/anchore/syft/syft/cataloging" +) + +type archiveConfig struct { + MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"` + MaxExtractionSizeBytes int64 `yaml:"max-extraction-size-bytes" json:"max-extraction-size-bytes" mapstructure:"max-extraction-size-bytes"` + MaxFileCount int `yaml:"max-file-count" json:"max-file-count" mapstructure:"max-file-count"` + MaxTotalExtractionBytes int64 `yaml:"max-total-extraction-bytes" json:"max-total-extraction-bytes" mapstructure:"max-total-extraction-bytes"` + ExcludeExtensions []string `yaml:"exclude-extensions" json:"exclude-extensions" mapstructure:"exclude-extensions"` +} + +var _ interface { + clio.FieldDescriber +} = (*archiveConfig)(nil) + +func defaultArchiveConfig() archiveConfig { + return archiveConfig{ + MaxDepth: cataloging.DefaultArchiveMaxDepth, + MaxExtractionSizeBytes: cataloging.DefaultArchiveMaxExtractionSizeBytes, + MaxFileCount: cataloging.DefaultArchiveMaxFileCount, + MaxTotalExtractionBytes: cataloging.DefaultArchiveMaxTotalExtractionBytes, + } +} + +func (o *archiveConfig) DescribeFields(descriptions clio.FieldDescriptionSet) { + descriptions.Add(&o.MaxDepth, `maximum depth of recursive archive extraction (0 = disabled, no recursive extraction) +enabling this allows syft to discover packages inside nested archives (e.g., a tar.gz containing Python packages)`) + descriptions.Add(&o.MaxExtractionSizeBytes, `maximum total bytes to extract from a single archive before stopping`) + descriptions.Add(&o.MaxFileCount, `maximum number of files to extract from a single archive before stopping`) + descriptions.Add(&o.MaxTotalExtractionBytes, `maximum total bytes to extract across all archives before stopping`) + descriptions.Add(&o.ExcludeExtensions, `archive file extensions to skip during recursive extraction (e.g., [".rpm", ".deb"])`) +} diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index bcb8ab66f..7483983d2 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -37,6 +37,7 @@ type Catalog struct { Package packageConfig `yaml:"package" json:"package" mapstructure:"package"` License licenseConfig `yaml:"license" json:"license" mapstructure:"license"` File fileConfig `yaml:"file" json:"file" mapstructure:"file"` + Archive archiveConfig `yaml:"archive" json:"archive" mapstructure:"archive"` Scope string `yaml:"scope" json:"scope" mapstructure:"scope"` Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel Relationships relationshipsConfig `yaml:"relationships" json:"relationships" mapstructure:"relationships"` @@ -76,6 +77,7 @@ func DefaultCatalog() Catalog { Scope: source.SquashedScope.String(), Package: defaultPackageConfig(), License: defaultLicenseConfig(), + Archive: defaultArchiveConfig(), LinuxKernel: defaultLinuxKernelConfig(), JavaScript: defaultJavaScriptConfig(), Python: defaultPythonConfig(), @@ -102,6 +104,8 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { WithPackagesConfig(cfg.ToPackagesConfig()). WithLicenseConfig(cfg.ToLicenseConfig()). WithFilesConfig(cfg.ToFilesConfig()). + WithArchiveConfig(cfg.ToArchiveConfig()). + WithExclusions(cfg.Exclusions). WithCatalogerSelection( cataloging.NewSelectionRequest(). WithDefaults(cfg.DefaultCatalogers...). @@ -109,6 +113,18 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { ) } +func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig { + return cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: cfg.Package.SearchIndexedArchives, + IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives, + MaxDepth: cfg.Archive.MaxDepth, + MaxExtractionSizeBytes: cfg.Archive.MaxExtractionSizeBytes, + MaxFileCount: cfg.Archive.MaxFileCount, + MaxTotalExtractionBytes: cfg.Archive.MaxTotalExtractionBytes, + ExcludeExtensions: cfg.Archive.ExcludeExtensions, + } +} + func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig { return cataloging.SearchConfig{ Scope: source.ParseScope(cfg.Scope), diff --git a/internal/archive/extractor.go b/internal/archive/extractor.go new file mode 100644 index 000000000..d11cce5ca --- /dev/null +++ b/internal/archive/extractor.go @@ -0,0 +1,285 @@ +package archive + +import ( + "context" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/mholt/archives" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/cataloging" +) + +// Extractor extracts archive contents to a destination directory. +type Extractor interface { + // CanExtract returns true if this extractor can handle the given file. + CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool + + // Extract extracts the archive contents to destDir, respecting the given limits. + // Returns the number of files extracted and total bytes written. + Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) +} + +// ExtractionLimits defines safety limits for archive extraction. +type ExtractionLimits struct { + MaxExtractionSizeBytes int64 + MaxFileCount int +} + +// ExtractionResult holds the result of an extraction operation. +type ExtractionResult struct { + FilesExtracted int + BytesWritten int64 +} + +// DefaultExtractionLimits returns limits from the given config. +func DefaultExtractionLimits(cfg cataloging.ArchiveSearchConfig) ExtractionLimits { + return ExtractionLimits{ + MaxExtractionSizeBytes: cfg.MaxExtractionSizeBytes, + MaxFileCount: cfg.MaxFileCount, + } +} + +// DefaultExtractors returns the set of built-in archive extractors. +func DefaultExtractors() []Extractor { + return []Extractor{ + &ZipExtractor{}, + &TarExtractor{}, + } +} + +// FindExtractor finds an extractor that can handle the given file, or nil if none can. +func FindExtractor(ctx context.Context, extractors []Extractor, path string) Extractor { + f, err := os.Open(path) + if err != nil { + log.Tracef("unable to open file for archive detection: %v", err) + return nil + } + defer f.Close() + + for _, ext := range extractors { + if ext.CanExtract(ctx, path, f) { + return ext + } + // reset for next extractor + if _, err := f.Seek(0, io.SeekStart); err != nil { + log.Tracef("unable to seek file for archive detection: %v", err) + return nil + } + } + return nil +} + +// IsExcludedExtension checks if a file path has an excluded extension. +func IsExcludedExtension(path string, excludeExtensions []string) bool { + for _, ext := range excludeExtensions { + if strings.HasSuffix(strings.ToLower(path), strings.ToLower(ext)) { + return true + } + } + return false +} + +// ZipExtractor extracts zip-based archives (zip, jar, war, ear, etc.). +type ZipExtractor struct{} + +func (z *ZipExtractor) CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool { + format, _, err := intFile.IdentifyArchive(ctx, path, reader) + if err != nil { + return false + } + _, ok := format.(archives.Zip) + return ok +} + +func (z *ZipExtractor) Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) { + var result ExtractionResult + + f, err := os.Open(path) + if err != nil { + return result, fmt.Errorf("unable to open zip archive %q: %w", path, err) + } + defer f.Close() + + return result, archives.Zip{}.Extract(ctx, f, func(_ context.Context, file archives.FileInfo) error { + if file.IsDir() { + joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive) + if err != nil { + return err + } + return os.MkdirAll(joinedPath, 0o755) + } + + if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount { + return fmt.Errorf("archive file count limit reached (%d files)", limits.MaxFileCount) + } + + joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive) + if err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(joinedPath), 0o755); err != nil { + return fmt.Errorf("unable to create parent dir: %w", err) + } + + return extractFileWithLimits(file, joinedPath, destDir, &result, limits) + }) +} + +// TarExtractor extracts tar-based archives (tar, tar.gz, tar.bz2, tar.xz, tar.zst). +type TarExtractor struct{} + +func (t *TarExtractor) CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool { + format, _, err := intFile.IdentifyArchive(ctx, path, reader) + if err != nil { + return false + } + // mholt/archives returns a compound type for tar+compression, check if it's an extractor but not a zip + if _, isZip := format.(archives.Zip); isZip { + return false + } + _, ok := format.(archives.Extractor) + return ok +} + +func (t *TarExtractor) Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) { + var result ExtractionResult + + f, err := os.Open(path) + if err != nil { + return result, fmt.Errorf("unable to open tar archive %q: %w", path, err) + } + defer f.Close() + + format, readerAfterIdentify, err := intFile.IdentifyArchive(ctx, path, f) + if err != nil { + return result, fmt.Errorf("unable to identify archive format for %q: %w", path, err) + } + + extractor, ok := format.(archives.Extractor) + if !ok { + return result, fmt.Errorf("file format does not support extraction: %s", path) + } + + // use the reader returned by IdentifyArchive since it may have consumed some bytes + extractReader := readerAfterIdentify + if extractReader == nil { + extractReader = f + } + + return result, extractor.Extract(ctx, extractReader, func(_ context.Context, file archives.FileInfo) error { + if file.IsDir() { + joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive) + if err != nil { + return err + } + return os.MkdirAll(joinedPath, 0o755) + } + + if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount { + return fmt.Errorf("archive file count limit reached (%d files)", limits.MaxFileCount) + } + + joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive) + if err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(joinedPath), 0o755); err != nil { + return fmt.Errorf("unable to create parent dir: %w", err) + } + + return extractFileWithLimits(file, joinedPath, destDir, &result, limits) + }) +} + +// extractFileWithLimits extracts a single file from an archive, respecting size limits. +// Symlinks are written as real symlinks (only when the target stays inside destDir); +// non-regular file types other than symlinks are skipped to avoid writing bogus content. +func extractFileWithLimits(file archives.FileInfo, destPath, destDir string, result *ExtractionResult, limits ExtractionLimits) error { + mode := file.Mode() + if mode.Type()&fs.ModeSymlink != 0 { + if err := writeSafeSymlink(file.LinkTarget, destPath, destDir); err != nil { + log.WithFields("entry", file.NameInArchive, "target", file.LinkTarget, "error", err).Debug("skipping unsafe symlink in archive") + return nil + } + result.FilesExtracted++ + return nil + } + if !mode.IsRegular() { + // skip device nodes, fifos, sockets, etc. — they can't safely round-trip + log.WithFields("entry", file.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry") + return nil + } + + src, err := file.Open() + if err != nil { + return fmt.Errorf("unable to open archive entry %q: %w", file.NameInArchive, err) + } + defer src.Close() + + dst, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("unable to create dest file %q: %w", destPath, err) + } + defer dst.Close() + + var reader io.Reader = src + + // enforce per-archive total size limit + if limits.MaxExtractionSizeBytes > 0 { + remaining := limits.MaxExtractionSizeBytes - result.BytesWritten + if remaining <= 0 { + return fmt.Errorf("archive extraction size limit reached (%d bytes)", limits.MaxExtractionSizeBytes) + } + reader = io.LimitReader(src, remaining+1) // +1 to detect exceeding the limit + } + + n, err := io.Copy(dst, reader) + result.BytesWritten += n + result.FilesExtracted++ + + if limits.MaxExtractionSizeBytes > 0 && result.BytesWritten > limits.MaxExtractionSizeBytes { + return fmt.Errorf("archive extraction size limit reached (%d bytes)", limits.MaxExtractionSizeBytes) + } + + return err +} + +// writeSafeSymlink creates a symlink at destPath only if target is relative and +// resolves — relative to destPath's directory — to a location inside destDir. +// +// Absolute targets are rejected outright: os.Symlink writes the literal target +// string, so an absolute target like "/etc/passwd" would resolve on the host +// filesystem when read, regardless of any safety check we performed against +// destDir at extraction time. +// +// NOTE: this guards against the single-symlink-escape pattern, but a malicious +// archive can still attempt a multi-step chain where two cooperating link +// entries together resolve outside destDir (e.g., one entry creates a benign +// link, a later entry writes through it). Fully preventing that requires +// either openat/O_NOFOLLOW per path component during extraction or +// transitively resolving every newly-created symlink against all prior ones +// — neither is implemented here. +func writeSafeSymlink(target, destPath, destDir string) error { + if target == "" { + return fmt.Errorf("empty link target") + } + if filepath.IsAbs(target) { + return fmt.Errorf("absolute symlink target not allowed") + } + resolved := filepath.Join(filepath.Dir(destPath), target) + cleanRoot := filepath.Clean(destDir) + cleanResolved := filepath.Clean(resolved) + if cleanResolved != cleanRoot && !strings.HasPrefix(cleanResolved, cleanRoot+string(filepath.Separator)) { + return fmt.Errorf("symlink target escapes extraction root") + } + return os.Symlink(target, destPath) +} diff --git a/internal/archive/extractor_test.go b/internal/archive/extractor_test.go new file mode 100644 index 000000000..715f90418 --- /dev/null +++ b/internal/archive/extractor_test.go @@ -0,0 +1,525 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cataloging" +) + +func createTestZip(t *testing.T, dir string, files map[string]string) string { + t.Helper() + zipPath := filepath.Join(dir, "test.zip") + f, err := os.Create(zipPath) + require.NoError(t, err) + defer f.Close() + + w := zip.NewWriter(f) + for name, content := range files { + fw, err := w.Create(name) + require.NoError(t, err) + _, err = fw.Write([]byte(content)) + require.NoError(t, err) + } + require.NoError(t, w.Close()) + return zipPath +} + +func createTestTarGz(t *testing.T, dir string, files map[string]string) string { + t.Helper() + tarPath := filepath.Join(dir, "test.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + defer f.Close() + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Mode: 0o644, + Size: int64(len(content)), + } + require.NoError(t, tw.WriteHeader(hdr)) + _, err = tw.Write([]byte(content)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + return tarPath +} + +func TestZipExtractor_CanExtract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) + + ext := &ZipExtractor{} + + f, err := os.Open(zipPath) + require.NoError(t, err) + defer f.Close() + + assert.True(t, ext.CanExtract(ctx, zipPath, f)) + + // non-zip file + nonZipPath := filepath.Join(dir, "notazip.txt") + require.NoError(t, os.WriteFile(nonZipPath, []byte("just text"), 0o644)) + + f2, err := os.Open(nonZipPath) + require.NoError(t, err) + defer f2.Close() + + assert.False(t, ext.CanExtract(ctx, nonZipPath, f2)) +} + +func TestZipExtractor_Extract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": "content1", + "dir/file2.txt": "content2", + } + zipPath := createTestZip(t, dir, files) + + ext := &ZipExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + result, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + assert.Equal(t, 2, result.FilesExtracted) + + // verify extracted files + content, err := os.ReadFile(filepath.Join(destDir, "file1.txt")) + require.NoError(t, err) + assert.Equal(t, "content1", string(content)) + + content, err = os.ReadFile(filepath.Join(destDir, "dir", "file2.txt")) + require.NoError(t, err) + assert.Equal(t, "content2", string(content)) +} + +func TestZipExtractor_Extract_FileLimitReached(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": "a", + "file2.txt": "b", + "file3.txt": "c", + } + zipPath := createTestZip(t, dir, files) + + ext := &ZipExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{MaxFileCount: 2}) + require.Error(t, err) + assert.Contains(t, err.Error(), "file count limit") +} + +func TestZipExtractor_Extract_SizeLimitReached(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": strings.Repeat("x", 100), + } + zipPath := createTestZip(t, dir, files) + + ext := &ZipExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{MaxExtractionSizeBytes: 50}) + require.Error(t, err) + assert.Contains(t, err.Error(), "size limit") +} + +func TestTarExtractor_CanExtract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) + + ext := &TarExtractor{} + + f, err := os.Open(tarPath) + require.NoError(t, err) + defer f.Close() + + assert.True(t, ext.CanExtract(ctx, tarPath, f)) +} + +func TestTarExtractor_Extract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": "content1", + "dir/file2.txt": "content2", + } + tarPath := createTestTarGz(t, dir, files) + + ext := &TarExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + result, err := ext.Extract(ctx, tarPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + assert.Equal(t, 2, result.FilesExtracted) + + // verify extracted files + content, err := os.ReadFile(filepath.Join(destDir, "file1.txt")) + require.NoError(t, err) + assert.Equal(t, "content1", string(content)) + + content, err = os.ReadFile(filepath.Join(destDir, "dir", "file2.txt")) + require.NoError(t, err) + assert.Equal(t, "content2", string(content)) +} + +func TestFindExtractor(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + extractors := DefaultExtractors() + + // zip file should match + zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) + ext := FindExtractor(ctx, extractors, zipPath) + require.NotNil(t, ext) + assert.IsType(t, &ZipExtractor{}, ext) + + // tar.gz should match + tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) + ext = FindExtractor(ctx, extractors, tarPath) + require.NotNil(t, ext) + assert.IsType(t, &TarExtractor{}, ext) + + // non-archive should return nil + textPath := filepath.Join(dir, "plain.txt") + require.NoError(t, os.WriteFile(textPath, []byte("just text"), 0o644)) + ext = FindExtractor(ctx, extractors, textPath) + assert.Nil(t, ext) +} + +func TestIsExcludedExtension(t *testing.T) { + assert.True(t, IsExcludedExtension("/path/to/file.rpm", []string{".rpm", ".deb"})) + assert.True(t, IsExcludedExtension("/path/to/file.deb", []string{".rpm", ".deb"})) + assert.False(t, IsExcludedExtension("/path/to/file.zip", []string{".rpm", ".deb"})) + assert.False(t, IsExcludedExtension("/path/to/file.zip", nil)) +} + +func TestFindExtractor_NoReaders(t *testing.T) { + ctx := context.Background() + ext := FindExtractor(ctx, DefaultExtractors(), "/nonexistent/path") + assert.Nil(t, ext) +} + +func TestTarExtractor_CanExtract_NonTarFile(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + // create a plain text file + path := filepath.Join(dir, "plain.txt") + require.NoError(t, os.WriteFile(path, []byte("not a tar"), 0o644)) + + ext := &TarExtractor{} + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + + assert.False(t, ext.CanExtract(ctx, path, f)) +} + +func TestTarExtractor_CanExtract_ZipFileReturnsFalse(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) + + ext := &TarExtractor{} + f, err := os.Open(zipPath) + require.NoError(t, err) + defer f.Close() + + // tar extractor should NOT claim to handle zip files + assert.False(t, ext.CanExtract(ctx, zipPath, f)) +} + +func TestDefaultExtractionLimits(t *testing.T) { + cfg := cataloging.ArchiveSearchConfig{ + MaxExtractionSizeBytes: 1024, + MaxFileCount: 50, + } + limits := DefaultExtractionLimits(cfg) + assert.Equal(t, int64(1024), limits.MaxExtractionSizeBytes) + assert.Equal(t, 50, limits.MaxFileCount) +} + +func TestZipExtractor_Extract_ZipSlipPrevented(t *testing.T) { + // Create a zip with a path traversal entry + dir := t.TempDir() + ctx := context.Background() + + zipPath := filepath.Join(dir, "evil.zip") + f, err := os.Create(zipPath) + require.NoError(t, err) + + w := zip.NewWriter(f) + fw, err := w.Create("../../etc/passwd") + require.NoError(t, err) + _, err = fw.Write([]byte("evil")) + require.NoError(t, err) + require.NoError(t, w.Close()) + require.NoError(t, f.Close()) + + ext := &ZipExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err = ext.Extract(ctx, zipPath, destDir, ExtractionLimits{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "path traversal") + + // verify evil file was not created + _, statErr := os.Stat(filepath.Join(dir, "etc", "passwd")) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestTarExtractor_Extract_SymlinkInsideRootIsCreated(t *testing.T) { + // A tar archive with a symlink whose target stays inside the extraction + // directory should be written as a real symlink, not as a regular file + // containing the link target as bytes. + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "with-link.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + target := []byte("real content") + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "real.txt", + Mode: 0o644, + Size: int64(len(target)), + })) + _, err = tw.Write(target) + require.NoError(t, err) + + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "link.txt", + Linkname: "real.txt", + Typeflag: tar.TypeSymlink, + Mode: 0o777, + })) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err = ext.Extract(ctx, tarPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + + // the link should be a real symlink + info, err := os.Lstat(filepath.Join(destDir, "link.txt")) + require.NoError(t, err) + assert.NotZero(t, info.Mode()&fs.ModeSymlink, "expected link.txt to be a symlink, got mode %v", info.Mode()) + + // reading through the link should return the target's content + readBack, err := os.ReadFile(filepath.Join(destDir, "link.txt")) + require.NoError(t, err) + assert.Equal(t, "real content", string(readBack)) +} + +func TestTarExtractor_Extract_SymlinkEscapingRootIsSkipped(t *testing.T) { + // A symlink whose target resolves outside the extraction directory must not + // be created — neither as a symlink nor as a regular file. Extraction itself + // should succeed (the unsafe entry is skipped, not fatal). + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "evil-link.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "passwd", + Linkname: "../../../../etc/passwd", + Typeflag: tar.TypeSymlink, + Mode: 0o777, + })) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err = ext.Extract(ctx, tarPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + + // nothing should have been created at the destination + _, statErr := os.Lstat(filepath.Join(destDir, "passwd")) + assert.True(t, os.IsNotExist(statErr), "expected no entry to be created for escaping symlink") +} + +func TestTarExtractor_Extract_AbsoluteSymlinkTargetIsSkipped(t *testing.T) { + // Absolute symlink targets must be rejected: os.Symlink writes the literal + // target, so once the symlink is read it resolves on the host filesystem + // regardless of any safety check at extraction time. Verifying both that + // the symlink itself is not created and (importantly) that nothing was + // written under destDir along the absolute path. + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "absolute-link.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "shadow", + Linkname: "/etc/shadow", + Typeflag: tar.TypeSymlink, + Mode: 0o777, + })) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + _, err = ext.Extract(ctx, tarPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + + // the symlink itself must not exist + _, statErr := os.Lstat(filepath.Join(destDir, "shadow")) + assert.True(t, os.IsNotExist(statErr), "expected absolute-target symlink to be skipped") +} + +func TestZipExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { + // JARs may contain directory entries with read-only permissions (e.g., META-INF/ with mode 0o555). + // The extractor must still be able to write files into those directories. + dir := t.TempDir() + ctx := context.Background() + + zipPath := filepath.Join(dir, "readonly-dirs.zip") + f, err := os.Create(zipPath) + require.NoError(t, err) + + w := zip.NewWriter(f) + + // create a directory entry with read-only permissions (0o555) + dirHeader := &zip.FileHeader{ + Name: "META-INF/", + } + dirHeader.SetMode(0o555) + _, err = w.CreateHeader(dirHeader) + require.NoError(t, err) + + // create a file inside that directory + fileHeader := &zip.FileHeader{ + Name: "META-INF/MANIFEST.MF", + } + fileHeader.SetMode(0o644) + fw, err := w.CreateHeader(fileHeader) + require.NoError(t, err) + _, err = fw.Write([]byte("Manifest-Version: 1.0")) + require.NoError(t, err) + + require.NoError(t, w.Close()) + require.NoError(t, f.Close()) + + ext := &ZipExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + result, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + assert.Equal(t, 1, result.FilesExtracted) + + content, err := os.ReadFile(filepath.Join(destDir, "META-INF", "MANIFEST.MF")) + require.NoError(t, err) + assert.Equal(t, "Manifest-Version: 1.0", string(content)) +} + +func TestTarExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { + // Tar archives may contain directory entries with read-only permissions. + // The extractor must still be able to write files into those directories. + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "readonly-dirs.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + // write a directory entry with read-only permissions + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "META-INF/", + Typeflag: tar.TypeDir, + Mode: 0o555, + })) + + // write a file inside that directory + content := []byte("Manifest-Version: 1.0") + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "META-INF/MANIFEST.MF", + Mode: 0o644, + Size: int64(len(content)), + })) + _, err = tw.Write(content) + require.NoError(t, err) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + destDir := filepath.Join(dir, "extracted") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + result, err := ext.Extract(ctx, tarPath, destDir, ExtractionLimits{}) + require.NoError(t, err) + assert.Equal(t, 1, result.FilesExtracted) + + extracted, err := os.ReadFile(filepath.Join(destDir, "META-INF", "MANIFEST.MF")) + require.NoError(t, err) + assert.Equal(t, "Manifest-Version: 1.0", string(extracted)) +} diff --git a/internal/archive/integration_test.go b/internal/archive/integration_test.go new file mode 100644 index 000000000..3566ac518 --- /dev/null +++ b/internal/archive/integration_test.go @@ -0,0 +1,524 @@ +package archive + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cataloging" +) + +// TestIntegration_TarGzContainingPythonPackage tests the scenario where a tar.gz +// archive contains Python package metadata files that would normally be found by +// the Python cataloger. This verifies the orchestrator makes those files visible. +func TestIntegration_TarGzContainingPythonPackage(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create a tar.gz containing Python dist-info metadata + tarPath := filepath.Join(dir, "python-packages.tar.gz") + createTarGzWithFiles(t, tarPath, map[string]string{ + "lib/python3.9/site-packages/requests-2.28.0.dist-info/METADATA": `Metadata-Version: 2.1 +Name: requests +Version: 2.28.0 +Summary: Python HTTP for Humans. +`, + "lib/python3.9/site-packages/requests-2.28.0.dist-info/RECORD": "requests/__init__.py,sha256=abc,1234", + "lib/python3.9/site-packages/requests/__init__.py": "# requests library", + }) + tarContent, err := os.ReadFile(tarPath) + require.NoError(t, err) + + // Base resolver sees the tar.gz as a single opaque file + parent := newMockResolver(map[string]string{ + "/app/python-packages.tar.gz": string(tarContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeUnindexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Extract the archive + count := orch.DiscoverAndExtract(ctx, 0) + require.Equal(t, 1, count, "should extract the tar.gz") + + // The Python METADATA file should now be visible through the composite resolver + resolver := orch.Resolver() + locs, err := resolver.FilesByPath("/lib/python3.9/site-packages/requests-2.28.0.dist-info/METADATA") + require.NoError(t, err) + require.Len(t, locs, 1, "METADATA file should be visible through the composite resolver") + + // Verify the access path shows it came from the archive + assert.Contains(t, locs[0].AccessPath, "python-packages.tar.gz:") + assert.Contains(t, locs[0].AccessPath, "METADATA") + + // Verify we can read the content + reader, err := resolver.FileContentsByLocation(locs[0]) + require.NoError(t, err) + defer reader.Close() + + buf := make([]byte, 1024) + n, _ := reader.Read(buf) + assert.Contains(t, string(buf[:n]), "Name: requests") +} + +// TestIntegration_ZipContainingJar tests a zip archive containing a JAR file, +// which itself should be extractable in a second round. +func TestIntegration_ZipContainingJar(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create an inner JAR (which is just a zip) with a MANIFEST.MF + innerJarPath := filepath.Join(dir, "inner.jar") + createZipWithFiles(t, innerJarPath, map[string]string{ + "META-INF/MANIFEST.MF": `Manifest-Version: 1.0 +Implementation-Title: example-lib +Implementation-Version: 1.2.3 +`, + }) + innerJarContent, err := os.ReadFile(innerJarPath) + require.NoError(t, err) + + // Create an outer zip containing the JAR and other files + outerZipPath := filepath.Join(dir, "app-bundle.zip") + createZipWithFiles(t, outerZipPath, map[string]string{ + "lib/example-lib-1.2.3.jar": string(innerJarContent), + "README.md": "# Application Bundle", + }) + outerZipContent, err := os.ReadFile(outerZipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/downloads/app-bundle.zip": string(outerZipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 2 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Round 1: extract outer zip + count1 := orch.DiscoverAndExtract(ctx, 0) + require.Equal(t, 1, count1, "should extract outer zip") + + // Round 2: extract inner JAR found inside the zip + count2 := orch.DiscoverAndExtract(ctx, 1) + require.Equal(t, 1, count2, "should extract inner JAR") + + resolver := orch.Resolver() + + // The MANIFEST.MF from inside the JAR should be visible + locs, err := resolver.FilesByPath("/META-INF/MANIFEST.MF") + require.NoError(t, err) + require.Len(t, locs, 1, "MANIFEST.MF should be visible from nested JAR") + + // The access path should show the full nesting chain + accessPath := locs[0].AccessPath + assert.Contains(t, accessPath, "app-bundle.zip:") + assert.Contains(t, accessPath, "example-lib-1.2.3.jar:") + assert.Contains(t, accessPath, "MANIFEST.MF") + + // Verify relationships + rels := orch.Relationships() + assert.Len(t, rels, 2, "should have relationships for both extracted archives") +} + +// TestIntegration_MixedArchiveTypes tests a directory containing both +// zip and tar.gz archives that should be extracted in a single round. +func TestIntegration_MixedArchiveTypes(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create a zip + zipPath := createTestZip(t, dir, map[string]string{ + "from-zip.txt": "zip content", + }) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + // Create a tar.gz + tarPath := createTestTarGz(t, dir, map[string]string{ + "from-tar.txt": "tar content", + }) + tarContent, err := os.ReadFile(tarPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/data/archive.zip": string(zipContent), + "/data/archive.tar.gz": string(tarContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.IncludeUnindexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + count := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 2, count, "should extract both archives") + + resolver := orch.Resolver() + + // Both files should be visible + zipLocs, err := resolver.FilesByPath("/from-zip.txt") + require.NoError(t, err) + assert.Len(t, zipLocs, 1, "file from zip should be visible") + + tarLocs, err := resolver.FilesByPath("/from-tar.txt") + require.NoError(t, err) + assert.Len(t, tarLocs, 1, "file from tar should be visible") +} + +// TestIntegration_DepthLimitPreventsDeepExtraction verifies that the +// depth limit correctly stops extraction at the configured level. +func TestIntegration_DepthLimitPreventsDeepExtraction(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create a deeply nested archive: zip -> zip -> zip -> file + innermostDir := filepath.Join(dir, "inner") + require.NoError(t, os.MkdirAll(innermostDir, 0o755)) + innermost := createTestZip(t, innermostDir, map[string]string{ + "secret.txt": "you found me!", + }) + innermostContent, err := os.ReadFile(innermost) + require.NoError(t, err) + + middleDir := filepath.Join(dir, "middle") + require.NoError(t, os.MkdirAll(middleDir, 0o755)) + middle := createTestZip(t, middleDir, map[string]string{ + "innermost.zip": string(innermostContent), + }) + middleContent, err := os.ReadFile(middle) + require.NoError(t, err) + + outerDir := filepath.Join(dir, "outer") + require.NoError(t, os.MkdirAll(outerDir, 0o755)) + outerZip := createTestZip(t, outerDir, map[string]string{ + "middle.zip": string(middleContent), + }) + outerContent, err := os.ReadFile(outerZip) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/nested.zip": string(outerContent), + }) + + // Only allow depth 1 — should only extract the outer zip + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + count := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, count, "should only extract the outer zip") + + // middle.zip should be visible but secret.txt should NOT be + resolver := orch.Resolver() + + middleLocs, err := resolver.FilesByPath("/middle.zip") + require.NoError(t, err) + assert.Len(t, middleLocs, 1, "middle.zip should be visible at depth 1") + + secretLocs, err := resolver.FilesByPath("/secret.txt") + require.NoError(t, err) + assert.Len(t, secretLocs, 0, "secret.txt should NOT be visible (depth limit)") +} + +// TestIntegration_ExcludeExtensionPreventsExtraction verifies that +// excluded extensions are respected. +func TestIntegration_ExcludeExtensionPreventsExtraction(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{ + "data.txt": "from zip", + }) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/archive.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.ExcludeExtensions = []string{".zip"} // exclude zip files + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + count := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, count, "zip should be excluded by extension") +} + +// TestIntegration_MultiLevelMixedFormats tests a 3-level nesting chain with +// different archive formats: tar.gz -> zip -> tar.gz -> files. +// This is the most realistic cross-format recursive extraction scenario. +func TestIntegration_MultiLevelMixedFormats(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Level 3 (deepest): a tar.gz containing application files + level3Dir := filepath.Join(dir, "level3") + require.NoError(t, os.MkdirAll(level3Dir, 0o755)) + level3Path := filepath.Join(level3Dir, "app-data.tar.gz") + createTarGzWithFiles(t, level3Path, map[string]string{ + "config.yaml": "app: myapp\nversion: 1.0", + "bin/run.sh": "#!/bin/sh\necho hello", + }) + level3Content, err := os.ReadFile(level3Path) + require.NoError(t, err) + + // Level 2: a zip containing the level3 tar.gz and some metadata + level2Dir := filepath.Join(dir, "level2") + require.NoError(t, os.MkdirAll(level2Dir, 0o755)) + level2Path := filepath.Join(level2Dir, "package.zip") + createZipWithFiles(t, level2Path, map[string]string{ + "app-data.tar.gz": string(level3Content), + "package-info.json": `{"name": "mypackage", "version": "2.0"}`, + }) + level2Content, err := os.ReadFile(level2Path) + require.NoError(t, err) + + // Level 1 (outermost): a tar.gz containing the level2 zip + level1Dir := filepath.Join(dir, "level1") + require.NoError(t, os.MkdirAll(level1Dir, 0o755)) + level1Path := filepath.Join(level1Dir, "distribution.tar.gz") + createTarGzWithFiles(t, level1Path, map[string]string{ + "package.zip": string(level2Content), + "LICENSE": "MIT License", + }) + level1Content, err := os.ReadFile(level1Path) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/distribution.tar.gz": string(level1Content), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 3 + cfg.IncludeIndexedArchives = true + cfg.IncludeUnindexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Round 1: extract distribution.tar.gz + count1 := orch.DiscoverAndExtract(ctx, 0) + require.Equal(t, 1, count1, "round 1: should extract distribution.tar.gz") + + // Round 2: extract package.zip from inside distribution.tar.gz + count2 := orch.DiscoverAndExtract(ctx, 1) + require.Equal(t, 1, count2, "round 2: should extract package.zip") + + // Round 3: extract app-data.tar.gz from inside package.zip + count3 := orch.DiscoverAndExtract(ctx, 2) + require.Equal(t, 1, count3, "round 3: should extract app-data.tar.gz") + + resolver := orch.Resolver() + + // Verify files from all levels are accessible: + + // Level 1: LICENSE from distribution.tar.gz + locs, err := resolver.FilesByPath("/LICENSE") + require.NoError(t, err) + require.Len(t, locs, 1, "LICENSE from level 1 should be visible") + assert.Contains(t, locs[0].AccessPath, "distribution.tar.gz:") + + // Level 2: package-info.json from package.zip + locs, err = resolver.FilesByPath("/package-info.json") + require.NoError(t, err) + require.Len(t, locs, 1, "package-info.json from level 2 should be visible") + assert.Contains(t, locs[0].AccessPath, "package.zip:") + + // Level 3: config.yaml from app-data.tar.gz + locs, err = resolver.FilesByPath("/config.yaml") + require.NoError(t, err) + require.Len(t, locs, 1, "config.yaml from level 3 should be visible") + assert.Contains(t, locs[0].AccessPath, "app-data.tar.gz:") + + // Level 3: bin/run.sh from app-data.tar.gz + locs, err = resolver.FilesByPath("/bin/run.sh") + require.NoError(t, err) + require.Len(t, locs, 1, "bin/run.sh from level 3 should be visible") + + // Verify we can read deeply nested content + reader, err := resolver.FileContentsByLocation(locs[0]) + require.NoError(t, err) + defer reader.Close() + buf := make([]byte, 1024) + n, _ := reader.Read(buf) + assert.Contains(t, string(buf[:n]), "echo hello") + + // Verify relationships cover all 3 levels + rels := orch.Relationships() + assert.Len(t, rels, 3, "should have 3 relationships (one per extracted archive)") +} + +// TestIntegration_TarInsideZipInsideTar tests tar -> zip -> tar nesting +// (the reverse of the above test), ensuring bidirectional format crossing. +func TestIntegration_TarInsideZipInsideTar(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Innermost: a tar.gz with a single file + innerTarDir := filepath.Join(dir, "innertar") + require.NoError(t, os.MkdirAll(innerTarDir, 0o755)) + innerTarPath := filepath.Join(innerTarDir, "inner.tar.gz") + createTarGzWithFiles(t, innerTarPath, map[string]string{ + "deeply-nested.txt": "found at the bottom", + }) + innerTarContent, err := os.ReadFile(innerTarPath) + require.NoError(t, err) + + // Middle: a zip containing the inner tar + middleZipDir := filepath.Join(dir, "middlezip") + require.NoError(t, os.MkdirAll(middleZipDir, 0o755)) + middleZipPath := filepath.Join(middleZipDir, "middle.zip") + createZipWithFiles(t, middleZipPath, map[string]string{ + "inner.tar.gz": string(innerTarContent), + "middle.txt": "middle level", + }) + middleZipContent, err := os.ReadFile(middleZipPath) + require.NoError(t, err) + + // Outermost: a tar.gz containing the middle zip + outerTarDir := filepath.Join(dir, "outertar") + require.NoError(t, os.MkdirAll(outerTarDir, 0o755)) + outerTarPath := filepath.Join(outerTarDir, "outer.tar.gz") + createTarGzWithFiles(t, outerTarPath, map[string]string{ + "middle.zip": string(middleZipContent), + "outer.txt": "outer level", + }) + outerTarContent, err := os.ReadFile(outerTarPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/outer.tar.gz": string(outerTarContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 3 + cfg.IncludeIndexedArchives = true + cfg.IncludeUnindexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Extract all 3 levels + count1 := orch.DiscoverAndExtract(ctx, 0) + require.Equal(t, 1, count1) + count2 := orch.DiscoverAndExtract(ctx, 1) + require.Equal(t, 1, count2) + count3 := orch.DiscoverAndExtract(ctx, 2) + require.Equal(t, 1, count3) + + resolver := orch.Resolver() + + // Check all levels are accessible + outerLocs, err := resolver.FilesByPath("/outer.txt") + require.NoError(t, err) + require.Len(t, outerLocs, 1) + + middleLocs, err := resolver.FilesByPath("/middle.txt") + require.NoError(t, err) + require.Len(t, middleLocs, 1) + + deepLocs, err := resolver.FilesByPath("/deeply-nested.txt") + require.NoError(t, err) + require.Len(t, deepLocs, 1) + + // Read the deeply nested content + reader, err := resolver.FileContentsByLocation(deepLocs[0]) + require.NoError(t, err) + defer reader.Close() + buf := make([]byte, 1024) + n, _ := reader.Read(buf) + assert.Equal(t, "found at the bottom", string(buf[:n])) + + // The access path should show the full chain: tar -> zip -> tar -> file + accessPath := deepLocs[0].AccessPath + assert.Contains(t, accessPath, "outer.tar.gz:") + assert.Contains(t, accessPath, "middle.zip:") + assert.Contains(t, accessPath, "inner.tar.gz:") + assert.Contains(t, accessPath, "deeply-nested.txt") +} + +// createZipWithFiles creates a zip at the given path with the given files. +func createZipWithFiles(t *testing.T, zipPath string, files map[string]string) { + t.Helper() + f, err := os.Create(zipPath) + require.NoError(t, err) + defer f.Close() + + w := zip.NewWriter(f) + for name, content := range files { + fw, err := w.Create(name) + require.NoError(t, err) + _, err = fw.Write([]byte(content)) + require.NoError(t, err) + } + require.NoError(t, w.Close()) +} + +// createTarGzWithFiles creates a tar.gz at the given path with the given files. +func createTarGzWithFiles(t *testing.T, tarPath string, files map[string]string) { + t.Helper() + f, err := os.Create(tarPath) + require.NoError(t, err) + defer f.Close() + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + // sort keys for deterministic output + for name, content := range files { + // create parent dirs as needed + dir := filepath.Dir(name) + if dir != "." { + hdr := &tar.Header{ + Typeflag: tar.TypeDir, + Name: dir + "/", + Mode: 0o755, + } + require.NoError(t, tw.WriteHeader(hdr)) + } + + hdr := &tar.Header{ + Name: name, + Mode: 0o644, + Size: int64(len(content)), + } + require.NoError(t, tw.WriteHeader(hdr)) + _, err = tw.Write([]byte(content)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) +} + diff --git a/internal/archive/orchestrator.go b/internal/archive/orchestrator.go new file mode 100644 index 000000000..0a8e1a934 --- /dev/null +++ b/internal/archive/orchestrator.go @@ -0,0 +1,362 @@ +package archive + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "sync" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/file" +) + +// ResolverFactory creates a file.Resolver for a directory path. +// This is injectable to avoid direct dependency on syft/internal/fileresolver. +type ResolverFactory func(root string) (file.Resolver, error) + +// Orchestrator manages iterative archive extraction and cataloging. It discovers +// archives in the file resolver, extracts them, builds child resolvers for their +// contents, and tracks the relationships between archives and their contents. +type Orchestrator struct { + composite *CompositeResolver + extractors []Extractor + config cataloging.ArchiveSearchConfig + tmpDir string + resolverFactory ResolverFactory + + // tracking state + processed map[processedKey]bool // archives we've already considered for extraction + totalBytesWritten int64 + archiveNodes []*archiveNode // for relationship tracking + cleanups []func() + mu sync.Mutex +} + +// archiveNode tracks an extracted archive for relationship generation. +type archiveNode struct { + location file.Location // where the archive is in the resolver + fsID string // the filesystem ID assigned to this archive's contents + depth int +} + +// processedKey identifies an archive location uniquely. We key the processed +// map on a struct (not a ":" string) because both fields can +// legally contain ':' — fsIDs are emitted as "archive:" and POSIX paths +// permit colons — so a string key would have an ambiguous boundary. +type processedKey struct { + fsID string + realPath string +} + +// NewOrchestrator creates an Orchestrator that manages archive extraction. +// The resolverFactory is used to build file.Resolvers for extracted archive directories. +func NewOrchestrator(baseResolver file.Resolver, cfg cataloging.ArchiveSearchConfig, tmpDir string, resolverFactory ResolverFactory) *Orchestrator { + return &Orchestrator{ + composite: NewCompositeResolver(baseResolver), + extractors: DefaultExtractors(), + config: cfg, + tmpDir: tmpDir, + resolverFactory: resolverFactory, + processed: make(map[processedKey]bool), + } +} + +// Resolver returns the composite resolver that includes all extracted archive contents. +func (o *Orchestrator) Resolver() *CompositeResolver { + return o.composite +} + +// DiscoverAndExtract finds archives in the current resolver view, extracts them, +// and adds their contents as child resolvers. Returns the number of new archives extracted. +// The depth parameter indicates the current nesting depth (0 = scanning base filesystem). +func (o *Orchestrator) DiscoverAndExtract(ctx context.Context, depth int) int { + if depth >= o.config.MaxDepth { + return 0 + } + + o.mu.Lock() + defer o.mu.Unlock() + + if o.config.MaxTotalExtractionBytes > 0 && o.totalBytesWritten >= o.config.MaxTotalExtractionBytes { + log.Debug("archive orchestrator: total extraction size limit reached, stopping") + return 0 + } + + // build the list of extractors actually permitted by config — every + // non-archive location in the resolver is going to pay a peek read against + // these, so excluding disabled types up-front avoids reading files we + // would only end up rejecting after detection. + enabled := o.enabledExtractors() + if len(enabled) == 0 { + return 0 + } + + newArchives := 0 + + // scan all locations in the current resolver for archives + for loc := range o.composite.AllLocations(ctx) { + if ctx.Err() != nil { + break + } + + // skip already-processed archives + key := processedKey{fsID: loc.FileSystemID, realPath: loc.RealPath} + if o.processed[key] { + continue + } + + // skip excluded extensions + if IsExcludedExtension(loc.RealPath, o.config.ExcludeExtensions) { + continue + } + + // check if this location is an archive we can extract — peek a small + // buffer rather than materializing the full file content, which would + // be a full disk-to-disk copy of every non-archive file in the resolver. + ext := o.detectFormat(ctx, loc, enabled) + if ext == nil { + continue + } + + // mark as processed before extraction to avoid re-processing + o.processed[key] = true + + // extract the archive + extracted, err := o.extractArchive(ctx, ext, loc, depth+1) + if err != nil { + log.WithFields("archive", loc.Path(), "error", err).Debug("failed to extract archive") + continue + } + if extracted { + newArchives++ + } + } + + return newArchives +} + +// enabledExtractors returns the subset of o.extractors whose corresponding +// config gate is on (IncludeIndexedArchives for ZipExtractor, +// IncludeUnindexedArchives for TarExtractor). +func (o *Orchestrator) enabledExtractors() []Extractor { + var out []Extractor + for _, ext := range o.extractors { + switch ext.(type) { + case *ZipExtractor: + if o.config.IncludeIndexedArchives { + out = append(out, ext) + } + case *TarExtractor: + if o.config.IncludeUnindexedArchives { + out = append(out, ext) + } + default: + // unknown extractor types: include by default so plugin extractors + // added in the future aren't silently disabled + out = append(out, ext) + } + } + return out +} + +// extractArchive extracts a single archive and registers its contents as a child resolver. +func (o *Orchestrator) extractArchive(ctx context.Context, ext Extractor, loc file.Location, depth int) (bool, error) { + // create a temp dir for this archive's contents + extractDir, err := os.MkdirTemp(o.tmpDir, fmt.Sprintf("archive-depth%d-*", depth)) + if err != nil { + return false, fmt.Errorf("unable to create temp dir: %w", err) + } + o.cleanups = append(o.cleanups, func() { + os.RemoveAll(extractDir) + }) + + // extractors need a file path, so materialize the archive contents to a temp file + tmpPath, err := o.copyArchiveToTempFile(loc) + if err != nil { + return false, err + } + defer os.Remove(tmpPath) + + // calculate remaining budget + limits := ExtractionLimits{ + MaxExtractionSizeBytes: o.config.MaxExtractionSizeBytes, + MaxFileCount: o.config.MaxFileCount, + } + if o.config.MaxTotalExtractionBytes > 0 { + remaining := o.config.MaxTotalExtractionBytes - o.totalBytesWritten + if remaining <= 0 { + return false, fmt.Errorf("total extraction budget exhausted") + } + if limits.MaxExtractionSizeBytes == 0 || remaining < limits.MaxExtractionSizeBytes { + limits.MaxExtractionSizeBytes = remaining + } + } + + result, err := ext.Extract(ctx, tmpPath, extractDir, limits) + if err != nil { + log.WithFields("archive", loc.Path(), "error", err).Debug("archive extraction stopped") + // partial extraction is still useful, continue if we got some files + if result.FilesExtracted == 0 { + return false, err + } + } + + o.totalBytesWritten += result.BytesWritten + + if result.FilesExtracted == 0 { + return false, nil + } + + // build a directory resolver for the extracted contents + childResolver, resolverErr := o.resolverFactory(extractDir) + if resolverErr != nil { + return false, fmt.Errorf("unable to build resolver for extracted archive: %w", resolverErr) + } + + fsID := o.composite.AddChild(childResolver, loc, depth) + + // track for relationship generation + o.archiveNodes = append(o.archiveNodes, &archiveNode{ + location: loc, + fsID: fsID, + depth: depth, + }) + + log.WithFields( + "archive", loc.Path(), + "depth", depth, + "files", result.FilesExtracted, + "bytes", result.BytesWritten, + ).Debug("extracted archive contents") + + return true, nil +} + +// formatProbeBytes is the size of the buffer peeked from each candidate file +// to decide whether it's an extractable archive. +// +// Single-step formats only need their magic bytes (ZIP: 4, gzip/bzip2/xz/zstd: +// 2-6, tar: a 512-byte first-block header). The size that matters is for +// compressed-tar without a recognized extension: there mholt/archives must +// gzip-decode our peek buffer and feed the result to tar.NewReader.Next(), +// which needs a full 512-byte tar header out the back. 4 KiB of compressed +// input gives gzip a comfortable margin and is still a single filesystem +// block read. +const formatProbeBytes = 4096 + +// detectFormat returns the first enabled extractor that can handle loc by +// peeking a small buffer of its contents (no full materialization). Returns +// nil when no extractor matches or when the file can't be opened. The caller +// is responsible for narrowing extractors to those permitted by config so we +// don't pay peek I/O for archive types that would only be rejected later. +func (o *Orchestrator) detectFormat(ctx context.Context, loc file.Location, extractors []Extractor) Extractor { + reader, err := o.composite.FileContentsByLocation(loc) + if err != nil { + return nil + } + defer internal.CloseAndLogError(reader, loc.RealPath) + + var peek [formatProbeBytes]byte + n, _ := io.ReadFull(reader, peek[:]) + if n == 0 { + return nil + } + for _, ext := range extractors { + if ext.CanExtract(ctx, loc.RealPath, bytes.NewReader(peek[:n])) { + return ext + } + } + return nil +} + +// copyArchiveToTempFile reads the archive contents from the resolver and writes them +// to a freshly-created temp file under o.tmpDir. The returned path must be removed +// by the caller (typically via defer). +func (o *Orchestrator) copyArchiveToTempFile(loc file.Location) (string, error) { + reader, err := o.composite.FileContentsByLocation(loc) + if err != nil { + return "", fmt.Errorf("unable to get archive contents: %w", err) + } + defer internal.CloseAndLogError(reader, loc.RealPath) + + tmpFile, err := os.CreateTemp(o.tmpDir, "archive-src-*") + if err != nil { + return "", fmt.Errorf("unable to create temp file: %w", err) + } + _, copyErr := safeIOCopy(tmpFile, reader) + tmpFile.Close() + if copyErr != nil { + os.Remove(tmpFile.Name()) + return "", fmt.Errorf("unable to copy archive to temp: %w", copyErr) + } + return tmpFile.Name(), nil +} + +// Relationships returns the archive containment relationships discovered during extraction. +func (o *Orchestrator) Relationships() []artifact.Relationship { + o.mu.Lock() + defer o.mu.Unlock() + + var rels []artifact.Relationship + for _, node := range o.archiveNodes { + // Create a "contains" relationship: the archive contains its extracted content + // The archive file (identified by its coordinates) contains the child filesystem + rels = append(rels, artifact.Relationship{ + From: node.location.Coordinates, + To: file.Coordinates{RealPath: "/", FileSystemID: node.fsID}, + Type: artifact.ContainsRelationship, + }) + } + return rels +} + +// Cleanup removes all temporary directories created during extraction. +func (o *Orchestrator) Cleanup() { + o.mu.Lock() + defer o.mu.Unlock() + for _, cleanup := range o.cleanups { + cleanup() + } + o.cleanups = nil +} + +// safeIOCopy copies with a size limit to prevent decompression bombs. +func safeIOCopy(dst *os.File, src interface{ Read([]byte) (int, error) }) (int64, error) { + const maxCopySize = 2 * 1024 * 1024 * 1024 // 2GB + return limitedCopy(dst, src, maxCopySize) +} + +func limitedCopy(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) { + n, err := copyN(dst, src, limit) + if n >= limit { + return n, fmt.Errorf("copy size limit reached (%d bytes)", limit) + } + return n, err +} + +func copyN(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) { + buf := make([]byte, 32*1024) + var written int64 + for written < limit { + nr, readErr := src.Read(buf) + if nr > 0 { + nw, writeErr := dst.Write(buf[:nr]) + written += int64(nw) + if writeErr != nil { + return written, writeErr + } + } + if readErr != nil { + if readErr == io.EOF { + return written, nil + } + return written, readErr + } + } + return written, nil +} diff --git a/internal/archive/orchestrator_test.go b/internal/archive/orchestrator_test.go new file mode 100644 index 000000000..bae414c5e --- /dev/null +++ b/internal/archive/orchestrator_test.go @@ -0,0 +1,537 @@ +package archive + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/file" +) + +// testResolverFactory creates a simple mock resolver factory for testing. +func testResolverFactory(root string) (file.Resolver, error) { + // Walk the directory and create a mock resolver from its contents + files := make(map[string]string) + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + relPath, _ := filepath.Rel(root, path) + content, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + files["/"+relPath] = string(content) + return nil + }) + if err != nil { + return nil, err + } + return newMockResolver(files), nil +} + +func TestOrchestrator_DiscoverAndExtract_ZipArchive(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create a zip file in the "source" directory + zipFiles := map[string]string{ + "inner.txt": "hello from inside the zip", + "lib/a.so": "library content", + } + zipPath := createTestZip(t, dir, zipFiles) + + // Create a parent resolver that sees the zip file + parentFiles := map[string]string{ + "/test.zip": "placeholder", // the resolver sees the file + } + parent := newMockResolver(parentFiles) + + // Override FileContentsByLocation to return actual zip content + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + parent.contents["/test.zip"] = string(zipContent) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, newCount) + + // the composite resolver should now have the child files + resolver := orch.Resolver() + assert.Equal(t, 1, resolver.ChildCount()) +} + +func TestOrchestrator_DiscoverAndExtract_TarGzArchive(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + tarFiles := map[string]string{ + "inner.txt": "hello from tar", + } + tarPath := createTestTarGz(t, dir, tarFiles) + + tarContent, err := os.ReadFile(tarPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.tar.gz": string(tarContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeUnindexedArchives = true // tar needs this enabled + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, newCount) + assert.Equal(t, 1, orch.Resolver().ChildCount()) +} + +func TestOrchestrator_DiscoverAndExtract_DepthLimit(t *testing.T) { + tmpDir := t.TempDir() + ctx := context.Background() + + parent := newMockResolver(map[string]string{ + "/file.txt": "just a file", + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // at depth 1 (== MaxDepth), should extract nothing + newCount := orch.DiscoverAndExtract(ctx, 1) + assert.Equal(t, 0, newCount) +} + +func TestOrchestrator_DiscoverAndExtract_NoArchives(t *testing.T) { + tmpDir := t.TempDir() + ctx := context.Background() + + parent := newMockResolver(map[string]string{ + "/file.txt": "plain text", + "/data.csv": "a,b,c", + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 3 + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) + assert.Equal(t, 0, orch.Resolver().ChildCount()) +} + +func TestOrchestrator_DiscoverAndExtract_ExcludeExtension(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.ExcludeExtensions = []string{".zip"} + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) +} + +func TestOrchestrator_DiscoverAndExtract_TotalSizeLimit(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.MaxTotalExtractionBytes = 1 // 1 byte total limit - will be exceeded + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // should still extract but hit the limit + _ = orch.DiscoverAndExtract(ctx, 0) + // second call should be blocked by total limit + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) +} + +func TestOrchestrator_Relationships(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + orch.DiscoverAndExtract(ctx, 0) + + rels := orch.Relationships() + assert.NotEmpty(t, rels) + + // should have a contains relationship for the archive + for _, rel := range rels { + assert.Equal(t, "contains", string(rel.Type)) + } +} + +func TestOrchestrator_Cleanup(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + + orch.DiscoverAndExtract(ctx, 0) + + // verify temp dirs exist + entries, err := os.ReadDir(tmpDir) + require.NoError(t, err) + initialCount := len(entries) + assert.True(t, initialCount > 0, "expected temp dirs to be created") + + orch.Cleanup() + + // verify temp dirs are cleaned up + entries, err = os.ReadDir(tmpDir) + require.NoError(t, err) + assert.True(t, len(entries) < initialCount, "expected temp dirs to be cleaned up") +} + +func TestOrchestrator_DoesNotReprocess(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 2 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // first call extracts the archive + first := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, first) + + // second call should not re-extract the same archive + second := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, second) + assert.Equal(t, 1, orch.Resolver().ChildCount()) +} + +func TestOrchestrator_NestedArchives_ZipInsideZip(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create inner zip + innerZipPath := createTestZip(t, dir, map[string]string{ + "deep.txt": "deeply nested content", + }) + innerZipContent, err := os.ReadFile(innerZipPath) + require.NoError(t, err) + + // Create outer zip containing the inner zip + outerDir := filepath.Join(dir, "outer") + require.NoError(t, os.MkdirAll(outerDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(outerDir, "inner.zip"), innerZipContent, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(outerDir, "readme.txt"), []byte("readme"), 0o644)) + outerZipPath := createTestZip(t, outerDir, map[string]string{ + "inner.zip": string(innerZipContent), + "readme.txt": "readme", + }) + outerZipContent, err := os.ReadFile(outerZipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/outer.zip": string(outerZipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 3 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Round 1: extract outer.zip + count1 := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 1, count1, "should extract outer.zip") + assert.Equal(t, 1, orch.Resolver().ChildCount()) + + // Round 2: extract inner.zip found inside outer.zip + count2 := orch.DiscoverAndExtract(ctx, 1) + assert.Equal(t, 1, count2, "should extract inner.zip from within outer.zip") + assert.Equal(t, 2, orch.Resolver().ChildCount()) + + // Round 3: no more archives to extract + count3 := orch.DiscoverAndExtract(ctx, 2) + assert.Equal(t, 0, count3, "no more archives to extract") +} + +func TestOrchestrator_ContextCancellation(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + // should handle gracefully without hanging + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.True(t, newCount >= 0) // may be 0 or 1 depending on timing +} + +func TestOrchestrator_MultipleArchivesSameLevel(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zip1Path := createTestZip(t, dir, map[string]string{"file1.txt": "from zip1"}) + zip1Content, err := os.ReadFile(zip1Path) + require.NoError(t, err) + + // Need a different dir to avoid overwriting test.zip + dir2 := t.TempDir() + zip2Path := createTestZip(t, dir2, map[string]string{"file2.txt": "from zip2"}) + zip2Content, err := os.ReadFile(zip2Path) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/archive1.zip": string(zip1Content), + "/archive2.zip": string(zip2Content), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 2, newCount, "should extract both archives") + assert.Equal(t, 2, orch.Resolver().ChildCount()) +} + +func TestOrchestrator_CorruptArchiveSkipped(t *testing.T) { + tmpDir := t.TempDir() + ctx := context.Background() + + // A file with .zip extension but invalid content + parent := newMockResolver(map[string]string{ + "/corrupt.zip": "this is not a valid zip file", + "/normal.txt": "just text", + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Should not panic or error out; corrupt archives should be skipped + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount, "corrupt archive should be skipped") +} + +func TestOrchestrator_EmptyArchive(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create an empty zip + emptyZipPath := createTestZip(t, dir, map[string]string{}) + emptyZipContent, err := os.ReadFile(emptyZipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/empty.zip": string(emptyZipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount, "empty archive should not count as extracted") +} + +func TestOrchestrator_PerArchiveFileLimitApplied(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + // Create a zip with many files + files := map[string]string{} + for i := 0; i < 10; i++ { + files[filepath.Join("dir", string(rune('a'+i))+".txt")] = "content" + } + zipPath := createTestZip(t, dir, files) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/big.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + cfg.MaxFileCount = 3 // only allow 3 files to be extracted + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + // Should partially extract but still register the archive + newCount := orch.DiscoverAndExtract(ctx, 0) + // Extraction will fail after 3 files, but partial extraction counts + assert.True(t, newCount >= 0) +} + +func TestOrchestrator_RelationshipsHaveArchiveCoordinates(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"file.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/archive.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = true + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + orch.DiscoverAndExtract(ctx, 0) + + rels := orch.Relationships() + require.Len(t, rels, 1) + + // verify the relationship structure + rel := rels[0] + assert.Equal(t, "contains", string(rel.Type)) + + // From should be the archive's coordinates + fromCoords, ok := rel.From.(file.Coordinates) + require.True(t, ok) + assert.Equal(t, "/archive.zip", fromCoords.RealPath) +} + +func TestOrchestrator_ZipDisabledByConfig(t *testing.T) { + dir := t.TempDir() + tmpDir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"}) + zipContent, err := os.ReadFile(zipPath) + require.NoError(t, err) + + parent := newMockResolver(map[string]string{ + "/test.zip": string(zipContent), + }) + + cfg := cataloging.DefaultArchiveSearchConfig() + cfg.MaxDepth = 1 + cfg.IncludeIndexedArchives = false // disable zip extraction + + orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory) + defer orch.Cleanup() + + newCount := orch.DiscoverAndExtract(ctx, 0) + assert.Equal(t, 0, newCount) +} diff --git a/internal/archive/resolver.go b/internal/archive/resolver.go new file mode 100644 index 000000000..a9baddd85 --- /dev/null +++ b/internal/archive/resolver.go @@ -0,0 +1,274 @@ +package archive + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "sync" + + "github.com/anchore/syft/syft/file" +) + +const ( + // ArchivePathSeparator is used to encode nested archive paths in AccessPath, + // matching the convention established by the Java cataloger. + ArchivePathSeparator = ":" +) + +// childResolver holds a resolver for an extracted archive along with metadata about the archive. +type childResolver struct { + resolver file.Resolver + archiveLocation file.Location // where the archive file lives in the parent resolver + fsID string // unique filesystem ID for this archive + depth int // nesting depth (0 = base filesystem) +} + +// CompositeResolver implements file.Resolver by compositing a parent resolver with +// child resolvers created from extracted archive contents. Files from child resolvers +// appear transparently alongside files from the parent. +type CompositeResolver struct { + parent file.Resolver + children []*childResolver + mu sync.RWMutex +} + +var _ file.Resolver = (*CompositeResolver)(nil) + +// NewCompositeResolver creates a new CompositeResolver wrapping the given parent resolver. +func NewCompositeResolver(parent file.Resolver) *CompositeResolver { + return &CompositeResolver{ + parent: parent, + } +} + +// AddChild registers a child resolver for an extracted archive. +func (r *CompositeResolver) AddChild(resolver file.Resolver, archiveLocation file.Location, depth int) string { + fsID := generateArchiveFSID(archiveLocation) + r.mu.Lock() + defer r.mu.Unlock() + r.children = append(r.children, &childResolver{ + resolver: resolver, + archiveLocation: archiveLocation, + fsID: fsID, + depth: depth, + }) + return fsID +} + +// ChildCount returns the number of child resolvers registered. +func (r *CompositeResolver) ChildCount() int { + r.mu.RLock() + defer r.mu.RUnlock() + return len(r.children) +} + +// HasPath checks if the path exists in the parent or any child resolver. +func (r *CompositeResolver) HasPath(path string) bool { + if r.parent.HasPath(path) { + return true + } + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + if child.resolver.HasPath(path) { + return true + } + } + return false +} + +// FilesByPath returns locations from the parent and all child resolvers. +func (r *CompositeResolver) FilesByPath(paths ...string) ([]file.Location, error) { + var results []file.Location + + parentLocs, err := r.parent.FilesByPath(paths...) + if err != nil { + return nil, err + } + results = append(results, parentLocs...) + + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + childLocs, err := child.resolver.FilesByPath(paths...) + if err != nil { + continue + } + for _, loc := range childLocs { + results = append(results, r.transformLocation(loc, child)) + } + } + + return results, nil +} + +// FilesByGlob returns locations from the parent and all child resolvers. +func (r *CompositeResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + var results []file.Location + + parentLocs, err := r.parent.FilesByGlob(patterns...) + if err != nil { + return nil, err + } + results = append(results, parentLocs...) + + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + childLocs, err := child.resolver.FilesByGlob(patterns...) + if err != nil { + continue + } + for _, loc := range childLocs { + results = append(results, r.transformLocation(loc, child)) + } + } + + return results, nil +} + +// FilesByMIMEType returns locations from the parent and all child resolvers. +func (r *CompositeResolver) FilesByMIMEType(types ...string) ([]file.Location, error) { + var results []file.Location + + parentLocs, err := r.parent.FilesByMIMEType(types...) + if err != nil { + return nil, err + } + results = append(results, parentLocs...) + + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + childLocs, err := child.resolver.FilesByMIMEType(types...) + if err != nil { + continue + } + for _, loc := range childLocs { + results = append(results, r.transformLocation(loc, child)) + } + } + + return results, nil +} + +// RelativeFileByPath looks up a file relative to a given location. +func (r *CompositeResolver) RelativeFileByPath(location file.Location, path string) *file.Location { + // try parent first + if loc := r.parent.RelativeFileByPath(location, path); loc != nil { + return loc + } + + // try children - match on the archive's FSID + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + if location.FileSystemID == child.fsID { + if loc := child.resolver.RelativeFileByPath(location, path); loc != nil { + transformed := r.transformLocation(*loc, child) + return &transformed + } + } + } + return nil +} + +// FileContentsByLocation routes to the appropriate resolver based on FileSystemID. +func (r *CompositeResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { + // check if this location belongs to a child resolver + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + if location.FileSystemID == child.fsID { + // look up the file by path in the child resolver to get a location with + // a valid internal reference (needed by directory resolvers that use + // stereoscope file references for content lookup) + childLocs, err := child.resolver.FilesByPath(location.RealPath) + if err != nil || len(childLocs) == 0 { + // fallback: try with a constructed location + childLoc := file.NewLocation(location.RealPath) + return child.resolver.FileContentsByLocation(childLoc) + } + return child.resolver.FileContentsByLocation(childLocs[0]) + } + } + + // fall through to parent + return r.parent.FileContentsByLocation(location) +} + +// AllLocations returns locations from the parent and all child resolvers. +func (r *CompositeResolver) AllLocations(ctx context.Context) <-chan file.Location { + results := make(chan file.Location) + go func() { + defer close(results) + + // parent locations + for loc := range r.parent.AllLocations(ctx) { + select { + case <-ctx.Done(): + return + case results <- loc: + } + } + + // child locations + r.mu.RLock() + children := make([]*childResolver, len(r.children)) + copy(children, r.children) + r.mu.RUnlock() + + for _, child := range children { + for loc := range child.resolver.AllLocations(ctx) { + transformed := r.transformLocation(loc, child) + select { + case <-ctx.Done(): + return + case results <- transformed: + } + } + } + }() + return results +} + +// FileMetadataByLocation routes to the appropriate resolver based on FileSystemID. +func (r *CompositeResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) { + r.mu.RLock() + defer r.mu.RUnlock() + for _, child := range r.children { + if location.FileSystemID == child.fsID { + childLocs, err := child.resolver.FilesByPath(location.RealPath) + if err != nil || len(childLocs) == 0 { + childLoc := file.NewLocation(location.RealPath) + return child.resolver.FileMetadataByLocation(childLoc) + } + return child.resolver.FileMetadataByLocation(childLocs[0]) + } + } + return r.parent.FileMetadataByLocation(location) +} + +// transformLocation transforms a location from a child resolver to include the archive context. +func (r *CompositeResolver) transformLocation(loc file.Location, child *childResolver) file.Location { + archivePath := child.archiveLocation.Path() + return file.Location{ + LocationData: file.LocationData{ + Coordinates: file.Coordinates{ + RealPath: loc.RealPath, + FileSystemID: child.fsID, + }, + AccessPath: archivePath + ArchivePathSeparator + loc.Path(), + }, + LocationMetadata: loc.LocationMetadata, + } +} + +// generateArchiveFSID creates a deterministic filesystem ID for an archive based on its location. +func generateArchiveFSID(archiveLocation file.Location) string { + h := sha256.New() + h.Write([]byte(archiveLocation.RealPath)) + h.Write([]byte(archiveLocation.FileSystemID)) + return fmt.Sprintf("archive:%x", h.Sum(nil)[:12]) +} diff --git a/internal/archive/resolver_test.go b/internal/archive/resolver_test.go new file mode 100644 index 000000000..05bf40bb3 --- /dev/null +++ b/internal/archive/resolver_test.go @@ -0,0 +1,584 @@ +package archive + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/file" +) + +// mockResolver implements file.Resolver for testing. +type mockResolver struct { + locations map[string]file.Location // path -> location + contents map[string]string // realPath -> content +} + +func newMockResolver(files map[string]string) *mockResolver { + r := &mockResolver{ + locations: make(map[string]file.Location), + contents: files, + } + for path := range files { + r.locations[path] = file.NewLocation(path) + } + return r +} + +func (m *mockResolver) HasPath(path string) bool { + _, ok := m.locations[path] + return ok +} + +func (m *mockResolver) FilesByPath(paths ...string) ([]file.Location, error) { + var results []file.Location + for _, p := range paths { + if loc, ok := m.locations[p]; ok { + results = append(results, loc) + } + } + return results, nil +} + +func (m *mockResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + var results []file.Location + for _, pattern := range patterns { + for path, loc := range m.locations { + matched, _ := filepath.Match(pattern, path) + if matched { + results = append(results, loc) + } + // also try with the basename for simple glob patterns + if !matched { + matched, _ = filepath.Match(pattern, filepath.Base(path)) + if matched { + results = append(results, loc) + } + } + } + } + return results, nil +} + +func (m *mockResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) { + return nil, nil +} + +func (m *mockResolver) RelativeFileByPath(_ file.Location, path string) *file.Location { + if loc, ok := m.locations[path]; ok { + return &loc + } + return nil +} + +func (m *mockResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { + if content, ok := m.contents[location.RealPath]; ok { + return io.NopCloser(strings.NewReader(content)), nil + } + return nil, os.ErrNotExist +} + +func (m *mockResolver) AllLocations(ctx context.Context) <-chan file.Location { + ch := make(chan file.Location) + go func() { + defer close(ch) + for _, loc := range m.locations { + select { + case <-ctx.Done(): + return + case ch <- loc: + } + } + }() + return ch +} + +func (m *mockResolver) FileMetadataByLocation(_ file.Location) (file.Metadata, error) { + return file.Metadata{}, nil +} + +// strictMockResolver mimics real directory resolvers (fileresolver.Directory) which +// require that FileContentsByLocation receives a location object that was originally +// returned by FilesByPath/FilesByGlob — not a freshly constructed file.NewLocation(). +// Real directory resolvers need the internal stereoscope file.Reference for index lookup. +// We simulate this by annotating returned locations and only accepting those back. +type strictMockResolver struct { + mockResolver + resolvedAnnotation string // annotation key that marks locations as "resolved" +} + +func newStrictMockResolver(files map[string]string) *strictMockResolver { + return &strictMockResolver{ + mockResolver: *newMockResolver(files), + resolvedAnnotation: "strict-resolved", + } +} + +func (s *strictMockResolver) annotate(loc file.Location) file.Location { + return loc.WithAnnotation(s.resolvedAnnotation, "true") +} + +func (s *strictMockResolver) isResolved(loc file.Location) bool { + return loc.Annotations[s.resolvedAnnotation] == "true" +} + +func (s *strictMockResolver) FilesByPath(paths ...string) ([]file.Location, error) { + locs, err := s.mockResolver.FilesByPath(paths...) + for i := range locs { + locs[i] = s.annotate(locs[i]) + } + return locs, err +} + +func (s *strictMockResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + locs, err := s.mockResolver.FilesByGlob(patterns...) + for i := range locs { + locs[i] = s.annotate(locs[i]) + } + return locs, err +} + +func (s *strictMockResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { + if !s.isResolved(location) { + return nil, fmt.Errorf("location %q was not resolved via FilesByPath/FilesByGlob first (simulating real directory resolver behavior)", location.RealPath) + } + return s.mockResolver.FileContentsByLocation(location) +} + +func (s *strictMockResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) { + if !s.isResolved(location) { + return file.Metadata{}, fmt.Errorf("location %q was not resolved via FilesByPath/FilesByGlob first", location.RealPath) + } + return s.mockResolver.FileMetadataByLocation(location) +} + +// TestCompositeResolver_FileContentsByLocation_ChildRequiresLookup tests that +// FileContentsByLocation works when the child resolver requires locations to be +// resolved via FilesByPath first (as real directory resolvers do). This catches +// the bug where we constructed a bare file.NewLocation() without a valid internal +// reference, which fails on real fileresolver.Directory implementations. +func TestCompositeResolver_FileContentsByLocation_ChildRequiresLookup(t *testing.T) { + parent := newMockResolver(map[string]string{}) + child := newStrictMockResolver(map[string]string{ + "/lib/data.txt": "important data", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.zip") + fsID := composite.AddChild(child, archiveLoc, 1) + + // Simulate what a cataloger does: find the file, then read its content. + // The location returned has the archive's fsID, not the child's internal ref. + locs, err := composite.FilesByPath("/lib/data.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + assert.Equal(t, fsID, locs[0].FileSystemID) + + // Now read content - this must work even though the location was transformed + reader, err := composite.FileContentsByLocation(locs[0]) + require.NoError(t, err, "FileContentsByLocation should resolve the file in the child resolver via FilesByPath before reading") + defer reader.Close() + + content, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "important data", string(content)) +} + +// TestCompositeResolver_FileMetadataByLocation_ChildRequiresLookup is the same +// test as above but for FileMetadataByLocation. +func TestCompositeResolver_FileMetadataByLocation_ChildRequiresLookup(t *testing.T) { + parent := newMockResolver(map[string]string{}) + child := newStrictMockResolver(map[string]string{ + "/lib/data.txt": "important data", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.zip") + fsID := composite.AddChild(child, archiveLoc, 1) + + locs, err := composite.FilesByPath("/lib/data.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + assert.Equal(t, fsID, locs[0].FileSystemID) + + // FileMetadataByLocation must also resolve via FilesByPath first + _, err = composite.FileMetadataByLocation(locs[0]) + require.NoError(t, err, "FileMetadataByLocation should resolve the file in the child resolver via FilesByPath before reading metadata") +} + +func TestCompositeResolver_FilesByPath_ParentOnly(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/parent.txt": "parent content", + }) + composite := NewCompositeResolver(parent) + + locs, err := composite.FilesByPath("/parent.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + assert.Equal(t, "/parent.txt", locs[0].RealPath) +} + +func TestCompositeResolver_FilesByPath_WithChild(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/parent.txt": "parent content", + }) + child := newMockResolver(map[string]string{ + "/child.txt": "child content", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.tar.gz") + composite.AddChild(child, archiveLoc, 1) + + locs, err := composite.FilesByPath("/child.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + assert.Contains(t, locs[0].AccessPath, "archive.tar.gz:") + assert.Contains(t, locs[0].AccessPath, "child.txt") +} + +func TestCompositeResolver_FileContentsByLocation_Parent(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/parent.txt": "parent content", + }) + composite := NewCompositeResolver(parent) + + locs, err := composite.FilesByPath("/parent.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + + reader, err := composite.FileContentsByLocation(locs[0]) + require.NoError(t, err) + defer reader.Close() + + content, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "parent content", string(content)) +} + +func TestCompositeResolver_FileContentsByLocation_Child(t *testing.T) { + parent := newMockResolver(map[string]string{}) + child := newMockResolver(map[string]string{ + "/child.txt": "child content", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.zip") + fsID := composite.AddChild(child, archiveLoc, 1) + + // construct a location as returned by FilesByPath + loc := file.Location{ + LocationData: file.LocationData{ + Coordinates: file.Coordinates{ + RealPath: "/child.txt", + FileSystemID: fsID, + }, + AccessPath: "/archive.zip:/child.txt", + }, + } + + reader, err := composite.FileContentsByLocation(loc) + require.NoError(t, err) + defer reader.Close() + + content, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "child content", string(content)) +} + +func TestCompositeResolver_HasPath(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/parent.txt": "p", + }) + child := newMockResolver(map[string]string{ + "/child.txt": "c", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.zip") + composite.AddChild(child, archiveLoc, 1) + + assert.True(t, composite.HasPath("/parent.txt")) + assert.True(t, composite.HasPath("/child.txt")) + assert.False(t, composite.HasPath("/nonexistent.txt")) +} + +func TestCompositeResolver_AllLocations(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/parent.txt": "p", + }) + child := newMockResolver(map[string]string{ + "/child.txt": "c", + }) + + composite := NewCompositeResolver(parent) + archiveLoc := file.NewLocation("/archive.zip") + composite.AddChild(child, archiveLoc, 1) + + ctx := context.Background() + var allLocs []file.Location + for loc := range composite.AllLocations(ctx) { + allLocs = append(allLocs, loc) + } + + assert.Len(t, allLocs, 2) + + hasArchivePrefix := false + for _, loc := range allLocs { + if strings.Contains(loc.AccessPath, "archive.zip:") { + hasArchivePrefix = true + break + } + } + assert.True(t, hasArchivePrefix) +} + +func TestCompositeResolver_ChildCount(t *testing.T) { + parent := newMockResolver(map[string]string{}) + composite := NewCompositeResolver(parent) + assert.Equal(t, 0, composite.ChildCount()) + + child := newMockResolver(map[string]string{"/a.txt": "a"}) + archiveLoc := file.NewLocation("/archive.zip") + composite.AddChild(child, archiveLoc, 1) + assert.Equal(t, 1, composite.ChildCount()) +} + +func TestCompositeResolver_TransformLocation(t *testing.T) { + composite := NewCompositeResolver(nil) + + archiveLoc := file.NewLocation("/path/to/archive.tar.gz") + child := &childResolver{ + archiveLocation: archiveLoc, + fsID: "archive:abc123", + depth: 1, + } + + loc := file.NewLocation("/inner/file.txt") + transformed := composite.transformLocation(loc, child) + + assert.Equal(t, "/inner/file.txt", transformed.RealPath) + assert.Equal(t, "archive:abc123", transformed.FileSystemID) + assert.Equal(t, "/path/to/archive.tar.gz:/inner/file.txt", transformed.AccessPath) +} + +func TestGenerateArchiveFSID(t *testing.T) { + loc1 := file.NewLocation("/archive1.zip") + loc2 := file.NewLocation("/archive2.zip") + + fsID1 := generateArchiveFSID(loc1) + fsID2 := generateArchiveFSID(loc2) + + assert.NotEqual(t, fsID1, fsID2) + assert.Equal(t, fsID1, generateArchiveFSID(loc1)) + assert.True(t, strings.HasPrefix(fsID1, "archive:")) +} + +func TestCompositeResolver_MultipleChildren(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/base.txt": "base", + }) + child1 := newMockResolver(map[string]string{ + "/lib.so": "library1", + }) + child2 := newMockResolver(map[string]string{ + "/lib.so": "library2", + }) + + composite := NewCompositeResolver(parent) + composite.AddChild(child1, file.NewLocation("/archive1.zip"), 1) + composite.AddChild(child2, file.NewLocation("/archive2.zip"), 1) + + assert.Equal(t, 2, composite.ChildCount()) + + // searching for /lib.so should find it in both children + locs, err := composite.FilesByPath("/lib.so") + require.NoError(t, err) + assert.Len(t, locs, 2) + + // each should have different archive prefixes + var accessPaths []string + for _, loc := range locs { + accessPaths = append(accessPaths, loc.AccessPath) + } + assert.Contains(t, accessPaths[0], "archive1.zip:") + assert.Contains(t, accessPaths[1], "archive2.zip:") +} + +func TestCompositeResolver_NestedChildren_ChildInChild(t *testing.T) { + // Simulates: base filesystem -> archive1.tar.gz -> archive2.zip -> inner file + // In practice the orchestrator adds children at different depths, but the resolver + // itself just has a flat list of children. The nesting is encoded in the AccessPath. + parent := newMockResolver(map[string]string{ + "/base.txt": "base", + }) + + // First level: contents of archive1.tar.gz + child1 := newMockResolver(map[string]string{ + "/inner-archive.zip": "zipdata", // this is an archive inside archive1 + "/readme.txt": "readme from archive1", + }) + + // Second level: contents of inner-archive.zip (which was inside archive1.tar.gz) + child2 := newMockResolver(map[string]string{ + "/deep-file.txt": "deeply nested content", + }) + + composite := NewCompositeResolver(parent) + + // Add child1 as extracted from archive1.tar.gz (depth 1) + archive1Loc := file.NewLocation("/archive1.tar.gz") + fsID1 := composite.AddChild(child1, archive1Loc, 1) + + // Add child2 as extracted from inner-archive.zip which was in archive1 (depth 2) + // The archive location for child2 refers to the inner-archive.zip as it appears in child1 + innerArchiveLoc := file.Location{ + LocationData: file.LocationData{ + Coordinates: file.Coordinates{ + RealPath: "/inner-archive.zip", + FileSystemID: fsID1, + }, + AccessPath: "/archive1.tar.gz:/inner-archive.zip", + }, + } + fsID2 := composite.AddChild(child2, innerArchiveLoc, 2) + + assert.Equal(t, 2, composite.ChildCount()) + assert.NotEqual(t, fsID1, fsID2) + + // Should find base file + locs, err := composite.FilesByPath("/base.txt") + require.NoError(t, err) + assert.Len(t, locs, 1) + assert.Equal(t, "/base.txt", locs[0].AccessPath) + + // Should find readme from archive1 + locs, err = composite.FilesByPath("/readme.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + assert.Equal(t, "/archive1.tar.gz:/readme.txt", locs[0].AccessPath) + assert.Equal(t, fsID1, locs[0].FileSystemID) + + // Should find deeply nested file + locs, err = composite.FilesByPath("/deep-file.txt") + require.NoError(t, err) + require.Len(t, locs, 1) + // The access path shows the full chain: outer-archive:inner-archive:file + assert.Equal(t, "/archive1.tar.gz:/inner-archive.zip:/deep-file.txt", locs[0].AccessPath) + assert.Equal(t, fsID2, locs[0].FileSystemID) + + // Should be able to read the deeply nested content + reader, err := composite.FileContentsByLocation(locs[0]) + require.NoError(t, err) + defer reader.Close() + content, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "deeply nested content", string(content)) +} + +func TestCompositeResolver_EmptyChild(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/file.txt": "content", + }) + child := newMockResolver(map[string]string{}) + + composite := NewCompositeResolver(parent) + composite.AddChild(child, file.NewLocation("/empty.zip"), 1) + + // should still find parent files + locs, err := composite.FilesByPath("/file.txt") + require.NoError(t, err) + assert.Len(t, locs, 1) + + // no extra locations from empty child + ctx := context.Background() + var all []file.Location + for loc := range composite.AllLocations(ctx) { + all = append(all, loc) + } + assert.Len(t, all, 1) +} + +func TestCompositeResolver_SameFileInParentAndChild(t *testing.T) { + // File with the same path exists in both parent and child + parent := newMockResolver(map[string]string{ + "/config.yaml": "parent version", + }) + child := newMockResolver(map[string]string{ + "/config.yaml": "child version", + }) + + composite := NewCompositeResolver(parent) + fsID := composite.AddChild(child, file.NewLocation("/archive.zip"), 1) + + locs, err := composite.FilesByPath("/config.yaml") + require.NoError(t, err) + // Should find both versions + assert.Len(t, locs, 2) + + // parent version has no archive prefix + parentLoc := locs[0] + assert.Equal(t, "/config.yaml", parentLoc.AccessPath) + assert.Empty(t, parentLoc.FileSystemID) + + // child version has archive prefix + childLoc := locs[1] + assert.Equal(t, "/archive.zip:/config.yaml", childLoc.AccessPath) + assert.Equal(t, fsID, childLoc.FileSystemID) + + // can read both + r1, err := composite.FileContentsByLocation(parentLoc) + require.NoError(t, err) + c1, _ := io.ReadAll(r1) + r1.Close() + assert.Equal(t, "parent version", string(c1)) + + r2, err := composite.FileContentsByLocation(childLoc) + require.NoError(t, err) + c2, _ := io.ReadAll(r2) + r2.Close() + assert.Equal(t, "child version", string(c2)) +} + +func TestCompositeResolver_AllLocations_ContextCancellation(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/a.txt": "a", + "/b.txt": "b", + "/c.txt": "c", + }) + + composite := NewCompositeResolver(parent) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + var count int + for range composite.AllLocations(ctx) { + count++ + } + // with cancelled context, we may get 0 or some locations, but shouldn't hang + assert.True(t, count <= 3) +} + +func TestCompositeResolver_NoChildren_BehavesLikeParent(t *testing.T) { + parent := newMockResolver(map[string]string{ + "/file.txt": "content", + }) + composite := NewCompositeResolver(parent) + + // FilesByPath + parentLocs, _ := parent.FilesByPath("/file.txt") + compositeLocs, _ := composite.FilesByPath("/file.txt") + assert.Equal(t, len(parentLocs), len(compositeLocs)) + + // HasPath + assert.Equal(t, parent.HasPath("/file.txt"), composite.HasPath("/file.txt")) + assert.Equal(t, parent.HasPath("/nope"), composite.HasPath("/nope")) +} diff --git a/syft/cataloging/archive_search.go b/syft/cataloging/archive_search.go index 56dff6cc6..464913909 100644 --- a/syft/cataloging/archive_search.go +++ b/syft/cataloging/archive_search.go @@ -1,17 +1,43 @@ package cataloging +const ( + DefaultArchiveMaxDepth = 0 // disabled by default for backwards compatibility + DefaultArchiveMaxExtractionSizeBytes = 500 * 1024 * 1024 // 500 MB per archive + DefaultArchiveMaxFileCount = 10000 + DefaultArchiveMaxTotalExtractionBytes = 2 * 1024 * 1024 * 1024 // 2 GB total +) + type ArchiveSearchConfig struct { // IncludeIndexedArchives indicates whether to search within indexed archive files (e.g., .zip). IncludeIndexedArchives bool `yaml:"include-indexed-archives" json:"include-indexed-archives" mapstructure:"include-indexed-archives"` // IncludeUnindexedArchives indicates whether to search within unindexed archive files (e.g., .tar*). IncludeUnindexedArchives bool `yaml:"include-unindexed-archives" json:"include-unindexed-archives" mapstructure:"include-unindexed-archives"` + + // MaxDepth is the maximum depth of recursive archive extraction (0 = disabled, no recursive extraction). + MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"` + + // MaxExtractionSizeBytes is the maximum total bytes to extract from a single archive before stopping. + MaxExtractionSizeBytes int64 `yaml:"max-extraction-size-bytes" json:"max-extraction-size-bytes" mapstructure:"max-extraction-size-bytes"` + + // MaxFileCount is the maximum number of files to extract from a single archive before stopping. + MaxFileCount int `yaml:"max-file-count" json:"max-file-count" mapstructure:"max-file-count"` + + // MaxTotalExtractionBytes is the maximum total bytes to extract across all archives before stopping. + MaxTotalExtractionBytes int64 `yaml:"max-total-extraction-bytes" json:"max-total-extraction-bytes" mapstructure:"max-total-extraction-bytes"` + + // ExcludeExtensions is a list of archive extensions to skip during recursive extraction (e.g., [".rpm"]). + ExcludeExtensions []string `yaml:"exclude-extensions" json:"exclude-extensions" mapstructure:"exclude-extensions"` } func DefaultArchiveSearchConfig() ArchiveSearchConfig { return ArchiveSearchConfig{ IncludeIndexedArchives: true, IncludeUnindexedArchives: false, + MaxDepth: DefaultArchiveMaxDepth, + MaxExtractionSizeBytes: DefaultArchiveMaxExtractionSizeBytes, + MaxFileCount: DefaultArchiveMaxFileCount, + MaxTotalExtractionBytes: DefaultArchiveMaxTotalExtractionBytes, } } @@ -24,3 +50,28 @@ func (c ArchiveSearchConfig) WithIncludeUnindexedArchives(include bool) ArchiveS c.IncludeUnindexedArchives = include return c } + +func (c ArchiveSearchConfig) WithMaxDepth(depth int) ArchiveSearchConfig { + c.MaxDepth = depth + return c +} + +func (c ArchiveSearchConfig) WithMaxExtractionSizeBytes(size int64) ArchiveSearchConfig { + c.MaxExtractionSizeBytes = size + return c +} + +func (c ArchiveSearchConfig) WithMaxFileCount(count int) ArchiveSearchConfig { + c.MaxFileCount = count + return c +} + +func (c ArchiveSearchConfig) WithMaxTotalExtractionBytes(size int64) ArchiveSearchConfig { + c.MaxTotalExtractionBytes = size + return c +} + +func (c ArchiveSearchConfig) WithExcludeExtensions(extensions []string) ArchiveSearchConfig { + c.ExcludeExtensions = extensions + return c +} diff --git a/syft/cataloging/archive_search_test.go b/syft/cataloging/archive_search_test.go new file mode 100644 index 000000000..f15e76b34 --- /dev/null +++ b/syft/cataloging/archive_search_test.go @@ -0,0 +1,44 @@ +package cataloging + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDefaultArchiveSearchConfig(t *testing.T) { + cfg := DefaultArchiveSearchConfig() + + assert.True(t, cfg.IncludeIndexedArchives) + assert.False(t, cfg.IncludeUnindexedArchives) + assert.Equal(t, DefaultArchiveMaxDepth, cfg.MaxDepth) + assert.Equal(t, int64(DefaultArchiveMaxExtractionSizeBytes), cfg.MaxExtractionSizeBytes) + assert.Equal(t, DefaultArchiveMaxFileCount, cfg.MaxFileCount) + assert.Equal(t, int64(DefaultArchiveMaxTotalExtractionBytes), cfg.MaxTotalExtractionBytes) + assert.Nil(t, cfg.ExcludeExtensions) +} + +func TestArchiveSearchConfig_WithMethods(t *testing.T) { + cfg := DefaultArchiveSearchConfig() + + cfg = cfg.WithMaxDepth(5) + assert.Equal(t, 5, cfg.MaxDepth) + + cfg = cfg.WithMaxExtractionSizeBytes(1024) + assert.Equal(t, int64(1024), cfg.MaxExtractionSizeBytes) + + cfg = cfg.WithMaxFileCount(100) + assert.Equal(t, 100, cfg.MaxFileCount) + + cfg = cfg.WithMaxTotalExtractionBytes(2048) + assert.Equal(t, int64(2048), cfg.MaxTotalExtractionBytes) + + cfg = cfg.WithExcludeExtensions([]string{".rpm", ".deb"}) + assert.Equal(t, []string{".rpm", ".deb"}, cfg.ExcludeExtensions) + + cfg = cfg.WithIncludeIndexedArchives(false) + assert.False(t, cfg.IncludeIndexedArchives) + + cfg = cfg.WithIncludeUnindexedArchives(true) + assert.True(t, cfg.IncludeUnindexedArchives) +} diff --git a/syft/configuration_audit_trail.go b/syft/configuration_audit_trail.go index aa7a51f4d..b63eb9be4 100644 --- a/syft/configuration_audit_trail.go +++ b/syft/configuration_audit_trail.go @@ -17,6 +17,7 @@ type configurationAuditTrail struct { Packages pkgcataloging.Config `json:"packages" yaml:"packages" mapstructure:"packages"` Files filecataloging.Config `json:"files" yaml:"files" mapstructure:"files"` Licenses cataloging.LicenseConfig `json:"licenses" yaml:"licenses" mapstructure:"licenses"` + Archive cataloging.ArchiveSearchConfig `json:"archive" yaml:"archive" mapstructure:"archive"` Catalogers catalogerManifest `json:"catalogers" yaml:"catalogers" mapstructure:"catalogers"` ExtraConfigs any `json:"extra,omitempty" yaml:"extra" mapstructure:"extra"` } diff --git a/syft/create_sbom.go b/syft/create_sbom.go index 817e698c6..ff4e1dc4b 100644 --- a/syft/create_sbom.go +++ b/syft/create_sbom.go @@ -10,6 +10,7 @@ import ( "github.com/scylladb/go-set/strset" "github.com/anchore/go-sync" + intArchive "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/log" @@ -19,9 +20,12 @@ import ( "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/event/monitor" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/fileresolver" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/source/directorysource" ) // CreateSBOM creates a software bill-of-materials from the given source. If the CreateSBOMConfig is nil, then @@ -60,6 +64,7 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( Packages: cfg.Packages, Files: cfg.Files, Licenses: cfg.Licenses, + Archive: cfg.Archive, Catalogers: *audit, ExtraConfigs: cfg.ToolConfiguration, }, @@ -92,9 +97,17 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( packageCatalogingProgress := monitorPackageCatalogingTask() builder := sbomsync.NewBuilder(&s, monitorPackageCount(packageCatalogingProgress)) + + // determine the effective resolver: if recursive archive extraction is enabled, + // wrap the resolver with the archive orchestrator + effectiveResolver, archiveCleanup, archiveRelationships := setupArchiveOrchestration(ctx, resolver, cfg) + if archiveCleanup != nil { + defer archiveCleanup() + } + for i := range taskGroups { err = sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(taskGroups[i]), func(t task.Task) (any, error) { - return nil, task.RunTask(ctx, t, resolver, builder, catalogingProgress) + return nil, task.RunTask(ctx, t, effectiveResolver, builder, catalogingProgress) }, nil) if err != nil { // TODO: tie this to the open progress monitors... @@ -102,12 +115,66 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( } } + // add archive containment relationships to the SBOM + if len(archiveRelationships) > 0 { + builder.AddRelationships(archiveRelationships...) + } + packageCatalogingProgress.SetCompleted() catalogingProgress.SetCompleted() return &s, nil } +// setupArchiveOrchestration wraps the resolver with an archive orchestrator when recursive +// archive extraction is enabled (MaxDepth > 0). It performs the iterative extract loop +// and returns the effective resolver, a cleanup function, and any archive relationships. +func setupArchiveOrchestration(ctx context.Context, baseResolver file.Resolver, cfg *CreateSBOMConfig) (file.Resolver, func(), []artifact.Relationship) { + if cfg.Archive.MaxDepth <= 0 { + return baseResolver, nil, nil + } + + td := tmpdir.FromContext(ctx) + if td == nil { + log.Warn("archive orchestration requested but no temp dir available") + return baseResolver, nil, nil + } + + archiveTmpDir, archiveTmpCleanup, err := td.NewChild("archive-extraction") //nolint:gocritic // cleanup is returned to caller, not deferred here + if err != nil { + log.WithFields("error", err).Warn("unable to create temp dir for archive extraction") + return baseResolver, nil, nil + } + + // build user-exclusion path filters once and reuse them for every child + // resolver, so that --exclude patterns also apply inside extracted archives + resolverFactory := func(root string) (file.Resolver, error) { + filters, err := directorysource.GetDirectoryExclusionFunctions(root, append([]string(nil), cfg.Exclusions...)) + if err != nil { + return nil, fmt.Errorf("invalid exclusion patterns for archive child resolver: %w", err) + } + return fileresolver.NewFromDirectory(root, "", filters...) + } + + orch := intArchive.NewOrchestrator(baseResolver, cfg.Archive, archiveTmpDir, resolverFactory) + + // iteratively discover and extract archives up to the configured max depth + for depth := 0; depth < cfg.Archive.MaxDepth; depth++ { + newArchives := orch.DiscoverAndExtract(ctx, depth) + if newArchives == 0 { + break + } + log.WithFields("depth", depth+1, "archives", newArchives).Debug("extracted archives for recursive cataloging") + } + + cleanup := func() { + orch.Cleanup() + archiveTmpCleanup() + } + + return orch.Resolver(), cleanup, orch.Relationships() +} + func setupContext(ctx context.Context, cfg *CreateSBOMConfig) (context.Context, error) { // configure parallel executors ctx = setContextExecutors(ctx, cfg) diff --git a/syft/create_sbom_config.go b/syft/create_sbom_config.go index 1a84c14b6..f08c914de 100644 --- a/syft/create_sbom_config.go +++ b/syft/create_sbom_config.go @@ -28,9 +28,16 @@ type CreateSBOMConfig struct { Packages pkgcataloging.Config Licenses cataloging.LicenseConfig Files filecataloging.Config + Archive cataloging.ArchiveSearchConfig Parallelism int CatalogerSelection cataloging.SelectionRequest + // Exclusions are user-provided glob patterns that the source resolver already + // applies; they are also propagated into child resolvers built during recursive + // archive extraction so that a "--exclude=**/foo" pattern keeps working inside + // extracted archive contents. + Exclusions []string + // audit what tool is being used to generate the SBOM ToolName string ToolVersion string @@ -49,6 +56,7 @@ func DefaultCreateSBOMConfig() *CreateSBOMConfig { Packages: pkgcataloging.DefaultConfig(), Licenses: cataloging.DefaultLicenseConfig(), Files: filecataloging.DefaultConfig(), + Archive: cataloging.DefaultArchiveSearchConfig(), Parallelism: 0, // use default: run in parallel based on number of CPUs packageTaskFactories: task.DefaultPackageTaskFactories(), @@ -153,6 +161,20 @@ func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig { return c } +// WithArchiveConfig allows for defining archive extraction and recursive cataloging parameters. +func (c *CreateSBOMConfig) WithArchiveConfig(cfg cataloging.ArchiveSearchConfig) *CreateSBOMConfig { + c.Archive = cfg + return c +} + +// WithExclusions sets the glob patterns that should be propagated to child resolvers +// constructed during recursive archive extraction. Patterns must follow the same +// "./", "*/", or "**/"-prefix convention as the syft --exclude flag. +func (c *CreateSBOMConfig) WithExclusions(exclusions []string) *CreateSBOMConfig { + c.Exclusions = exclusions + return c +} + // WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag. func (c *CreateSBOMConfig) WithCatalogerSelection(selection cataloging.SelectionRequest) *CreateSBOMConfig { c.CatalogerSelection = selection diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 572dfed8e..357b426a6 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -3,6 +3,7 @@ package java import ( "context" "errors" + "path/filepath" "strings" "github.com/anchore/syft/internal" @@ -40,6 +41,9 @@ func (p pomXMLCataloger) Catalog(ctx context.Context, fileResolver file.Resolver var poms []*maven.Project pomLocations := map[*maven.Project]file.Location{} for _, pomLocation := range locations { + if isArchiveMetaPom(pomLocation) { + continue + } pom, err := readPomFromLocation(fileResolver, pomLocation) if err != nil || pom == nil { log.WithFields("error", err, "pomLocation", pomLocation).Debug("error while reading pom") @@ -299,6 +303,20 @@ func pomParent(ctx context.Context, r *maven.Resolver, pom *maven.Project) *pkg. } } +// isArchiveMetaPom returns true if the pom.xml location is inside a META-INF/maven directory, +// indicating it is archive metadata embedded by the Maven build process rather than a project +// pom.xml. These embedded pom.xml files are verbatim copies of the original build POM with +// potentially unresolved ${property} references and dependency versions inherited from parent +// POMs that are not available inside the archive. The java-archive-cataloger already uses +// pom.properties (which always contains resolved values) to identify these archives, so the +// pom cataloger should skip them to avoid creating phantom dependency packages. +func isArchiveMetaPom(location file.Location) bool { + p := location.Path() + // normalize to forward slashes for consistent matching + p = filepath.ToSlash(p) + return strings.Contains(p, "META-INF/maven/") +} + func cleanDescription(original string) (cleaned string) { descriptionLines := strings.Split(original, "\n") for _, line := range descriptionLines { diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 44dc5da05..94fde2542 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -690,6 +690,123 @@ func getCommonsTextExpectedPackages(resolved bool) expected { return expected{pkgs, relationships} } +func Test_isArchiveMetaPom(t *testing.T) { + tests := []struct { + name string + path string + expected bool + }{ + { + name: "META-INF maven pom", + path: "META-INF/maven/com.example/my-lib/pom.xml", + expected: true, + }, + { + name: "nested META-INF maven pom", + path: "some/path/META-INF/maven/org.apache/commons/pom.xml", + expected: true, + }, + { + name: "project pom.xml", + path: "pom.xml", + expected: false, + }, + { + name: "module pom.xml", + path: "submodule/pom.xml", + expected: false, + }, + { + name: "META-INF but not maven", + path: "META-INF/pom.xml", + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loc := file.NewLocation(test.path) + assert.Equal(t, test.expected, isArchiveMetaPom(loc)) + }) + } +} + +func Test_pomCatalogerSkipsMetaInfPoms(t *testing.T) { + // A directory that only contains a pom.xml under META-INF/maven/ should produce + // no packages, since these are archive metadata not project files. + cat := NewPomCataloger(ArchiveCatalogerConfig{ + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: true, + }, + }) + + pkgtest.NewCatalogTester(). + FromDirectory(t, "testdata/pom/meta-inf-archive"). + Expects(nil, nil). + TestCataloger(t, cat) +} + +func Test_pomCatalogerSkipsMetaInfButKeepsProjectPom(t *testing.T) { + // When a directory contains both a project pom.xml and a META-INF/maven pom.xml, + // only the project pom.xml should be cataloged. The META-INF pom.xml and its + // dependencies should be ignored. + pomLocation := file.NewLocationSet(file.NewLocation("pom.xml")) + + myApp := pkg.Package{ + Name: "my-app", + Version: "2.0.0", + PURL: "pkg:maven/org.anchore/my-app@2.0.0", + Language: pkg.Java, + Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, + Locations: pomLocation, + Metadata: pkg.JavaArchive{ + PomProject: &pkg.JavaPomProject{ + GroupID: "org.anchore", + ArtifactID: "my-app", + Version: "2.0.0", + }, + }, + } + finalizePackage(&myApp) + + guava := pkg.Package{ + Name: "guava", + Version: "31.1-jre", + PURL: "pkg:maven/com.google.guava/guava@31.1-jre", + Language: pkg.Java, + Type: pkg.JavaPkg, + FoundBy: pomCatalogerName, + Locations: pomLocation, + Metadata: pkg.JavaArchive{ + PomProperties: &pkg.JavaPomProperties{ + GroupID: "com.google.guava", + ArtifactID: "guava", + }, + }, + } + finalizePackage(&guava) + + expectedPkgs := []pkg.Package{myApp, guava} + expectedRelationships := []artifact.Relationship{ + { + From: guava, + To: myApp, + Type: artifact.DependencyOfRelationship, + }, + } + + cat := NewPomCataloger(ArchiveCatalogerConfig{ + ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ + IncludeIndexedArchives: true, + IncludeUnindexedArchives: true, + }, + }) + + pkgtest.TestCataloger(t, "testdata/pom/mixed-meta-inf-and-project", cat, expectedPkgs, expectedRelationships) +} + func expectedTransientPackageData() expected { epl2 := pkg.NewLicenseSet(pkg.License{ Value: "Eclipse Public License v2.0", diff --git a/syft/pkg/cataloger/java/testdata/pom/meta-inf-archive/META-INF/maven/com.example/my-lib/pom.xml b/syft/pkg/cataloger/java/testdata/pom/meta-inf-archive/META-INF/maven/com.example/my-lib/pom.xml new file mode 100644 index 000000000..ff3d28844 --- /dev/null +++ b/syft/pkg/cataloger/java/testdata/pom/meta-inf-archive/META-INF/maven/com.example/my-lib/pom.xml @@ -0,0 +1,24 @@ + + + 4.0.0 + + com.example + my-lib + 1.0.0 + + + + org.phantom + phantom-dep + ${phantom.version} + + + org.testonly + test-dep + 2.0.0 + test + + + diff --git a/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/META-INF/maven/com.example/embedded-lib/pom.xml b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/META-INF/maven/com.example/embedded-lib/pom.xml new file mode 100644 index 000000000..cf2ab1b72 --- /dev/null +++ b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/META-INF/maven/com.example/embedded-lib/pom.xml @@ -0,0 +1,18 @@ + + + 4.0.0 + + com.example + embedded-lib + 3.0.0 + + + + org.phantom + should-not-appear + ${unresolvable.version} + + + diff --git a/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/pom.xml b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/pom.xml new file mode 100644 index 000000000..32b8e2426 --- /dev/null +++ b/syft/pkg/cataloger/java/testdata/pom/mixed-meta-inf-and-project/pom.xml @@ -0,0 +1,18 @@ + + + 4.0.0 + + org.anchore + my-app + 2.0.0 + + + + com.google.guava + guava + 31.1-jre + + +