mirror of
https://github.com/anchore/syft.git
synced 2026-11-01 23:16:30 +01:00
Merge PR #4761: recursive archive cataloging (orchestrator + composite resolver)
Adds internal/archive package (extractor, CompositeResolver, Orchestrator), wires it into CreateSBOM, plus java pom-cataloger fixes. Original work by @toabctl in anchore/syft#4761.
This commit is contained in:
@@ -0,0 +1,285 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
)
|
||||
|
||||
// Extractor extracts archive contents to a destination directory.
|
||||
type Extractor interface {
|
||||
// CanExtract returns true if this extractor can handle the given file.
|
||||
CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool
|
||||
|
||||
// Extract extracts the archive contents to destDir, respecting the given limits.
|
||||
// Returns the number of files extracted and total bytes written.
|
||||
Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error)
|
||||
}
|
||||
|
||||
// ExtractionLimits defines safety limits for archive extraction.
|
||||
type ExtractionLimits struct {
|
||||
MaxExtractionSizeBytes int64
|
||||
MaxFileCount int
|
||||
}
|
||||
|
||||
// ExtractionResult holds the result of an extraction operation.
|
||||
type ExtractionResult struct {
|
||||
FilesExtracted int
|
||||
BytesWritten int64
|
||||
}
|
||||
|
||||
// DefaultExtractionLimits returns limits from the given config.
|
||||
func DefaultExtractionLimits(cfg cataloging.ArchiveSearchConfig) ExtractionLimits {
|
||||
return ExtractionLimits{
|
||||
MaxExtractionSizeBytes: cfg.MaxExtractionSizeBytes,
|
||||
MaxFileCount: cfg.MaxFileCount,
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultExtractors returns the set of built-in archive extractors.
|
||||
func DefaultExtractors() []Extractor {
|
||||
return []Extractor{
|
||||
&ZipExtractor{},
|
||||
&TarExtractor{},
|
||||
}
|
||||
}
|
||||
|
||||
// FindExtractor finds an extractor that can handle the given file, or nil if none can.
|
||||
func FindExtractor(ctx context.Context, extractors []Extractor, path string) Extractor {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
log.Tracef("unable to open file for archive detection: %v", err)
|
||||
return nil
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
for _, ext := range extractors {
|
||||
if ext.CanExtract(ctx, path, f) {
|
||||
return ext
|
||||
}
|
||||
// reset for next extractor
|
||||
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||
log.Tracef("unable to seek file for archive detection: %v", err)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsExcludedExtension checks if a file path has an excluded extension.
|
||||
func IsExcludedExtension(path string, excludeExtensions []string) bool {
|
||||
for _, ext := range excludeExtensions {
|
||||
if strings.HasSuffix(strings.ToLower(path), strings.ToLower(ext)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ZipExtractor extracts zip-based archives (zip, jar, war, ear, etc.).
|
||||
type ZipExtractor struct{}
|
||||
|
||||
func (z *ZipExtractor) CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool {
|
||||
format, _, err := intFile.IdentifyArchive(ctx, path, reader)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_, ok := format.(archives.Zip)
|
||||
return ok
|
||||
}
|
||||
|
||||
func (z *ZipExtractor) Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) {
|
||||
var result ExtractionResult
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("unable to open zip archive %q: %w", path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
return result, archives.Zip{}.Extract(ctx, f, func(_ context.Context, file archives.FileInfo) error {
|
||||
if file.IsDir() {
|
||||
joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.MkdirAll(joinedPath, 0o755)
|
||||
}
|
||||
|
||||
if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount {
|
||||
return fmt.Errorf("archive file count limit reached (%d files)", limits.MaxFileCount)
|
||||
}
|
||||
|
||||
joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(joinedPath), 0o755); err != nil {
|
||||
return fmt.Errorf("unable to create parent dir: %w", err)
|
||||
}
|
||||
|
||||
return extractFileWithLimits(file, joinedPath, destDir, &result, limits)
|
||||
})
|
||||
}
|
||||
|
||||
// TarExtractor extracts tar-based archives (tar, tar.gz, tar.bz2, tar.xz, tar.zst).
|
||||
type TarExtractor struct{}
|
||||
|
||||
func (t *TarExtractor) CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool {
|
||||
format, _, err := intFile.IdentifyArchive(ctx, path, reader)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
// mholt/archives returns a compound type for tar+compression, check if it's an extractor but not a zip
|
||||
if _, isZip := format.(archives.Zip); isZip {
|
||||
return false
|
||||
}
|
||||
_, ok := format.(archives.Extractor)
|
||||
return ok
|
||||
}
|
||||
|
||||
func (t *TarExtractor) Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) {
|
||||
var result ExtractionResult
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("unable to open tar archive %q: %w", path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
format, readerAfterIdentify, err := intFile.IdentifyArchive(ctx, path, f)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("unable to identify archive format for %q: %w", path, err)
|
||||
}
|
||||
|
||||
extractor, ok := format.(archives.Extractor)
|
||||
if !ok {
|
||||
return result, fmt.Errorf("file format does not support extraction: %s", path)
|
||||
}
|
||||
|
||||
// use the reader returned by IdentifyArchive since it may have consumed some bytes
|
||||
extractReader := readerAfterIdentify
|
||||
if extractReader == nil {
|
||||
extractReader = f
|
||||
}
|
||||
|
||||
return result, extractor.Extract(ctx, extractReader, func(_ context.Context, file archives.FileInfo) error {
|
||||
if file.IsDir() {
|
||||
joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.MkdirAll(joinedPath, 0o755)
|
||||
}
|
||||
|
||||
if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount {
|
||||
return fmt.Errorf("archive file count limit reached (%d files)", limits.MaxFileCount)
|
||||
}
|
||||
|
||||
joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(joinedPath), 0o755); err != nil {
|
||||
return fmt.Errorf("unable to create parent dir: %w", err)
|
||||
}
|
||||
|
||||
return extractFileWithLimits(file, joinedPath, destDir, &result, limits)
|
||||
})
|
||||
}
|
||||
|
||||
// extractFileWithLimits extracts a single file from an archive, respecting size limits.
|
||||
// Symlinks are written as real symlinks (only when the target stays inside destDir);
|
||||
// non-regular file types other than symlinks are skipped to avoid writing bogus content.
|
||||
func extractFileWithLimits(file archives.FileInfo, destPath, destDir string, result *ExtractionResult, limits ExtractionLimits) error {
|
||||
mode := file.Mode()
|
||||
if mode.Type()&fs.ModeSymlink != 0 {
|
||||
if err := writeSafeSymlink(file.LinkTarget, destPath, destDir); err != nil {
|
||||
log.WithFields("entry", file.NameInArchive, "target", file.LinkTarget, "error", err).Debug("skipping unsafe symlink in archive")
|
||||
return nil
|
||||
}
|
||||
result.FilesExtracted++
|
||||
return nil
|
||||
}
|
||||
if !mode.IsRegular() {
|
||||
// skip device nodes, fifos, sockets, etc. — they can't safely round-trip
|
||||
log.WithFields("entry", file.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry")
|
||||
return nil
|
||||
}
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open archive entry %q: %w", file.NameInArchive, err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
dst, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create dest file %q: %w", destPath, err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
var reader io.Reader = src
|
||||
|
||||
// enforce per-archive total size limit
|
||||
if limits.MaxExtractionSizeBytes > 0 {
|
||||
remaining := limits.MaxExtractionSizeBytes - result.BytesWritten
|
||||
if remaining <= 0 {
|
||||
return fmt.Errorf("archive extraction size limit reached (%d bytes)", limits.MaxExtractionSizeBytes)
|
||||
}
|
||||
reader = io.LimitReader(src, remaining+1) // +1 to detect exceeding the limit
|
||||
}
|
||||
|
||||
n, err := io.Copy(dst, reader)
|
||||
result.BytesWritten += n
|
||||
result.FilesExtracted++
|
||||
|
||||
if limits.MaxExtractionSizeBytes > 0 && result.BytesWritten > limits.MaxExtractionSizeBytes {
|
||||
return fmt.Errorf("archive extraction size limit reached (%d bytes)", limits.MaxExtractionSizeBytes)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// writeSafeSymlink creates a symlink at destPath only if target is relative and
|
||||
// resolves — relative to destPath's directory — to a location inside destDir.
|
||||
//
|
||||
// Absolute targets are rejected outright: os.Symlink writes the literal target
|
||||
// string, so an absolute target like "/etc/passwd" would resolve on the host
|
||||
// filesystem when read, regardless of any safety check we performed against
|
||||
// destDir at extraction time.
|
||||
//
|
||||
// NOTE: this guards against the single-symlink-escape pattern, but a malicious
|
||||
// archive can still attempt a multi-step chain where two cooperating link
|
||||
// entries together resolve outside destDir (e.g., one entry creates a benign
|
||||
// link, a later entry writes through it). Fully preventing that requires
|
||||
// either openat/O_NOFOLLOW per path component during extraction or
|
||||
// transitively resolving every newly-created symlink against all prior ones
|
||||
// — neither is implemented here.
|
||||
func writeSafeSymlink(target, destPath, destDir string) error {
|
||||
if target == "" {
|
||||
return fmt.Errorf("empty link target")
|
||||
}
|
||||
if filepath.IsAbs(target) {
|
||||
return fmt.Errorf("absolute symlink target not allowed")
|
||||
}
|
||||
resolved := filepath.Join(filepath.Dir(destPath), target)
|
||||
cleanRoot := filepath.Clean(destDir)
|
||||
cleanResolved := filepath.Clean(resolved)
|
||||
if cleanResolved != cleanRoot && !strings.HasPrefix(cleanResolved, cleanRoot+string(filepath.Separator)) {
|
||||
return fmt.Errorf("symlink target escapes extraction root")
|
||||
}
|
||||
return os.Symlink(target, destPath)
|
||||
}
|
||||
@@ -0,0 +1,525 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
)
|
||||
|
||||
func createTestZip(t *testing.T, dir string, files map[string]string) string {
|
||||
t.Helper()
|
||||
zipPath := filepath.Join(dir, "test.zip")
|
||||
f, err := os.Create(zipPath)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
w := zip.NewWriter(f)
|
||||
for name, content := range files {
|
||||
fw, err := w.Create(name)
|
||||
require.NoError(t, err)
|
||||
_, err = fw.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, w.Close())
|
||||
return zipPath
|
||||
}
|
||||
|
||||
func createTestTarGz(t *testing.T, dir string, files map[string]string) string {
|
||||
t.Helper()
|
||||
tarPath := filepath.Join(dir, "test.tar.gz")
|
||||
f, err := os.Create(tarPath)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
gw := gzip.NewWriter(f)
|
||||
tw := tar.NewWriter(gw)
|
||||
|
||||
for name, content := range files {
|
||||
hdr := &tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(content)),
|
||||
}
|
||||
require.NoError(t, tw.WriteHeader(hdr))
|
||||
_, err = tw.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
return tarPath
|
||||
}
|
||||
|
||||
func TestZipExtractor_CanExtract(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"})
|
||||
|
||||
ext := &ZipExtractor{}
|
||||
|
||||
f, err := os.Open(zipPath)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
assert.True(t, ext.CanExtract(ctx, zipPath, f))
|
||||
|
||||
// non-zip file
|
||||
nonZipPath := filepath.Join(dir, "notazip.txt")
|
||||
require.NoError(t, os.WriteFile(nonZipPath, []byte("just text"), 0o644))
|
||||
|
||||
f2, err := os.Open(nonZipPath)
|
||||
require.NoError(t, err)
|
||||
defer f2.Close()
|
||||
|
||||
assert.False(t, ext.CanExtract(ctx, nonZipPath, f2))
|
||||
}
|
||||
|
||||
func TestZipExtractor_Extract(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
files := map[string]string{
|
||||
"file1.txt": "content1",
|
||||
"dir/file2.txt": "content2",
|
||||
}
|
||||
zipPath := createTestZip(t, dir, files)
|
||||
|
||||
ext := &ZipExtractor{}
|
||||
destDir := filepath.Join(dir, "extracted")
|
||||
require.NoError(t, os.MkdirAll(destDir, 0o755))
|
||||
|
||||
result, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, result.FilesExtracted)
|
||||
|
||||
// verify extracted files
|
||||
content, err := os.ReadFile(filepath.Join(destDir, "file1.txt"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "content1", string(content))
|
||||
|
||||
content, err = os.ReadFile(filepath.Join(destDir, "dir", "file2.txt"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "content2", string(content))
|
||||
}
|
||||
|
||||
func TestZipExtractor_Extract_FileLimitReached(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
files := map[string]string{
|
||||
"file1.txt": "a",
|
||||
"file2.txt": "b",
|
||||
"file3.txt": "c",
|
||||
}
|
||||
zipPath := createTestZip(t, dir, files)
|
||||
|
||||
ext := &ZipExtractor{}
|
||||
destDir := filepath.Join(dir, "extracted")
|
||||
require.NoError(t, os.MkdirAll(destDir, 0o755))
|
||||
|
||||
_, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{MaxFileCount: 2})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "file count limit")
|
||||
}
|
||||
|
||||
func TestZipExtractor_Extract_SizeLimitReached(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
files := map[string]string{
|
||||
"file1.txt": strings.Repeat("x", 100),
|
||||
}
|
||||
zipPath := createTestZip(t, dir, files)
|
||||
|
||||
ext := &ZipExtractor{}
|
||||
destDir := filepath.Join(dir, "extracted")
|
||||
require.NoError(t, os.MkdirAll(destDir, 0o755))
|
||||
|
||||
_, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{MaxExtractionSizeBytes: 50})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "size limit")
|
||||
}
|
||||
|
||||
func TestTarExtractor_CanExtract(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"})
|
||||
|
||||
ext := &TarExtractor{}
|
||||
|
||||
f, err := os.Open(tarPath)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
assert.True(t, ext.CanExtract(ctx, tarPath, f))
|
||||
}
|
||||
|
||||
func TestTarExtractor_Extract(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
files := map[string]string{
|
||||
"file1.txt": "content1",
|
||||
"dir/file2.txt": "content2",
|
||||
}
|
||||
tarPath := createTestTarGz(t, dir, files)
|
||||
|
||||
ext := &TarExtractor{}
|
||||
destDir := filepath.Join(dir, "extracted")
|
||||
require.NoError(t, os.MkdirAll(destDir, 0o755))
|
||||
|
||||
result, err := ext.Extract(ctx, tarPath, destDir, ExtractionLimits{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, result.FilesExtracted)
|
||||
|
||||
// verify extracted files
|
||||
content, err := os.ReadFile(filepath.Join(destDir, "file1.txt"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "content1", string(content))
|
||||
|
||||
content, err = os.ReadFile(filepath.Join(destDir, "dir", "file2.txt"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "content2", string(content))
|
||||
}
|
||||
|
||||
func TestFindExtractor(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
extractors := DefaultExtractors()
|
||||
|
||||
// zip file should match
|
||||
zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"})
|
||||
ext := FindExtractor(ctx, extractors, zipPath)
|
||||
require.NotNil(t, ext)
|
||||
assert.IsType(t, &ZipExtractor{}, ext)
|
||||
|
||||
// tar.gz should match
|
||||
tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"})
|
||||
ext = FindExtractor(ctx, extractors, tarPath)
|
||||
require.NotNil(t, ext)
|
||||
assert.IsType(t, &TarExtractor{}, ext)
|
||||
|
||||
// non-archive should return nil
|
||||
textPath := filepath.Join(dir, "plain.txt")
|
||||
require.NoError(t, os.WriteFile(textPath, []byte("just text"), 0o644))
|
||||
ext = FindExtractor(ctx, extractors, textPath)
|
||||
assert.Nil(t, ext)
|
||||
}
|
||||
|
||||
func TestIsExcludedExtension(t *testing.T) {
|
||||
assert.True(t, IsExcludedExtension("/path/to/file.rpm", []string{".rpm", ".deb"}))
|
||||
assert.True(t, IsExcludedExtension("/path/to/file.deb", []string{".rpm", ".deb"}))
|
||||
assert.False(t, IsExcludedExtension("/path/to/file.zip", []string{".rpm", ".deb"}))
|
||||
assert.False(t, IsExcludedExtension("/path/to/file.zip", nil))
|
||||
}
|
||||
|
||||
func TestFindExtractor_NoReaders(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ext := FindExtractor(ctx, DefaultExtractors(), "/nonexistent/path")
|
||||
assert.Nil(t, ext)
|
||||
}
|
||||
|
||||
func TestTarExtractor_CanExtract_NonTarFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// create a plain text file
|
||||
path := filepath.Join(dir, "plain.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("not a tar"), 0o644))
|
||||
|
||||
ext := &TarExtractor{}
|
||||
f, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
assert.False(t, ext.CanExtract(ctx, path, f))
|
||||
}
|
||||
|
||||
func TestTarExtractor_CanExtract_ZipFileReturnsFalse(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"})
|
||||
|
||||
ext := &TarExtractor{}
|
||||
f, err := os.Open(zipPath)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
// tar extractor should NOT claim to handle zip files
|
||||
assert.False(t, ext.CanExtract(ctx, zipPath, f))
|
||||
}
|
||||
|
||||
func TestDefaultExtractionLimits(t *testing.T) {
|
||||
cfg := cataloging.ArchiveSearchConfig{
|
||||
MaxExtractionSizeBytes: 1024,
|
||||
MaxFileCount: 50,
|
||||
}
|
||||
limits := DefaultExtractionLimits(cfg)
|
||||
assert.Equal(t, int64(1024), limits.MaxExtractionSizeBytes)
|
||||
assert.Equal(t, 50, limits.MaxFileCount)
|
||||
}
|
||||
|
||||
func TestZipExtractor_Extract_ZipSlipPrevented(t *testing.T) {
|
||||
// Create a zip with a path traversal entry
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := filepath.Join(dir, "evil.zip")
|
||||
f, err := os.Create(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
w := zip.NewWriter(f)
|
||||
fw, err := w.Create("../../etc/passwd")
|
||||
require.NoError(t, err)
|
||||
_, err = fw.Write([]byte("evil"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, w.Close())
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
ext := &ZipExtractor{}
|
||||
destDir := filepath.Join(dir, "extracted")
|
||||
require.NoError(t, os.MkdirAll(destDir, 0o755))
|
||||
|
||||
_, err = ext.Extract(ctx, zipPath, destDir, ExtractionLimits{})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "path traversal")
|
||||
|
||||
// verify evil file was not created
|
||||
_, statErr := os.Stat(filepath.Join(dir, "etc", "passwd"))
|
||||
assert.True(t, os.IsNotExist(statErr))
|
||||
}
|
||||
|
||||
func TestTarExtractor_Extract_SymlinkInsideRootIsCreated(t *testing.T) {
|
||||
// A tar archive with a symlink whose target stays inside the extraction
|
||||
// directory should be written as a real symlink, not as a regular file
|
||||
// containing the link target as bytes.
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
tarPath := filepath.Join(dir, "with-link.tar.gz")
|
||||
f, err := os.Create(tarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
gw := gzip.NewWriter(f)
|
||||
tw := tar.NewWriter(gw)
|
||||
|
||||
target := []byte("real content")
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{
|
||||
Name: "real.txt",
|
||||
Mode: 0o644,
|
||||
Size: int64(len(target)),
|
||||
}))
|
||||
_, err = tw.Write(target)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{
|
||||
Name: "link.txt",
|
||||
Linkname: "real.txt",
|
||||
Typeflag: tar.TypeSymlink,
|
||||
Mode: 0o777,
|
||||
}))
|
||||
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
ext := &TarExtractor{}
|
||||
destDir := filepath.Join(dir, "extracted")
|
||||
require.NoError(t, os.MkdirAll(destDir, 0o755))
|
||||
|
||||
_, err = ext.Extract(ctx, tarPath, destDir, ExtractionLimits{})
|
||||
require.NoError(t, err)
|
||||
|
||||
// the link should be a real symlink
|
||||
info, err := os.Lstat(filepath.Join(destDir, "link.txt"))
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, info.Mode()&fs.ModeSymlink, "expected link.txt to be a symlink, got mode %v", info.Mode())
|
||||
|
||||
// reading through the link should return the target's content
|
||||
readBack, err := os.ReadFile(filepath.Join(destDir, "link.txt"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "real content", string(readBack))
|
||||
}
|
||||
|
||||
func TestTarExtractor_Extract_SymlinkEscapingRootIsSkipped(t *testing.T) {
|
||||
// A symlink whose target resolves outside the extraction directory must not
|
||||
// be created — neither as a symlink nor as a regular file. Extraction itself
|
||||
// should succeed (the unsafe entry is skipped, not fatal).
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
tarPath := filepath.Join(dir, "evil-link.tar.gz")
|
||||
f, err := os.Create(tarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
gw := gzip.NewWriter(f)
|
||||
tw := tar.NewWriter(gw)
|
||||
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{
|
||||
Name: "passwd",
|
||||
Linkname: "../../../../etc/passwd",
|
||||
Typeflag: tar.TypeSymlink,
|
||||
Mode: 0o777,
|
||||
}))
|
||||
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
ext := &TarExtractor{}
|
||||
destDir := filepath.Join(dir, "extracted")
|
||||
require.NoError(t, os.MkdirAll(destDir, 0o755))
|
||||
|
||||
_, err = ext.Extract(ctx, tarPath, destDir, ExtractionLimits{})
|
||||
require.NoError(t, err)
|
||||
|
||||
// nothing should have been created at the destination
|
||||
_, statErr := os.Lstat(filepath.Join(destDir, "passwd"))
|
||||
assert.True(t, os.IsNotExist(statErr), "expected no entry to be created for escaping symlink")
|
||||
}
|
||||
|
||||
func TestTarExtractor_Extract_AbsoluteSymlinkTargetIsSkipped(t *testing.T) {
|
||||
// Absolute symlink targets must be rejected: os.Symlink writes the literal
|
||||
// target, so once the symlink is read it resolves on the host filesystem
|
||||
// regardless of any safety check at extraction time. Verifying both that
|
||||
// the symlink itself is not created and (importantly) that nothing was
|
||||
// written under destDir along the absolute path.
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
tarPath := filepath.Join(dir, "absolute-link.tar.gz")
|
||||
f, err := os.Create(tarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
gw := gzip.NewWriter(f)
|
||||
tw := tar.NewWriter(gw)
|
||||
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{
|
||||
Name: "shadow",
|
||||
Linkname: "/etc/shadow",
|
||||
Typeflag: tar.TypeSymlink,
|
||||
Mode: 0o777,
|
||||
}))
|
||||
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
ext := &TarExtractor{}
|
||||
destDir := filepath.Join(dir, "extracted")
|
||||
require.NoError(t, os.MkdirAll(destDir, 0o755))
|
||||
|
||||
_, err = ext.Extract(ctx, tarPath, destDir, ExtractionLimits{})
|
||||
require.NoError(t, err)
|
||||
|
||||
// the symlink itself must not exist
|
||||
_, statErr := os.Lstat(filepath.Join(destDir, "shadow"))
|
||||
assert.True(t, os.IsNotExist(statErr), "expected absolute-target symlink to be skipped")
|
||||
}
|
||||
|
||||
func TestZipExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) {
|
||||
// JARs may contain directory entries with read-only permissions (e.g., META-INF/ with mode 0o555).
|
||||
// The extractor must still be able to write files into those directories.
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := filepath.Join(dir, "readonly-dirs.zip")
|
||||
f, err := os.Create(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
w := zip.NewWriter(f)
|
||||
|
||||
// create a directory entry with read-only permissions (0o555)
|
||||
dirHeader := &zip.FileHeader{
|
||||
Name: "META-INF/",
|
||||
}
|
||||
dirHeader.SetMode(0o555)
|
||||
_, err = w.CreateHeader(dirHeader)
|
||||
require.NoError(t, err)
|
||||
|
||||
// create a file inside that directory
|
||||
fileHeader := &zip.FileHeader{
|
||||
Name: "META-INF/MANIFEST.MF",
|
||||
}
|
||||
fileHeader.SetMode(0o644)
|
||||
fw, err := w.CreateHeader(fileHeader)
|
||||
require.NoError(t, err)
|
||||
_, err = fw.Write([]byte("Manifest-Version: 1.0"))
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, w.Close())
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
ext := &ZipExtractor{}
|
||||
destDir := filepath.Join(dir, "extracted")
|
||||
require.NoError(t, os.MkdirAll(destDir, 0o755))
|
||||
|
||||
result, err := ext.Extract(ctx, zipPath, destDir, ExtractionLimits{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, result.FilesExtracted)
|
||||
|
||||
content, err := os.ReadFile(filepath.Join(destDir, "META-INF", "MANIFEST.MF"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Manifest-Version: 1.0", string(content))
|
||||
}
|
||||
|
||||
func TestTarExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) {
|
||||
// Tar archives may contain directory entries with read-only permissions.
|
||||
// The extractor must still be able to write files into those directories.
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
tarPath := filepath.Join(dir, "readonly-dirs.tar.gz")
|
||||
f, err := os.Create(tarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
gw := gzip.NewWriter(f)
|
||||
tw := tar.NewWriter(gw)
|
||||
|
||||
// write a directory entry with read-only permissions
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{
|
||||
Name: "META-INF/",
|
||||
Typeflag: tar.TypeDir,
|
||||
Mode: 0o555,
|
||||
}))
|
||||
|
||||
// write a file inside that directory
|
||||
content := []byte("Manifest-Version: 1.0")
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{
|
||||
Name: "META-INF/MANIFEST.MF",
|
||||
Mode: 0o644,
|
||||
Size: int64(len(content)),
|
||||
}))
|
||||
_, err = tw.Write(content)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
ext := &TarExtractor{}
|
||||
destDir := filepath.Join(dir, "extracted")
|
||||
require.NoError(t, os.MkdirAll(destDir, 0o755))
|
||||
|
||||
result, err := ext.Extract(ctx, tarPath, destDir, ExtractionLimits{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, result.FilesExtracted)
|
||||
|
||||
extracted, err := os.ReadFile(filepath.Join(destDir, "META-INF", "MANIFEST.MF"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Manifest-Version: 1.0", string(extracted))
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
)
|
||||
|
||||
// TestIntegration_TarGzContainingPythonPackage tests the scenario where a tar.gz
|
||||
// archive contains Python package metadata files that would normally be found by
|
||||
// the Python cataloger. This verifies the orchestrator makes those files visible.
|
||||
func TestIntegration_TarGzContainingPythonPackage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a tar.gz containing Python dist-info metadata
|
||||
tarPath := filepath.Join(dir, "python-packages.tar.gz")
|
||||
createTarGzWithFiles(t, tarPath, map[string]string{
|
||||
"lib/python3.9/site-packages/requests-2.28.0.dist-info/METADATA": `Metadata-Version: 2.1
|
||||
Name: requests
|
||||
Version: 2.28.0
|
||||
Summary: Python HTTP for Humans.
|
||||
`,
|
||||
"lib/python3.9/site-packages/requests-2.28.0.dist-info/RECORD": "requests/__init__.py,sha256=abc,1234",
|
||||
"lib/python3.9/site-packages/requests/__init__.py": "# requests library",
|
||||
})
|
||||
tarContent, err := os.ReadFile(tarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Base resolver sees the tar.gz as a single opaque file
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/app/python-packages.tar.gz": string(tarContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeUnindexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Extract the archive
|
||||
count := orch.DiscoverAndExtract(ctx, 0)
|
||||
require.Equal(t, 1, count, "should extract the tar.gz")
|
||||
|
||||
// The Python METADATA file should now be visible through the composite resolver
|
||||
resolver := orch.Resolver()
|
||||
locs, err := resolver.FilesByPath("/lib/python3.9/site-packages/requests-2.28.0.dist-info/METADATA")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "METADATA file should be visible through the composite resolver")
|
||||
|
||||
// Verify the access path shows it came from the archive
|
||||
assert.Contains(t, locs[0].AccessPath, "python-packages.tar.gz:")
|
||||
assert.Contains(t, locs[0].AccessPath, "METADATA")
|
||||
|
||||
// Verify we can read the content
|
||||
reader, err := resolver.FileContentsByLocation(locs[0])
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
|
||||
buf := make([]byte, 1024)
|
||||
n, _ := reader.Read(buf)
|
||||
assert.Contains(t, string(buf[:n]), "Name: requests")
|
||||
}
|
||||
|
||||
// TestIntegration_ZipContainingJar tests a zip archive containing a JAR file,
|
||||
// which itself should be extractable in a second round.
|
||||
func TestIntegration_ZipContainingJar(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an inner JAR (which is just a zip) with a MANIFEST.MF
|
||||
innerJarPath := filepath.Join(dir, "inner.jar")
|
||||
createZipWithFiles(t, innerJarPath, map[string]string{
|
||||
"META-INF/MANIFEST.MF": `Manifest-Version: 1.0
|
||||
Implementation-Title: example-lib
|
||||
Implementation-Version: 1.2.3
|
||||
`,
|
||||
})
|
||||
innerJarContent, err := os.ReadFile(innerJarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create an outer zip containing the JAR and other files
|
||||
outerZipPath := filepath.Join(dir, "app-bundle.zip")
|
||||
createZipWithFiles(t, outerZipPath, map[string]string{
|
||||
"lib/example-lib-1.2.3.jar": string(innerJarContent),
|
||||
"README.md": "# Application Bundle",
|
||||
})
|
||||
outerZipContent, err := os.ReadFile(outerZipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/downloads/app-bundle.zip": string(outerZipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 2
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Round 1: extract outer zip
|
||||
count1 := orch.DiscoverAndExtract(ctx, 0)
|
||||
require.Equal(t, 1, count1, "should extract outer zip")
|
||||
|
||||
// Round 2: extract inner JAR found inside the zip
|
||||
count2 := orch.DiscoverAndExtract(ctx, 1)
|
||||
require.Equal(t, 1, count2, "should extract inner JAR")
|
||||
|
||||
resolver := orch.Resolver()
|
||||
|
||||
// The MANIFEST.MF from inside the JAR should be visible
|
||||
locs, err := resolver.FilesByPath("/META-INF/MANIFEST.MF")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "MANIFEST.MF should be visible from nested JAR")
|
||||
|
||||
// The access path should show the full nesting chain
|
||||
accessPath := locs[0].AccessPath
|
||||
assert.Contains(t, accessPath, "app-bundle.zip:")
|
||||
assert.Contains(t, accessPath, "example-lib-1.2.3.jar:")
|
||||
assert.Contains(t, accessPath, "MANIFEST.MF")
|
||||
|
||||
// Verify relationships
|
||||
rels := orch.Relationships()
|
||||
assert.Len(t, rels, 2, "should have relationships for both extracted archives")
|
||||
}
|
||||
|
||||
// TestIntegration_MixedArchiveTypes tests a directory containing both
|
||||
// zip and tar.gz archives that should be extracted in a single round.
|
||||
func TestIntegration_MixedArchiveTypes(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a zip
|
||||
zipPath := createTestZip(t, dir, map[string]string{
|
||||
"from-zip.txt": "zip content",
|
||||
})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a tar.gz
|
||||
tarPath := createTestTarGz(t, dir, map[string]string{
|
||||
"from-tar.txt": "tar content",
|
||||
})
|
||||
tarContent, err := os.ReadFile(tarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/data/archive.zip": string(zipContent),
|
||||
"/data/archive.tar.gz": string(tarContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.IncludeUnindexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
count := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 2, count, "should extract both archives")
|
||||
|
||||
resolver := orch.Resolver()
|
||||
|
||||
// Both files should be visible
|
||||
zipLocs, err := resolver.FilesByPath("/from-zip.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, zipLocs, 1, "file from zip should be visible")
|
||||
|
||||
tarLocs, err := resolver.FilesByPath("/from-tar.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, tarLocs, 1, "file from tar should be visible")
|
||||
}
|
||||
|
||||
// TestIntegration_DepthLimitPreventsDeepExtraction verifies that the
|
||||
// depth limit correctly stops extraction at the configured level.
|
||||
func TestIntegration_DepthLimitPreventsDeepExtraction(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a deeply nested archive: zip -> zip -> zip -> file
|
||||
innermostDir := filepath.Join(dir, "inner")
|
||||
require.NoError(t, os.MkdirAll(innermostDir, 0o755))
|
||||
innermost := createTestZip(t, innermostDir, map[string]string{
|
||||
"secret.txt": "you found me!",
|
||||
})
|
||||
innermostContent, err := os.ReadFile(innermost)
|
||||
require.NoError(t, err)
|
||||
|
||||
middleDir := filepath.Join(dir, "middle")
|
||||
require.NoError(t, os.MkdirAll(middleDir, 0o755))
|
||||
middle := createTestZip(t, middleDir, map[string]string{
|
||||
"innermost.zip": string(innermostContent),
|
||||
})
|
||||
middleContent, err := os.ReadFile(middle)
|
||||
require.NoError(t, err)
|
||||
|
||||
outerDir := filepath.Join(dir, "outer")
|
||||
require.NoError(t, os.MkdirAll(outerDir, 0o755))
|
||||
outerZip := createTestZip(t, outerDir, map[string]string{
|
||||
"middle.zip": string(middleContent),
|
||||
})
|
||||
outerContent, err := os.ReadFile(outerZip)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/nested.zip": string(outerContent),
|
||||
})
|
||||
|
||||
// Only allow depth 1 — should only extract the outer zip
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
count := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 1, count, "should only extract the outer zip")
|
||||
|
||||
// middle.zip should be visible but secret.txt should NOT be
|
||||
resolver := orch.Resolver()
|
||||
|
||||
middleLocs, err := resolver.FilesByPath("/middle.zip")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, middleLocs, 1, "middle.zip should be visible at depth 1")
|
||||
|
||||
secretLocs, err := resolver.FilesByPath("/secret.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, secretLocs, 0, "secret.txt should NOT be visible (depth limit)")
|
||||
}
|
||||
|
||||
// TestIntegration_ExcludeExtensionPreventsExtraction verifies that
|
||||
// excluded extensions are respected.
|
||||
func TestIntegration_ExcludeExtensionPreventsExtraction(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{
|
||||
"data.txt": "from zip",
|
||||
})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/archive.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.ExcludeExtensions = []string{".zip"} // exclude zip files
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
count := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, count, "zip should be excluded by extension")
|
||||
}
|
||||
|
||||
// TestIntegration_MultiLevelMixedFormats tests a 3-level nesting chain with
|
||||
// different archive formats: tar.gz -> zip -> tar.gz -> files.
|
||||
// This is the most realistic cross-format recursive extraction scenario.
|
||||
func TestIntegration_MultiLevelMixedFormats(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Level 3 (deepest): a tar.gz containing application files
|
||||
level3Dir := filepath.Join(dir, "level3")
|
||||
require.NoError(t, os.MkdirAll(level3Dir, 0o755))
|
||||
level3Path := filepath.Join(level3Dir, "app-data.tar.gz")
|
||||
createTarGzWithFiles(t, level3Path, map[string]string{
|
||||
"config.yaml": "app: myapp\nversion: 1.0",
|
||||
"bin/run.sh": "#!/bin/sh\necho hello",
|
||||
})
|
||||
level3Content, err := os.ReadFile(level3Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Level 2: a zip containing the level3 tar.gz and some metadata
|
||||
level2Dir := filepath.Join(dir, "level2")
|
||||
require.NoError(t, os.MkdirAll(level2Dir, 0o755))
|
||||
level2Path := filepath.Join(level2Dir, "package.zip")
|
||||
createZipWithFiles(t, level2Path, map[string]string{
|
||||
"app-data.tar.gz": string(level3Content),
|
||||
"package-info.json": `{"name": "mypackage", "version": "2.0"}`,
|
||||
})
|
||||
level2Content, err := os.ReadFile(level2Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Level 1 (outermost): a tar.gz containing the level2 zip
|
||||
level1Dir := filepath.Join(dir, "level1")
|
||||
require.NoError(t, os.MkdirAll(level1Dir, 0o755))
|
||||
level1Path := filepath.Join(level1Dir, "distribution.tar.gz")
|
||||
createTarGzWithFiles(t, level1Path, map[string]string{
|
||||
"package.zip": string(level2Content),
|
||||
"LICENSE": "MIT License",
|
||||
})
|
||||
level1Content, err := os.ReadFile(level1Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/distribution.tar.gz": string(level1Content),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 3
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.IncludeUnindexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Round 1: extract distribution.tar.gz
|
||||
count1 := orch.DiscoverAndExtract(ctx, 0)
|
||||
require.Equal(t, 1, count1, "round 1: should extract distribution.tar.gz")
|
||||
|
||||
// Round 2: extract package.zip from inside distribution.tar.gz
|
||||
count2 := orch.DiscoverAndExtract(ctx, 1)
|
||||
require.Equal(t, 1, count2, "round 2: should extract package.zip")
|
||||
|
||||
// Round 3: extract app-data.tar.gz from inside package.zip
|
||||
count3 := orch.DiscoverAndExtract(ctx, 2)
|
||||
require.Equal(t, 1, count3, "round 3: should extract app-data.tar.gz")
|
||||
|
||||
resolver := orch.Resolver()
|
||||
|
||||
// Verify files from all levels are accessible:
|
||||
|
||||
// Level 1: LICENSE from distribution.tar.gz
|
||||
locs, err := resolver.FilesByPath("/LICENSE")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "LICENSE from level 1 should be visible")
|
||||
assert.Contains(t, locs[0].AccessPath, "distribution.tar.gz:")
|
||||
|
||||
// Level 2: package-info.json from package.zip
|
||||
locs, err = resolver.FilesByPath("/package-info.json")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "package-info.json from level 2 should be visible")
|
||||
assert.Contains(t, locs[0].AccessPath, "package.zip:")
|
||||
|
||||
// Level 3: config.yaml from app-data.tar.gz
|
||||
locs, err = resolver.FilesByPath("/config.yaml")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "config.yaml from level 3 should be visible")
|
||||
assert.Contains(t, locs[0].AccessPath, "app-data.tar.gz:")
|
||||
|
||||
// Level 3: bin/run.sh from app-data.tar.gz
|
||||
locs, err = resolver.FilesByPath("/bin/run.sh")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "bin/run.sh from level 3 should be visible")
|
||||
|
||||
// Verify we can read deeply nested content
|
||||
reader, err := resolver.FileContentsByLocation(locs[0])
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
buf := make([]byte, 1024)
|
||||
n, _ := reader.Read(buf)
|
||||
assert.Contains(t, string(buf[:n]), "echo hello")
|
||||
|
||||
// Verify relationships cover all 3 levels
|
||||
rels := orch.Relationships()
|
||||
assert.Len(t, rels, 3, "should have 3 relationships (one per extracted archive)")
|
||||
}
|
||||
|
||||
// TestIntegration_TarInsideZipInsideTar tests tar -> zip -> tar nesting
|
||||
// (the reverse of the above test), ensuring bidirectional format crossing.
|
||||
func TestIntegration_TarInsideZipInsideTar(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Innermost: a tar.gz with a single file
|
||||
innerTarDir := filepath.Join(dir, "innertar")
|
||||
require.NoError(t, os.MkdirAll(innerTarDir, 0o755))
|
||||
innerTarPath := filepath.Join(innerTarDir, "inner.tar.gz")
|
||||
createTarGzWithFiles(t, innerTarPath, map[string]string{
|
||||
"deeply-nested.txt": "found at the bottom",
|
||||
})
|
||||
innerTarContent, err := os.ReadFile(innerTarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Middle: a zip containing the inner tar
|
||||
middleZipDir := filepath.Join(dir, "middlezip")
|
||||
require.NoError(t, os.MkdirAll(middleZipDir, 0o755))
|
||||
middleZipPath := filepath.Join(middleZipDir, "middle.zip")
|
||||
createZipWithFiles(t, middleZipPath, map[string]string{
|
||||
"inner.tar.gz": string(innerTarContent),
|
||||
"middle.txt": "middle level",
|
||||
})
|
||||
middleZipContent, err := os.ReadFile(middleZipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Outermost: a tar.gz containing the middle zip
|
||||
outerTarDir := filepath.Join(dir, "outertar")
|
||||
require.NoError(t, os.MkdirAll(outerTarDir, 0o755))
|
||||
outerTarPath := filepath.Join(outerTarDir, "outer.tar.gz")
|
||||
createTarGzWithFiles(t, outerTarPath, map[string]string{
|
||||
"middle.zip": string(middleZipContent),
|
||||
"outer.txt": "outer level",
|
||||
})
|
||||
outerTarContent, err := os.ReadFile(outerTarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/outer.tar.gz": string(outerTarContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 3
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.IncludeUnindexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Extract all 3 levels
|
||||
count1 := orch.DiscoverAndExtract(ctx, 0)
|
||||
require.Equal(t, 1, count1)
|
||||
count2 := orch.DiscoverAndExtract(ctx, 1)
|
||||
require.Equal(t, 1, count2)
|
||||
count3 := orch.DiscoverAndExtract(ctx, 2)
|
||||
require.Equal(t, 1, count3)
|
||||
|
||||
resolver := orch.Resolver()
|
||||
|
||||
// Check all levels are accessible
|
||||
outerLocs, err := resolver.FilesByPath("/outer.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, outerLocs, 1)
|
||||
|
||||
middleLocs, err := resolver.FilesByPath("/middle.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, middleLocs, 1)
|
||||
|
||||
deepLocs, err := resolver.FilesByPath("/deeply-nested.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deepLocs, 1)
|
||||
|
||||
// Read the deeply nested content
|
||||
reader, err := resolver.FileContentsByLocation(deepLocs[0])
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
buf := make([]byte, 1024)
|
||||
n, _ := reader.Read(buf)
|
||||
assert.Equal(t, "found at the bottom", string(buf[:n]))
|
||||
|
||||
// The access path should show the full chain: tar -> zip -> tar -> file
|
||||
accessPath := deepLocs[0].AccessPath
|
||||
assert.Contains(t, accessPath, "outer.tar.gz:")
|
||||
assert.Contains(t, accessPath, "middle.zip:")
|
||||
assert.Contains(t, accessPath, "inner.tar.gz:")
|
||||
assert.Contains(t, accessPath, "deeply-nested.txt")
|
||||
}
|
||||
|
||||
// createZipWithFiles creates a zip at the given path with the given files.
|
||||
func createZipWithFiles(t *testing.T, zipPath string, files map[string]string) {
|
||||
t.Helper()
|
||||
f, err := os.Create(zipPath)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
w := zip.NewWriter(f)
|
||||
for name, content := range files {
|
||||
fw, err := w.Create(name)
|
||||
require.NoError(t, err)
|
||||
_, err = fw.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, w.Close())
|
||||
}
|
||||
|
||||
// createTarGzWithFiles creates a tar.gz at the given path with the given files.
|
||||
func createTarGzWithFiles(t *testing.T, tarPath string, files map[string]string) {
|
||||
t.Helper()
|
||||
f, err := os.Create(tarPath)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
gw := gzip.NewWriter(f)
|
||||
tw := tar.NewWriter(gw)
|
||||
|
||||
// sort keys for deterministic output
|
||||
for name, content := range files {
|
||||
// create parent dirs as needed
|
||||
dir := filepath.Dir(name)
|
||||
if dir != "." {
|
||||
hdr := &tar.Header{
|
||||
Typeflag: tar.TypeDir,
|
||||
Name: dir + "/",
|
||||
Mode: 0o755,
|
||||
}
|
||||
require.NoError(t, tw.WriteHeader(hdr))
|
||||
}
|
||||
|
||||
hdr := &tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(content)),
|
||||
}
|
||||
require.NoError(t, tw.WriteHeader(hdr))
|
||||
_, err = tw.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,362 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// ResolverFactory creates a file.Resolver for a directory path.
|
||||
// This is injectable to avoid direct dependency on syft/internal/fileresolver.
|
||||
type ResolverFactory func(root string) (file.Resolver, error)
|
||||
|
||||
// Orchestrator manages iterative archive extraction and cataloging. It discovers
|
||||
// archives in the file resolver, extracts them, builds child resolvers for their
|
||||
// contents, and tracks the relationships between archives and their contents.
|
||||
type Orchestrator struct {
|
||||
composite *CompositeResolver
|
||||
extractors []Extractor
|
||||
config cataloging.ArchiveSearchConfig
|
||||
tmpDir string
|
||||
resolverFactory ResolverFactory
|
||||
|
||||
// tracking state
|
||||
processed map[processedKey]bool // archives we've already considered for extraction
|
||||
totalBytesWritten int64
|
||||
archiveNodes []*archiveNode // for relationship tracking
|
||||
cleanups []func()
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// archiveNode tracks an extracted archive for relationship generation.
|
||||
type archiveNode struct {
|
||||
location file.Location // where the archive is in the resolver
|
||||
fsID string // the filesystem ID assigned to this archive's contents
|
||||
depth int
|
||||
}
|
||||
|
||||
// processedKey identifies an archive location uniquely. We key the processed
|
||||
// map on a struct (not a "<fsID>:<realPath>" string) because both fields can
|
||||
// legally contain ':' — fsIDs are emitted as "archive:<hex>" and POSIX paths
|
||||
// permit colons — so a string key would have an ambiguous boundary.
|
||||
type processedKey struct {
|
||||
fsID string
|
||||
realPath string
|
||||
}
|
||||
|
||||
// NewOrchestrator creates an Orchestrator that manages archive extraction.
|
||||
// The resolverFactory is used to build file.Resolvers for extracted archive directories.
|
||||
func NewOrchestrator(baseResolver file.Resolver, cfg cataloging.ArchiveSearchConfig, tmpDir string, resolverFactory ResolverFactory) *Orchestrator {
|
||||
return &Orchestrator{
|
||||
composite: NewCompositeResolver(baseResolver),
|
||||
extractors: DefaultExtractors(),
|
||||
config: cfg,
|
||||
tmpDir: tmpDir,
|
||||
resolverFactory: resolverFactory,
|
||||
processed: make(map[processedKey]bool),
|
||||
}
|
||||
}
|
||||
|
||||
// Resolver returns the composite resolver that includes all extracted archive contents.
|
||||
func (o *Orchestrator) Resolver() *CompositeResolver {
|
||||
return o.composite
|
||||
}
|
||||
|
||||
// DiscoverAndExtract finds archives in the current resolver view, extracts them,
|
||||
// and adds their contents as child resolvers. Returns the number of new archives extracted.
|
||||
// The depth parameter indicates the current nesting depth (0 = scanning base filesystem).
|
||||
func (o *Orchestrator) DiscoverAndExtract(ctx context.Context, depth int) int {
|
||||
if depth >= o.config.MaxDepth {
|
||||
return 0
|
||||
}
|
||||
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
|
||||
if o.config.MaxTotalExtractionBytes > 0 && o.totalBytesWritten >= o.config.MaxTotalExtractionBytes {
|
||||
log.Debug("archive orchestrator: total extraction size limit reached, stopping")
|
||||
return 0
|
||||
}
|
||||
|
||||
// build the list of extractors actually permitted by config — every
|
||||
// non-archive location in the resolver is going to pay a peek read against
|
||||
// these, so excluding disabled types up-front avoids reading files we
|
||||
// would only end up rejecting after detection.
|
||||
enabled := o.enabledExtractors()
|
||||
if len(enabled) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
newArchives := 0
|
||||
|
||||
// scan all locations in the current resolver for archives
|
||||
for loc := range o.composite.AllLocations(ctx) {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
// skip already-processed archives
|
||||
key := processedKey{fsID: loc.FileSystemID, realPath: loc.RealPath}
|
||||
if o.processed[key] {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip excluded extensions
|
||||
if IsExcludedExtension(loc.RealPath, o.config.ExcludeExtensions) {
|
||||
continue
|
||||
}
|
||||
|
||||
// check if this location is an archive we can extract — peek a small
|
||||
// buffer rather than materializing the full file content, which would
|
||||
// be a full disk-to-disk copy of every non-archive file in the resolver.
|
||||
ext := o.detectFormat(ctx, loc, enabled)
|
||||
if ext == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// mark as processed before extraction to avoid re-processing
|
||||
o.processed[key] = true
|
||||
|
||||
// extract the archive
|
||||
extracted, err := o.extractArchive(ctx, ext, loc, depth+1)
|
||||
if err != nil {
|
||||
log.WithFields("archive", loc.Path(), "error", err).Debug("failed to extract archive")
|
||||
continue
|
||||
}
|
||||
if extracted {
|
||||
newArchives++
|
||||
}
|
||||
}
|
||||
|
||||
return newArchives
|
||||
}
|
||||
|
||||
// enabledExtractors returns the subset of o.extractors whose corresponding
|
||||
// config gate is on (IncludeIndexedArchives for ZipExtractor,
|
||||
// IncludeUnindexedArchives for TarExtractor).
|
||||
func (o *Orchestrator) enabledExtractors() []Extractor {
|
||||
var out []Extractor
|
||||
for _, ext := range o.extractors {
|
||||
switch ext.(type) {
|
||||
case *ZipExtractor:
|
||||
if o.config.IncludeIndexedArchives {
|
||||
out = append(out, ext)
|
||||
}
|
||||
case *TarExtractor:
|
||||
if o.config.IncludeUnindexedArchives {
|
||||
out = append(out, ext)
|
||||
}
|
||||
default:
|
||||
// unknown extractor types: include by default so plugin extractors
|
||||
// added in the future aren't silently disabled
|
||||
out = append(out, ext)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// extractArchive extracts a single archive and registers its contents as a child resolver.
|
||||
func (o *Orchestrator) extractArchive(ctx context.Context, ext Extractor, loc file.Location, depth int) (bool, error) {
|
||||
// create a temp dir for this archive's contents
|
||||
extractDir, err := os.MkdirTemp(o.tmpDir, fmt.Sprintf("archive-depth%d-*", depth))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("unable to create temp dir: %w", err)
|
||||
}
|
||||
o.cleanups = append(o.cleanups, func() {
|
||||
os.RemoveAll(extractDir)
|
||||
})
|
||||
|
||||
// extractors need a file path, so materialize the archive contents to a temp file
|
||||
tmpPath, err := o.copyArchiveToTempFile(loc)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
// calculate remaining budget
|
||||
limits := ExtractionLimits{
|
||||
MaxExtractionSizeBytes: o.config.MaxExtractionSizeBytes,
|
||||
MaxFileCount: o.config.MaxFileCount,
|
||||
}
|
||||
if o.config.MaxTotalExtractionBytes > 0 {
|
||||
remaining := o.config.MaxTotalExtractionBytes - o.totalBytesWritten
|
||||
if remaining <= 0 {
|
||||
return false, fmt.Errorf("total extraction budget exhausted")
|
||||
}
|
||||
if limits.MaxExtractionSizeBytes == 0 || remaining < limits.MaxExtractionSizeBytes {
|
||||
limits.MaxExtractionSizeBytes = remaining
|
||||
}
|
||||
}
|
||||
|
||||
result, err := ext.Extract(ctx, tmpPath, extractDir, limits)
|
||||
if err != nil {
|
||||
log.WithFields("archive", loc.Path(), "error", err).Debug("archive extraction stopped")
|
||||
// partial extraction is still useful, continue if we got some files
|
||||
if result.FilesExtracted == 0 {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
o.totalBytesWritten += result.BytesWritten
|
||||
|
||||
if result.FilesExtracted == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// build a directory resolver for the extracted contents
|
||||
childResolver, resolverErr := o.resolverFactory(extractDir)
|
||||
if resolverErr != nil {
|
||||
return false, fmt.Errorf("unable to build resolver for extracted archive: %w", resolverErr)
|
||||
}
|
||||
|
||||
fsID := o.composite.AddChild(childResolver, loc, depth)
|
||||
|
||||
// track for relationship generation
|
||||
o.archiveNodes = append(o.archiveNodes, &archiveNode{
|
||||
location: loc,
|
||||
fsID: fsID,
|
||||
depth: depth,
|
||||
})
|
||||
|
||||
log.WithFields(
|
||||
"archive", loc.Path(),
|
||||
"depth", depth,
|
||||
"files", result.FilesExtracted,
|
||||
"bytes", result.BytesWritten,
|
||||
).Debug("extracted archive contents")
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// formatProbeBytes is the size of the buffer peeked from each candidate file
|
||||
// to decide whether it's an extractable archive.
|
||||
//
|
||||
// Single-step formats only need their magic bytes (ZIP: 4, gzip/bzip2/xz/zstd:
|
||||
// 2-6, tar: a 512-byte first-block header). The size that matters is for
|
||||
// compressed-tar without a recognized extension: there mholt/archives must
|
||||
// gzip-decode our peek buffer and feed the result to tar.NewReader.Next(),
|
||||
// which needs a full 512-byte tar header out the back. 4 KiB of compressed
|
||||
// input gives gzip a comfortable margin and is still a single filesystem
|
||||
// block read.
|
||||
const formatProbeBytes = 4096
|
||||
|
||||
// detectFormat returns the first enabled extractor that can handle loc by
|
||||
// peeking a small buffer of its contents (no full materialization). Returns
|
||||
// nil when no extractor matches or when the file can't be opened. The caller
|
||||
// is responsible for narrowing extractors to those permitted by config so we
|
||||
// don't pay peek I/O for archive types that would only be rejected later.
|
||||
func (o *Orchestrator) detectFormat(ctx context.Context, loc file.Location, extractors []Extractor) Extractor {
|
||||
reader, err := o.composite.FileContentsByLocation(loc)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer internal.CloseAndLogError(reader, loc.RealPath)
|
||||
|
||||
var peek [formatProbeBytes]byte
|
||||
n, _ := io.ReadFull(reader, peek[:])
|
||||
if n == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, ext := range extractors {
|
||||
if ext.CanExtract(ctx, loc.RealPath, bytes.NewReader(peek[:n])) {
|
||||
return ext
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyArchiveToTempFile reads the archive contents from the resolver and writes them
|
||||
// to a freshly-created temp file under o.tmpDir. The returned path must be removed
|
||||
// by the caller (typically via defer).
|
||||
func (o *Orchestrator) copyArchiveToTempFile(loc file.Location) (string, error) {
|
||||
reader, err := o.composite.FileContentsByLocation(loc)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to get archive contents: %w", err)
|
||||
}
|
||||
defer internal.CloseAndLogError(reader, loc.RealPath)
|
||||
|
||||
tmpFile, err := os.CreateTemp(o.tmpDir, "archive-src-*")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to create temp file: %w", err)
|
||||
}
|
||||
_, copyErr := safeIOCopy(tmpFile, reader)
|
||||
tmpFile.Close()
|
||||
if copyErr != nil {
|
||||
os.Remove(tmpFile.Name())
|
||||
return "", fmt.Errorf("unable to copy archive to temp: %w", copyErr)
|
||||
}
|
||||
return tmpFile.Name(), nil
|
||||
}
|
||||
|
||||
// Relationships returns the archive containment relationships discovered during extraction.
|
||||
func (o *Orchestrator) Relationships() []artifact.Relationship {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
|
||||
var rels []artifact.Relationship
|
||||
for _, node := range o.archiveNodes {
|
||||
// Create a "contains" relationship: the archive contains its extracted content
|
||||
// The archive file (identified by its coordinates) contains the child filesystem
|
||||
rels = append(rels, artifact.Relationship{
|
||||
From: node.location.Coordinates,
|
||||
To: file.Coordinates{RealPath: "/", FileSystemID: node.fsID},
|
||||
Type: artifact.ContainsRelationship,
|
||||
})
|
||||
}
|
||||
return rels
|
||||
}
|
||||
|
||||
// Cleanup removes all temporary directories created during extraction.
|
||||
func (o *Orchestrator) Cleanup() {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
for _, cleanup := range o.cleanups {
|
||||
cleanup()
|
||||
}
|
||||
o.cleanups = nil
|
||||
}
|
||||
|
||||
// safeIOCopy copies with a size limit to prevent decompression bombs.
|
||||
func safeIOCopy(dst *os.File, src interface{ Read([]byte) (int, error) }) (int64, error) {
|
||||
const maxCopySize = 2 * 1024 * 1024 * 1024 // 2GB
|
||||
return limitedCopy(dst, src, maxCopySize)
|
||||
}
|
||||
|
||||
func limitedCopy(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) {
|
||||
n, err := copyN(dst, src, limit)
|
||||
if n >= limit {
|
||||
return n, fmt.Errorf("copy size limit reached (%d bytes)", limit)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func copyN(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) {
|
||||
buf := make([]byte, 32*1024)
|
||||
var written int64
|
||||
for written < limit {
|
||||
nr, readErr := src.Read(buf)
|
||||
if nr > 0 {
|
||||
nw, writeErr := dst.Write(buf[:nr])
|
||||
written += int64(nw)
|
||||
if writeErr != nil {
|
||||
return written, writeErr
|
||||
}
|
||||
}
|
||||
if readErr != nil {
|
||||
if readErr == io.EOF {
|
||||
return written, nil
|
||||
}
|
||||
return written, readErr
|
||||
}
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
@@ -0,0 +1,537 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// testResolverFactory creates a simple mock resolver factory for testing.
|
||||
func testResolverFactory(root string) (file.Resolver, error) {
|
||||
// Walk the directory and create a mock resolver from its contents
|
||||
files := make(map[string]string)
|
||||
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info.IsDir() {
|
||||
return err
|
||||
}
|
||||
relPath, _ := filepath.Rel(root, path)
|
||||
content, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
files["/"+relPath] = string(content)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newMockResolver(files), nil
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_ZipArchive(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a zip file in the "source" directory
|
||||
zipFiles := map[string]string{
|
||||
"inner.txt": "hello from inside the zip",
|
||||
"lib/a.so": "library content",
|
||||
}
|
||||
zipPath := createTestZip(t, dir, zipFiles)
|
||||
|
||||
// Create a parent resolver that sees the zip file
|
||||
parentFiles := map[string]string{
|
||||
"/test.zip": "placeholder", // the resolver sees the file
|
||||
}
|
||||
parent := newMockResolver(parentFiles)
|
||||
|
||||
// Override FileContentsByLocation to return actual zip content
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
parent.contents["/test.zip"] = string(zipContent)
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 1, newCount)
|
||||
|
||||
// the composite resolver should now have the child files
|
||||
resolver := orch.Resolver()
|
||||
assert.Equal(t, 1, resolver.ChildCount())
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_TarGzArchive(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
tarFiles := map[string]string{
|
||||
"inner.txt": "hello from tar",
|
||||
}
|
||||
tarPath := createTestTarGz(t, dir, tarFiles)
|
||||
|
||||
tarContent, err := os.ReadFile(tarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.tar.gz": string(tarContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeUnindexedArchives = true // tar needs this enabled
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 1, newCount)
|
||||
assert.Equal(t, 1, orch.Resolver().ChildCount())
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_DepthLimit(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/file.txt": "just a file",
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// at depth 1 (== MaxDepth), should extract nothing
|
||||
newCount := orch.DiscoverAndExtract(ctx, 1)
|
||||
assert.Equal(t, 0, newCount)
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_NoArchives(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/file.txt": "plain text",
|
||||
"/data.csv": "a,b,c",
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 3
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount)
|
||||
assert.Equal(t, 0, orch.Resolver().ChildCount())
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_ExcludeExtension(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.ExcludeExtensions = []string{".zip"}
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount)
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_TotalSizeLimit(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.MaxTotalExtractionBytes = 1 // 1 byte total limit - will be exceeded
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// should still extract but hit the limit
|
||||
_ = orch.DiscoverAndExtract(ctx, 0)
|
||||
// second call should be blocked by total limit
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount)
|
||||
}
|
||||
|
||||
func TestOrchestrator_Relationships(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
orch.DiscoverAndExtract(ctx, 0)
|
||||
|
||||
rels := orch.Relationships()
|
||||
assert.NotEmpty(t, rels)
|
||||
|
||||
// should have a contains relationship for the archive
|
||||
for _, rel := range rels {
|
||||
assert.Equal(t, "contains", string(rel.Type))
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrchestrator_Cleanup(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
|
||||
orch.DiscoverAndExtract(ctx, 0)
|
||||
|
||||
// verify temp dirs exist
|
||||
entries, err := os.ReadDir(tmpDir)
|
||||
require.NoError(t, err)
|
||||
initialCount := len(entries)
|
||||
assert.True(t, initialCount > 0, "expected temp dirs to be created")
|
||||
|
||||
orch.Cleanup()
|
||||
|
||||
// verify temp dirs are cleaned up
|
||||
entries, err = os.ReadDir(tmpDir)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, len(entries) < initialCount, "expected temp dirs to be cleaned up")
|
||||
}
|
||||
|
||||
func TestOrchestrator_DoesNotReprocess(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 2
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// first call extracts the archive
|
||||
first := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 1, first)
|
||||
|
||||
// second call should not re-extract the same archive
|
||||
second := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, second)
|
||||
assert.Equal(t, 1, orch.Resolver().ChildCount())
|
||||
}
|
||||
|
||||
func TestOrchestrator_NestedArchives_ZipInsideZip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create inner zip
|
||||
innerZipPath := createTestZip(t, dir, map[string]string{
|
||||
"deep.txt": "deeply nested content",
|
||||
})
|
||||
innerZipContent, err := os.ReadFile(innerZipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create outer zip containing the inner zip
|
||||
outerDir := filepath.Join(dir, "outer")
|
||||
require.NoError(t, os.MkdirAll(outerDir, 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(outerDir, "inner.zip"), innerZipContent, 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(outerDir, "readme.txt"), []byte("readme"), 0o644))
|
||||
outerZipPath := createTestZip(t, outerDir, map[string]string{
|
||||
"inner.zip": string(innerZipContent),
|
||||
"readme.txt": "readme",
|
||||
})
|
||||
outerZipContent, err := os.ReadFile(outerZipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/outer.zip": string(outerZipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 3
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Round 1: extract outer.zip
|
||||
count1 := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 1, count1, "should extract outer.zip")
|
||||
assert.Equal(t, 1, orch.Resolver().ChildCount())
|
||||
|
||||
// Round 2: extract inner.zip found inside outer.zip
|
||||
count2 := orch.DiscoverAndExtract(ctx, 1)
|
||||
assert.Equal(t, 1, count2, "should extract inner.zip from within outer.zip")
|
||||
assert.Equal(t, 2, orch.Resolver().ChildCount())
|
||||
|
||||
// Round 3: no more archives to extract
|
||||
count3 := orch.DiscoverAndExtract(ctx, 2)
|
||||
assert.Equal(t, 0, count3, "no more archives to extract")
|
||||
}
|
||||
|
||||
func TestOrchestrator_ContextCancellation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel immediately
|
||||
|
||||
// should handle gracefully without hanging
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.True(t, newCount >= 0) // may be 0 or 1 depending on timing
|
||||
}
|
||||
|
||||
func TestOrchestrator_MultipleArchivesSameLevel(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zip1Path := createTestZip(t, dir, map[string]string{"file1.txt": "from zip1"})
|
||||
zip1Content, err := os.ReadFile(zip1Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Need a different dir to avoid overwriting test.zip
|
||||
dir2 := t.TempDir()
|
||||
zip2Path := createTestZip(t, dir2, map[string]string{"file2.txt": "from zip2"})
|
||||
zip2Content, err := os.ReadFile(zip2Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/archive1.zip": string(zip1Content),
|
||||
"/archive2.zip": string(zip2Content),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 2, newCount, "should extract both archives")
|
||||
assert.Equal(t, 2, orch.Resolver().ChildCount())
|
||||
}
|
||||
|
||||
func TestOrchestrator_CorruptArchiveSkipped(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// A file with .zip extension but invalid content
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/corrupt.zip": "this is not a valid zip file",
|
||||
"/normal.txt": "just text",
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Should not panic or error out; corrupt archives should be skipped
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount, "corrupt archive should be skipped")
|
||||
}
|
||||
|
||||
func TestOrchestrator_EmptyArchive(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an empty zip
|
||||
emptyZipPath := createTestZip(t, dir, map[string]string{})
|
||||
emptyZipContent, err := os.ReadFile(emptyZipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/empty.zip": string(emptyZipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount, "empty archive should not count as extracted")
|
||||
}
|
||||
|
||||
func TestOrchestrator_PerArchiveFileLimitApplied(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a zip with many files
|
||||
files := map[string]string{}
|
||||
for i := 0; i < 10; i++ {
|
||||
files[filepath.Join("dir", string(rune('a'+i))+".txt")] = "content"
|
||||
}
|
||||
zipPath := createTestZip(t, dir, files)
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/big.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.MaxFileCount = 3 // only allow 3 files to be extracted
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Should partially extract but still register the archive
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
// Extraction will fail after 3 files, but partial extraction counts
|
||||
assert.True(t, newCount >= 0)
|
||||
}
|
||||
|
||||
func TestOrchestrator_RelationshipsHaveArchiveCoordinates(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"file.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/archive.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
orch.DiscoverAndExtract(ctx, 0)
|
||||
|
||||
rels := orch.Relationships()
|
||||
require.Len(t, rels, 1)
|
||||
|
||||
// verify the relationship structure
|
||||
rel := rels[0]
|
||||
assert.Equal(t, "contains", string(rel.Type))
|
||||
|
||||
// From should be the archive's coordinates
|
||||
fromCoords, ok := rel.From.(file.Coordinates)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "/archive.zip", fromCoords.RealPath)
|
||||
}
|
||||
|
||||
func TestOrchestrator_ZipDisabledByConfig(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = false // disable zip extraction
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount)
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
const (
|
||||
// ArchivePathSeparator is used to encode nested archive paths in AccessPath,
|
||||
// matching the convention established by the Java cataloger.
|
||||
ArchivePathSeparator = ":"
|
||||
)
|
||||
|
||||
// childResolver holds a resolver for an extracted archive along with metadata about the archive.
|
||||
type childResolver struct {
|
||||
resolver file.Resolver
|
||||
archiveLocation file.Location // where the archive file lives in the parent resolver
|
||||
fsID string // unique filesystem ID for this archive
|
||||
depth int // nesting depth (0 = base filesystem)
|
||||
}
|
||||
|
||||
// CompositeResolver implements file.Resolver by compositing a parent resolver with
|
||||
// child resolvers created from extracted archive contents. Files from child resolvers
|
||||
// appear transparently alongside files from the parent.
|
||||
type CompositeResolver struct {
|
||||
parent file.Resolver
|
||||
children []*childResolver
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
var _ file.Resolver = (*CompositeResolver)(nil)
|
||||
|
||||
// NewCompositeResolver creates a new CompositeResolver wrapping the given parent resolver.
|
||||
func NewCompositeResolver(parent file.Resolver) *CompositeResolver {
|
||||
return &CompositeResolver{
|
||||
parent: parent,
|
||||
}
|
||||
}
|
||||
|
||||
// AddChild registers a child resolver for an extracted archive.
|
||||
func (r *CompositeResolver) AddChild(resolver file.Resolver, archiveLocation file.Location, depth int) string {
|
||||
fsID := generateArchiveFSID(archiveLocation)
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.children = append(r.children, &childResolver{
|
||||
resolver: resolver,
|
||||
archiveLocation: archiveLocation,
|
||||
fsID: fsID,
|
||||
depth: depth,
|
||||
})
|
||||
return fsID
|
||||
}
|
||||
|
||||
// ChildCount returns the number of child resolvers registered.
|
||||
func (r *CompositeResolver) ChildCount() int {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return len(r.children)
|
||||
}
|
||||
|
||||
// HasPath checks if the path exists in the parent or any child resolver.
|
||||
func (r *CompositeResolver) HasPath(path string) bool {
|
||||
if r.parent.HasPath(path) {
|
||||
return true
|
||||
}
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
if child.resolver.HasPath(path) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FilesByPath returns locations from the parent and all child resolvers.
|
||||
func (r *CompositeResolver) FilesByPath(paths ...string) ([]file.Location, error) {
|
||||
var results []file.Location
|
||||
|
||||
parentLocs, err := r.parent.FilesByPath(paths...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, parentLocs...)
|
||||
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
childLocs, err := child.resolver.FilesByPath(paths...)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, loc := range childLocs {
|
||||
results = append(results, r.transformLocation(loc, child))
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// FilesByGlob returns locations from the parent and all child resolvers.
|
||||
func (r *CompositeResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
|
||||
var results []file.Location
|
||||
|
||||
parentLocs, err := r.parent.FilesByGlob(patterns...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, parentLocs...)
|
||||
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
childLocs, err := child.resolver.FilesByGlob(patterns...)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, loc := range childLocs {
|
||||
results = append(results, r.transformLocation(loc, child))
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// FilesByMIMEType returns locations from the parent and all child resolvers.
|
||||
func (r *CompositeResolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
|
||||
var results []file.Location
|
||||
|
||||
parentLocs, err := r.parent.FilesByMIMEType(types...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, parentLocs...)
|
||||
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
childLocs, err := child.resolver.FilesByMIMEType(types...)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, loc := range childLocs {
|
||||
results = append(results, r.transformLocation(loc, child))
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// RelativeFileByPath looks up a file relative to a given location.
|
||||
func (r *CompositeResolver) RelativeFileByPath(location file.Location, path string) *file.Location {
|
||||
// try parent first
|
||||
if loc := r.parent.RelativeFileByPath(location, path); loc != nil {
|
||||
return loc
|
||||
}
|
||||
|
||||
// try children - match on the archive's FSID
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
if location.FileSystemID == child.fsID {
|
||||
if loc := child.resolver.RelativeFileByPath(location, path); loc != nil {
|
||||
transformed := r.transformLocation(*loc, child)
|
||||
return &transformed
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FileContentsByLocation routes to the appropriate resolver based on FileSystemID.
|
||||
func (r *CompositeResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
|
||||
// check if this location belongs to a child resolver
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
if location.FileSystemID == child.fsID {
|
||||
// look up the file by path in the child resolver to get a location with
|
||||
// a valid internal reference (needed by directory resolvers that use
|
||||
// stereoscope file references for content lookup)
|
||||
childLocs, err := child.resolver.FilesByPath(location.RealPath)
|
||||
if err != nil || len(childLocs) == 0 {
|
||||
// fallback: try with a constructed location
|
||||
childLoc := file.NewLocation(location.RealPath)
|
||||
return child.resolver.FileContentsByLocation(childLoc)
|
||||
}
|
||||
return child.resolver.FileContentsByLocation(childLocs[0])
|
||||
}
|
||||
}
|
||||
|
||||
// fall through to parent
|
||||
return r.parent.FileContentsByLocation(location)
|
||||
}
|
||||
|
||||
// AllLocations returns locations from the parent and all child resolvers.
|
||||
func (r *CompositeResolver) AllLocations(ctx context.Context) <-chan file.Location {
|
||||
results := make(chan file.Location)
|
||||
go func() {
|
||||
defer close(results)
|
||||
|
||||
// parent locations
|
||||
for loc := range r.parent.AllLocations(ctx) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case results <- loc:
|
||||
}
|
||||
}
|
||||
|
||||
// child locations
|
||||
r.mu.RLock()
|
||||
children := make([]*childResolver, len(r.children))
|
||||
copy(children, r.children)
|
||||
r.mu.RUnlock()
|
||||
|
||||
for _, child := range children {
|
||||
for loc := range child.resolver.AllLocations(ctx) {
|
||||
transformed := r.transformLocation(loc, child)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case results <- transformed:
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return results
|
||||
}
|
||||
|
||||
// FileMetadataByLocation routes to the appropriate resolver based on FileSystemID.
|
||||
func (r *CompositeResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
if location.FileSystemID == child.fsID {
|
||||
childLocs, err := child.resolver.FilesByPath(location.RealPath)
|
||||
if err != nil || len(childLocs) == 0 {
|
||||
childLoc := file.NewLocation(location.RealPath)
|
||||
return child.resolver.FileMetadataByLocation(childLoc)
|
||||
}
|
||||
return child.resolver.FileMetadataByLocation(childLocs[0])
|
||||
}
|
||||
}
|
||||
return r.parent.FileMetadataByLocation(location)
|
||||
}
|
||||
|
||||
// transformLocation transforms a location from a child resolver to include the archive context.
|
||||
func (r *CompositeResolver) transformLocation(loc file.Location, child *childResolver) file.Location {
|
||||
archivePath := child.archiveLocation.Path()
|
||||
return file.Location{
|
||||
LocationData: file.LocationData{
|
||||
Coordinates: file.Coordinates{
|
||||
RealPath: loc.RealPath,
|
||||
FileSystemID: child.fsID,
|
||||
},
|
||||
AccessPath: archivePath + ArchivePathSeparator + loc.Path(),
|
||||
},
|
||||
LocationMetadata: loc.LocationMetadata,
|
||||
}
|
||||
}
|
||||
|
||||
// generateArchiveFSID creates a deterministic filesystem ID for an archive based on its location.
|
||||
func generateArchiveFSID(archiveLocation file.Location) string {
|
||||
h := sha256.New()
|
||||
h.Write([]byte(archiveLocation.RealPath))
|
||||
h.Write([]byte(archiveLocation.FileSystemID))
|
||||
return fmt.Sprintf("archive:%x", h.Sum(nil)[:12])
|
||||
}
|
||||
@@ -0,0 +1,584 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// mockResolver implements file.Resolver for testing.
|
||||
type mockResolver struct {
|
||||
locations map[string]file.Location // path -> location
|
||||
contents map[string]string // realPath -> content
|
||||
}
|
||||
|
||||
func newMockResolver(files map[string]string) *mockResolver {
|
||||
r := &mockResolver{
|
||||
locations: make(map[string]file.Location),
|
||||
contents: files,
|
||||
}
|
||||
for path := range files {
|
||||
r.locations[path] = file.NewLocation(path)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (m *mockResolver) HasPath(path string) bool {
|
||||
_, ok := m.locations[path]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (m *mockResolver) FilesByPath(paths ...string) ([]file.Location, error) {
|
||||
var results []file.Location
|
||||
for _, p := range paths {
|
||||
if loc, ok := m.locations[p]; ok {
|
||||
results = append(results, loc)
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (m *mockResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
|
||||
var results []file.Location
|
||||
for _, pattern := range patterns {
|
||||
for path, loc := range m.locations {
|
||||
matched, _ := filepath.Match(pattern, path)
|
||||
if matched {
|
||||
results = append(results, loc)
|
||||
}
|
||||
// also try with the basename for simple glob patterns
|
||||
if !matched {
|
||||
matched, _ = filepath.Match(pattern, filepath.Base(path))
|
||||
if matched {
|
||||
results = append(results, loc)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (m *mockResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockResolver) RelativeFileByPath(_ file.Location, path string) *file.Location {
|
||||
if loc, ok := m.locations[path]; ok {
|
||||
return &loc
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
|
||||
if content, ok := m.contents[location.RealPath]; ok {
|
||||
return io.NopCloser(strings.NewReader(content)), nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
func (m *mockResolver) AllLocations(ctx context.Context) <-chan file.Location {
|
||||
ch := make(chan file.Location)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, loc := range m.locations {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case ch <- loc:
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
func (m *mockResolver) FileMetadataByLocation(_ file.Location) (file.Metadata, error) {
|
||||
return file.Metadata{}, nil
|
||||
}
|
||||
|
||||
// strictMockResolver mimics real directory resolvers (fileresolver.Directory) which
|
||||
// require that FileContentsByLocation receives a location object that was originally
|
||||
// returned by FilesByPath/FilesByGlob — not a freshly constructed file.NewLocation().
|
||||
// Real directory resolvers need the internal stereoscope file.Reference for index lookup.
|
||||
// We simulate this by annotating returned locations and only accepting those back.
|
||||
type strictMockResolver struct {
|
||||
mockResolver
|
||||
resolvedAnnotation string // annotation key that marks locations as "resolved"
|
||||
}
|
||||
|
||||
func newStrictMockResolver(files map[string]string) *strictMockResolver {
|
||||
return &strictMockResolver{
|
||||
mockResolver: *newMockResolver(files),
|
||||
resolvedAnnotation: "strict-resolved",
|
||||
}
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) annotate(loc file.Location) file.Location {
|
||||
return loc.WithAnnotation(s.resolvedAnnotation, "true")
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) isResolved(loc file.Location) bool {
|
||||
return loc.Annotations[s.resolvedAnnotation] == "true"
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) FilesByPath(paths ...string) ([]file.Location, error) {
|
||||
locs, err := s.mockResolver.FilesByPath(paths...)
|
||||
for i := range locs {
|
||||
locs[i] = s.annotate(locs[i])
|
||||
}
|
||||
return locs, err
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
|
||||
locs, err := s.mockResolver.FilesByGlob(patterns...)
|
||||
for i := range locs {
|
||||
locs[i] = s.annotate(locs[i])
|
||||
}
|
||||
return locs, err
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
|
||||
if !s.isResolved(location) {
|
||||
return nil, fmt.Errorf("location %q was not resolved via FilesByPath/FilesByGlob first (simulating real directory resolver behavior)", location.RealPath)
|
||||
}
|
||||
return s.mockResolver.FileContentsByLocation(location)
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
|
||||
if !s.isResolved(location) {
|
||||
return file.Metadata{}, fmt.Errorf("location %q was not resolved via FilesByPath/FilesByGlob first", location.RealPath)
|
||||
}
|
||||
return s.mockResolver.FileMetadataByLocation(location)
|
||||
}
|
||||
|
||||
// TestCompositeResolver_FileContentsByLocation_ChildRequiresLookup tests that
|
||||
// FileContentsByLocation works when the child resolver requires locations to be
|
||||
// resolved via FilesByPath first (as real directory resolvers do). This catches
|
||||
// the bug where we constructed a bare file.NewLocation() without a valid internal
|
||||
// reference, which fails on real fileresolver.Directory implementations.
|
||||
func TestCompositeResolver_FileContentsByLocation_ChildRequiresLookup(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{})
|
||||
child := newStrictMockResolver(map[string]string{
|
||||
"/lib/data.txt": "important data",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
fsID := composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
// Simulate what a cataloger does: find the file, then read its content.
|
||||
// The location returned has the archive's fsID, not the child's internal ref.
|
||||
locs, err := composite.FilesByPath("/lib/data.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, fsID, locs[0].FileSystemID)
|
||||
|
||||
// Now read content - this must work even though the location was transformed
|
||||
reader, err := composite.FileContentsByLocation(locs[0])
|
||||
require.NoError(t, err, "FileContentsByLocation should resolve the file in the child resolver via FilesByPath before reading")
|
||||
defer reader.Close()
|
||||
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "important data", string(content))
|
||||
}
|
||||
|
||||
// TestCompositeResolver_FileMetadataByLocation_ChildRequiresLookup is the same
|
||||
// test as above but for FileMetadataByLocation.
|
||||
func TestCompositeResolver_FileMetadataByLocation_ChildRequiresLookup(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{})
|
||||
child := newStrictMockResolver(map[string]string{
|
||||
"/lib/data.txt": "important data",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
fsID := composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
locs, err := composite.FilesByPath("/lib/data.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, fsID, locs[0].FileSystemID)
|
||||
|
||||
// FileMetadataByLocation must also resolve via FilesByPath first
|
||||
_, err = composite.FileMetadataByLocation(locs[0])
|
||||
require.NoError(t, err, "FileMetadataByLocation should resolve the file in the child resolver via FilesByPath before reading metadata")
|
||||
}
|
||||
|
||||
func TestCompositeResolver_FilesByPath_ParentOnly(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/parent.txt": "parent content",
|
||||
})
|
||||
composite := NewCompositeResolver(parent)
|
||||
|
||||
locs, err := composite.FilesByPath("/parent.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, "/parent.txt", locs[0].RealPath)
|
||||
}
|
||||
|
||||
func TestCompositeResolver_FilesByPath_WithChild(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/parent.txt": "parent content",
|
||||
})
|
||||
child := newMockResolver(map[string]string{
|
||||
"/child.txt": "child content",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.tar.gz")
|
||||
composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
locs, err := composite.FilesByPath("/child.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Contains(t, locs[0].AccessPath, "archive.tar.gz:")
|
||||
assert.Contains(t, locs[0].AccessPath, "child.txt")
|
||||
}
|
||||
|
||||
func TestCompositeResolver_FileContentsByLocation_Parent(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/parent.txt": "parent content",
|
||||
})
|
||||
composite := NewCompositeResolver(parent)
|
||||
|
||||
locs, err := composite.FilesByPath("/parent.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
|
||||
reader, err := composite.FileContentsByLocation(locs[0])
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "parent content", string(content))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_FileContentsByLocation_Child(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{})
|
||||
child := newMockResolver(map[string]string{
|
||||
"/child.txt": "child content",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
fsID := composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
// construct a location as returned by FilesByPath
|
||||
loc := file.Location{
|
||||
LocationData: file.LocationData{
|
||||
Coordinates: file.Coordinates{
|
||||
RealPath: "/child.txt",
|
||||
FileSystemID: fsID,
|
||||
},
|
||||
AccessPath: "/archive.zip:/child.txt",
|
||||
},
|
||||
}
|
||||
|
||||
reader, err := composite.FileContentsByLocation(loc)
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "child content", string(content))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_HasPath(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/parent.txt": "p",
|
||||
})
|
||||
child := newMockResolver(map[string]string{
|
||||
"/child.txt": "c",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
assert.True(t, composite.HasPath("/parent.txt"))
|
||||
assert.True(t, composite.HasPath("/child.txt"))
|
||||
assert.False(t, composite.HasPath("/nonexistent.txt"))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_AllLocations(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/parent.txt": "p",
|
||||
})
|
||||
child := newMockResolver(map[string]string{
|
||||
"/child.txt": "c",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
ctx := context.Background()
|
||||
var allLocs []file.Location
|
||||
for loc := range composite.AllLocations(ctx) {
|
||||
allLocs = append(allLocs, loc)
|
||||
}
|
||||
|
||||
assert.Len(t, allLocs, 2)
|
||||
|
||||
hasArchivePrefix := false
|
||||
for _, loc := range allLocs {
|
||||
if strings.Contains(loc.AccessPath, "archive.zip:") {
|
||||
hasArchivePrefix = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, hasArchivePrefix)
|
||||
}
|
||||
|
||||
func TestCompositeResolver_ChildCount(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{})
|
||||
composite := NewCompositeResolver(parent)
|
||||
assert.Equal(t, 0, composite.ChildCount())
|
||||
|
||||
child := newMockResolver(map[string]string{"/a.txt": "a"})
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
composite.AddChild(child, archiveLoc, 1)
|
||||
assert.Equal(t, 1, composite.ChildCount())
|
||||
}
|
||||
|
||||
func TestCompositeResolver_TransformLocation(t *testing.T) {
|
||||
composite := NewCompositeResolver(nil)
|
||||
|
||||
archiveLoc := file.NewLocation("/path/to/archive.tar.gz")
|
||||
child := &childResolver{
|
||||
archiveLocation: archiveLoc,
|
||||
fsID: "archive:abc123",
|
||||
depth: 1,
|
||||
}
|
||||
|
||||
loc := file.NewLocation("/inner/file.txt")
|
||||
transformed := composite.transformLocation(loc, child)
|
||||
|
||||
assert.Equal(t, "/inner/file.txt", transformed.RealPath)
|
||||
assert.Equal(t, "archive:abc123", transformed.FileSystemID)
|
||||
assert.Equal(t, "/path/to/archive.tar.gz:/inner/file.txt", transformed.AccessPath)
|
||||
}
|
||||
|
||||
func TestGenerateArchiveFSID(t *testing.T) {
|
||||
loc1 := file.NewLocation("/archive1.zip")
|
||||
loc2 := file.NewLocation("/archive2.zip")
|
||||
|
||||
fsID1 := generateArchiveFSID(loc1)
|
||||
fsID2 := generateArchiveFSID(loc2)
|
||||
|
||||
assert.NotEqual(t, fsID1, fsID2)
|
||||
assert.Equal(t, fsID1, generateArchiveFSID(loc1))
|
||||
assert.True(t, strings.HasPrefix(fsID1, "archive:"))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_MultipleChildren(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/base.txt": "base",
|
||||
})
|
||||
child1 := newMockResolver(map[string]string{
|
||||
"/lib.so": "library1",
|
||||
})
|
||||
child2 := newMockResolver(map[string]string{
|
||||
"/lib.so": "library2",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
composite.AddChild(child1, file.NewLocation("/archive1.zip"), 1)
|
||||
composite.AddChild(child2, file.NewLocation("/archive2.zip"), 1)
|
||||
|
||||
assert.Equal(t, 2, composite.ChildCount())
|
||||
|
||||
// searching for /lib.so should find it in both children
|
||||
locs, err := composite.FilesByPath("/lib.so")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, locs, 2)
|
||||
|
||||
// each should have different archive prefixes
|
||||
var accessPaths []string
|
||||
for _, loc := range locs {
|
||||
accessPaths = append(accessPaths, loc.AccessPath)
|
||||
}
|
||||
assert.Contains(t, accessPaths[0], "archive1.zip:")
|
||||
assert.Contains(t, accessPaths[1], "archive2.zip:")
|
||||
}
|
||||
|
||||
func TestCompositeResolver_NestedChildren_ChildInChild(t *testing.T) {
|
||||
// Simulates: base filesystem -> archive1.tar.gz -> archive2.zip -> inner file
|
||||
// In practice the orchestrator adds children at different depths, but the resolver
|
||||
// itself just has a flat list of children. The nesting is encoded in the AccessPath.
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/base.txt": "base",
|
||||
})
|
||||
|
||||
// First level: contents of archive1.tar.gz
|
||||
child1 := newMockResolver(map[string]string{
|
||||
"/inner-archive.zip": "zipdata", // this is an archive inside archive1
|
||||
"/readme.txt": "readme from archive1",
|
||||
})
|
||||
|
||||
// Second level: contents of inner-archive.zip (which was inside archive1.tar.gz)
|
||||
child2 := newMockResolver(map[string]string{
|
||||
"/deep-file.txt": "deeply nested content",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
|
||||
// Add child1 as extracted from archive1.tar.gz (depth 1)
|
||||
archive1Loc := file.NewLocation("/archive1.tar.gz")
|
||||
fsID1 := composite.AddChild(child1, archive1Loc, 1)
|
||||
|
||||
// Add child2 as extracted from inner-archive.zip which was in archive1 (depth 2)
|
||||
// The archive location for child2 refers to the inner-archive.zip as it appears in child1
|
||||
innerArchiveLoc := file.Location{
|
||||
LocationData: file.LocationData{
|
||||
Coordinates: file.Coordinates{
|
||||
RealPath: "/inner-archive.zip",
|
||||
FileSystemID: fsID1,
|
||||
},
|
||||
AccessPath: "/archive1.tar.gz:/inner-archive.zip",
|
||||
},
|
||||
}
|
||||
fsID2 := composite.AddChild(child2, innerArchiveLoc, 2)
|
||||
|
||||
assert.Equal(t, 2, composite.ChildCount())
|
||||
assert.NotEqual(t, fsID1, fsID2)
|
||||
|
||||
// Should find base file
|
||||
locs, err := composite.FilesByPath("/base.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, locs, 1)
|
||||
assert.Equal(t, "/base.txt", locs[0].AccessPath)
|
||||
|
||||
// Should find readme from archive1
|
||||
locs, err = composite.FilesByPath("/readme.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, "/archive1.tar.gz:/readme.txt", locs[0].AccessPath)
|
||||
assert.Equal(t, fsID1, locs[0].FileSystemID)
|
||||
|
||||
// Should find deeply nested file
|
||||
locs, err = composite.FilesByPath("/deep-file.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
// The access path shows the full chain: outer-archive:inner-archive:file
|
||||
assert.Equal(t, "/archive1.tar.gz:/inner-archive.zip:/deep-file.txt", locs[0].AccessPath)
|
||||
assert.Equal(t, fsID2, locs[0].FileSystemID)
|
||||
|
||||
// Should be able to read the deeply nested content
|
||||
reader, err := composite.FileContentsByLocation(locs[0])
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "deeply nested content", string(content))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_EmptyChild(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/file.txt": "content",
|
||||
})
|
||||
child := newMockResolver(map[string]string{})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
composite.AddChild(child, file.NewLocation("/empty.zip"), 1)
|
||||
|
||||
// should still find parent files
|
||||
locs, err := composite.FilesByPath("/file.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, locs, 1)
|
||||
|
||||
// no extra locations from empty child
|
||||
ctx := context.Background()
|
||||
var all []file.Location
|
||||
for loc := range composite.AllLocations(ctx) {
|
||||
all = append(all, loc)
|
||||
}
|
||||
assert.Len(t, all, 1)
|
||||
}
|
||||
|
||||
func TestCompositeResolver_SameFileInParentAndChild(t *testing.T) {
|
||||
// File with the same path exists in both parent and child
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/config.yaml": "parent version",
|
||||
})
|
||||
child := newMockResolver(map[string]string{
|
||||
"/config.yaml": "child version",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
fsID := composite.AddChild(child, file.NewLocation("/archive.zip"), 1)
|
||||
|
||||
locs, err := composite.FilesByPath("/config.yaml")
|
||||
require.NoError(t, err)
|
||||
// Should find both versions
|
||||
assert.Len(t, locs, 2)
|
||||
|
||||
// parent version has no archive prefix
|
||||
parentLoc := locs[0]
|
||||
assert.Equal(t, "/config.yaml", parentLoc.AccessPath)
|
||||
assert.Empty(t, parentLoc.FileSystemID)
|
||||
|
||||
// child version has archive prefix
|
||||
childLoc := locs[1]
|
||||
assert.Equal(t, "/archive.zip:/config.yaml", childLoc.AccessPath)
|
||||
assert.Equal(t, fsID, childLoc.FileSystemID)
|
||||
|
||||
// can read both
|
||||
r1, err := composite.FileContentsByLocation(parentLoc)
|
||||
require.NoError(t, err)
|
||||
c1, _ := io.ReadAll(r1)
|
||||
r1.Close()
|
||||
assert.Equal(t, "parent version", string(c1))
|
||||
|
||||
r2, err := composite.FileContentsByLocation(childLoc)
|
||||
require.NoError(t, err)
|
||||
c2, _ := io.ReadAll(r2)
|
||||
r2.Close()
|
||||
assert.Equal(t, "child version", string(c2))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_AllLocations_ContextCancellation(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/a.txt": "a",
|
||||
"/b.txt": "b",
|
||||
"/c.txt": "c",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel immediately
|
||||
|
||||
var count int
|
||||
for range composite.AllLocations(ctx) {
|
||||
count++
|
||||
}
|
||||
// with cancelled context, we may get 0 or some locations, but shouldn't hang
|
||||
assert.True(t, count <= 3)
|
||||
}
|
||||
|
||||
func TestCompositeResolver_NoChildren_BehavesLikeParent(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/file.txt": "content",
|
||||
})
|
||||
composite := NewCompositeResolver(parent)
|
||||
|
||||
// FilesByPath
|
||||
parentLocs, _ := parent.FilesByPath("/file.txt")
|
||||
compositeLocs, _ := composite.FilesByPath("/file.txt")
|
||||
assert.Equal(t, len(parentLocs), len(compositeLocs))
|
||||
|
||||
// HasPath
|
||||
assert.Equal(t, parent.HasPath("/file.txt"), composite.HasPath("/file.txt"))
|
||||
assert.Equal(t, parent.HasPath("/nope"), composite.HasPath("/nope"))
|
||||
}
|
||||
Reference in New Issue
Block a user