From 47cafae01b4e1a86f0bcd2514bda60c404245343 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 22 Sep 2026 00:23:36 -0400 Subject: [PATCH] chore: simplify and cleanup Signed-off-by: Keith Zantow --- cmd/syft/internal/options/catalog.go | 5 +- cmd/syft/internal/options/pkg_test.go | 35 - cmd/syft/internal/ui/ui.go | 3 +- internal/archive/README.md | 62 + internal/archive/content.go | 172 +-- internal/archive/content_test.go | 363 +++-- internal/archive/entry.go | 75 + internal/archive/entry_header.go | 47 - internal/archive/entrystore.go | 355 ++--- internal/archive/entrystore_test.go | 176 +-- internal/archive/exclusions.go | 38 + internal/archive/exclusions_test.go | 56 + internal/archive/extract.go | 231 +++ internal/archive/extract_test.go | 467 +++++++ internal/archive/extractor.go | 340 ----- internal/archive/extractor_test.go | 1241 ----------------- internal/archive/helpers_test.go | 259 +++- internal/archive/index.go | 350 +++++ .../archive/index_bench_test.go | 29 +- .../archive/index_glob_test.go | 25 +- .../archive/index_links_test.go | 120 +- .../archive/index_test.go | 168 +-- internal/archive/limits.go | 136 +- internal/archive/limits_test.go | 31 +- internal/archive/notify.go | 76 - internal/archive/notify_test.go | 76 - internal/archive/path_guards_test.go | 102 -- internal/archive/resolver.go | 250 ---- internal/archive/resolver_test.go | 213 --- internal/archive/storage_test.go | 144 +- internal/archive/traversal.go | 70 +- internal/archive/traversal_test.go | 171 +-- internal/archive/workdir.go | 91 +- internal/archive/workdir_test.go | 66 +- internal/bus/helpers.go | 32 +- internal/file/zip_file_manifest.go | 13 +- internal/file/zip_file_manifest_test.go | 6 +- internal/index/README.md | 39 + internal/index/key_split.go | 44 +- internal/index/prefix_suffix.go | 10 - .../task}/archive_nesting_test.go | 606 +++----- internal/task/archive_tasks.go | 391 ++---- internal/task/archive_tasks_test.go | 825 ++++------- syft/archive_cataloger_test.go | 276 +--- syft/archive_exclusions.go | 58 - syft/archive_exclusions_test.go | 116 -- syft/cataloging/archive_exclusions.go | 32 - syft/cataloging/archive_exclusions_test.go | 57 - syft/cataloging/archive_search.go | 49 +- syft/cataloging/archive_search_test.go | 31 - syft/create_sbom.go | 30 +- syft/create_sbom_config.go | 55 +- syft/create_sbom_config_test.go | 16 +- syft/file/coordinates.go | 8 +- syft/internal/fileresolver/archive_entry.go | 110 -- .../fileresolver/archive_entry_test.go | 83 -- .../fileresolver/archive_glob_bounds.go | 401 ------ .../fileresolver/archive_glob_bounds_test.go | 462 ------ syft/internal/fileresolver/archive_index.go | 546 -------- .../fileresolver/archive_index_charge_test.go | 228 --- .../fileresolver/archive_index_glob.go | 544 -------- syft/pkg/cataloger/java/archive_entries.go | 100 +- syft/pkg/cataloger/java/archive_parser.go | 140 +- .../pkg/cataloger/java/archive_parser_test.go | 2 +- .../java/archive_parser_traversal_test.go | 128 +- syft/pkg/cataloger/java/archive_source.go | 119 -- .../pkg/cataloger/java/archive_source_test.go | 29 +- syft/pkg/cataloger/java/cataloger.go | 67 +- syft/pkg/cataloger/java/config.go | 9 - syft/pkg/cataloger/java/parse_pom_xml.go | 4 +- .../cataloger/java/version_properties_test.go | 4 +- .../directorysource/directory_source.go | 17 +- syft/source/exclude.go | 10 +- syft/source/filesource/file_source.go | 4 +- syft/source/stereoscopesource/image_source.go | 7 +- 75 files changed, 3128 insertions(+), 8623 deletions(-) create mode 100644 internal/archive/README.md create mode 100644 internal/archive/entry.go delete mode 100644 internal/archive/entry_header.go create mode 100644 internal/archive/exclusions.go create mode 100644 internal/archive/exclusions_test.go create mode 100644 internal/archive/extract.go create mode 100644 internal/archive/extract_test.go delete mode 100644 internal/archive/extractor.go delete mode 100644 internal/archive/extractor_test.go create mode 100644 internal/archive/index.go rename syft/internal/fileresolver/archive_index_bench_test.go => internal/archive/index_bench_test.go (56%) rename syft/internal/fileresolver/archive_index_glob_test.go => internal/archive/index_glob_test.go (66%) rename syft/internal/fileresolver/archive_index_links_test.go => internal/archive/index_links_test.go (64%) rename syft/internal/fileresolver/archive_index_test.go => internal/archive/index_test.go (50%) delete mode 100644 internal/archive/notify.go delete mode 100644 internal/archive/notify_test.go delete mode 100644 internal/archive/path_guards_test.go delete mode 100644 internal/archive/resolver.go delete mode 100644 internal/archive/resolver_test.go create mode 100644 internal/index/README.md rename {syft => internal/task}/archive_nesting_test.go (53%) delete mode 100644 syft/archive_exclusions.go delete mode 100644 syft/archive_exclusions_test.go delete mode 100644 syft/cataloging/archive_exclusions.go delete mode 100644 syft/cataloging/archive_exclusions_test.go delete mode 100644 syft/internal/fileresolver/archive_entry.go delete mode 100644 syft/internal/fileresolver/archive_entry_test.go delete mode 100644 syft/internal/fileresolver/archive_glob_bounds.go delete mode 100644 syft/internal/fileresolver/archive_glob_bounds_test.go delete mode 100644 syft/internal/fileresolver/archive_index.go delete mode 100644 syft/internal/fileresolver/archive_index_charge_test.go delete mode 100644 syft/internal/fileresolver/archive_index_glob.go delete mode 100644 syft/pkg/cataloger/java/archive_source.go diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index 1701be24a..95f2fe1ce 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -120,15 +120,12 @@ func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig { } func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig { - // exclusion patterns come from --exclude, narrowed to those that reach inside an archive. CreateSBOM - // fills this field from the source only when empty, so this value wins. return cataloging.DefaultArchiveSearchConfig(). WithIncludeIndexedArchives(cfg.Package.SearchIndexedArchives). WithIncludeUnindexedArchives(cfg.Package.SearchUnindexedArchives). WithMaxDepth(cfg.Package.NestedArchiveMaxDepth). WithMaxMemoryBytes(cfg.Package.NestedArchiveMaxMemoryBytes). - WithMaxDiskBytes(cfg.Package.NestedArchiveMaxDiskBytes). - WithExclusionPatterns(cataloging.ArchiveExclusionPatterns(cfg.Exclusions)) + WithMaxDiskBytes(cfg.Package.NestedArchiveMaxDiskBytes) } func (cfg Catalog) ToRelationshipsConfig() cataloging.RelationshipsConfig { diff --git a/cmd/syft/internal/options/pkg_test.go b/cmd/syft/internal/options/pkg_test.go index eb4a3f354..37a39d7e0 100644 --- a/cmd/syft/internal/options/pkg_test.go +++ b/cmd/syft/internal/options/pkg_test.go @@ -1,7 +1,6 @@ package options import ( - "reflect" "testing" "github.com/stretchr/testify/assert" @@ -21,26 +20,6 @@ func Test_packageConfigDefaultsMatchArchiveSearchConfig(t *testing.T) { require.Zero(t, got.NestedArchiveMaxDepth, "nested archive cataloging must be off by default") } -func Test_packageConfigNamesNoArchiveExclusionSetting(t *testing.T) { - // nested archive cataloging honors the scan's own --exclude patterns, so the application config must - // name no archive-specific exclusion key. Asserted over the yaml tags rather than the Go field names: - // the key is what a user writes and what a config dump shows. - var keys []string - typ := reflect.TypeOf(packageConfig{}) - for i := 0; i < typ.NumField(); i++ { - keys = append(keys, typ.Field(i).Tag.Get("yaml")) - } - - assert.Equal(t, []string{ - "search-unindexed-archives", - "search-indexed-archives", - "exclude-binary-overlap-by-ownership", - "nested-archive-max-depth", - "nested-archive-max-memory-bytes", - "nested-archive-max-disk-bytes", - }, keys) -} - func Test_ToArchiveConfig(t *testing.T) { defaults := cataloging.DefaultArchiveSearchConfig() @@ -71,18 +50,4 @@ func Test_ToArchiveConfig(t *testing.T) { assert.True(t, got.IncludeIndexedArchives) assert.True(t, got.IncludeUnindexedArchives) }) - - t.Run("--exclude patterns arrive already filtered to the in-scope subset, with no source involved", func(t *testing.T) { - // every-boundary-populates-the-patterns: the CLI is one of the two boundaries filling - // ExclusionPatterns, from Catalog.Exclusions, applying the same shape rule the library entry - // point applies to what the source publishes - only an any-depth ("**/") pattern is in scope. - // A root-anchored or one-level pattern is not, and no source is consulted: Catalog carries none. - cfg := Catalog{Exclusions: []string{"./root-anchored", "**/*.rpm", "*/one-level", "**/vendor"}} - assert.Equal(t, []string{"**/*.rpm", "**/vendor"}, cfg.ToArchiveConfig().ExclusionPatterns) - }) - - t.Run("no --exclude patterns leaves the field empty", func(t *testing.T) { - cfg := Catalog{} - assert.Empty(t, cfg.ToArchiveConfig().ExclusionPatterns) - }) } diff --git a/cmd/syft/internal/ui/ui.go b/cmd/syft/internal/ui/ui.go index 9cb80fa21..b426fd867 100644 --- a/cmd/syft/internal/ui/ui.go +++ b/cmd/syft/internal/ui/ui.go @@ -186,8 +186,7 @@ func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) { func (m UI) View() string { v := m.frame.View() - // bubbletea erases the line the cursor rests on at exit, so end on a newline or a frame's last line - // of content is lost + // bubbletea erases the line the cursor rests on at exit, which would drop the frame's last line if !strings.HasSuffix(v, "\n") { return v + "\n" } diff --git a/internal/archive/README.md b/internal/archive/README.md new file mode 100644 index 000000000..2110b5df1 --- /dev/null +++ b/internal/archive/README.md @@ -0,0 +1,62 @@ +# internal/archive + +Opens one archive as its own filesystem so the ordinary catalogers can run against its contents. +The archive cataloger task (`internal/task/archive_tasks.go`) drives the recursion; this package +does everything for a single archive. + +## Flow + +``` +Extract(ctx, reader, fileSystemID, archivePath, limiter, exclusions) + acquireContent read in place if the reader is already seekable, else hold in memory / write to disk + identifyFormat mholt/archives sniffing first, end-of-central-directory fallback for stub-prefixed zips + extractInto every entry -> EntryStore (excluded entries are never stored) + NewIndex file.Resolver over the store's entries + digestsOf SHA-1 of the archive itself +``` + +`Extract` returns `nil` for content that is not an archive, `ErrDiskLimitReached` when the archive's +own bytes cannot be placed, and otherwise an `Extracted` whose `Cleanup` releases everything. + +## Where bytes live + +| what | in memory while... | otherwise | +|---|---|---| +| the archive's own bytes | the reader is not seekable and memory admits them | `/archive` | +| its entries' content | memory admits them | `/entries`, one file for all entries, located by offset | + +Anything already random-access (an `*os.File`, or a nested archive read out of its parent's store) +is used in place and costs nothing. The work directory is created only when something is written, +so most archives never touch the filesystem. Held archive bytes are released as soon as extraction +and digesting finish. + +## Limits + +`Limits` bound what one scan holds *at once*; each falls as archives are released. For both limits, +positive is the bound, zero forbids the resource, negative is unbounded. + +- `Limiter` is scan-wide; `Charge` is one archive's draw on it. Bytes are charged as they land, + never from declared sizes. +- Memory refused → content moves to disk. Disk refused → the archive's own bytes cannot be placed + (`ErrDiskLimitReached`, archive skipped) or an entry cannot be stored (`Extracted.Truncated`, + the entries stored so far are still cataloged). +- Index bookkeeping is approximated as `approxIndexBytesPerEntry` (2 KiB) plus name length per + entry, charged to memory and falling back to disk so a zero memory limit still indexes. + +## Index (the resolver) + +Paths are archive-relative with no leading slash. Directories implied by entry paths exist for +`HasPath`. Links resolve inside the archive only; a glob matching a link and its target returns one +location. `FilesByGlob` narrows by the pattern's last segment through a base-name index +(`internal/index`) when it is a literal, `*suffix` or `prefix*`, and otherwise matches every file +with doublestar. + +## Other pieces + +- `Exclusions`: the scan's `**/` exclusion patterns, matched against each entry path and its parent + directories at extraction time. Sourced from `source.PathExcluder`. +- `Traversal`: put on the context by the task so a cataloger (java) knows which archive it is inside. + `VirtualPath(location)` builds the `outer.war:WEB-INF/lib/inner.jar` style chain from the + `ArchivePath` the index stamps on every location. +- `entryHeader`: cleans entry names to archive-relative paths, drops device nodes and fifos, and + refuses names longer than PATH_MAX. diff --git a/internal/archive/content.go b/internal/archive/content.go index c73a57f9a..b2c66a52a 100644 --- a/internal/archive/content.go +++ b/internal/archive/content.go @@ -6,147 +6,153 @@ import ( "fmt" "io" "os" + "path/filepath" - intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" ) -// copyChunkSize is the read/charge/write chunk size, bounding how far a charge runs ahead of what is -// written. Matches io.Copy's buffer size. +// copyChunkSize is how many bytes are read and charged at a time, bounding how far a charge runs +// ahead of what is actually held. const copyChunkSize = 32 * 1024 -// ReaderAtSeeker is the random access an archive format needs: a zip reads its central directory from -// the end of the stream, so a plain io.Reader will not do. +// ReaderAtSeeker is the random access an archive format needs: a zip is read from its central +// directory at the end of the stream. type ReaderAtSeeker interface { io.Reader io.ReaderAt io.Seeker } -// Content is one archive's bytes, wherever the limiter routed them. +// Content is one archive's bytes, ready for random access. type Content struct { - Name string - Reader ReaderAtSeeker + ReaderAtSeeker - // closer releases the overflow file's handle; nil for content held in memory. - closer io.Closer + // release gives back what holding the bytes cost; nil when they are read in place + release func() } -// Close releases the handle on overflow content. It is a no-op for content held in memory. -func (c Content) Close() error { - if c.closer == nil { - return nil +// Release drops the held bytes and refunds their charge. Safe to call more than once. +func (c *Content) Release() { + if c.release != nil { + c.release() + c.release = nil } - return c.closer.Close() } -// holdContent holds one archive's bytes in memory while the memory limit admits them, and spills into -// workDir against the disk limit once it does not. Reads and charges copyChunkSize at a time so an -// oversized archive is never held past the limit. A zero memory limit spills in full; a negative one -// never refuses. -// -// On a refused chunk, held bytes and the rest of the stream spill without re-reading, and the memory -// charge is refunded as it is charged to disk. The disk limit is terminal: exceeding it yields -// ErrDiskLimitReached and the archive is skipped (see ExtractToResolver). -func holdContent(r io.Reader, workDir *WorkDir, name string, charge *Charge, notify Notify) (Content, error) { - var held bytes.Buffer +// acquireContent makes an archive's bytes random-access. Bytes that already are, such as a file on +// disk or an entry of a parent archive, are read in place and cost nothing. Anything else is held +// in memory or written to disk within the limits. +func acquireContent(r io.Reader, name string, workDir *WorkDir, charge *Charge) (Content, error) { + if ras, ok := r.(ReaderAtSeeker); ok { + return Content{ReaderAtSeeker: ras}, nil + } + return holdContent(r, name, workDir, charge) +} + +// holdContent holds an archive's bytes in memory while the memory limit admits them and writes them +// to disk once it does not. Reaching the disk limit yields ErrDiskLimitReached. +func holdContent(r io.Reader, name string, workDir *WorkDir, charge *Charge) (Content, error) { + held, rest, err := readWhileMemoryAdmits(r, charge) + if err != nil { + return Content{}, fmt.Errorf("unable to read archive content: %w", err) + } + if rest == nil { + return Content{ + ReaderAtSeeker: bytes.NewReader(held), + release: func() { charge.RefundMemory(int64(len(held))) }, + }, nil + } + + log.WithFields("archive", name).Debug("archive content does not fit in memory; writing it to disk") + charge.RefundMemory(int64(len(held))) + return writeContentToDisk(io.MultiReader(bytes.NewReader(held), rest), workDir, charge) +} + +// readWhileMemoryAdmits reads r into memory, charging each chunk before holding it so a stream larger +// than the memory limit is never held in full. When a chunk is refused it returns what is held so +// far, still charged, and a reader over the refused chunk followed by the rest of r. rest is nil when +// all of r fit. +func readWhileMemoryAdmits(r io.Reader, charge *Charge) (held []byte, rest io.Reader, err error) { chunk := make([]byte, copyChunkSize) for { n, readErr := r.Read(chunk) if n > 0 { if !charge.Memory(int64(n)) { - // spill held bytes, this chunk, and the rest without re-reading - if notify != nil { - notify(ContentOverflowed{Archive: name, Bytes: int64(held.Len() + n), Reason: overflowReason(charge)}) - } - charge.RefundMemory(int64(held.Len())) - return overflowContent(io.MultiReader(&held, bytes.NewReader(chunk[:n]), r), workDir, name, charge) + return held, io.MultiReader(bytes.NewReader(chunk[:n]), r), nil } - held.Write(chunk[:n]) + held = append(held, chunk[:n]...) + } + if errors.Is(readErr, io.EOF) { + return held, nil, nil } if readErr != nil { - if errors.Is(readErr, io.EOF) { - return Content{Name: name, Reader: bytes.NewReader(held.Bytes())}, nil - } - return Content{}, fmt.Errorf("unable to read archive content: %w", readErr) + return nil, nil, readErr } } } -// overflowContent writes an archive's bytes into its work directory, charging the disk limit as they -// land, and returns a reader over the file positioned at its start. The work directory is created -// here on first write, so an in-memory archive never creates one. -// -// This is the only filesystem write in the extraction path, and the name derives from where the -// archive was found - attacker-controlled inside another archive - so the destination is checked -// both lexically and against where it really lands. -func overflowContent(r io.Reader, workDir *WorkDir, name string, charge *Charge) (Content, error) { +// contentFileName is the file an archive's own bytes are written to when they do not fit in memory. +const contentFileName = "archive" + +// writeContentToDisk writes r into the work directory, charging the disk limit as bytes land, and +// returns a reader over the file. Reaching the disk limit removes the file and yields +// ErrDiskLimitReached. +func writeContentToDisk(r io.Reader, workDir *WorkDir, charge *Charge) (Content, error) { dir, err := workDir.Path() if err != nil { return Content{}, err } - - dest, err := intFile.SafeJoin(dir, name) + path := filepath.Join(dir, contentFileName) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { - return Content{}, fmt.Errorf("refusing to overflow archive content to %q: %w", name, err) - } - if inside, err := resolvesInsideRoot(dir, dest); err != nil || !inside { - return Content{}, fmt.Errorf("refusing to overflow archive content outside its work directory: %q", name) - } - f, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) - if err != nil { - return Content{}, fmt.Errorf("unable to create temp archive file: %w", err) + return Content{}, fmt.Errorf("unable to create archive content file: %w", err) } - written, limitReached, err := copyCharged(f, r, charge) - if err == nil && limitReached { - err = ErrDiskLimitReached + written, err := copyCharged(f, r, charge) + release := func() { + charge.RefundDisk(written) + if err := f.Close(); err != nil { + log.WithFields("path", path, "error", err).Trace("unable to close archive content file") + } + if err := os.Remove(path); err != nil { + log.WithFields("path", path, "error", err).Trace("unable to remove archive content file") + } } if err == nil { _, err = f.Seek(0, io.SeekStart) } if err != nil { - charge.RefundDisk(written) - if closeErr := f.Close(); closeErr != nil { - log.WithFields("path", dest, "error", closeErr).Trace("unable to close overflow archive file") - } - if rmErr := os.Remove(dest); rmErr != nil { - log.WithFields("path", dest, "error", rmErr).Trace("unable to remove overflow archive file") - } + release() if errors.Is(err, ErrDiskLimitReached) { return Content{}, err } - return Content{}, fmt.Errorf("unable to write temp archive file: %w", err) + return Content{}, fmt.Errorf("unable to write archive content file: %w", err) } - - return Content{Name: name, Reader: f, closer: f}, nil + return Content{ReaderAtSeeker: f, release: release}, nil } -// copyCharged copies src into dst, charging each chunk to the disk limit before writing, so the copy -// is bounded by what lands rather than any declared size. It stops at the first refused charge and -// reports it; the caller decides whether that truncates one entry or skips the archive. -func copyCharged(dst io.Writer, src io.Reader, charge *Charge) (written int64, limitReached bool, err error) { - buf := make([]byte, copyChunkSize) +// copyCharged copies src to dst, charging each chunk to the disk limit before writing it. It stops +// with ErrDiskLimitReached at the first refused chunk; bytes written before that stay charged. +func copyCharged(dst io.Writer, src io.Reader, charge *Charge) (written int64, err error) { + chunk := make([]byte, copyChunkSize) for { - n, readErr := src.Read(buf) + n, readErr := src.Read(chunk) if n > 0 { if !charge.Disk(int64(n)) { - return written, true, nil + return written, ErrDiskLimitReached } - w, writeErr := dst.Write(buf[:n]) + w, writeErr := dst.Write(chunk[:n]) written += int64(w) - if w < n { - charge.RefundDisk(int64(n - w)) - } + charge.RefundDisk(int64(n - w)) if writeErr != nil { - return written, false, writeErr + return written, writeErr } } + if errors.Is(readErr, io.EOF) { + return written, nil + } if readErr != nil { - if errors.Is(readErr, io.EOF) { - return written, false, nil - } - return written, false, readErr + return written, readErr } } } diff --git a/internal/archive/content_test.go b/internal/archive/content_test.go index ed701beb5..d6b8ab946 100644 --- a/internal/archive/content_test.go +++ b/internal/archive/content_test.go @@ -3,8 +3,6 @@ package archive import ( "bytes" "io" - "os" - "path/filepath" "strings" "testing" @@ -12,234 +10,68 @@ import ( "github.com/stretchr/testify/require" ) +func TestAcquireContent_randomAccessReadersAreUsedInPlace(t *testing.T) { + workDir := t.TempDir() + charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).Charge() + reader := bytes.NewReader([]byte("already random access")) + + content, err := acquireContent(reader, "app.zip", WorkDirAt(workDir), charge) + require.NoError(t, err) + + assert.Same(t, reader, content.ReaderAtSeeker) + assert.Empty(t, filesIn(t, workDir)) + mem, disk := charge.held() + assert.Zero(t, mem, "nothing is held, so nothing is charged") + assert.Zero(t, disk) + content.Release() +} + func TestHoldContent_heldWhileTheLimitAdmitsIt(t *testing.T) { - // an archive small enough to hold needs no file on disk at all workDir := t.TempDir() body := []byte(strings.Repeat("a", 100)) - limiter := NewLimiter(Limits{MaxMemoryBytes: 1000, MaxDiskBytes: 1000}) - charge := limiter.Charge() + charge := NewLimiter(Limits{MaxMemoryBytes: 1000, MaxDiskBytes: 1000}).Charge() - held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) + content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.IsType(t, &bytes.Reader{}, held.Reader) - assert.Equal(t, body, readAll(t, held)) - assert.Empty(t, filesIn(t, workDir), "content held in memory must write nothing to disk") + assert.Equal(t, body, readAll(t, content)) + assert.Empty(t, filesIn(t, workDir)) mem, disk := charge.held() assert.Equal(t, int64(100), mem) assert.Zero(t, disk) + + content.Release() + mem, _ = charge.held() + assert.Zero(t, mem, "releasing held content refunds its memory") } -func TestHoldContent_overflowsWhenTheLimitDoesNotAdmitIt(t *testing.T) { - // the memory limit alone decides where the boundary falls: an archive that does not fit spills in - // full, and the bytes already read to discover that lead the spill rather than being lost +func TestHoldContent_writtenToDiskWhenTheLimitDoesNotAdmitIt(t *testing.T) { workDir := t.TempDir() body := []byte(strings.Repeat("b", 250)) - limiter := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 10_000}) - charge := limiter.Charge() + charge := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 10_000}).Charge() - held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) + content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir)) - assert.Equal(t, body, readAll(t, held), - "the bytes read to decide the routing must lead the overflow rather than being lost") + assert.Equal(t, []string{contentFileName}, filesIn(t, workDir)) + assert.Equal(t, body, readAll(t, content), "the bytes read before the limit refused must not be lost") mem, disk := charge.held() - assert.Zero(t, mem, "nothing is charged to a limit that refused") + assert.Zero(t, mem, "nothing stays charged to a limit that refused") assert.Equal(t, int64(250), disk) + + content.Release() + assert.Empty(t, filesIn(t, workDir), "releasing removes the file") + _, disk = charge.held() + assert.Zero(t, disk, "and refunds it") } -// peakTrackingReader records the highest memory charge observed while reading, which is how a test -// sees that a large archive was never buffered beyond what the limit admits; the routing decision is -// not otherwise observable from outside holdContent. +// peakTrackingReader records the highest memory charge observed while being read. type peakTrackingReader struct { r io.Reader charge *Charge peak int64 } -func TestHoldContent_largeArchiveNeverBuffersBeyondTheLimit(t *testing.T) { - // an archive many times the memory limit must spill, and at no point may more than the limit admits - // be held while the routing is decided: it is never read speculatively - workDir := t.TempDir() - const limit = int64(3 * copyChunkSize) - body := []byte(strings.Repeat("c", 10*copyChunkSize)) - limiter := NewLimiter(Limits{MaxMemoryBytes: limit, MaxDiskBytes: int64(len(body)) + 1}) - charge := limiter.Charge() - tracked := &peakTrackingReader{r: bytes.NewReader(body), charge: charge} - - held, err := holdContent(tracked, WorkDirAt(workDir), "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir), "an archive this much larger than the limit must overflow") - assert.Equal(t, body, readAll(t, held)) - assert.LessOrEqual(t, tracked.peak, limit, - "memory charged must never exceed the limit, even transiently, while routing is decided") - - mem, disk := charge.held() - assert.Zero(t, mem) - assert.Equal(t, int64(len(body)), disk) -} - -func TestHoldContent_diskLimitIsTerminal(t *testing.T) { - // there is nowhere further to spill, so content that will not fit is refused and the caller skips the - // archive rather than waiting for capacity nothing will release - workDir := t.TempDir() - body := []byte(strings.Repeat("d", 500)) - limiter := NewLimiter(Limits{MaxDiskBytes: 100}) - charge := limiter.Charge() - - _, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) - require.ErrorIs(t, err, ErrDiskLimitReached) - - assert.Empty(t, filesIn(t, workDir), "the partial overflow must not be left behind") - mem, disk := charge.held() - assert.Zero(t, mem) - assert.Zero(t, disk, "bytes no longer on disk must be refunded, or the limit only ever rises") -} - -func TestHoldContent_zeroDiskLimitSkipsRatherThanFails(t *testing.T) { - // a zero disk limit behaves like a positive one exceeded: ErrDiskLimitReached, so the caller skips - // the archive rather than the scan failing - workDir := t.TempDir() - body := []byte(strings.Repeat("f", 500)) - limiter := NewLimiter(Limits{MaxMemoryBytes: 10, MaxDiskBytes: 0}) - charge := limiter.Charge() - - _, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) - require.ErrorIs(t, err, ErrDiskLimitReached) - - assert.Empty(t, filesIn(t, workDir), "no partial overflow is left behind") - mem, disk := charge.held() - assert.Zero(t, mem) - assert.Zero(t, disk) -} - -func TestHoldContent_threeStateLimits(t *testing.T) { - // both limits read positive/zero/negative the same way - the limit, none of that resource, unbounded - // - and a caller can bound one without the other - body := []byte(strings.Repeat("e", 500)) - - t.Run("a zero memory limit sends everything to disk", func(t *testing.T) { - // zero does not mean unlimited memory: the limit decides a routing question, so zero means the - // memory side of that routing is never taken - workDir := t.TempDir() - limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10_000}) - charge := limiter.Charge() - - held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir), "a zero memory limit holds nothing in memory") - mem, disk := charge.held() - assert.Zero(t, mem) - assert.Equal(t, int64(500), disk, "and the disk limit still applies to what it overflowed") - }) - - t.Run("a negative memory limit holds content regardless of how much is already held", func(t *testing.T) { - workDir := t.TempDir() - limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: 10_000}) - charge := limiter.Charge() - require.True(t, charge.Memory(1_000_000), "a negative memory limit admits any amount already") - - held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Empty(t, filesIn(t, workDir), "nothing overflows for want of memory when the limit is negative") - mem, disk := charge.held() - assert.Equal(t, int64(1_000_000+len(body)), mem) - assert.Zero(t, disk) - }) - - t.Run("a zero disk limit writes nothing to disk while content the memory limit admits is still cataloged", func(t *testing.T) { - workDir := t.TempDir() - limiter := NewLimiter(Limits{MaxMemoryBytes: 10_000, MaxDiskBytes: 0}) - charge := limiter.Charge() - - held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Empty(t, filesIn(t, workDir), "content the memory limit admits never touches disk") - mem, disk := charge.held() - assert.Equal(t, int64(len(body)), mem) - assert.Zero(t, disk) - }) - - t.Run("a negative disk limit overflows with no ceiling", func(t *testing.T) { - workDir := t.TempDir() - limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) - charge := limiter.Charge() - - held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir)) - assert.Equal(t, body, readAll(t, held)) - }) -} - -func TestHoldContent_noLimitsAtAll(t *testing.T) { - // a nil charge is what an extraction with no configured bounds gets: it must route content rather - // than panic, holding everything in memory since nothing refuses it - workDir := t.TempDir() - body := []byte("small enough") - - held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", nil, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Equal(t, body, readAll(t, held)) - assert.Empty(t, filesIn(t, workDir)) -} - -func TestOpenFileContent(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "app.zip") - require.NoError(t, os.WriteFile(path, []byte("body"), 0o600)) - - content, err := openFileContent(path) - require.NoError(t, err) - assert.Equal(t, "app.zip", content.Name) - assert.Equal(t, []byte("body"), readAll(t, content)) - require.NoError(t, content.Close()) - - _, err = openFileContent(filepath.Join(dir, "missing.zip")) - assert.Error(t, err) -} - -// filesIn lists the regular files directly in dir, which is how a test tells content held in memory -// from content spilled to disk. -func filesIn(t *testing.T, dir string) []string { - t.Helper() - entries, err := os.ReadDir(dir) - require.NoError(t, err) - var names []string - for _, e := range entries { - if !e.IsDir() { - names = append(names, e.Name()) - } - } - return names -} - -func readAll(t *testing.T, c Content) []byte { - t.Helper() - _, err := c.Reader.Seek(0, io.SeekStart) - require.NoError(t, err) - body, err := io.ReadAll(c.Reader) - require.NoError(t, err) - return body -} - func (p *peakTrackingReader) Read(b []byte) (int, error) { n, err := p.r.Read(b) if mem, _ := p.charge.held(); mem > p.peak { @@ -247,3 +79,126 @@ func (p *peakTrackingReader) Read(b []byte) (int, error) { } return n, err } + +func TestHoldContent_largeArchiveNeverBuffersBeyondTheLimit(t *testing.T) { + workDir := t.TempDir() + const limit = int64(3 * copyChunkSize) + body := []byte(strings.Repeat("c", 10*copyChunkSize)) + charge := NewLimiter(Limits{MaxMemoryBytes: limit, MaxDiskBytes: int64(len(body)) + 1}).Charge() + tracked := &peakTrackingReader{r: bytes.NewReader(body), charge: charge} + + content, err := holdContent(tracked, "app.zip", WorkDirAt(workDir), charge) + require.NoError(t, err) + t.Cleanup(content.Release) + + assert.Equal(t, []string{contentFileName}, filesIn(t, workDir)) + assert.Equal(t, body, readAll(t, content)) + assert.LessOrEqual(t, tracked.peak, limit, "memory charged must never exceed the limit, even transiently") + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Equal(t, int64(len(body)), disk) +} + +func TestHoldContent_diskLimitIsTerminal(t *testing.T) { + tests := []struct { + name string + limits Limits + }{ + {"a disk limit that is exceeded", Limits{MaxDiskBytes: 100}}, + {"a zero disk limit", Limits{MaxMemoryBytes: 10, MaxDiskBytes: 0}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + workDir := t.TempDir() + charge := NewLimiter(tt.limits).Charge() + + _, err := holdContent(bytes.NewReader([]byte(strings.Repeat("d", 500))), "app.zip", WorkDirAt(workDir), charge) + require.ErrorIs(t, err, ErrDiskLimitReached) + + assert.Empty(t, filesIn(t, workDir), "the partial file must not be left behind") + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Zero(t, disk, "bytes no longer on disk must be refunded") + }) + } +} + +func TestHoldContent_threeStateLimits(t *testing.T) { + body := []byte(strings.Repeat("e", 500)) + + t.Run("a zero memory limit sends everything to disk", func(t *testing.T) { + workDir := t.TempDir() + charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10_000}).Charge() + + content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) + require.NoError(t, err) + t.Cleanup(content.Release) + + assert.Equal(t, []string{contentFileName}, filesIn(t, workDir)) + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Equal(t, int64(500), disk) + }) + + t.Run("a negative memory limit holds content regardless of how much is already held", func(t *testing.T) { + workDir := t.TempDir() + charge := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: 10_000}).Charge() + require.True(t, charge.Memory(1_000_000)) + + content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) + require.NoError(t, err) + t.Cleanup(content.Release) + + assert.Empty(t, filesIn(t, workDir)) + mem, disk := charge.held() + assert.Equal(t, int64(1_000_000+len(body)), mem) + assert.Zero(t, disk) + }) + + t.Run("a zero disk limit still holds what memory admits", func(t *testing.T) { + workDir := t.TempDir() + charge := NewLimiter(Limits{MaxMemoryBytes: 10_000, MaxDiskBytes: 0}).Charge() + + content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) + require.NoError(t, err) + t.Cleanup(content.Release) + + assert.Empty(t, filesIn(t, workDir)) + mem, disk := charge.held() + assert.Equal(t, int64(len(body)), mem) + assert.Zero(t, disk) + }) + + t.Run("a negative disk limit writes with no ceiling", func(t *testing.T) { + workDir := t.TempDir() + charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}).Charge() + + content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) + require.NoError(t, err) + t.Cleanup(content.Release) + + assert.Equal(t, []string{contentFileName}, filesIn(t, workDir)) + assert.Equal(t, body, readAll(t, content)) + }) +} + +func TestHoldContent_nilChargeHoldsEverythingInMemory(t *testing.T) { + workDir := t.TempDir() + body := []byte("small enough") + + content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), nil) + require.NoError(t, err) + t.Cleanup(content.Release) + + assert.Equal(t, body, readAll(t, content)) + assert.Empty(t, filesIn(t, workDir)) +} + +func readAll(t *testing.T, c Content) []byte { + t.Helper() + _, err := c.Seek(0, io.SeekStart) + require.NoError(t, err) + body, err := io.ReadAll(c) + require.NoError(t, err) + return body +} diff --git a/internal/archive/entry.go b/internal/archive/entry.go new file mode 100644 index 000000000..9d629d68d --- /dev/null +++ b/internal/archive/entry.go @@ -0,0 +1,75 @@ +package archive + +import ( + "archive/tar" + "io/fs" + "path" + "strings" + "time" + + "github.com/mholt/archives" + + "github.com/anchore/syft/internal/log" +) + +// Entry is one archive entry: its header and where its content is. Content starts in memory and +// moves to the store's overflow file when memory runs out; the move changes nothing an index over +// the entry holds. +type Entry struct { + Header tar.Header + + mem []byte // content while held in memory + offset int64 // where content starts in the overflow file once written there + size int64 +} + +// maxEntryNameBytes is the longest entry name stored: PATH_MAX. Every path component becomes an index +// node, so a longer name is a cheap way to inflate the index and no filesystem could hold it anyway. +const maxEntryNameBytes = 4096 + +// entryHeader builds the header stored for one archive entry, or reports false for an entry not worth +// carrying: device nodes, fifos and sockets, which no cataloger reads, and names that cannot be a path +// inside the archive. +// +// The name is cleaned to an archive-relative path with no leading slash, so "../x" names x at the +// archive root rather than anything outside it. Symlinks are stored as headers, never followed on the +// host; the index resolves their targets inside the archive. +func entryHeader(f archives.FileInfo) (tar.Header, bool) { + mode := f.Mode() + if !mode.IsRegular() && !mode.IsDir() && mode.Type()&fs.ModeSymlink == 0 { + log.WithFields("entry", f.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry") + return tar.Header{}, false + } + + name, ok := entryName(f.NameInArchive) + if !ok { + log.WithFields("entry-name-bytes", len(f.NameInArchive)).Trace("skipping archive entry with an unusable name") + return tar.Header{}, false + } + + hdr, err := tar.FileInfoHeader(f.FileInfo, f.LinkTarget) + if err != nil { + log.WithFields("entry", f.NameInArchive, "error", err).Debug("skipping archive entry with an unusable header") + return tar.Header{}, false + } + hdr.Name = name + hdr.Linkname = f.LinkTarget + + // nothing reads these, and the header is held for the life of the archive + hdr.PAXRecords = nil + hdr.Uname, hdr.Gname = "", "" + hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} + hdr.ModTime = hdr.ModTime.Truncate(time.Second) + + return *hdr, true +} + +// entryName cleans an archive entry's name to a path relative to the archive root, reporting false +// when it names nothing inside the archive. +func entryName(name string) (string, bool) { + if len(name) > maxEntryNameBytes || strings.ContainsRune(name, 0) { + return "", false + } + cleaned := strings.TrimPrefix(path.Clean("/"+name), "/") + return cleaned, cleaned != "" +} diff --git a/internal/archive/entry_header.go b/internal/archive/entry_header.go deleted file mode 100644 index febe562ef..000000000 --- a/internal/archive/entry_header.go +++ /dev/null @@ -1,47 +0,0 @@ -package archive - -import ( - "archive/tar" - "io/fs" - "strings" - "time" - - "github.com/mholt/archives" - - "github.com/anchore/syft/internal/log" -) - -// entryHeader builds the tar header for one archive entry's metadata, or reports false when the entry -// is not worth carrying. The entry's bytes are held separately. -// -// Symlinks are recorded as headers, not real links; the target is resolved later inside the archive's -// own filetree, so one naming the host filesystem resolves to nothing. Device nodes, fifos and -// sockets are skipped - no cataloger reads them. -func entryHeader(f archives.FileInfo) (*tar.Header, bool) { - mode := f.Mode() - if !mode.IsRegular() && !mode.IsDir() && mode.Type()&fs.ModeSymlink == 0 { - log.WithFields("entry", f.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry") - return nil, false - } - - hdr, err := tar.FileInfoHeader(f.FileInfo, f.LinkTarget) - if err != nil { - log.WithFields("entry", f.NameInArchive, "error", err).Debug("skipping archive entry with an unusable header") - return nil, false - } - - // carried through as-is: nothing is written through this name, and the filetree sanitizes it - hdr.Name = f.NameInArchive - if f.IsDir() && !strings.HasSuffix(hdr.Name, "/") { - hdr.Name += "/" - } - hdr.Linkname = f.LinkTarget - - // dropped: no cataloger reads them, and the header is charged against the memory budget - hdr.PAXRecords = nil - hdr.Uname, hdr.Gname = "", "" - hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} - hdr.ModTime = hdr.ModTime.Truncate(time.Second) - - return hdr, true -} diff --git a/internal/archive/entrystore.go b/internal/archive/entrystore.go index d50f2327b..b7d0c5cdc 100644 --- a/internal/archive/entrystore.go +++ b/internal/archive/entrystore.go @@ -8,247 +8,158 @@ import ( "io" "os" "path/filepath" - "reflect" - "sync" - - "github.com/mholt/archives" "github.com/anchore/syft/internal/log" ) -// Entry is one archive entry: its header and a reference to its bytes. The bytes move, the entry does -// not, so an index built over these entries stays valid when content spills to disk. -type Entry struct { - Header tar.Header +// approxIndexBytesPerEntry estimates what holding one entry costs beyond its content: the entry and +// header themselves, and the index node, path map slot and name index slot the resolver builds over +// it. It is charged so an archive of many tiny entries is still bounded by the limits. +const approxIndexBytesPerEntry = 2 * 1024 - // ref locates this entry's bytes. Guarded by the store's mutex: a spill rewrites it while readers - // may be asking for it. - ref blobRef +func approxIndexBytes(hdr tar.Header) int64 { + return approxIndexBytesPerEntry + int64(len(hdr.Name)+len(hdr.Linkname)) } -// blobRef is either bytes held in memory or a region of the store's overflow file; mem is nil once -// the entry has been written out. -type blobRef struct { - mem []byte - offset int64 - length int64 -} +// entriesFileName is the single file an archive's entries overflow into. Entries are located by +// offset; the file has no framing of its own. +const entriesFileName = "entries" -// EntryStore holds one extracted archive's entries in memory while the memory limit admits them, and -// in a single overflow blob once it does not; an archive that fits in memory never touches disk. The -// store owns the overflow file and closes it on Close; readers from Open are valid only until then. +// EntryStore holds one archive's entries: in memory while the memory limit admits them, and in one +// overflow file once it does not. Everything held is charged to one Charge. type EntryStore struct { - mu sync.RWMutex + name string + workDir *WorkDir + charge *Charge entries []*Entry - - workDir *WorkDir file *os.File written int64 - - // archive names this store's archive in the events it sends, attributing a spill to its cause. - archive string - notify Notify } -// overflowBlobName is the file entries spill into. It has no framing: entry offsets live in the store. -const overflowBlobName = "contents.blob" - -// indexRecordOverhead is the fixed memory kept per entry regardless of content: the *Entry (tar.Header -// plus blobRef), its slot in the entries slice, and append's headroom. -// -// It covers only what the store keeps. The resolver built over these entries keeps several times as -// much in nodes the store cannot count, and charges for that itself; see Charge.IndexRecord. -var indexRecordOverhead = int64(reflect.TypeFor[Entry]().Size()) + 24 - -// indexRecordCost is what one entry's record charges: the fixed overhead plus the two header strings. -func indexRecordCost(hdr tar.Header) int64 { - return indexRecordOverhead + int64(len(hdr.Name)) + int64(len(hdr.Linkname)) +// NewEntryStore returns an empty store for the named archive that overflows into workDir. +func NewEntryStore(name string, workDir *WorkDir, charge *Charge) *EntryStore { + return &EntryStore{name: name, workDir: workDir, charge: charge} } -var _ EntrySink = (*EntryStore)(nil) - -// NewEntryStore returns a store that spills into workDir, naming archive in its events. The directory -// is created only on spill, so an in-memory store creates nothing. -// -// The store has no memory budget of its own: it asks the limiter entry by entry and spills everything -// it holds on the first refusal. -func NewEntryStore(workDir *WorkDir, archive string, notify Notify) *EntryStore { - return &EntryStore{workDir: workDir, archive: archive, notify: notify} -} - -// Add stores one entry's header and content. Content is read now, not referenced: the archive's -// reader is sequential. -// -// It reads and charges copyChunkSize at a time rather than into a header-sized buffer (whose size is -// the archive's own claim), so an entry larger than the memory limit is not held in full before the -// limiter is asked. The first refusal spills this entry and everything else the store holds. -func (s *EntryStore) Add(hdr tar.Header, contents io.Reader, charge *Charge) (*Entry, error) { - s.mu.Lock() - defer s.mu.Unlock() - - // charge the record before holding the entry, so an archive of many near-empty entries is bounded - // by the same budgets its content is. Refused only when neither budget admits it, truncating the - // archive at the entries already stored. - if !charge.IndexRecord(indexRecordCost(hdr)) { - return nil, ErrDiskLimitReached +// Add stores one entry, reading its content in full now since archive readers are sequential. +// Reaching the disk limit stores nothing for this entry and returns ErrDiskLimitReached; entries +// stored before it remain usable. +func (s *EntryStore) Add(hdr tar.Header, content io.Reader) error { + if !s.charge.Index(approxIndexBytes(hdr)) { + return ErrDiskLimitReached } entry := &Entry{Header: hdr} + if hdr.Typeflag == tar.TypeReg && content != nil { + held, rest, err := readWhileMemoryAdmits(content, s.charge) + if err != nil { + return fmt.Errorf("unable to read archive entry %q: %w", hdr.Name, err) + } + entry.mem, entry.size = held, int64(len(held)) + if rest != nil { + if err := s.overflow(entry, rest); err != nil { + return err + } + } + } + s.entries = append(s.entries, entry) - - if hdr.Typeflag != tar.TypeReg || contents == nil { - return entry, nil - } - - var held bytes.Buffer - chunk := make([]byte, copyChunkSize) - for { - n, readErr := contents.Read(chunk) - if n > 0 { - if !charge.Memory(int64(n)) { - // memory full: held bytes join the spill, the rest streams straight into the blob - entry.ref = memRef(&held) - rest := io.MultiReader(bytes.NewReader(chunk[:n]), contents) - return entry, s.spillThroughLocked(entry, rest, charge) - } - held.Write(chunk[:n]) - } - if readErr != nil { - if errors.Is(readErr, io.EOF) { - break - } - return nil, fmt.Errorf("unable to read archive entry %q: %w", hdr.Name, readErr) - } - } - - entry.ref = memRef(&held) - return entry, nil -} - -// memRef references bytes held in memory. The slice stays non-nil even when empty, since a nil -// blobRef.mem means the bytes are on disk - which a zero-length entry's are not. -func memRef(held *bytes.Buffer) blobRef { - body := held.Bytes() - if body == nil { - body = []byte{} - } - return blobRef{mem: body, length: int64(len(body))} -} - -// spillThroughLocked spills everything the store holds to the overflow blob and writes the rest of -// entry's content in behind it, charging the disk limit as it lands. -// -// entry is the newest, so spillLocked writes it last, ending its region right where the rest of its -// content goes. The two are contiguous, so the entry keeps one reference. -func (s *EntryStore) spillThroughLocked(entry *Entry, rest io.Reader, charge *Charge) error { - spilled := entry.ref.length - moved, movedBytes, err := s.spillLocked(charge) - if err != nil { - return err - } - - written, limitReached, err := copyCharged(io.NewOffsetWriter(s.file, s.written), rest, charge) - s.written += written - entry.ref = blobRef{offset: s.written - written - spilled, length: spilled + written} - - // reported after the move so the event counts streamed bytes as well as held ones - if s.notify != nil && moved > 0 { - s.notify(EntriesOverflowed{ - Archive: s.archive, - Entries: moved, - Bytes: movedBytes + written, - Reason: overflowReason(charge), - }) - } - - if err != nil { - return fmt.Errorf("unable to write archive entry %q to the overflow blob: %w", entry.Header.Name, err) - } - if limitReached { - return ErrDiskLimitReached - } return nil } -// spillLocked writes every memory-backed entry into the overflow blob and repoints it there, -// reporting what it moved. Any index over these entries is untouched: only blobRef changes. It sends -// no event; spillThroughLocked sends one once the triggering entry has also landed. -func (s *EntryStore) spillLocked(charge *Charge) (moved int, movedBytes int64, err error) { - if s.file == nil { - // first spill creates the work directory - dir, err := s.workDir.Path() - if err != nil { - return 0, 0, err - } - f, err := os.OpenFile(filepath.Join(dir, overflowBlobName), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) - if err != nil { - return 0, 0, fmt.Errorf("unable to create overflow blob: %w", err) - } - s.file = f +// overflow writes every entry held in memory into the overflow file, followed by the new entry's held +// bytes and the rest of its content, charging the disk limit as bytes land and refunding the memory +// they leave. +func (s *EntryStore) overflow(entry *Entry, rest io.Reader) error { + if err := s.openFile(); err != nil { + return err } - for _, entry := range s.entries { - if entry.ref.mem == nil { + moved := 0 + for _, held := range s.entries { + if held.mem == nil { continue } - body := entry.ref.mem - if !charge.Disk(int64(len(body))) { - return moved, movedBytes, ErrDiskLimitReached + if err := s.moveToDisk(held); err != nil { + return err } - if _, err := s.file.WriteAt(body, s.written); err != nil { - return moved, movedBytes, fmt.Errorf("unable to write archive entry %q to the overflow blob: %w", entry.Header.Name, err) - } - entry.ref = blobRef{offset: s.written, length: int64(len(body))} - s.written += int64(len(body)) moved++ - movedBytes += int64(len(body)) - // on disk now, so refund the memory it was charged - charge.RefundMemory(int64(len(body))) } - return moved, movedBytes, nil + + if err := s.moveToDisk(entry); err != nil { + return err + } + written, err := copyCharged(io.NewOffsetWriter(s.file, s.written), rest, s.charge) + s.written += written + entry.size += written + + log.WithFields("archive", s.name, "entries", moved+1, "bytes", s.written). + Debug("archive entries do not fit in memory; written to disk") + + if err != nil && !errors.Is(err, ErrDiskLimitReached) { + return fmt.Errorf("unable to write archive entry %q to disk: %w", entry.Header.Name, err) + } + return err } -// overflowReason distinguishes an archive spilled under memory pressure from one never held because -// the memory limit is zero. -func overflowReason(charge *Charge) OverflowReason { - if charge != nil && charge.limiter != nil && charge.limiter.limits.MaxMemoryBytes == 0 { - return MemoryHoldsNothing +func (s *EntryStore) openFile() error { + if s.file != nil { + return nil } - return MemoryLimitReached + dir, err := s.workDir.Path() + if err != nil { + return err + } + f, err := os.OpenFile(filepath.Join(dir, entriesFileName), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return fmt.Errorf("unable to create archive entries file: %w", err) + } + s.file = f + return nil } -// Entries returns the entries stored so far, in the order they were added. +// moveToDisk appends an entry's in-memory content to the overflow file and points the entry there. +func (s *EntryStore) moveToDisk(entry *Entry) error { + n := int64(len(entry.mem)) + if !s.charge.Disk(n) { + return ErrDiskLimitReached + } + if _, err := s.file.WriteAt(entry.mem, s.written); err != nil { + return fmt.Errorf("unable to write archive entry %q to disk: %w", entry.Header.Name, err) + } + entry.offset, entry.size = s.written, n + s.written += n + s.charge.RefundMemory(n) + entry.mem = nil + return nil +} + +// Entries returns the stored entries in the order they were added. func (s *EntryStore) Entries() []*Entry { - s.mu.RLock() - defer s.mu.RUnlock() - out := make([]*Entry, len(s.entries)) - copy(out, s.entries) - return out + return s.entries } -// Open returns a reader over one entry's content, from memory or the overflow blob. It supports Read, -// Seek and ReadAt either way, so a nested archive is extracted in place rather than copied out. -func (s *EntryStore) Open(entry *Entry) (ReaderAtSeeker, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - if entry.ref.mem != nil { - return bytes.NewReader(entry.ref.mem), nil +// Open returns a reader over an entry's content, wherever it is. The reader supports Read, Seek and +// ReadAt, so a nested archive can be read where it lies. +func (s *EntryStore) Open(entry *Entry) ReaderAtSeeker { + switch { + case entry.mem != nil: + return bytes.NewReader(entry.mem) + case entry.size == 0: + return bytes.NewReader(nil) + default: + return io.NewSectionReader(s.file, entry.offset, entry.size) } - if entry.ref.length == 0 { - return bytes.NewReader(nil), nil - } - if s.file == nil { - return nil, fmt.Errorf("archive entry %q has no content", entry.Header.Name) - } - return io.NewSectionReader(s.file, entry.ref.offset, entry.ref.length), nil } -// Close releases the overflow blob. Readers handed out by Open do not outlive it. +// OnDisk reports how many bytes of entry content have been written to the overflow file. +func (s *EntryStore) OnDisk() int64 { + return s.written +} + +// Close releases the overflow file. Readers from Open must not be used after this. func (s *EntryStore) Close() error { - s.mu.Lock() - defer s.mu.Unlock() if s.file == nil { return nil } @@ -256,51 +167,3 @@ func (s *EntryStore) Close() error { s.file = nil return err } - -// AddEntry stores one archive entry. There is no framing, so the disk limit is charged only when -// content reaches disk - never, for an archive that fits in memory. -func (s *EntryStore) AddEntry(f archives.FileInfo, result *ExtractionResult, charge *Charge) error { - hdr, ok := entryHeader(f) - if !ok { - return nil - } - - var contents io.ReadCloser - if f.Mode().IsRegular() { - opened, err := f.Open() - if err != nil { - log.WithFields("entry", f.NameInArchive, "error", err). - Trace("unable to open archive entry, skipping it") - return nil - } - contents = opened - } - - _, err := s.Add(*hdr, contents, charge) - if contents != nil { - if closeErr := contents.Close(); closeErr != nil { - log.WithFields("entry", f.NameInArchive, "error", closeErr).Trace("unable to close archive entry") - } - } - - switch { - case errors.Is(err, ErrDiskLimitReached): - // keep the entries stored so far rather than discard the archive - result.Truncation = TruncatedByDiskLimit - return errTruncated - case err != nil: - return err - } - - return nil -} - -// Finish is a no-op: entries are complete as added, with no framing to close off. -func (s *EntryStore) Finish(*ExtractionResult) {} - -// OnDisk reports the entry content written out so far, which is what the disk limit was charged. -func (s *EntryStore) OnDisk() int64 { - s.mu.RLock() - defer s.mu.RUnlock() - return s.written -} diff --git a/internal/archive/entrystore_test.go b/internal/archive/entrystore_test.go index 6b5c56004..1323daad6 100644 --- a/internal/archive/entrystore_test.go +++ b/internal/archive/entrystore_test.go @@ -13,187 +13,131 @@ import ( func TestEntryStore_holdsSmallContentInMemory(t *testing.T) { dir := t.TempDir() - s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) - charge := memCharge(1024) + s := NewEntryStore("app.zip", WorkDirAt(dir), memCharge(1024)) t.Cleanup(func() { require.NoError(t, s.Close()) }) - entry, err := s.Add(regularHeader("a.txt", 5), bytes.NewReader([]byte("hello")), charge) - require.NoError(t, err) + require.NoError(t, s.Add(regularHeader("a.txt", 5), bytes.NewReader([]byte("hello")))) assert.Equal(t, int64(5), s.heldInMemory()) - assert.NoFileExists(t, filepath.Join(dir, overflowBlobName), - "a small archive must not touch the disk at all - that is the whole point of the store") - - r, err := s.Open(entry) - require.NoError(t, err) - assert.Equal(t, "hello", readEntry(t, r)) + assert.NoFileExists(t, filepath.Join(dir, entriesFileName)) + assert.Equal(t, "hello", readEntry(t, s.Open(s.Entries()[0]))) } -func TestEntryStore_spillsToDiskAndKeepsTheSameEntries(t *testing.T) { - // the index over these entries is not rebuilt when content moves, so the entry pointers and their - // contents must survive it +func TestEntryStore_overflowsToDiskAndKeepsTheSameEntries(t *testing.T) { + // an index over these entries is not rebuilt when content moves dir := t.TempDir() - s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) - charge := memCharge(8) + s := NewEntryStore("app.zip", WorkDirAt(dir), memCharge(8)) t.Cleanup(func() { require.NoError(t, s.Close()) }) - first, err := s.Add(regularHeader("a.txt", 4), bytes.NewReader([]byte("aaaa")), charge) - require.NoError(t, err) + require.NoError(t, s.Add(regularHeader("a.txt", 4), bytes.NewReader([]byte("aaaa")))) + first := s.Entries()[0] require.Equal(t, int64(4), s.heldInMemory()) - // the second entry takes the store past its memory bound, so everything held moves out - second, err := s.Add(regularHeader("b.txt", 6), bytes.NewReader([]byte("bbbbbb")), charge) - require.NoError(t, err) + require.NoError(t, s.Add(regularHeader("b.txt", 6), bytes.NewReader([]byte("bbbbbb")))) - assert.Zero(t, s.heldInMemory(), "everything must have moved out, not just the entry that did not fit") - assert.FileExists(t, filepath.Join(dir, overflowBlobName)) + assert.Zero(t, s.heldInMemory(), "everything moves out, not just the entry that did not fit") + assert.FileExists(t, filepath.Join(dir, entriesFileName)) + assert.Equal(t, int64(10), s.OnDisk()) - // the same entry pointers still read their own content, from their new home - firstReader, err := s.Open(first) - require.NoError(t, err) - assert.Equal(t, "aaaa", readEntry(t, firstReader)) - - secondReader, err := s.Open(second) - require.NoError(t, err) - assert.Equal(t, "bbbbbb", readEntry(t, secondReader)) - - assert.Equal(t, []*Entry{first, second}, s.Entries(), "order and identity must be unchanged by the move") + entries := s.Entries() + require.Len(t, entries, 2) + assert.Same(t, first, entries[0], "entry identity survives the move") + assert.Equal(t, "aaaa", readEntry(t, s.Open(entries[0]))) + assert.Equal(t, "bbbbbb", readEntry(t, s.Open(entries[1]))) } func TestEntryStore_zeroMemoryHoldsNothing(t *testing.T) { - // the reading a zero memory limit has everywhere else: none of that resource may be used dir := t.TempDir() - s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) - charge := memCharge(0) + s := NewEntryStore("app.zip", WorkDirAt(dir), memCharge(0)) t.Cleanup(func() { require.NoError(t, s.Close()) }) - entry, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), charge) - require.NoError(t, err) + require.NoError(t, s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")))) assert.Zero(t, s.heldInMemory()) - assert.FileExists(t, filepath.Join(dir, overflowBlobName)) - - r, err := s.Open(entry) - require.NoError(t, err) - assert.Equal(t, "abc", readEntry(t, r)) + assert.FileExists(t, filepath.Join(dir, entriesFileName)) + assert.Equal(t, "abc", readEntry(t, s.Open(s.Entries()[0]))) } -func TestEntryStore_negativeMemoryNeverSpills(t *testing.T) { +func TestEntryStore_negativeMemoryNeverOverflows(t *testing.T) { dir := t.TempDir() - s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) - charge := memCharge(-1) + s := NewEntryStore("app.zip", WorkDirAt(dir), memCharge(-1)) t.Cleanup(func() { require.NoError(t, s.Close()) }) for _, body := range []string{"one", "two", "three"} { - _, err := s.Add(regularHeader(body+".txt", int64(len(body))), bytes.NewReader([]byte(body)), charge) - require.NoError(t, err) + require.NoError(t, s.Add(regularHeader(body+".txt", int64(len(body))), bytes.NewReader([]byte(body)))) } assert.Equal(t, int64(11), s.heldInMemory()) - assert.NoFileExists(t, filepath.Join(dir, overflowBlobName)) + assert.NoFileExists(t, filepath.Join(dir, entriesFileName)) } -func TestEntryStore_spillIsRefusedWhenDiskIsNotAvailable(t *testing.T) { - // content that does not fit in memory and cannot be written has nowhere to go, so the archive is - // skipped rather than cataloged as if it were empty - dir := t.TempDir() - s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) - t.Cleanup(func() { require.NoError(t, s.Close()) }) +func TestEntryStore_refusedEntryIsNotStored(t *testing.T) { + // a half-stored pom a cataloger parses is worse than an absent one + s := storeFor(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).Charge()) - nowhereToPutIt := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).Charge() - _, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), nowhereToPutIt) + err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc"))) assert.ErrorIs(t, err, ErrDiskLimitReached) + assert.Empty(t, s.Entries()) } func TestEntryStore_directoryEntriesHoldNoContent(t *testing.T) { - dir := t.TempDir() - s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) - charge := memCharge(1024) - t.Cleanup(func() { require.NoError(t, s.Close()) }) + s := storeFor(t, memCharge(1024)) - entry, err := s.Add(tar.Header{Name: "lib/", Typeflag: tar.TypeDir, Mode: 0o755}, nil, charge) - require.NoError(t, err) + require.NoError(t, s.Add(tar.Header{Name: "lib", Typeflag: tar.TypeDir, Mode: 0o755}, nil)) assert.Zero(t, s.heldInMemory()) - r, err := s.Open(entry) - require.NoError(t, err) - assert.Empty(t, readEntry(t, r)) + assert.Empty(t, readEntry(t, s.Open(s.Entries()[0]))) } -func TestEntryStore_chargesIndexRecordEvenForEmptyContent(t *testing.T) { - // an empty entry still charges its index record - the header and node held to reach it - which bounds - // entry count, the dimension the content-byte budgets do not cover - dir := t.TempDir() - s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) +func TestEntryStore_indexCostIsChargedEvenForEmptyContent(t *testing.T) { + // the index estimate is what bounds an archive of many empty entries limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}) - charge := limiter.Charge() - t.Cleanup(func() { require.NoError(t, s.Close()) }) + s := storeFor(t, limiter.Charge()) hdr := regularHeader("a.txt", 0) - _, err := s.Add(hdr, bytes.NewReader(nil), charge) - require.NoError(t, err) + require.NoError(t, s.Add(hdr, bytes.NewReader(nil))) mem, _ := limiter.InUse() - assert.Equal(t, indexRecordCost(hdr), mem, "the index record is charged to memory") - assert.Equal(t, int64(0), s.heldInMemory(), "no content was held") + assert.Equal(t, approxIndexBytes(hdr), mem) + assert.Zero(t, s.heldInMemory()) } -func TestEntryStore_indexOverflowsToDiskWhenMemoryIsZero(t *testing.T) { - // the index cannot be refused just because memory is zero: a resolver needs the records to reach - // content living entirely on disk. With disk unbounded, every entry is admitted and its record - // charged to disk instead. - dir := t.TempDir() - s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) +func TestEntryStore_indexCostFallsBackToDiskWhenMemoryIsZero(t *testing.T) { limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) - charge := limiter.Charge() - t.Cleanup(func() { require.NoError(t, s.Close()) }) + s := storeFor(t, limiter.Charge()) hdr := regularHeader("a.txt", 0) - _, err := s.Add(hdr, bytes.NewReader(nil), charge) - require.NoError(t, err) + require.NoError(t, s.Add(hdr, bytes.NewReader(nil))) mem, disk := limiter.InUse() - assert.Equal(t, int64(0), mem, "memory budget is zero, so nothing is charged there") - assert.Equal(t, indexRecordCost(hdr), disk, "the index record overflows to the disk budget") + assert.Zero(t, mem) + assert.Equal(t, approxIndexBytes(hdr), disk) } func TestEntryStore_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) { - // with memory zero and only a few index records' worth of disk, the store admits a bounded number of - // entries and then refuses, so an archive of millions of empty entries cannot grow the index without - // limit - dir := t.TempDir() - s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) - room := indexRecordCost(regularHeader("x", 0)) * 3 - charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).Charge() - t.Cleanup(func() { require.NoError(t, s.Close()) }) + room := approxIndexBytes(regularHeader("x", 0)) * 3 + s := storeFor(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).Charge()) var err error admitted := 0 - for i := 0; i < 100; i++ { - if _, err = s.Add(regularHeader("x", 0), bytes.NewReader(nil), charge); err != nil { - break + for i := 0; i < 100 && err == nil; i++ { + if err = s.Add(regularHeader("x", 0), bytes.NewReader(nil)); err == nil { + admitted++ } - admitted++ } require.ErrorIs(t, err, ErrDiskLimitReached) - assert.Equal(t, 3, admitted, "exactly the index records that fit the disk budget were admitted") + assert.Equal(t, 3, admitted) } -func regularHeader(name string, size int64) tar.Header { - return tar.Header{Name: name, Size: size, Mode: 0o600, Typeflag: tar.TypeReg} -} +func TestEntryStore_readsAfterCloseFail(t *testing.T) { + s := storeFor(t, memCharge(0)) + require.NoError(t, s.Add(regularHeader("a.txt", 5), bytes.NewReader([]byte("hello")))) + entry := s.Entries()[0] + require.Equal(t, "hello", readEntry(t, s.Open(entry))) -func readEntry(t *testing.T, r ReaderAtSeeker) string { - t.Helper() - _, err := r.Seek(0, io.SeekStart) - require.NoError(t, err) - b, err := io.ReadAll(r) - require.NoError(t, err) - return string(b) -} + require.NoError(t, s.Close()) -// memCharge is a charge against a memory bound with disk unbounded, the shape the store reads: hold -// while memory admits, move out when it does not. -func memCharge(maxMemory int64) *Charge { - return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).Charge() + _, err := io.ReadAll(s.Open(entry)) + assert.Error(t, err) + assert.NoError(t, s.Close(), "Close is idempotent") } diff --git a/internal/archive/exclusions.go b/internal/archive/exclusions.go new file mode 100644 index 000000000..b8cef0681 --- /dev/null +++ b/internal/archive/exclusions.go @@ -0,0 +1,38 @@ +package archive + +import ( + "strings" + + "github.com/bmatcuk/doublestar/v4" +) + +// Exclusions are the scan's exclusion patterns that apply inside archives. +// +// A scan accepts three pattern shapes: `./x` anchored at the scan root, `*/x` exactly one level below +// it, and `**/x` at any depth. Only the last means the same thing when re-rooted at an archive, so +// only it reaches inside one. +type Exclusions []string + +func NewExclusions(scanPatterns []string) Exclusions { + var out Exclusions + for _, pattern := range scanPatterns { + if strings.HasPrefix(pattern, "**/") { + // a trailing slash reads as "a directory" but makes doublestar match nothing (issue #4839) + out = append(out, strings.TrimSuffix(pattern, "/")) + } + } + return out +} + +// Excludes reports whether an archive-relative path, or any directory above it, matches a pattern. +// Directories are checked because an archive need not list a directory before the files in it. +func (e Exclusions) Excludes(entryPath string) bool { + for _, pattern := range e { + for end := len(entryPath); end > 0; end = strings.LastIndexByte(entryPath[:end], '/') { + if matched, _ := doublestar.Match(pattern, entryPath[:end]); matched { + return true + } + } + } + return false +} diff --git a/internal/archive/exclusions_test.go b/internal/archive/exclusions_test.go new file mode 100644 index 000000000..e2e31893b --- /dev/null +++ b/internal/archive/exclusions_test.go @@ -0,0 +1,56 @@ +package archive + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewExclusions(t *testing.T) { + tests := []struct { + name string + given []string + want Exclusions + }{ + {name: "nothing configured"}, + {name: "root-anchored does not reach inside", given: []string{"./nested/marker.txt"}}, + {name: "one level down is anchored too", given: []string{"*/vendor"}}, + {name: "any depth reaches inside", given: []string{"**/*.rpm"}, want: Exclusions{"**/*.rpm"}}, + {name: "mixed keeps only the any-depth patterns", given: []string{"./a", "**/b", "*/c", "**/d"}, want: Exclusions{"**/b", "**/d"}}, + // a trailing slash makes doublestar match nothing (issue #4839) + {name: "a trailing slash is trimmed", given: []string{"**/vendor/"}, want: Exclusions{"**/vendor"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, NewExclusions(tt.given)) + }) + } +} + +func TestExclusions_Excludes(t *testing.T) { + exclusions := NewExclusions([]string{"**/vendor", "**/*.rpm"}) + + tests := []struct { + path string + want bool + }{ + {"vendor", true}, + {"a/vendor", true}, + {"vendor/x/y.go", true}, + {"a/vendor/y.go", true}, + {"pkg.rpm", true}, + {"deep/down/pkg.rpm", true}, + {"keep/vendored.txt", false}, + {"vendor-ish", false}, + {"keep.txt", false}, + {"PKG.RPM", false}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + assert.Equal(t, tt.want, exclusions.Excludes(tt.path)) + }) + } + + assert.False(t, NewExclusions(nil).Excludes("anything")) + assert.False(t, Exclusions{"["}.Excludes("keep.txt"), "a malformed pattern excludes nothing") +} diff --git a/internal/archive/extract.go b/internal/archive/extract.go new file mode 100644 index 000000000..01ae1ef4a --- /dev/null +++ b/internal/archive/extract.go @@ -0,0 +1,231 @@ +package archive + +import ( + "archive/tar" + "bytes" + "context" + "crypto" + "errors" + "fmt" + "io" + "path" + "strings" + + "github.com/mholt/archives" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/file" +) + +// Extracted is one archive opened as its own filesystem. +type Extracted struct { + // Resolver reads the archive's entries. Its locations carry the archive's FileSystemID and + // ArchivePath. + Resolver file.Resolver + + // Digests are of the archive file itself. + Digests []file.Digest + + // Truncated reports that the disk limit stopped extraction early. Resolver covers the entries + // stored before that. + Truncated bool + + cleanup func() +} + +// Cleanup removes everything the archive holds on disk and releases its charge against the limiter. +// Safe to call more than once. +func (e *Extracted) Cleanup() { + if e == nil || e.cleanup == nil { + return + } + e.cleanup() + e.cleanup = nil +} + +// Extract opens the archive read from r as its own filesystem, holding its bytes and entries within +// the limiter's bounds. It returns nil when the content is not an archive it can read, and +// ErrDiskLimitReached when the archive's own bytes cannot be placed within the disk limit. The caller +// must Cleanup the result. +// +// fileSystemID and archivePath are stamped onto every location the resolver returns: the filesystem +// the archive was found on, and the colon-delimited chain of archives from the scan root to this one. +func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string, limiter *Limiter, exclusions Exclusions) (*Extracted, error) { + workDir := NewWorkDir(ctx) + charge := limiter.Charge() + store := NewEntryStore(archivePath, workDir, charge) + var content Content + release := func() { + content.Release() + if err := store.Close(); err != nil { + log.WithFields("archive", archivePath, "error", err).Trace("unable to close archive entries file") + } + workDir.Remove() + charge.Release() + } + + content, err := acquireContent(r, archivePath, workDir, charge) + if err != nil { + release() + return nil, err + } + // the archive's own bytes are needed only until they are extracted and digested + defer content.Release() + + format := identifyFormat(ctx, archiveFileName(archivePath), content) + if format == nil { + release() + return nil, nil + } + + truncated, err := extractInto(ctx, format, content, store, exclusions) + if err != nil { + release() + return nil, fmt.Errorf("unable to extract archive %q: %w", archivePath, err) + } + + return &Extracted{ + Resolver: NewIndex(store, fileSystemID, archivePath), + Digests: digestsOf(ctx, content, archivePath), + Truncated: truncated, + cleanup: release, + }, nil +} + +// archiveFileName returns the archive's own file name from the end of its archive path, which format +// identification reads for its extension. +func archiveFileName(archivePath string) string { + return path.Base(archivePath[strings.LastIndexByte(archivePath, ':')+1:]) +} + +// identifyFormat returns the format able to extract this content, or nil when it is not an archive. +// +// Identification from the head of the stream wins: a tar holding one jar ends in that jar's central +// directory, and reading the tar as a zip would lose the rest. The end of the stream is consulted +// only when the head identifies nothing, which is how a zip appended to a launcher script (a Spring +// Boot executable jar) is found. +func identifyFormat(ctx context.Context, name string, content ReaderAtSeeker) archives.Extractor { + if _, err := content.Seek(0, io.SeekStart); err != nil { + return nil + } + format, _, err := intFile.IdentifyArchive(ctx, name, content) + if err == nil { + // a bare compression format, such as a gzipped file that is not a tar, is not extractable + extractor, _ := format.(archives.Extractor) + return extractor + } + if HasZipEndOfCentralDirectory(content) { + return archives.Zip{} + } + return nil +} + +// extractInto reads every entry of the archive into the store. Reaching the disk limit stops the +// walk and reports truncation rather than an error. +func extractInto(ctx context.Context, format archives.Extractor, content ReaderAtSeeker, store *EntryStore, exclusions Exclusions) (truncated bool, err error) { + if _, err := content.Seek(0, io.SeekStart); err != nil { + return false, err + } + err = format.Extract(ctx, content, func(_ context.Context, f archives.FileInfo) error { + hdr, ok := entryHeader(f) + if !ok || exclusions.Excludes(hdr.Name) { + return nil + } + return storeEntry(store, hdr, f) + }) + if errors.Is(err, ErrDiskLimitReached) { + return true, nil + } + return false, err +} + +func storeEntry(store *EntryStore, hdr tar.Header, f archives.FileInfo) error { + if !f.Mode().IsRegular() { + return store.Add(hdr, nil) + } + content, err := f.Open() + if err != nil { + log.WithFields("entry", hdr.Name, "error", err).Trace("unable to open archive entry, skipping it") + return nil + } + defer func() { + if err := content.Close(); err != nil { + log.WithFields("entry", hdr.Name, "error", err).Trace("unable to close archive entry") + } + }() + return store.Add(hdr, content) +} + +// archiveDigestHashes are the digests taken of an archive as a whole: SHA-1, matching what the java +// cataloger reports for a jar. +var archiveDigestHashes = []crypto.Hash{crypto.SHA1} + +// digestsOf digests the archive's own bytes while they are in hand. A failure is logged and returns +// nothing rather than discarding an archive that extracted fine. +func digestsOf(ctx context.Context, content ReaderAtSeeker, archivePath string) []file.Digest { + if _, err := content.Seek(0, io.SeekStart); err != nil { + log.WithFields("archive", archivePath, "error", err).Trace("unable to rewind archive content for digests") + return nil + } + digests, err := intFile.NewDigestsFromFile(ctx, io.NopCloser(content), archiveDigestHashes) + if err != nil { + log.WithFields("archive", archivePath, "error", err).Trace("unable to digest archive") + return nil + } + return digests +} + +var ( + // zipLocalFileHeaderSignature starts every entry in a zip. + zipLocalFileHeaderSignature = []byte{'P', 'K', 0x03, 0x04} + + // zipEOCDSignature starts the end-of-central-directory record that closes every zip. Readers + // locate the central directory from it, which is why a zip with data prepended still opens. + zipEOCDSignature = []byte{'P', 'K', 0x05, 0x06} +) + +const ( + // maxAppendedArchiveStubBytes is how far into a file an appended zip's first entry is looked for. A + // Spring Boot launcher stub is about 9 KB. + maxAppendedArchiveStubBytes = 64 * 1024 + + // maxZipEOCDSearch is how far from the end the end-of-central-directory record can start: the 22 + // byte record plus a comment of up to 65535 bytes. + maxZipEOCDSearch = 22 + 65535 +) + +// MayHideAnAppendedArchive reports whether a zip entry signature appears in the head of the stream, +// which a zip appended to a launcher stub does and a file that merely has an archive's extension +// almost never does. The returned reader replays what was consumed. +func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) { + head := make([]byte, maxAppendedArchiveStubBytes) + n, err := io.ReadFull(r, head) + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + return r, false + } + head = head[:n] + found := bytes.Contains(head, zipLocalFileHeaderSignature) + + if seeker, ok := r.(io.Seeker); ok { + if _, err := seeker.Seek(0, io.SeekStart); err == nil { + return r, found + } + } + return io.MultiReader(bytes.NewReader(head), r), found +} + +// HasZipEndOfCentralDirectory reports whether the content ends in a zip end-of-central-directory +// record, the one part of a zip whose position is known without parsing it. +func HasZipEndOfCentralDirectory(r ReaderAtSeeker) bool { + size, err := r.Seek(0, io.SeekEnd) + if err != nil || size < int64(len(zipEOCDSignature)) { + return false + } + window := min(size, int64(maxZipEOCDSearch)) + buf := make([]byte, window) + if _, err := r.ReadAt(buf, size-window); err != nil && !errors.Is(err, io.EOF) { + return false + } + return bytes.Contains(buf, zipEOCDSignature) +} diff --git a/internal/archive/extract_test.go b/internal/archive/extract_test.go new file mode 100644 index 000000000..714f704db --- /dev/null +++ b/internal/archive/extract_test.go @@ -0,0 +1,467 @@ +package archive + +import ( + "archive/tar" + "bytes" + "context" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mholt/archives" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/file" +) + +var sampleFiles = map[string]string{ + "file1.txt": "content1", + "dir/file2.txt": "content2", +} + +func Test_identifyFormat(t *testing.T) { + ctx := context.Background() + launcher := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"), zipBytes(t, sampleFiles)...) + + tests := []struct { + name string + data []byte + want archives.Extractor + }{ + {"test.zip", zipBytes(t, sampleFiles), archives.Zip{}}, + {"test.tar.gz", tarGzBytes(t, sampleFiles), archives.CompressedArchive{}}, + {"plain.txt", []byte("just text"), nil}, + // the name can be claimed by identification, so the probe alone must not admit prose + {"app.jar", []byte("plain text, no central directory anywhere in it"), nil}, + {"bundle", []byte("plain text, no central directory anywhere in it"), nil}, + // content sniffing types this as a shell script; only its tail says zip + {"app.jar", launcher, archives.Zip{}}, + } + for _, tt := range tests { + t.Run(fmt.Sprintf("%s/%d bytes", tt.name, len(tt.data)), func(t *testing.T) { + got := identifyFormat(ctx, tt.name, bytes.NewReader(tt.data)) + if tt.want == nil { + assert.Nil(t, got) + return + } + assert.IsType(t, tt.want, got) + }) + } +} + +func Test_extractInto_storesEveryEntryInMemoryWhenUnbounded(t *testing.T) { + for name, data := range map[string][]byte{ + "test.zip": zipBytes(t, sampleFiles), + "test.tar.gz": tarGzBytes(t, sampleFiles), + } { + t.Run(name, func(t *testing.T) { + store, truncated := extractBytes(t, data, name, nil) + assert.False(t, truncated) + assert.Zero(t, store.OnDisk()) + assert.Empty(t, filesIn(t, storeDir(t, store))) + + entries := readStore(t, store) + assert.Equal(t, "content1", entries["file1.txt"].body) + assert.Equal(t, "content2", entries["dir/file2.txt"].body) + }) + } +} + +func Test_extractInto_diskLimitTruncatesAtTheEntryThatDoesNotFit(t *testing.T) { + // each entry costs its index estimate plus its content on disk + record := approxIndexBytes(tar.Header{Name: "file1.txt"}) + data := zipBytes(t, map[string]string{"file1.txt": "a", "file2.txt": "b", "file3.txt": "c"}) + + store, truncated := extractBytes(t, data, "test.zip", diskCharge(2*(record+1)+record)) + + assert.True(t, truncated) + assert.Equal(t, []string{"file1.txt", "file2.txt"}, storedNames(t, store), "the refused entry is not stored") + entries := readStore(t, store) + assert.Equal(t, "a", entries["file1.txt"].body) + assert.Equal(t, "b", entries["file2.txt"].body) +} + +func Test_extractInto_anEntryLargerThanTheBudgetIsDroppedWhole(t *testing.T) { + data := zipBytes(t, map[string]string{"file1.txt": strings.Repeat("x", 100)}) + + store, truncated := extractBytes(t, data, "test.zip", diskCharge(50)) + + assert.True(t, truncated) + assert.Empty(t, store.Entries()) + assert.Zero(t, store.OnDisk()) +} + +func Test_extractInto_diskLimitBitesMidEntry(t *testing.T) { + // a tar is walked entry by entry, so the limit can bite part way through an entry + data := tarGzBytes(t, map[string]string{ + "a/first.txt": strings.Repeat("a", 40*1024), + "b/second.txt": strings.Repeat("b", 40*1024), + }) + const budget = 80 * 1024 + charge := diskCharge(budget) + + store, truncated := extractBytes(t, data, "test.tar.gz", charge) + + assert.True(t, truncated) + assert.Equal(t, []string{"a/first.txt"}, storedNames(t, store)) + assert.Greater(t, store.OnDisk(), int64(40*1024), "the first entry landed whole and the second part way") + _, disk := charge.held() + assert.LessOrEqual(t, disk, int64(budget)) + assert.Greater(t, disk, store.OnDisk(), "the index estimate is charged to disk too") + + info, err := os.Stat(filepath.Join(storeDir(t, store), entriesFileName)) + require.NoError(t, err) + assert.Equal(t, store.OnDisk(), info.Size()) +} + +func Test_extractInto_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) { + data := zipBytes(t, map[string]string{"a.txt": strings.Repeat("a", 200)}) + oneArchive := approxIndexBytes(tar.Header{Name: "a.txt"}) + 200 + limiter := NewLimiter(Limits{MaxDiskBytes: oneArchive * 3 / 2}) + + first := limiter.Charge() + store, truncated := extractBytes(t, data, "test.zip", first) + require.False(t, truncated) + require.Equal(t, int64(200), store.OnDisk()) + + second := limiter.Charge() + _, truncated = extractBytes(t, data, "test.zip", second) + assert.True(t, truncated, "a second archive is bounded while the first is still held") + second.Release() + + first.Release() + _, disk := limiter.InUse() + require.Zero(t, disk) + + store, truncated = extractBytes(t, data, "test.zip", limiter.Charge()) + assert.False(t, truncated, "and extracts in full once the first is released") + assert.Equal(t, int64(200), store.OnDisk()) +} + +func Test_extractInto_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) { + var entries []testEntry + for i := range 50 { + entries = append(entries, testEntry{name: fmt.Sprintf("d%04d/", i), dir: true}) + } + room := approxIndexBytes(tar.Header{Name: "d0000"}) * 10 + + for name, data := range map[string][]byte{ + "dirs.zip": zipFromEntries(t, entries), + "dirs.tar.gz": tarGzFromEntries(t, entries), + } { + t.Run(name, func(t *testing.T) { + store, truncated := extractBytes(t, data, name, diskCharge(room)) + assert.True(t, truncated) + assert.Len(t, store.Entries(), 10) + }) + } +} + +func Test_extractInto_entryNamesAreCleanedAndNothingIsWrittenUnderThem(t *testing.T) { + entries := []testEntry{ + {name: "../../etc/passwd", body: "evil"}, + {name: "/etc/hosts", body: "pwned"}, + {name: "META-INF/", dir: true, mode: 0o555}, + {name: "META-INF/MANIFEST.MF", body: "Manifest-Version: 1.0"}, + {name: "passwd", link: "../../../../etc/passwd"}, + {name: "shadow", link: "/etc/shadow"}, + {name: "bin/link.txt", link: "../lib/real.txt"}, + } + + for name, data := range map[string][]byte{ + "evil.zip": zipFromEntries(t, entries), + "evil.tar.gz": tarGzFromEntries(t, entries), + } { + t.Run(name, func(t *testing.T) { + store, truncated := extractBytes(t, data, name, nil) + require.False(t, truncated) + + assert.Equal(t, []string{"META-INF", "META-INF/MANIFEST.MF", "bin/link.txt", "etc/hosts", "etc/passwd", "passwd", "shadow"}, + storedNames(t, store)) + assert.Empty(t, filesIn(t, storeDir(t, store)), "entries live in the store, never under their own names") + assert.NoFileExists(t, filepath.Join(t.TempDir(), "..", "etc", "passwd")) + + got := readStore(t, store) + assert.Equal(t, "evil", got["etc/passwd"].body) + assert.Equal(t, "pwned", got["etc/hosts"].body) + assert.Equal(t, byte(tar.TypeDir), got["META-INF"].typeflag) + assert.Equal(t, fs.FileMode(0o555), got["META-INF"].mode.Perm()) + assert.Equal(t, "Manifest-Version: 1.0", got["META-INF/MANIFEST.MF"].body) + + // links are headers carrying their target verbatim; the index resolves them inside the archive + for _, link := range []string{"passwd", "shadow", "bin/link.txt"} { + assert.Equal(t, byte(tar.TypeSymlink), got[link].typeflag) + assert.Empty(t, got[link].body) + } + assert.Equal(t, "../../../../etc/passwd", got["passwd"].linkTarget) + assert.Equal(t, "/etc/shadow", got["shadow"].linkTarget) + assert.Equal(t, "../lib/real.txt", got["bin/link.txt"].linkTarget) + }) + } +} + +func Test_extractInto_dataEdgeCases(t *testing.T) { + t.Run("an empty archive is not an error", func(t *testing.T) { + store, truncated := extractBytes(t, zipBytes(t, map[string]string{}), "test.zip", nil) + assert.Empty(t, store.Entries()) + assert.False(t, truncated) + }) + + t.Run("a zero-byte entry costs no disk", func(t *testing.T) { + store, _ := extractBytes(t, zipBytes(t, map[string]string{"empty.txt": ""}), "test.zip", memCharge(0)) + assert.Equal(t, []string{"empty.txt"}, storedNames(t, store)) + assert.Zero(t, store.OnDisk()) + assert.Empty(t, readStore(t, store)["empty.txt"].body) + }) + + t.Run("unicode entry names survive", func(t *testing.T) { + names := map[string]string{ + "café/naïve.txt": "accents", + "日本語/ファイル.txt": "japanese", + "emoji-🎉.txt": "emoji", + } + store, _ := extractBytes(t, zipBytes(t, names), "test.zip", nil) + entries := readStore(t, store) + for name, want := range names { + assert.Equal(t, want, entries[name].body, name) + } + }) + + t.Run("an over-long entry name is skipped", func(t *testing.T) { + data := zipBytes(t, map[string]string{ + strings.Repeat("a/", maxEntryNameBytes) + "f": "deep", + "ok.txt": "fine", + }) + store, _ := extractBytes(t, data, "test.zip", nil) + assert.Equal(t, []string{"ok.txt"}, storedNames(t, store)) + }) +} + +func Test_extractInto_excludedEntriesAreNeverStored(t *testing.T) { + data := zipBytes(t, map[string]string{ + "keep.txt": "kept", + "vendor/lib.go": "excluded by directory", + "deep/down/pkg.rpm": "excluded by name", + }) + limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}) + store := storeFor(t, limiter.Charge()) + content := bytes.NewReader(data) + + truncated, err := extractInto(context.Background(), identifyFormat(context.Background(), "test.zip", content), content, store, + NewExclusions([]string{"**/vendor", "**/*.rpm"})) + require.NoError(t, err) + require.False(t, truncated) + + assert.Equal(t, []string{"keep.txt"}, storedNames(t, store)) + mem, _ := limiter.InUse() + assert.Equal(t, approxIndexBytes(tar.Header{Name: "keep.txt"})+4, mem, "an excluded entry costs nothing") +} + +func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) { + data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"}) + + extracted, err := Extract(context.Background(), bytes.NewReader(data), "parentFS", "app.war:some/path/app.zip", nil, nil) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + assert.False(t, extracted.Truncated) + require.Len(t, extracted.Digests, 1) + assert.Equal(t, "sha1", extracted.Digests[0].Algorithm) + + locations, err := extracted.Resolver.FilesByGlob("**/*.txt") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, file.Coordinates{RealPath: "dir/hello.txt", FileSystemID: "parentFS", ArchivePath: "app.war:some/path/app.zip"}, + locations[0].Coordinates) + + reader, err := extracted.Resolver.FileContentsByLocation(locations[0]) + require.NoError(t, err) + body, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "hello world", string(body)) +} + +func TestExtract_notAnArchive(t *testing.T) { + extracted, err := Extract(context.Background(), strings.NewReader("this is not an archive"), "", "notes.txt", nil, nil) + require.NoError(t, err) + assert.Nil(t, extracted) +} + +func TestExtract_writesUnderTheScansTempRootAndCleansUp(t *testing.T) { + ctx, root := scanContext(t) + limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) + + extracted, err := Extract(ctx, bytes.NewReader(zipBytes(t, sampleFiles)), "", "app.zip", limiter, nil) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + workDirs := workDirsUnder(t, root) + require.Len(t, workDirs, 1) + assert.Equal(t, []string{entriesFileName}, filesIn(t, workDirs[0])) + _, disk := limiter.InUse() + assert.Positive(t, disk) + + extracted.Cleanup() + assert.NoDirExists(t, workDirs[0]) + _, disk = limiter.InUse() + assert.Zero(t, disk, "cleanup releases everything the archive held") + extracted.Cleanup() +} + +func TestExtract_anArchiveThatNeverWritesCreatesNoWorkDir(t *testing.T) { + ctx, root := scanContext(t) + + extracted, err := Extract(ctx, bytes.NewReader(zipBytes(t, sampleFiles)), "", "app.zip", + NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), nil) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + assert.Empty(t, workDirsUnder(t, root)) + extracted.Cleanup() + assert.Empty(t, workDirsUnder(t, root)) +} + +func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) { + // the archive's own bytes are only needed to extract and digest it; afterwards the limiter holds + // the entries and the index estimate alone + data := zipBytes(t, map[string]string{"hello.txt": "hi"}) + limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) + + extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil) + require.NoError(t, err) + t.Cleanup(extracted.Cleanup) + + mem, disk := limiter.InUse() + assert.Equal(t, approxIndexBytes(tar.Header{Name: "hello.txt"})+2, mem) + assert.Zero(t, disk) +} + +func TestExtract_heldContentWrittenToDiskIsRemovedOnceExtracted(t *testing.T) { + ctx, root := scanContext(t) + data := zipBytes(t, map[string]string{"hello.txt": "hi"}) + limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) + + extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil) + require.NoError(t, err) + t.Cleanup(extracted.Cleanup) + + workDirs := workDirsUnder(t, root) + require.Len(t, workDirs, 1) + assert.Equal(t, []string{entriesFileName}, filesIn(t, workDirs[0]), "the archive's own bytes are gone") + _, disk := limiter.InUse() + assert.Equal(t, approxIndexBytes(tar.Header{Name: "hello.txt"})+2, disk) +} + +func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) { + data := zipBytes(t, sampleFiles) + limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}) + + _, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil) + assert.ErrorIs(t, err, ErrDiskLimitReached) + + // read in place, the same bytes cost nothing; only the entries are refused + extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "app.zip", limiter, nil) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + assert.True(t, extracted.Truncated) + locations, err := extracted.Resolver.FilesByGlob("**/*") + require.NoError(t, err) + assert.Empty(t, locations) +} + +func TestExtract_truncationKeepsWhatWasStored(t *testing.T) { + data := zipBytes(t, map[string]string{ + "a/small.txt": strings.Repeat("s", 50), + "b/big.txt": strings.Repeat("b", 500), + }) + records := approxIndexBytes(tar.Header{Name: "a/small.txt"}) + approxIndexBytes(tar.Header{Name: "b/big.txt"}) + limiter := NewLimiter(Limits{MaxDiskBytes: records + 50 + 100}) + + extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "app.zip", limiter, nil) + require.NoError(t, err) + t.Cleanup(extracted.Cleanup) + + assert.True(t, extracted.Truncated) + locations, err := extracted.Resolver.FilesByGlob("**/*.txt") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, "a/small.txt", locations[0].RealPath) +} + +func TestExtract_aZipBehindALauncherScript(t *testing.T) { + // a Spring Boot executable jar sniffs as a shell script; archive/zip still finds the central directory + data := append([]byte("#!/bin/bash\necho launcher\nexit 0\n"), zipBytes(t, map[string]string{ + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", + "BOOT-INF/lib/dep.jar": "not really a jar, just bytes", + })...) + + extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "/opt/app.jar", nil, nil) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + assert.False(t, extracted.Truncated) + locations, err := extracted.Resolver.FilesByGlob("**/*") + require.NoError(t, err) + assert.Len(t, locations, 2) +} + +func Test_HasZipEndOfCentralDirectory(t *testing.T) { + tests := []struct { + name string + data []byte + want bool + }{ + {"a plain zip", zipBytes(t, sampleFiles), true}, + {"a zip behind a launcher script", append([]byte("#!/bin/bash\nexit 0\n"), zipBytes(t, sampleFiles)...), true}, + {"prose", []byte("this is not an archive, it is a sentence"), false}, + {"a tar.gz", tarGzBytes(t, sampleFiles), false}, + {"empty", nil, false}, + {"shorter than the signature", []byte("PK"), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, HasZipEndOfCentralDirectory(bytes.NewReader(tt.data))) + }) + } +} + +func Test_MayHideAnAppendedArchive(t *testing.T) { + launcher := append([]byte("#!/bin/bash\nexit 0\n"), zipBytes(t, sampleFiles)...) + prose := []byte("plain text, whatever the extension claims") + + t.Run("a seekable reader is rewound and returned as is", func(t *testing.T) { + reader := bytes.NewReader(launcher) + got, found := MayHideAnAppendedArchive(reader) + assert.True(t, found) + assert.Same(t, reader, got) + rest, err := io.ReadAll(got) + require.NoError(t, err) + assert.Equal(t, launcher, rest) + }) + + t.Run("an unseekable reader replays what was consumed", func(t *testing.T) { + got, found := MayHideAnAppendedArchive(unseekable{bytes.NewReader(launcher)}) + assert.True(t, found) + rest, err := io.ReadAll(got) + require.NoError(t, err) + assert.Equal(t, launcher, rest) + }) + + t.Run("prose hides nothing", func(t *testing.T) { + got, found := MayHideAnAppendedArchive(unseekable{bytes.NewReader(prose)}) + assert.False(t, found) + rest, err := io.ReadAll(got) + require.NoError(t, err) + assert.Equal(t, prose, rest) + }) +} diff --git a/internal/archive/extractor.go b/internal/archive/extractor.go deleted file mode 100644 index 488910c61..000000000 --- a/internal/archive/extractor.go +++ /dev/null @@ -1,340 +0,0 @@ -package archive - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "strings" - - "github.com/mholt/archives" - - intFile "github.com/anchore/syft/internal/file" - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/cataloging" -) - -// Extractor reads the entries of one archive into an EntrySink. -type Extractor interface { - CanExtract(ctx context.Context, content Content) bool - - Extract(ctx context.Context, content Content, sink EntrySink, charge *Charge) (ExtractionResult, error) -} - -// EntrySink is where an extractor puts the entries it reads out of an archive. -type EntrySink interface { - AddEntry(f archives.FileInfo, result *ExtractionResult, charge *Charge) error - - // Finish is called once the last entry has been offered. - Finish(result *ExtractionResult) -} - -// TruncationReason names the bound that stopped an extraction early; empty means extracted in full. -type TruncationReason string - -const ( - // TruncatedByDiskLimit: storing more content would exceed the scan's disk limit. - TruncatedByDiskLimit TruncationReason = "disk limit" - - // TruncatedByIndexLimit: indexing more entries would exceed the record budget. Distinct from the - // disk limit because the index is never written to disk, though the disk budget backstops it - see - // Charge.IndexRecord. - TruncatedByIndexLimit TruncationReason = "index limit" - - // TruncatedByLimit: a bound stopped extraction but the walker replaced the error naming which one. - TruncatedByLimit TruncationReason = "configured limit" -) - -// errTruncated stops an in-progress walk at a configured limit. It never escapes this package: -// extractors convert it into a successful, truncated ExtractionResult. -var errTruncated = errors.New("archive extraction stopped at a configured limit") - -// ExtractionResult holds the result of an extraction. Reaching a limit is a truncation, not an error: -// content stored before the limit stays usable. A non-nil error from Extract means nothing usable was -// produced. -type ExtractionResult struct { - // Truncation names the limit that stopped extraction early, or "" if none did. - Truncation TruncationReason -} - -func (r ExtractionResult) Truncated() bool { - return r.Truncation != "" -} - -// finishExtraction turns a walk stopped at a configured limit into a successful, truncated result. -// Both result and error are checked: walkers may wrap or replace the handler's error, so -// errors.Is(err, errTruncated) alone would report some truncations as failures. -func finishExtraction(result ExtractionResult, err error) (ExtractionResult, error) { - if result.Truncated() || errors.Is(err, errTruncated) { - if !result.Truncated() { - // the walker swallowed the reason but passed the sentinel through - result.Truncation = TruncatedByLimit - } - return result, nil - } - return result, err -} - -// DefaultLimits returns the scan-wide in-use limits from the given config. -func DefaultLimits(cfg cataloging.ArchiveSearchConfig) Limits { - return Limits{ - MaxMemoryBytes: cfg.MaxMemoryBytes, - MaxDiskBytes: cfg.MaxDiskBytes, - } -} - -// DefaultExtractors returns the set of built-in archive extractors. -func DefaultExtractors() []Extractor { - return []Extractor{ - &ZipExtractor{}, - &TarExtractor{}, - } -} - -// FindExtractor finds an extractor that can handle the given archive content, or nil if none can. -func FindExtractor(ctx context.Context, extractors []Extractor, content Content) Extractor { - for _, ext := range extractors { - // every extractor sniffs from the start, whether the content is a file or a buffer - if _, err := content.Reader.Seek(0, io.SeekStart); err != nil { - log.Tracef("unable to seek archive content for detection: %v", err) - return nil - } - if ext.CanExtract(ctx, content) { - return ext - } - } - return nil -} - -// ZipExtractor extracts zip-based archives (zip, jar, war, ear, etc.). -type ZipExtractor struct{} - -// CanExtract reports whether this content is a zip. Test order matters: head identification wins, -// because a tar holding one jar ends in that jar's EOCD record, so reading it as a zip would lose the -// rest of the tar. The EOCD record is consulted only when nothing identifies from the head, where it -// is the only test that sees a zip concatenated onto a launcher stub (e.g. a Spring Boot executable -// jar). -// -// IdentifyArchive matches on filename as well as content; dropping the name would run every format's -// content matcher against every candidate, far slower. The name cannot admit a non-archive, since -// files reach an extractor only after content sniffing (see -// Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError). -func (z *ZipExtractor) CanExtract(ctx context.Context, content Content) bool { - format, _, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader) - if err == nil { - _, isZip := format.(archives.Zip) - return isZip - } - - if _, seekErr := content.Reader.Seek(0, io.SeekStart); seekErr != nil { - return false - } - return HasZipEndOfCentralDirectory(content.Reader) -} - -func (z *ZipExtractor) Extract(ctx context.Context, content Content, sink EntrySink, charge *Charge) (ExtractionResult, error) { - var result ExtractionResult - - if _, err := content.Reader.Seek(0, io.SeekStart); err != nil { - return result, fmt.Errorf("unable to seek zip archive %q: %w", content.Name, err) - } - - err := archives.Zip{}.Extract(ctx, content.Reader, func(_ context.Context, file archives.FileInfo) error { - return sink.AddEntry(file, &result, charge) - }) - - sink.Finish(&result) - - return finishExtraction(result, err) -} - -// TarExtractor extracts tar-based archives (tar, tar.gz, tar.bz2, tar.xz, tar.zst). -type TarExtractor struct{} - -// CanExtract reports whether the content identifies as an extractable non-zip format. -func (t *TarExtractor) CanExtract(ctx context.Context, content Content) bool { - format, _, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader) - if err != nil { - return false - } - if _, isZip := format.(archives.Zip); isZip { - return false - } - // tar+compression comes back as a compound type, so test for the Extractor interface - _, ok := format.(archives.Extractor) - return ok -} - -func (t *TarExtractor) Extract(ctx context.Context, content Content, sink EntrySink, charge *Charge) (ExtractionResult, error) { - var result ExtractionResult - - if _, err := content.Reader.Seek(0, io.SeekStart); err != nil { - return result, fmt.Errorf("unable to seek tar archive %q: %w", content.Name, err) - } - - format, readerAfterIdentify, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader) - if err != nil { - return result, fmt.Errorf("unable to identify archive format for %q: %w", content.Name, err) - } - - extractor, ok := format.(archives.Extractor) - if !ok { - return result, fmt.Errorf("file format does not support extraction: %s", content.Name) - } - - // use the reader returned by IdentifyArchive since it may have consumed some bytes - var extractReader = readerAfterIdentify - if extractReader == nil { - extractReader = content.Reader - } - - err = extractor.Extract(ctx, extractReader, func(_ context.Context, file archives.FileInfo) error { - return sink.AddEntry(file, &result, charge) - }) - - sink.Finish(&result) - - return finishExtraction(result, err) -} - -// resolvesInsideRoot reports whether writing to path lands inside root, following existing symlinks -// as the filesystem would. It guards the one filesystem write left in this package: an archive -// spilling into its work directory under an attacker-controlled name. -// -// The walk is component by component rather than one EvalSymlinks call because the leaf does not yet -// exist, which makes EvalSymlinks fail rather than answer. The first missing component ends the walk: -// anything created below it lands under an already-checked path. -func resolvesInsideRoot(root, path string) (bool, error) { - // the root itself may sit under a symlinked directory (e.g. a macOS temp dir) - realRoot, err := filepath.EvalSymlinks(root) - if err != nil { - return false, fmt.Errorf("unable to resolve extraction root: %w", err) - } - - rel, err := filepath.Rel(root, path) - if err != nil { - return false, err - } - if !isWithin(filepath.Join(realRoot, rel), realRoot) { - return false, nil - } - - current := realRoot - for _, part := range strings.Split(filepath.ToSlash(rel), "/") { - if part == "" || part == "." { - continue - } - current = filepath.Join(current, part) - - info, err := os.Lstat(current) - if err != nil { - if os.IsNotExist(err) { - break - } - return false, err - } - if info.Mode()&fs.ModeSymlink == 0 { - continue - } - - resolved, err := resolveLink(current) - if err != nil { - return false, err - } - if !isWithin(resolved, realRoot) { - return false, nil - } - current = resolved - } - - return isWithin(current, realRoot), nil -} - -// resolveLink returns where a symlink points, falling back to the lexically cleaned target when it -// does not exist yet - a dangling link still redirects a write. -func resolveLink(path string) (string, error) { - target, err := os.Readlink(path) - if err != nil { - return "", err - } - if !filepath.IsAbs(target) { - target = filepath.Join(filepath.Dir(path), target) - } - resolved, err := filepath.EvalSymlinks(target) - if err != nil { - if !os.IsNotExist(err) { - return "", err - } - return filepath.Clean(target), nil - } - return resolved, nil -} - -// isWithin reports whether path is root or beneath it, compared on path boundaries so a sibling -// sharing the prefix does not pass. -func isWithin(path, root string) bool { - rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path)) - if err != nil { - return false - } - return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))) -} - -// zipEOCDSignature is the end-of-central-directory record's signature. Readers seek to this record -// to find the central directory, which is why a zip with something prepended still opens. -var zipEOCDSignature = []byte{'P', 'K', 0x05, 0x06} - -// maxZipEOCDSearch is how far back from the end the EOCD record can start: a 22 byte record plus a -// comment of up to 65535 bytes. -const maxZipEOCDSearch = 22 + 65535 - -// zipLocalFileHeaderSignature starts every entry in a zip. In a self-extracting archive the first -// one follows the stub, within reach of a bounded read of the head. -var zipLocalFileHeaderSignature = []byte{'P', 'K', 0x03, 0x04} - -// maxAppendedArchiveStubBytes is how far into a file an appended archive is looked for. A Spring -// Boot launcher stub is about 9 KB; a longer stub hides its archive from the walk. -const maxAppendedArchiveStubBytes = 64 * 1024 - -// MayHideAnAppendedArchive reports whether the head of this stream contains a zip local file header, -// and returns a reader that replays what it consumed. -// -// A cheap, bounded pre-filter for HasZipEndOfCentralDirectory: reading every script, executable and -// unrecognized file to its end roughly doubles a scan, so only files showing an entry signature in -// their head are read in full. It narrows what is examined; it admits nothing on its own. -func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) { - head := make([]byte, maxAppendedArchiveStubBytes) - n, err := io.ReadFull(r, head) - if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { - return io.MultiReader(bytes.NewReader(head[:n]), r), false - } - head = head[:n] - return io.MultiReader(bytes.NewReader(head), r), bytes.Contains(head, zipLocalFileHeaderSignature) -} - -// HasZipEndOfCentralDirectory reports whether the content ends in a zip end-of-central-directory -// record - the only part of a zip whose position is knowable without parsing. It identifies archives -// that head-based sniffing types as something else, such as a launcher stub with a zip appended, -// which mholt/archives extracts correctly via archive/zip. -func HasZipEndOfCentralDirectory(r ReaderAtSeeker) bool { - size, err := r.Seek(0, io.SeekEnd) - if err != nil || size < int64(len(zipEOCDSignature)) { - return false - } - - window := int64(maxZipEOCDSearch) - if size < window { - window = size - } - - buf := make([]byte, window) - if _, err := r.ReadAt(buf, size-window); err != nil && !errors.Is(err, io.EOF) { - return false - } - - return bytes.LastIndex(buf, zipEOCDSignature) >= 0 -} diff --git a/internal/archive/extractor_test.go b/internal/archive/extractor_test.go deleted file mode 100644 index 97393a29f..000000000 --- a/internal/archive/extractor_test.go +++ /dev/null @@ -1,1241 +0,0 @@ -package archive - -import ( - "archive/tar" - "archive/zip" - "bytes" - "compress/gzip" - "context" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "sort" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/anchore/syft/syft/cataloging" -) - -// storedEntry is one entry read back out of an EntryStore. Assertions run against these rather than -// files in a directory, since an archive's entries are headers and bytes in a store. -type storedEntry struct { - name string - body string - linkTarget string - mode fs.FileMode - typeflag byte -} - -func TestZipExtractor_CanExtract(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) - - ext := &ZipExtractor{} - - assert.True(t, ext.CanExtract(ctx, fileContent(t, zipPath))) - - nonZipPath := filepath.Join(dir, "notazip.txt") - require.NoError(t, os.WriteFile(nonZipPath, []byte("just text"), 0o644)) - - assert.False(t, ext.CanExtract(ctx, fileContent(t, nonZipPath))) -} - -func TestZipExtractor_Extract(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - files := map[string]string{ - "file1.txt": "content1", - "dir/file2.txt": "content2", - } - zipPath := createTestZip(t, dir, files) - - ext := &ZipExtractor{} - overflow := storeFor(t) - - _, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, nil) - require.NoError(t, err) - assert.Len(t, overflow.Entries(), 2) - - // both entries are in the one store, under their own names and with their own content - entries := readStore(t, overflow) - assert.Equal(t, "content1", entries["file1.txt"].body) - assert.Equal(t, "content2", entries["dir/file2.txt"].body) - - // and with no bound to push them out, nothing was written to disk at all - assert.Zero(t, overflow.OnDisk()) - assert.NoFileExists(t, filepath.Join(storeDir(t, overflow), overflowBlobName)) -} - -func TestZipExtractor_Extract_FileLimitReached(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - files := map[string]string{ - "file1.txt": "a", - "file2.txt": "b", - "file3.txt": "c", - } - zipPath := createTestZip(t, dir, files) - - ext := &ZipExtractor{} - overflow := storeFor(t) - - // reaching a limit truncates rather than fails: what was stored stays catalogable. Each entry costs - // its index record plus its one byte of content, so two entries' worth of room refuses the third. - record := indexRecordCost(tar.Header{Name: "file1.txt"}) - result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, diskCharge(2*(record+1)+record)) - require.NoError(t, err) - assert.True(t, result.Truncated()) - assert.Equal(t, TruncatedByDiskLimit, result.Truncation) - - entries := readStore(t, overflow) - assert.Equal(t, "a", entries["file1.txt"].body) - assert.Equal(t, "b", entries["file2.txt"].body, - "the two entries stored before the limit must still be readable") - assert.Empty(t, entries["file3.txt"].body, "the entry the limit refused holds no content") -} - -func TestZipExtractor_Extract_SizeLimitReached(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - files := map[string]string{ - "file1.txt": strings.Repeat("x", 100), - } - zipPath := createTestZip(t, dir, files) - - ext := &ZipExtractor{} - overflow := storeFor(t) - - result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, diskCharge(50)) - require.NoError(t, err) - assert.True(t, result.Truncated()) - assert.Equal(t, TruncatedByDiskLimit, result.Truncation) - - // the single entry blew the budget on its own, so it is dropped rather than left half-written: a - // partial jar or pom a cataloger parses anyway is worse than an absent one. The budget goes on the - // entry's header before any content, a header being bytes on disk too. - assert.Empty(t, overflow.Entries()) - assert.Zero(t, overflow.OnDisk()) - assert.Empty(t, storedNames(t, overflow), "the over-limit entry must not be left behind") -} - -func TestTarExtractor_CanExtract(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) - - ext := &TarExtractor{} - assert.True(t, ext.CanExtract(ctx, fileContent(t, tarPath))) -} - -func TestTarExtractor_Extract(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - files := map[string]string{ - "file1.txt": "content1", - "dir/file2.txt": "content2", - } - tarPath := createTestTarGz(t, dir, files) - - ext := &TarExtractor{} - overflow := storeFor(t) - - _, err := ext.Extract(ctx, fileContent(t, tarPath), overflow, nil) - require.NoError(t, err) - - // a tar-family archive is walked entry by entry like any other, and with no bound pushing them out - // its entries stay in memory - assert.Len(t, overflow.Entries(), 2) - assert.Zero(t, overflow.OnDisk()) - - entries := readStore(t, overflow) - assert.Equal(t, "content1", entries["file1.txt"].body) - assert.Equal(t, "content2", entries["dir/file2.txt"].body) -} - -func TestFindExtractor(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - extractors := DefaultExtractors() - - zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) - ext := FindExtractor(ctx, extractors, fileContent(t, zipPath)) - require.NotNil(t, ext) - assert.IsType(t, &ZipExtractor{}, ext) - - tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) - ext = FindExtractor(ctx, extractors, fileContent(t, tarPath)) - require.NotNil(t, ext) - assert.IsType(t, &TarExtractor{}, ext) - - textPath := filepath.Join(dir, "plain.txt") - require.NoError(t, os.WriteFile(textPath, []byte("just text"), 0o644)) - ext = FindExtractor(ctx, extractors, fileContent(t, textPath)) - assert.Nil(t, ext) -} - -func TestFindExtractor_NoReaders(t *testing.T) { - // content no extractor claims yields nil rather than an error, so a file matching a broad mime - // filter but not an archive is skipped - ctx := context.Background() - dir := t.TempDir() - path := filepath.Join(dir, "plain.txt") - require.NoError(t, os.WriteFile(path, []byte("just text"), 0o644)) - - ext := FindExtractor(ctx, DefaultExtractors(), fileContent(t, path)) - assert.Nil(t, ext) -} - -func TestTarExtractor_CanExtract_NonTarFile(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - path := filepath.Join(dir, "plain.txt") - require.NoError(t, os.WriteFile(path, []byte("not a tar"), 0o644)) - - ext := &TarExtractor{} - assert.False(t, ext.CanExtract(ctx, fileContent(t, path))) -} - -func TestTarExtractor_CanExtract_ZipFileReturnsFalse(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) - - ext := &TarExtractor{} - - assert.False(t, ext.CanExtract(ctx, fileContent(t, zipPath))) -} - -func TestDefaultLimits(t *testing.T) { - cfg := cataloging.ArchiveSearchConfig{ - MaxMemoryBytes: 111, - MaxDiskBytes: 222, - } - limits := DefaultLimits(cfg) - assert.Equal(t, int64(111), limits.MaxMemoryBytes) - assert.Equal(t, int64(222), limits.MaxDiskBytes) -} - -func TestZipExtractor_Extract_ZipSlipPrevented(t *testing.T) { - // an entry named "../../etc/passwd" becomes a field in a tar header, and writing a header traverses - // no path, so extraction succeeds and nothing is created anywhere. The name still matters to the - // path the SBOM reports, which is clamped at index time - see the archive resolver's own tests. - dir := t.TempDir() - ctx := context.Background() - - zipPath := filepath.Join(dir, "evil.zip") - f, err := os.Create(zipPath) - require.NoError(t, err) - - w := zip.NewWriter(f) - fw, err := w.Create("../../etc/passwd") - require.NoError(t, err) - _, err = fw.Write([]byte("evil")) - require.NoError(t, err) - require.NoError(t, w.Close()) - require.NoError(t, f.Close()) - - ext := &ZipExtractor{} - overflow := storeFor(t) - - _, err = ext.Extract(ctx, fileContent(t, zipPath), overflow, nil) - require.NoError(t, err) - assert.Len(t, overflow.Entries(), 1) - - // nothing is created outside the tar, at the traversed path or anywhere else - _, statErr := os.Stat(filepath.Join(dir, "etc", "passwd")) - assert.True(t, os.IsNotExist(statErr)) - _, statErr = os.Stat(filepath.Join(storeDir(t, overflow), "etc")) - assert.True(t, os.IsNotExist(statErr)) - - // the name is carried through unsanitized, this being the archive's own record of what it held; - // sanitizing happens where the logical filetree is built - assert.Equal(t, []string{"../../etc/passwd"}, storedNames(t, overflow)) -} - -func TestTarExtractor_Extract_SymlinkIsRecordedAsALink(t *testing.T) { - // a symlink entry is recorded as a symlink header carrying its target - not a regular file holding - // the target as bytes, and not a link on the real filesystem. Nothing is created, so nothing can be - // redirected; where the link points is decided in the archive's own filetree. - dir := t.TempDir() - ctx := context.Background() - - tarPath := filepath.Join(dir, "with-link.tar.gz") - f, err := os.Create(tarPath) - require.NoError(t, err) - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - target := []byte("real content") - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "real.txt", - Mode: 0o644, - Size: int64(len(target)), - })) - _, err = tw.Write(target) - require.NoError(t, err) - - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "link.txt", - Linkname: "real.txt", - Typeflag: tar.TypeSymlink, - Mode: 0o777, - })) - - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - require.NoError(t, f.Close()) - - ext := &TarExtractor{} - overflow := storeFor(t) - - _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, nil) - require.NoError(t, err) - - entries := readStore(t, overflow) - link := entries["link.txt"] - assert.Equal(t, byte(tar.TypeSymlink), link.typeflag, "link.txt must stay a link, not become a file") - assert.Equal(t, "real.txt", link.linkTarget) - assert.Empty(t, link.body, "a link carries a target, not content") - assert.Equal(t, "real content", entries["real.txt"].body) - - // and no link exists on the real filesystem beside the tar - _, statErr := os.Lstat(filepath.Join(storeDir(t, overflow), "link.txt")) - assert.True(t, os.IsNotExist(statErr)) -} - -func TestTarExtractor_Extract_SymlinkEscapingRootCreatesNothing(t *testing.T) { - // a link target climbing out of the archive is recorded as the archive gave it, the entry being a - // header with nothing created for a write to be redirected through. The target is clamped to the - // archive's own root when the filetree is built - see the archive resolver's own tests. - dir := t.TempDir() - ctx := context.Background() - - tarPath := filepath.Join(dir, "evil-link.tar.gz") - f, err := os.Create(tarPath) - require.NoError(t, err) - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "passwd", - Linkname: "../../../../etc/passwd", - Typeflag: tar.TypeSymlink, - Mode: 0o777, - })) - - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - require.NoError(t, f.Close()) - - ext := &TarExtractor{} - overflow := storeFor(t) - - _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, nil) - require.NoError(t, err) - - // nothing on the real filesystem beside the tar - _, statErr := os.Lstat(filepath.Join(storeDir(t, overflow), "passwd")) - assert.True(t, os.IsNotExist(statErr), "no link may be created for an escaping symlink") - - entries := readStore(t, overflow) - assert.Equal(t, byte(tar.TypeSymlink), entries["passwd"].typeflag) - assert.Equal(t, "../../../../etc/passwd", entries["passwd"].linkTarget) -} - -func TestTarExtractor_Extract_AbsoluteSymlinkTargetCreatesNothing(t *testing.T) { - // an absolute target creates no link either: a header write resolves nowhere, and the target is - // re-rooted inside the archive when the filetree is built - dir := t.TempDir() - ctx := context.Background() - - tarPath := filepath.Join(dir, "absolute-link.tar.gz") - f, err := os.Create(tarPath) - require.NoError(t, err) - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "shadow", - Linkname: "/etc/shadow", - Typeflag: tar.TypeSymlink, - Mode: 0o777, - })) - - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - require.NoError(t, f.Close()) - - ext := &TarExtractor{} - overflow := storeFor(t) - - _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, nil) - require.NoError(t, err) - - _, statErr := os.Lstat(filepath.Join(storeDir(t, overflow), "shadow")) - assert.True(t, os.IsNotExist(statErr), "no link may be created for an absolute target") - - entries := readStore(t, overflow) - assert.Equal(t, "/etc/shadow", entries["shadow"].linkTarget) -} - -func TestZipExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { - // jars may hold directory entries with read-only permissions (e.g. META-INF/ at mode 0o555); the - // extractor must still write files into them - dir := t.TempDir() - ctx := context.Background() - - zipPath := filepath.Join(dir, "readonly-dirs.zip") - f, err := os.Create(zipPath) - require.NoError(t, err) - - w := zip.NewWriter(f) - - dirHeader := &zip.FileHeader{ - Name: "META-INF/", - } - dirHeader.SetMode(0o555) - _, err = w.CreateHeader(dirHeader) - require.NoError(t, err) - - fileHeader := &zip.FileHeader{ - Name: "META-INF/MANIFEST.MF", - } - fileHeader.SetMode(0o644) - fw, err := w.CreateHeader(fileHeader) - require.NoError(t, err) - _, err = fw.Write([]byte("Manifest-Version: 1.0")) - require.NoError(t, err) - - require.NoError(t, w.Close()) - require.NoError(t, f.Close()) - - ext := &ZipExtractor{} - overflow := storeFor(t) - - _, err = ext.Extract(ctx, fileContent(t, zipPath), overflow, nil) - require.NoError(t, err) - // two entries: the directory and the file. A directory carries no bytes of its own, but its index - // record is charged, which is what bounds an archive of nothing but directories. - assert.Len(t, overflow.Entries(), 2) - - // nothing is written into a directory, so its mode decides nothing, but both entries must be - // recorded, modes and all - entries := readStore(t, overflow) - assert.Equal(t, "Manifest-Version: 1.0", entries["META-INF/MANIFEST.MF"].body) - assert.Equal(t, fs.FileMode(0o555), entries["META-INF/"].mode.Perm()) - assert.Equal(t, byte(tar.TypeDir), entries["META-INF/"].typeflag) -} - -func TestTarExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { - // tars may hold directory entries with read-only permissions; the extractor must still write files - // into them - dir := t.TempDir() - ctx := context.Background() - - tarPath := filepath.Join(dir, "readonly-dirs.tar.gz") - f, err := os.Create(tarPath) - require.NoError(t, err) - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "META-INF/", - Typeflag: tar.TypeDir, - Mode: 0o555, - })) - - content := []byte("Manifest-Version: 1.0") - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "META-INF/MANIFEST.MF", - Mode: 0o644, - Size: int64(len(content)), - })) - _, err = tw.Write(content) - require.NoError(t, err) - - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - require.NoError(t, f.Close()) - - ext := &TarExtractor{} - overflow := storeFor(t) - - _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, nil) - require.NoError(t, err) - - entries := readStore(t, overflow) - assert.Equal(t, "Manifest-Version: 1.0", entries["META-INF/MANIFEST.MF"].body) - assert.Equal(t, fs.FileMode(0o555), entries["META-INF/"].mode.Perm()) -} - -func TestZipExtractor_Extract_nonPositiveLimitDisablesOnlyThatLimit(t *testing.T) { - // <= 0 means "no limit", per limit, so a caller can opt out of one without the rest; substituting a - // default would re-impose a limit the caller turned off - dir := t.TempDir() - ctx := context.Background() - - files := map[string]string{} - for i := range 20 { - files[fmt.Sprintf("file%02d.txt", i)] = "x" - } - files["big.txt"] = strings.Repeat("y", 500) - zipPath := createTestZip(t, dir, files) - - t.Run("the disk limit bites", func(t *testing.T) { - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), storeFor(t), diskCharge(100)) - require.NoError(t, err) - assert.Equal(t, TruncatedByDiskLimit, result.Truncation, "the disk limit must still bite") - }) - - t.Run("unbounded extracts everything", func(t *testing.T) { - overflow := storeFor(t) - - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, nil) - require.NoError(t, err) - assert.False(t, result.Truncated()) - assert.Len(t, overflow.Entries(), len(files)) - }) -} - -func TestZipExtractor_Extract_dataEdgeCases(t *testing.T) { - ctx := context.Background() - - t.Run("empty archive is a non-event", func(t *testing.T) { - dir := t.TempDir() - zipPath := createTestZip(t, dir, map[string]string{}) - - overflow := storeFor(t) - - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, nil) - require.NoError(t, err, "a valid archive containing nothing is not a failure") - assert.Empty(t, overflow.Entries()) - assert.False(t, result.Truncated()) - }) - - t.Run("zero-byte entry is recorded", func(t *testing.T) { - dir := t.TempDir() - zipPath := createTestZip(t, dir, map[string]string{"empty.txt": ""}) - overflow := storeFor(t) - - _, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, nil) - require.NoError(t, err) - assert.Len(t, overflow.Entries(), 1) - - // a store has no framing to write, so an entry carrying no content costs no disk at all - assert.Zero(t, overflow.OnDisk()) - - entries := readStore(t, overflow) - require.Contains(t, entries, "empty.txt") - assert.Empty(t, entries["empty.txt"].body) - }) - - t.Run("absolute entry path is contained", func(t *testing.T) { - // distinct from ../ traversal: an entry literally named /etc/hosts must land inside the - // extraction dir, never at the host path - dir := t.TempDir() - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - w, err := zw.Create("/etc/hosts") - require.NoError(t, err) - _, err = w.Write([]byte("pwned")) - require.NoError(t, err) - require.NoError(t, zw.Close()) - - zipPath := filepath.Join(dir, "abs.zip") - require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0o600)) - - overflow := storeFor(t) - - _, err = (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, nil) - require.NoError(t, err) - - // one entry in the tar and nothing at the host path; the leading slash is dropped when the - // filetree is built, so it reads as a path inside this archive - assert.Equal(t, "pwned", readStore(t, overflow)["/etc/hosts"].body) - _, statErr := os.Stat(filepath.Join(dir, "etc", "hosts")) - assert.True(t, os.IsNotExist(statErr)) - }) - - t.Run("unicode and non-ascii entry names survive", func(t *testing.T) { - dir := t.TempDir() - names := map[string]string{ - "café/naïve.txt": "accents", - "日本語/ファイル.txt": "japanese", - "emoji-🎉.txt": "emoji", - } - zipPath := createTestZip(t, dir, names) - overflow := storeFor(t) - - _, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, nil) - require.NoError(t, err) - assert.Len(t, overflow.Entries(), len(names)) - - entries := readStore(t, overflow) - for name, want := range names { - require.Contains(t, entries, name, "entry %q must be recorded under its own name", name) - assert.Equal(t, want, entries[name].body) - } - }) -} - -func TestExtract_diskLimitTruncatesOneArchive(t *testing.T) { - // the disk limit is enforced as entries land, not between archives: a bound checked only between - // archives lets one archive overshoot by everything it writes. - // - // The budget is built from what a store charges, there being no framing: each entry's index record - // plus the content bytes that reach the blob. It admits both records and the small entry whole, and - // refuses the big one. - ctx := context.Background() - files := map[string]string{ - "a/small.txt": strings.Repeat("s", 50), - "b/big.txt": strings.Repeat("b", 500), - } - - dir := t.TempDir() - archivePath := createTestZip(t, dir, files) - overflow := storeFor(t) - - records := indexRecordCost(tar.Header{Name: "a/small.txt"}) + indexRecordCost(tar.Header{Name: "b/big.txt"}) - charge := diskCharge(records + 50 + 100) - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, archivePath), overflow, charge) - require.NoError(t, err, "reaching a limit is a truncation, not a failure") - - assert.Equal(t, TruncatedByDiskLimit, result.Truncation, - "a limit breach reports its own reason, so the log can name which bound was reached") - assert.Equal(t, int64(50), overflow.OnDisk(), - "only the small entry was stored whole, and a store writes content and no framing") - - _, disk := charge.held() - assert.Equal(t, records+50, disk, "the limiter holds the index records plus what content landed") - - // what was stored before the limit filled stays readable - entries := readStore(t, overflow) - assert.Len(t, entries["a/small.txt"].body, 50) - - // KNOWN GAP: the entry the limit refused stays in the store, its header claiming the size the - // archive declared with none of the content behind it. A half-written pom a cataloger parses and - // believes is worse than an absent one. Asserted as it stands, so a fix changes a test. - require.Contains(t, entries, "b/big.txt") - assert.Empty(t, entries["b/big.txt"].body, "none of the refused entry's content landed") -} - -func TestTarExtractor_Extract_diskLimitStopsMidEntry(t *testing.T) { - // a tar-family archive is walked entry by entry, so the disk limit can bite mid-entry rather than on - // a boundary: the walk stops, says why, and the limiter holds no more than the bound - ctx := context.Background() - dir := t.TempDir() - - // two entries either side of a copy chunk, so the limit can bite mid-stream rather than refusing the - // first chunk and writing nothing - files := map[string]string{ - "a/first.txt": strings.Repeat("a", 40*1024), - "b/second.txt": strings.Repeat("b", 40*1024), - } - archivePath := createTestTarGz(t, dir, files) - overflow := storeFor(t) - - const budget = 64 * 1024 - charge := diskCharge(budget) - result, err := (&TarExtractor{}).Extract(ctx, fileContent(t, archivePath), overflow, charge) - require.NoError(t, err) - - assert.Equal(t, TruncatedByDiskLimit, result.Truncation) - assert.Greater(t, overflow.OnDisk(), int64(40*1024), - "the first entry landed whole and the second got part way in") - - _, disk := charge.held() - assert.LessOrEqual(t, disk, int64(budget), "the limiter never holds more than the bound") - assert.Greater(t, disk, overflow.OnDisk(), - "the charge also carries the index records, which are held in the store rather than written") - - info, err := os.Stat(filepath.Join(storeDir(t, overflow), overflowBlobName)) - require.NoError(t, err) - assert.Equal(t, overflow.OnDisk(), info.Size(), "the blob holds exactly what the store reports") -} - -func TestExtract_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) { - // an in-use limit rather than a counter: two archives of the same size both extract in full when the - // first is released between them, and the second is bounded when it is not - ctx := context.Background() - dir := t.TempDir() - zipPath := createTestZip(t, dir, map[string]string{"a.txt": strings.Repeat("a", 200)}) - - // one entry of 200 bytes costs its index record plus its content, there being no framing. A limit of - // one and a half archives leaves no room for a second while the first is held. - oneArchive := indexRecordCost(tar.Header{Name: "a.txt"}) + 200 - limiter := NewLimiter(Limits{MaxDiskBytes: oneArchive * 3 / 2}) - - first := limiter.Charge() - firstStore := storeFor(t) - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), firstStore, first) - require.NoError(t, err) - require.False(t, result.Truncated()) - require.Equal(t, int64(200), firstStore.OnDisk()) - - t.Run("a second archive is bounded while the first is still held", func(t *testing.T) { - second := limiter.Charge() - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), storeFor(t), second) - require.NoError(t, err) - assert.Equal(t, TruncatedByDiskLimit, result.Truncation) - second.Release() - }) - - t.Run("and extracts in full once the first is released", func(t *testing.T) { - first.Release() - _, disk := limiter.InUse() - require.Zero(t, disk, "releasing every charge must take the limiter back to nothing") - - third := limiter.Charge() - thirdStore := storeFor(t) - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), thirdStore, third) - require.NoError(t, err) - assert.False(t, result.Truncated(), "a counter would have refused this; a limiter does not") - assert.Equal(t, int64(200), thirdStore.OnDisk()) - }) -} - -func TestExtract_directoryEntriesAreBoundedByDisk(t *testing.T) { - // an archive of directory entries alone carries no content, so what bounds it is the disk limit - // charging each entry's header and the tar's padding - ctx := context.Background() - - t.Run("zip", func(t *testing.T) { - dir := t.TempDir() - zipPath := createDirOnlyZip(t, dir, 50) - overflow := storeFor(t) - - // each directory entry is one header block; room for twelve admits ten entries plus the two - // end-of-archive blocks - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, diskCharge(12*512)) - require.NoError(t, err, "a bound is a truncation, not a failure") - assert.Equal(t, TruncatedByDiskLimit, result.Truncation) - assert.Less(t, len(overflow.Entries()), 50, "the bound must stop the archive short") - assert.NotEmpty(t, storedNames(t, overflow), "what was written before the bound stays readable") - }) - - t.Run("tar.gz is copied whole, so the disk limit is what stops it", func(t *testing.T) { - // a tar is not walked entry by entry: the copy takes the whole archive, and the disk limit is - // charged as those bytes land - dir := t.TempDir() - tarPath := createDirOnlyTarGz(t, dir, 50) - overflow := storeFor(t) - - result, err := (&TarExtractor{}).Extract(ctx, fileContent(t, tarPath), overflow, diskCharge(-1)) - require.NoError(t, err) - assert.False(t, result.Truncated()) - assert.Len(t, storedNames(t, overflow), 50) - }) -} - -// symlinkEntry describes one archive entry for the escape tests: a symlink when linkTarget is set, -// a regular file otherwise. -type symlinkEntry struct { - name string - linkTarget string - body string -} - -func TestExtract_symlinkChainEscapeWritesNothing(t *testing.T) { - // two cooperating link entries that each pass a lexical check and together resolve outside the - // extraction directory: "d" -> "." lands on the root, so "d/up" -> ".." is created as "/up" - // -> ".." pointing above it, and a third entry is written through it. - // - // The chain needs real links on a real filesystem. Entries are headers, so no link is created and - // nothing can be written through one. Kept as the input that would violate "nothing outside the - // store is created". - chain := []symlinkEntry{ - {name: "d", linkTarget: "."}, - {name: "d/up", linkTarget: ".."}, - {name: "up/PWNED", body: "arbitrary write"}, - } - - // three link pairs, each climbing one more level: a check walking only one level would pass a - // one-level test and still let this through - escalating := []symlinkEntry{ - {name: "a", linkTarget: "."}, - {name: "a/b", linkTarget: ".."}, - {name: "b/c", linkTarget: "."}, - {name: "b/c/d", linkTarget: ".."}, - {name: "b/d/e", linkTarget: "."}, - {name: "b/d/e/f", linkTarget: ".."}, - {name: "b/d/f/PWNED", body: "arbitrary write, three levels up"}, - } - - for _, tc := range []struct { - name string - entries []symlinkEntry - }{ - {name: "one pair", entries: chain}, - {name: "three pairs escalating", entries: escalating}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Run("tar.gz", func(t *testing.T) { - assertNoEscape(t, createSymlinkChainTarGz, func() Extractor { return &TarExtractor{} }, "chain.tar.gz", tc.entries) - }) - t.Run("zip", func(t *testing.T) { - assertNoEscape(t, createSymlinkChainZip, func() Extractor { return &ZipExtractor{} }, "chain.zip", tc.entries) - }) - }) - } -} - -func TestTarExtractor_Extract_legitimateRelativeSymlinkIsCarriedThrough(t *testing.T) { - // containing an escape must not cost what real archives contain: a relative link into a sibling - // directory of the archive's own tree keeps its target verbatim so the filetree can resolve it. - // That it does resolve is asserted in the archive resolver's tests. - dir := t.TempDir() - - archivePath := createSymlinkChainTarGz(t, dir, "legit.tar.gz", []symlinkEntry{ - {name: "lib/real.txt", body: "real content"}, - {name: "bin/link.txt", linkTarget: "../lib/real.txt"}, - {name: "lib/sub/deep.txt", body: "deep"}, - {name: "bin/deep.txt", linkTarget: "../lib/sub/deep.txt"}, - }) - overflow := storeFor(t) - - _, err := (&TarExtractor{}).Extract(context.Background(), fileContent(t, archivePath), overflow, nil) - require.NoError(t, err) - - entries := readStore(t, overflow) - assert.Equal(t, "../lib/real.txt", entries["bin/link.txt"].linkTarget) - assert.Equal(t, "../lib/sub/deep.txt", entries["bin/deep.txt"].linkTarget) - assert.Equal(t, "real content", entries["lib/real.txt"].body) - assert.Equal(t, "deep", entries["lib/sub/deep.txt"].body) -} - -func TestExtract_siblingOfExtractionDirectoryWritesNothing(t *testing.T) { - // the realistic form of the string-prefix hole SafeJoin was fixed for: an entry naming - // "../contents-evil/..." shares the extraction directory's prefix and is outside it. - // - // Nothing joins an entry name to a filesystem path, so the entry is recorded as the name it claims - // and no path is derived from it. The assertion: nothing is written beside the extraction directory. - ctx := context.Background() - - for _, tc := range []struct { - name string - build func(t *testing.T, dir string, files map[string]string) string - extract func(path string, sink EntrySink) (ExtractionResult, error) - }{ - { - name: "zip", - build: createTestZip, - extract: func(path string, sink EntrySink) (ExtractionResult, error) { - return (&ZipExtractor{}).Extract(ctx, fileContent(t, path), sink, nil) - }, - }, - { - name: "tar.gz", - build: createTestTarGz, - extract: func(path string, sink EntrySink) (ExtractionResult, error) { - return (&TarExtractor{}).Extract(ctx, fileContent(t, path), sink, nil) - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - base := t.TempDir() - destDir := filepath.Join(base, "contents") - require.NoError(t, os.MkdirAll(destDir, 0o755)) - - archivePath := tc.build(t, base, map[string]string{"../contents-evil/x.txt": "escaped"}) - - store := NewEntryStore(WorkDirAt(base), "evil", nil) - t.Cleanup(func() { _ = store.Close() }) - - _, err := tc.extract(archivePath, store) - require.NoError(t, err, "a hostile name is a header field, not a failure") - - _, statErr := os.Stat(filepath.Join(base, "contents-evil", "x.txt")) - assert.True(t, os.IsNotExist(statErr), - "nothing may be written to a sibling of the extraction directory") - - entries, err := os.ReadDir(destDir) - require.NoError(t, err) - assert.Empty(t, entries, "the extraction directory stays empty: entries live in the store") - - assert.Equal(t, "escaped", readStore(t, store)["../contents-evil/x.txt"].body) - }) - } -} - -func Test_HasZipEndOfCentralDirectory(t *testing.T) { - var plainZip bytes.Buffer - zw := zip.NewWriter(&plainZip) - f, err := zw.Create("a.txt") - require.NoError(t, err) - _, err = f.Write([]byte("hello")) - require.NoError(t, err) - require.NoError(t, zw.Close()) - - var tarGz bytes.Buffer - gw := gzip.NewWriter(&tarGz) - tw := tar.NewWriter(gw) - require.NoError(t, tw.WriteHeader(&tar.Header{Name: "a.txt", Size: 5, Mode: 0o600})) - _, err = tw.Write([]byte("hello")) - require.NoError(t, err) - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - - tests := []struct { - name string - data []byte - want bool - }{ - {"a plain zip", plainZip.Bytes(), true}, - { - "a zip behind a launcher script - the case content sniffing types by its head", - scriptPrefixedZip(t, "#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n", - map[string]string{"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n"}), - true, - }, - {"prose, whatever it is named", []byte("this is not an archive, it is a sentence"), false}, - {"a tar.gz, which is an archive but not a zip", tarGz.Bytes(), false}, - {"empty", nil, false}, - {"shorter than the signature", []byte("PK"), false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, HasZipEndOfCentralDirectory(bytes.NewReader(tt.data))) - }) - } -} - -func Test_ZipExtractor_canExtractAZipBehindALauncherScript(t *testing.T) { - // a Spring Boot executable jar sniffs as text/x-shellscript, which format identification refuses, - // losing every package inside - data := scriptPrefixedZip(t, "#!/bin/bash\necho launcher\nexit 0\n", map[string]string{ - "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", - "BOOT-INF/lib/dep.jar": "not really a jar, just bytes", - }) - - extractor := &ZipExtractor{} - require.True(t, extractor.CanExtract(context.Background(), bytesContent("app.jar", data)), - "a zip behind a launcher script must be recognized") - - // and it extracts, because archive/zip finds the central directory from the end - store := storeFor(t) - result, err := extractor.Extract(context.Background(), bytesContent("app.jar", data), store, - diskCharge(-1)) - require.NoError(t, err) - assert.Len(t, store.Entries(), 2, "the entries behind the script prefix must actually be stored") - assert.Empty(t, string(result.Truncation)) - assert.Contains(t, readStore(t, store), "BOOT-INF/lib/dep.jar") -} - -func Test_ZipExtractor_refusesANonArchiveThatNothingIdentifies(t *testing.T) { - // the probe must not turn a file with no central directory into an archive. The name matters here in - // a way it does not in a scan, since identification matches on it and `notes.zip` is claimed by name - // before the probe is reached; the `misnamed non-archive is not extracted` scenario is therefore - // asserted at the task level - see Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError. - extractor := &ZipExtractor{} - for _, name := range []string{"app.jar", "bundle", "launcher"} { - t.Run(name, func(t *testing.T) { - content := bytesContent(name, []byte("plain text, no central directory anywhere in it")) - assert.False(t, extractor.CanExtract(context.Background(), content)) - }) - } -} - -// fileContent opens an archive already on disk as Content, which is what the extractors take. -// Registered for close so a test that opens several does not leak handles. -func fileContent(t *testing.T, path string) Content { - t.Helper() - c, err := openFileContent(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, c.Close()) }) - return c -} - -// diskCharge returns a charge against a disk limit of the given size, for a test wanting the byte -// bound to bite without standing up a whole scan. -func diskCharge(max int64) *Charge { - return NewLimiter(Limits{MaxDiskBytes: max}).Charge() -} - -// sortedNames fixes the entry order of a test archive: limits are enforced as the walk proceeds, so -// entry order decides which entries land before a truncation, and map order is random. -func sortedNames(files map[string]string) []string { - names := make([]string, 0, len(files)) - for name := range files { - names = append(names, name) - } - sort.Strings(names) - return names -} - -// storeFor makes the sink an archive's entries are written into, in the test's own temp space. -func storeFor(t *testing.T) *EntryStore { - t.Helper() - s := NewEntryStore(WorkDirAt(t.TempDir()), "test-archive", nil) - t.Cleanup(func() { _ = s.Close() }) - return s -} - -// storeDir is the directory a store would spill into, for tests asserting what is and is not on disk. -func storeDir(t *testing.T, s *EntryStore) string { - t.Helper() - dir, err := s.workDir.Path() - require.NoError(t, err) - return dir -} - -func readStore(t *testing.T, s *EntryStore) map[string]storedEntry { - t.Helper() - - out := map[string]storedEntry{} - for _, entry := range s.Entries() { - reader, err := s.Open(entry) - require.NoError(t, err) - body, err := io.ReadAll(reader) - require.NoError(t, err) - out[entry.Header.Name] = storedEntry{ - name: entry.Header.Name, - body: string(body), - linkTarget: entry.Header.Linkname, - mode: entry.Header.FileInfo().Mode(), - typeflag: entry.Header.Typeflag, - } - } - return out -} - -func storedNames(t *testing.T, s *EntryStore) []string { - t.Helper() - entries := readStore(t, s) - names := make([]string, 0, len(entries)) - for name := range entries { - names = append(names, name) - } - sort.Strings(names) - return names -} - -func createTestZip(t *testing.T, dir string, files map[string]string) string { - t.Helper() - zipPath := filepath.Join(dir, "test.zip") - f, err := os.Create(zipPath) - require.NoError(t, err) - defer f.Close() - - w := zip.NewWriter(f) - for _, name := range sortedNames(files) { - fw, err := w.Create(name) - require.NoError(t, err) - _, err = fw.Write([]byte(files[name])) - require.NoError(t, err) - } - require.NoError(t, w.Close()) - return zipPath -} - -func createTestTarGz(t *testing.T, dir string, files map[string]string) string { - t.Helper() - tarPath := filepath.Join(dir, "test.tar.gz") - f, err := os.Create(tarPath) - require.NoError(t, err) - defer f.Close() - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - for _, name := range sortedNames(files) { - content := files[name] - hdr := &tar.Header{ - Name: name, - Mode: 0o644, - Size: int64(len(content)), - } - require.NoError(t, tw.WriteHeader(hdr)) - _, err = tw.Write([]byte(content)) - require.NoError(t, err) - } - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - return tarPath -} - -func createDirOnlyZip(t *testing.T, dir string, count int) string { - t.Helper() - zipPath := filepath.Join(dir, "dirs.zip") - f, err := os.Create(zipPath) - require.NoError(t, err) - defer f.Close() - - w := zip.NewWriter(f) - for i := range count { - hdr := &zip.FileHeader{Name: fmt.Sprintf("d%04d/", i)} - hdr.SetMode(fs.ModeDir | 0o755) - _, err := w.CreateHeader(hdr) - require.NoError(t, err) - } - require.NoError(t, w.Close()) - return zipPath -} - -func createDirOnlyTarGz(t *testing.T, dir string, count int) string { - t.Helper() - tarPath := filepath.Join(dir, "dirs.tar.gz") - f, err := os.Create(tarPath) - require.NoError(t, err) - defer f.Close() - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - for i := range count { - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: fmt.Sprintf("d%04d/", i), - Typeflag: tar.TypeDir, - Mode: 0o755, - })) - } - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - return tarPath -} - -func createSymlinkChainTarGz(t *testing.T, dir, name string, entries []symlinkEntry) string { - t.Helper() - path := filepath.Join(dir, name) - f, err := os.Create(path) - require.NoError(t, err) - defer f.Close() - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - for _, e := range entries { - if e.linkTarget != "" { - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: e.name, - Typeflag: tar.TypeSymlink, - Linkname: e.linkTarget, - Mode: 0o777, - })) - continue - } - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: e.name, - Mode: 0o644, - Size: int64(len(e.body)), - })) - _, err := tw.Write([]byte(e.body)) - require.NoError(t, err) - } - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - return path -} - -func createSymlinkChainZip(t *testing.T, dir, name string, entries []symlinkEntry) string { - t.Helper() - path := filepath.Join(dir, name) - f, err := os.Create(path) - require.NoError(t, err) - defer f.Close() - - w := zip.NewWriter(f) - for _, e := range entries { - hdr := &zip.FileHeader{Name: e.name} - body := e.body - if e.linkTarget != "" { - // a zip stores a symlink as an entry whose mode carries the symlink bit and whose content - // is the link target - hdr.SetMode(fs.ModeSymlink | 0o777) - body = e.linkTarget - } else { - hdr.SetMode(0o644) - } - fw, err := w.CreateHeader(hdr) - require.NoError(t, err) - _, err = fw.Write([]byte(body)) - require.NoError(t, err) - } - require.NoError(t, w.Close()) - return path -} - -func findEscapedFiles(t *testing.T, base, destDir, name string) []string { - t.Helper() - var found []string - require.NoError(t, filepath.WalkDir(base, func(path string, entry fs.DirEntry, err error) error { - if err != nil { - return nil // an escaping symlink can make a walk stumble; that is not the assertion - } - if entry.IsDir() { - return nil - } - if filepath.Base(path) != name { - return nil - } - if rel, relErr := filepath.Rel(destDir, path); relErr == nil && !strings.HasPrefix(rel, "..") { - return nil - } - found = append(found, path) - return nil - })) - return found -} - -func assertNoEscape(t *testing.T, build func(*testing.T, string, string, []symlinkEntry) string, newExtractor func() Extractor, archiveName string, entries []symlinkEntry) { - t.Helper() - - // several levels of headroom above the extraction directory, all inside the test's own temp tree, so - // an escape has somewhere to land and is still cleaned up - base := t.TempDir() - destDir := filepath.Join(base, "one", "two", "three", "contents") - require.NoError(t, os.MkdirAll(destDir, 0o755)) - - archivePath := build(t, base, archiveName, entries) - - store := NewEntryStore(WorkDirAt(destDir), archiveName, nil) - t.Cleanup(func() { _ = store.Close() }) - - _, err := newExtractor().Extract(context.Background(), fileContent(t, archivePath), store, nil) - require.NoError(t, err, "an unsafe entry is skipped, not a failure of the whole archive") - - assert.Empty(t, findEscapedFiles(t, base, destDir, "PWNED"), - "nothing may be written outside the extraction directory") - - // and nothing was created inside it either: entries are headers and bytes in the store, and with no - // bound pushing them out not even the overflow blob exists - created, err := os.ReadDir(destDir) - require.NoError(t, err) - assert.Empty(t, created) -} - -// scriptPrefixedZip is a self-extracting archive in the shape Spring Boot produces: a launcher script -// with a complete zip appended. -func scriptPrefixedZip(t *testing.T, prefix string, files map[string]string) []byte { - t.Helper() - var zbuf bytes.Buffer - w := zip.NewWriter(&zbuf) - for _, name := range sortedNames(files) { - f, err := w.Create(name) - require.NoError(t, err) - _, err = f.Write([]byte(files[name])) - require.NoError(t, err) - } - require.NoError(t, w.Close()) - return append([]byte(prefix), zbuf.Bytes()...) -} - -func bytesContent(name string, data []byte) Content { - return Content{Name: name, Reader: bytes.NewReader(data)} -} diff --git a/internal/archive/helpers_test.go b/internal/archive/helpers_test.go index 3d6f99343..c64f00210 100644 --- a/internal/archive/helpers_test.go +++ b/internal/archive/helpers_test.go @@ -1,47 +1,46 @@ package archive import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "io" + "io/fs" "os" "path/filepath" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/tmpdir" ) -// Seams the package itself does not need, kept here so no production file carries a function only -// tests call. - -// WorkDirAt returns a WorkDir over a directory that already exists. Production always creates its own -// (NewWorkDir); a test lends one it can inspect afterwards, so Remove leaves it alone. +// WorkDirAt returns a WorkDir over a directory that already exists, so a test can inspect it +// afterwards; Remove leaves it alone. func WorkDirAt(dir string) *WorkDir { - return &WorkDir{path: dir, done: true} + return &WorkDir{created: true, path: dir} } -// heldInMemory reports the entry content this store is still holding in memory. Production never asks: -// the limiter is what bounds it, and the store charges as it goes rather than tracking a total. +func (w *WorkDir) wasCreated() bool { + return w != nil && w.remove != nil +} + +// heldInMemory reports the entry content the store is still holding in memory. func (s *EntryStore) heldInMemory() int64 { - s.mu.RLock() - defer s.mu.RUnlock() var total int64 for _, entry := range s.entries { - if entry.ref.mem != nil { - total += entry.ref.length - } + total += int64(len(entry.mem)) } return total } -// created reports whether anything ever asked for the directory, and so whether there is anything to -// remove. -func (w *WorkDir) created() bool { - if w == nil { - return false - } - w.mu.Lock() - defer w.mu.Unlock() - return w.remove != nil -} - // held reports what this charge is currently holding, in memory and on disk. func (c *Charge) held() (memory, disk int64) { - if c == nil || c.limiter == nil { + if c == nil { return 0, 0 } c.limiter.mu.Lock() @@ -49,11 +48,209 @@ func (c *Charge) held() (memory, disk int64) { return c.memory, c.disk } -// openFileContent opens an archive already on disk as Content. The caller closes it. -func openFileContent(path string) (Content, error) { - f, err := os.Open(path) - if err != nil { - return Content{}, err +// unseekable hides a reader's Seek and ReadAt, forcing content through the held path. +type unseekable struct{ io.Reader } + +// testEntry is one entry of a fixture archive: a directory when dir is set, a symlink when link is +// set, otherwise a regular file. +type testEntry struct { + name string + body string + link string + dir bool + mode fs.FileMode +} + +func fileEntries(files map[string]string) []testEntry { + names := make([]string, 0, len(files)) + for name := range files { + names = append(names, name) } - return Content{Name: filepath.Base(path), Reader: f, closer: f}, nil + // limits bite as the walk proceeds, so entry order decides what lands before a truncation + sort.Strings(names) + entries := make([]testEntry, 0, len(names)) + for _, name := range names { + entries = append(entries, testEntry{name: name, body: files[name]}) + } + return entries +} + +func zipBytes(t testing.TB, files map[string]string) []byte { + return zipFromEntries(t, fileEntries(files)) +} + +func zipFromEntries(t testing.TB, entries []testEntry) []byte { + t.Helper() + var buf bytes.Buffer + w := zip.NewWriter(&buf) + for _, e := range entries { + hdr := &zip.FileHeader{Name: e.name} + body := e.body + switch { + case e.dir: + hdr.SetMode(fs.ModeDir | e.modeOr(0o755)) + case e.link != "": + hdr.SetMode(fs.ModeSymlink | 0o777) + body = e.link + default: + hdr.SetMode(e.modeOr(0o644)) + } + fw, err := w.CreateHeader(hdr) + require.NoError(t, err) + _, err = fw.Write([]byte(body)) + require.NoError(t, err) + } + require.NoError(t, w.Close()) + return buf.Bytes() +} + +func tarGzBytes(t testing.TB, files map[string]string) []byte { + return tarGzFromEntries(t, fileEntries(files)) +} + +func tarGzFromEntries(t testing.TB, entries []testEntry) []byte { + t.Helper() + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + for _, e := range entries { + hdr := &tar.Header{Name: e.name, Mode: int64(e.modeOr(0o644)), Size: int64(len(e.body))} + switch { + case e.dir: + hdr.Typeflag, hdr.Mode, hdr.Size = tar.TypeDir, int64(e.modeOr(0o755)), 0 + case e.link != "": + hdr.Typeflag, hdr.Linkname, hdr.Mode, hdr.Size = tar.TypeSymlink, e.link, 0o777, 0 + } + require.NoError(t, tw.WriteHeader(hdr)) + _, err := tw.Write([]byte(e.body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + return buf.Bytes() +} + +func (e testEntry) modeOr(fallback fs.FileMode) fs.FileMode { + if e.mode != 0 { + return e.mode + } + return fallback +} + +func regularHeader(name string, size int64) tar.Header { + return tar.Header{Name: name, Size: size, Mode: 0o600, Typeflag: tar.TypeReg} +} + +// storeFor makes a store in the test's own temp space with the given charge. +func storeFor(t testing.TB, charge *Charge) *EntryStore { + t.Helper() + s := NewEntryStore("test-archive", WorkDirAt(t.TempDir()), charge) + t.Cleanup(func() { _ = s.Close() }) + return s +} + +// storeDir is the directory a store spills into. +func storeDir(t testing.TB, s *EntryStore) string { + t.Helper() + dir, err := s.workDir.Path() + require.NoError(t, err) + return dir +} + +func readEntry(t testing.TB, r ReaderAtSeeker) string { + t.Helper() + _, err := r.Seek(0, io.SeekStart) + require.NoError(t, err) + b, err := io.ReadAll(r) + require.NoError(t, err) + return string(b) +} + +// storedEntry is one entry read back out of a store. +type storedEntry struct { + body string + linkTarget string + mode fs.FileMode + typeflag byte +} + +func readStore(t testing.TB, s *EntryStore) map[string]storedEntry { + t.Helper() + out := map[string]storedEntry{} + for _, entry := range s.Entries() { + out[entry.Header.Name] = storedEntry{ + body: readEntry(t, s.Open(entry)), + linkTarget: entry.Header.Linkname, + mode: entry.Header.FileInfo().Mode(), + typeflag: entry.Header.Typeflag, + } + } + return out +} + +func storedNames(t testing.TB, s *EntryStore) []string { + t.Helper() + var names []string + for _, entry := range s.Entries() { + names = append(names, entry.Header.Name) + } + sort.Strings(names) + return names +} + +// memCharge is a charge against a memory bound with disk unbounded. +func memCharge(maxMemory int64) *Charge { + return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).Charge() +} + +// diskCharge is a charge against a disk bound with memory refusing everything. +func diskCharge(maxDisk int64) *Charge { + return NewLimiter(Limits{MaxDiskBytes: maxDisk}).Charge() +} + +// filesIn lists the regular files directly in dir. +func filesIn(t testing.TB, dir string) []string { + t.Helper() + entries, err := os.ReadDir(dir) + require.NoError(t, err) + var names []string + for _, e := range entries { + if !e.IsDir() { + names = append(names, e.Name()) + } + } + return names +} + +// scanContext returns a context whose temp root is a fresh directory, and that directory. +func scanContext(t testing.TB) (context.Context, string) { + t.Helper() + root := t.TempDir() + return tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)), root +} + +// workDirsUnder returns the archive work directories directly under a scan temp root. +func workDirsUnder(t testing.TB, root string) []string { + t.Helper() + children, err := os.ReadDir(root) + require.NoError(t, err) + var out []string + for _, child := range children { + if child.IsDir() && strings.HasPrefix(child.Name(), workDirName) { + out = append(out, filepath.Join(root, child.Name())) + } + } + return out +} + +// extractBytes extracts an archive into a store built over the given charge. +func extractBytes(t testing.TB, data []byte, name string, charge *Charge) (*EntryStore, bool) { + t.Helper() + store := storeFor(t, charge) + content := bytes.NewReader(data) + format := identifyFormat(context.Background(), name, content) + require.NotNil(t, format, "fixture must identify as an archive") + truncated, err := extractInto(context.Background(), format, content, store, nil) + require.NoError(t, err) + return store, truncated } diff --git a/internal/archive/index.go b/internal/archive/index.go new file mode 100644 index 000000000..8567d8585 --- /dev/null +++ b/internal/archive/index.go @@ -0,0 +1,350 @@ +package archive + +import ( + "archive/tar" + "context" + "fmt" + "io" + "path" + "sort" + "strings" + + "github.com/bmatcuk/doublestar/v4" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" + syftindex "github.com/anchore/syft/internal/index" + "github.com/anchore/syft/syft/file" +) + +// Index is a file.Resolver over one extracted archive's entries. +// +// Paths are archive-relative. Nearly every cataloger glob is `**/` or `**/*`, so files +// are indexed by base name and a pattern whose last segment fixes a name, prefix or suffix is +// narrowed to those files before doublestar decides; any other pattern is matched against every file. +type Index struct { + fileSystemID string + archivePath string + store *EntryStore + + // byPath holds every file and directory by rooted path, including directories the archive never + // listed but that entry paths imply + byPath map[string]*node + + // files holds every file the archive listed, in path order + files []*node + + // names indexes files by base name + names syftindex.PrefixSuffix[[]*node] +} + +type node struct { + path string // rooted, e.g. "/META-INF/MANIFEST.MF" + isDir bool + entry *Entry // nil for a directory the archive never listed + metadata file.Metadata + + // target is the node whose entry holds this node's content: itself for a regular file, the + // resolved node for a link, and nil for a link that leads nowhere inside this archive + target *node +} + +var _ file.Resolver = (*Index)(nil) + +// NewIndex builds a resolver over the store's entries whose locations carry the given fileSystemID +// and archivePath. +func NewIndex(store *EntryStore, fileSystemID, archivePath string) *Index { + r := &Index{ + fileSystemID: fileSystemID, + archivePath: archivePath, + store: store, + byPath: map[string]*node{"/": {path: "/", isDir: true}}, + } + for _, entry := range store.Entries() { + r.add(entry) + } + sort.Slice(r.files, func(i, j int) bool { return r.files[i].path < r.files[j].path }) + for _, n := range r.byPath { + n.target = r.linkTarget(n) + } + return r +} + +// add places an entry at its path, creating the directories the path implies. The first entry at a +// path wins. +func (r *Index) add(entry *Entry) { + entryPath := path.Clean("/" + entry.Header.Name) + if entryPath == "/" { + return + } + for i := 1; i < len(entryPath); i++ { + if entryPath[i] == '/' { + r.nodeAt(entryPath[:i], true) + } + } + + n := r.nodeAt(entryPath, entry.Header.FileInfo().IsDir()) + if n.entry != nil { + return + } + n.entry = entry + n.metadata = r.metadataOf(entry, entryPath) + if n.isDir { + return + } + r.files = append(r.files, n) + r.names.Update(path.Base(entryPath), func(current *syftindex.Node[[]*node]) []*node { + return append(current.Value(), n) + }) +} + +func (r *Index) nodeAt(p string, isDir bool) *node { + if n, ok := r.byPath[p]; ok { + return n + } + n := &node{path: p, isDir: isDir} + r.byPath[p] = n + return n +} + +// maxLinkHops bounds a chain of links pointing at links, which is also what ends a cycle. +const maxLinkHops = 8 + +func (r *Index) linkTarget(n *node) *node { + for hops := 0; hops <= maxLinkHops; hops++ { + if n.entry == nil || n.isDir { + return nil + } + if n.entry.Header.Linkname == "" { + return n + } + next, ok := r.byPath[linkDestination(n.path, n.entry.Header)] + if !ok { + return nil + } + n = next + } + return nil +} + +// linkDestination is the rooted path a link points at inside the archive. A hard link names its +// target from the archive root, as does an absolute symlink, so "/etc/passwd" is an entry of this +// archive and never the host's. A relative symlink is joined onto the link's own directory, and +// cleaning against the root keeps "../../x" inside the archive. +func linkDestination(linkPath string, hdr tar.Header) string { + if hdr.Typeflag == tar.TypeLink || path.IsAbs(hdr.Linkname) { + return path.Join("/", hdr.Linkname) + } + return path.Join(path.Dir(linkPath), hdr.Linkname) +} + +func (r *Index) metadataOf(entry *Entry, entryPath string) file.Metadata { + metadata := stereoscopeFile.NewMetadata(entry.Header, r.store.Open(entry)) + // NewMetadata copies the header and points FileInfo into the copy; share the one the store holds + metadata.FileInfo = entry.Header.FileInfo() + metadata.Path = reportedPath(entryPath) + if entry.Header.Linkname != "" { + metadata.LinkDestination = reportedPath(linkDestination(entryPath, entry.Header)) + } + return metadata +} + +// reportedPath is how paths leave the resolver: archive-relative with no leading slash. +func reportedPath(rooted string) string { + return strings.TrimPrefix(rooted, "/") +} + +func rootedPath(userPath string) string { + return path.Clean("/" + userPath) +} + +func (r *Index) coordinates(n *node) file.Coordinates { + return file.Coordinates{ + RealPath: reportedPath(n.path), + FileSystemID: r.fileSystemID, + ArchivePath: r.archivePath, + } +} + +// locationOf names a node by its own path. +func (r *Index) locationOf(n *node) file.Location { + return file.NewLocationFromCoordinates(r.coordinates(n)) +} + +// resolvedLocation names the file behind a node: a link is reported at its target's path with the +// link's own path as the access path, and a link leading nowhere is reported as itself. +func (r *Index) resolvedLocation(n *node) file.Location { + if n.target == nil || n.target == n { + return r.locationOf(n) + } + return file.NewVirtualLocationFromCoordinates(r.coordinates(n.target), reportedPath(n.path)) +} + +func (r *Index) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { + n, ok := r.byPath[rootedPath(location.RealPath)] + if !ok || n.entry == nil { + return nil, fmt.Errorf("no archive entry for path=%q", location.RealPath) + } + return entryReader{r.store.Open(n.entry)}, nil +} + +func (r *Index) FileMetadataByLocation(location file.Location) (file.Metadata, error) { + n, ok := r.byPath[rootedPath(location.RealPath)] + if !ok { + return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.RealPath) + } + return n.metadata, nil +} + +func (r *Index) HasPath(userPath string) bool { + _, ok := r.byPath[rootedPath(userPath)] + return ok +} + +func (r *Index) FilesByPath(paths ...string) ([]file.Location, error) { + var out []file.Location + seen := map[*node]struct{}{} + for _, userPath := range paths { + n, ok := r.byPath[rootedPath(userPath)] + if !ok || n.isDir || n.entry == nil { + continue + } + if _, dup := seen[n]; dup { + continue + } + seen[n] = struct{}{} + out = append(out, r.resolvedLocation(n)) + } + return out, nil +} + +func (r *Index) FilesByGlob(patterns ...string) ([]file.Location, error) { + matched := map[*node]struct{}{} + for _, pattern := range patterns { + pattern = strings.TrimLeft(pattern, "/") + if pattern == "" { + continue + } + for _, n := range r.globCandidates(pattern) { + ok, err := doublestar.Match(pattern, reportedPath(n.path)) + if err != nil { + return nil, fmt.Errorf("invalid glob %q: %w", pattern, err) + } + if ok { + matched[n] = struct{}{} + } + } + } + + var out []file.Location + for _, n := range onePathPerFile(matched) { + out = append(out, r.resolvedLocation(n)) + } + sort.Slice(out, func(i, j int) bool { + if out[i].RealPath != out[j].RealPath { + return out[i].RealPath < out[j].RealPath + } + return out[i].AccessPath < out[j].AccessPath + }) + return out, nil +} + +// globMetacharacters are the glob metacharacters other than `*` itself. +const globMetacharacters = `?[]{}\` + +// globCandidates narrows the files a pattern can match by its last segment: an exact name, `*suffix` +// or `prefix*` is looked up in the name index. Any other shape has every file as a candidate. +func (r *Index) globCandidates(pattern string) []*node { + last := pattern[strings.LastIndexByte(pattern, '/')+1:] + if strings.ContainsAny(last, globMetacharacters) || strings.Count(last, "*") > 1 { + return r.files + } + switch star := strings.IndexByte(last, '*'); { + case star < 0: + return r.names.Get(last) + case star == 0: + return flatten(r.names.BySuffix(last[1:])) + case star == len(last)-1: + return flatten(r.names.ByPrefix(last[:star])) + } + return r.files +} + +func flatten(groups [][]*node) []*node { + var out []*node + for _, group := range groups { + out = append(out, group...) + } + return out +} + +// onePathPerFile keeps one matched path per file. Several paths can name the same content, such as +// a link and its target, and a resolver answers with one: the file's own path when it matched, +// otherwise the lowest-sorting link. A link leading nowhere stands as its own file. +func onePathPerFile(matched map[*node]struct{}) []*node { + best := map[*node]*node{} + for n := range matched { + key := n.target + if key == nil { + key = n + } + current, seen := best[key] + if !seen || n == key || (current != key && n.path < current.path) { + best[key] = n + } + } + out := make([]*node, 0, len(best)) + for _, n := range best { + out = append(out, n) + } + return out +} + +func (r *Index) FilesByMIMEType(types ...string) ([]file.Location, error) { + wanted := map[string]struct{}{} + for _, t := range types { + wanted[t] = struct{}{} + } + var out []file.Location + for _, n := range r.files { + // only a file holding bytes has a type of its own; a link's target answers for itself + if n.target != n { + continue + } + if _, ok := wanted[n.metadata.MIMEType]; ok { + out = append(out, r.locationOf(n)) + } + } + return out, nil +} + +// RelativeFileByPath resolves against the archive root, the only frame an archive has. +func (r *Index) RelativeFileByPath(_ file.Location, userPath string) *file.Location { + locations, _ := r.FilesByPath(userPath) + if len(locations) == 0 { + return nil + } + return &locations[0] +} + +func (r *Index) AllLocations(ctx context.Context) <-chan file.Location { + out := make(chan file.Location) + go func() { + defer close(out) + for _, n := range r.files { + select { + case out <- r.locationOf(n): + case <-ctx.Done(): + return + } + } + }() + return out +} + +// entryReader gives an entry's reader the Close the resolver interface requires. The store owns the +// bytes, so there is nothing to close. +type entryReader struct { + ReaderAtSeeker +} + +func (entryReader) Close() error { return nil } diff --git a/syft/internal/fileresolver/archive_index_bench_test.go b/internal/archive/index_bench_test.go similarity index 56% rename from syft/internal/fileresolver/archive_index_bench_test.go rename to internal/archive/index_bench_test.go index 0ac9c2b18..f4a4c6eb5 100644 --- a/syft/internal/fileresolver/archive_index_bench_test.go +++ b/internal/archive/index_bench_test.go @@ -1,20 +1,12 @@ -package fileresolver +package archive import ( "fmt" "testing" ) -// BenchmarkArchiveIndex_FilesByGlob covers the shapes a cataloger asks for: the ones the name index -// answers outright, the ones reducing to a finite set of lookups, and the ones reaching doublestar -// over whatever the literal bounds narrow them to. -// -// Every pattern but the last two is one syft's own catalogers register, verbatim. That matters because -// a benchmark of invented patterns measures the wrong thing: no cataloger glob uses a character class, -// so `class-in-extension` is kept for coverage rather than as a cost anyone pays, and `many-matches` -// is a control - it matches thousands of files, so it measures building and sorting locations rather -// than finding them, and is expected to stay flat while the others improve. -func BenchmarkArchiveIndex_FilesByGlob(b *testing.B) { +// every pattern but the last two is one syft's own catalogers register +func BenchmarkIndex_FilesByGlob(b *testing.B) { r := benchArchive(b) patterns := map[string]string{ @@ -43,14 +35,9 @@ func BenchmarkArchiveIndex_FilesByGlob(b *testing.B) { } } -// benchArchive is shaped like a fat application image layer: thousands of entries with thousands of -// distinct base names, spread over nested directories. -// -// The names must be distinct: an archive whose files are all called README.md puts two keys in the -// name index and makes every lookup look free. The decoys serve the same purpose - names ending "ar" -// that are not archives, names beginning "go" that are not the go binary - separating a lookup keyed -// on a whole extension from one keyed on the two characters a weak bound fixes. -func benchArchive(tb testing.TB) *ArchiveIndex { +// benchArchive is shaped like a fat application image layer: thousands of entries with distinct base +// names, plus decoys sharing the prefixes and suffixes the patterns fix, so no lookup looks free. +func benchArchive(tb testing.TB) *Index { tb.Helper() entries := map[string]string{ "usr/local/go/bin/go": "binary", @@ -64,16 +51,12 @@ func benchArchive(tb testing.TB) *ArchiveIndex { "usr/lib/jvm/java-21/release": "jvm", } for i := range 1500 { - // decoys for the weak tail bound "ar" that `*.[jw]ar` and `*.{jar,war}` would share entries[fmt.Sprintf("var/cache/archive%04d.tar", i)] = "data" entries[fmt.Sprintf("var/cache/bundle%04d.rar", i)] = "data" entries[fmt.Sprintf("usr/share/cal%04dendar", i)] = "data" - // decoys for the head bound "go" entries[fmt.Sprintf("usr/share/gopher%04d.md", i)] = "docs" - // decoys for the head bounds "l" and "m", and for "libstd-" entries[fmt.Sprintf("usr/lib/libstd-%04d.a", i)] = "archive" entries[fmt.Sprintf("usr/lib/mod%04d.py", i)] = "code" - // ordinary bulk, with distinct names entries[fmt.Sprintf("usr/share/doc/pkg%04d/NOTES%04d.md", i, i)] = "docs" } return indexOver(tb, 1<<26, entries) diff --git a/syft/internal/fileresolver/archive_index_glob_test.go b/internal/archive/index_glob_test.go similarity index 66% rename from syft/internal/fileresolver/archive_index_glob_test.go rename to internal/archive/index_glob_test.go index 871356471..1cb978a19 100644 --- a/syft/internal/fileresolver/archive_index_glob_test.go +++ b/internal/archive/index_glob_test.go @@ -1,4 +1,4 @@ -package fileresolver +package archive import ( "sort" @@ -9,12 +9,8 @@ import ( "github.com/stretchr/testify/require" ) -// Test_FilesByGlob_agreesWithBruteForce is the safety net under every index shortcut. -// -// Each shortcut - the exact-name lookup, the prefix and suffix lookups, and the literal bounds that -// narrow what doublestar is shown - answers a pattern from the index rather than the archive. A -// shortcut that is too eager fails silently, returning fewer files, so every shape is checked against -// matching every path the slow way. +// every pattern shape is checked against matching every path the slow way, since a name-index +// shortcut that is too eager fails silently by returning fewer files func Test_FilesByGlob_agreesWithBruteForce(t *testing.T) { paths := []string{ "LICENSE", @@ -58,9 +54,7 @@ func Test_FilesByGlob_agreesWithBruteForce(t *testing.T) { r := indexOver(t, 1<<20, entries) patterns := []string{ - // the shapes the index answers outright "**/pom.xml", "**/*.jar", "**/pom*", "**/*", "/*", "/opt/*", - // the shapes that reach doublestar, with and without usable bounds "**/{go,go.exe}", "**/libstd-????????????????.so", "**/*.[jw]ar", @@ -69,7 +63,6 @@ func Test_FilesByGlob_agreesWithBruteForce(t *testing.T) { "**/*o*", "**/?o", "**/{a,b}.{jar,war}", - // segments that reduce to a finite set of names, answered by exact lookups "**/{mariadb,mysql}", "**/[lm]*", "**/*.{jar,war}", @@ -78,34 +71,28 @@ func Test_FilesByGlob_agreesWithBruteForce(t *testing.T) { "**/[a-c]*", "**/libstd-[s0]*.so", "**/x[!y]z", - // alternations where only some branches reduce to a lookup: dropping the rest would silently - // lose every file those branches match "**/{go,*}", "**/{lib,l}*", "**/{lo,m}*", "**/{*.jar,pom.xml}", - // metacharacters in a middle segment, where only the leading bound is usable "/{META-INF,WEB-INF}/**/*.xml", "/usr/{bin,lib}/*", "/opt/*/lib/*.jar", "/us?/lib/libstd-*.so", "**/lib/**/*.jar", - // a `**` below a selected directory, where the name it looks up also exists outside that - // directory - the lookup is archive-wide, so only the subtree filter keeps the stranger out + // the name looked up also exists outside the selected directory "/opt/**/*.jar", "/opt/**/lib/*.jar", "/usr/**/go", "/{opt,WEB-INF}/**/*.jar", "/META-INF/**/pom.*", "/usr/**/lib*.so", - // the alternations syft's catalogers actually write, whose branches carry separators of their - // own - splitting the pattern into segments before expanding these cuts a group in half + // alternations whose branches carry separators, as syft's catalogers write them "**/var/lib/dpkg/{status,status.d/**}", "**/var/lib/dpkg/{status,status.d/*}", "**/{var/lib,usr/share,usr/lib/sysimage}/rpm/{Packages,Packages.db,rpmdb.sqlite}", "**/{firefox,firefox.exe}", "**/{java,jvm}/*/release", - // and the degenerate ones "**", "**/**", "/**/*.so", } @@ -125,8 +112,6 @@ func Test_FilesByGlob_agreesWithBruteForce(t *testing.T) { } } -// bruteForceMatches is what the index computes faster: every path in the archive, matched one at a -// time. Patterns and paths are both archive-relative, so the leading slash is dropped from both. func bruteForceMatches(t *testing.T, pattern string, paths []string) []string { t.Helper() trimmed := trimArchiveRoot(pattern) diff --git a/syft/internal/fileresolver/archive_index_links_test.go b/internal/archive/index_links_test.go similarity index 64% rename from syft/internal/fileresolver/archive_index_links_test.go rename to internal/archive/index_links_test.go index 02d789808..c7af26e84 100644 --- a/syft/internal/fileresolver/archive_index_links_test.go +++ b/internal/archive/index_links_test.go @@ -1,4 +1,4 @@ -package fileresolver +package archive import ( "archive/tar" @@ -10,13 +10,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/anchore/syft/internal/archive" - "github.com/anchore/syft/internal/tmpdir" ) -// linkEntry is one entry of a fixture archive: a regular file when linkname is empty, otherwise a -// symlink or, with hard set, a hard link. +// linkEntry is a regular file when linkname is empty, otherwise a symlink or, with hard set, a hard link. type linkEntry struct { name string body string @@ -24,32 +20,23 @@ type linkEntry struct { hard bool } -func indexWithLinks(t testing.TB, entries ...linkEntry) *ArchiveIndex { +func indexWithLinks(t testing.TB, entries ...linkEntry) *Index { t.Helper() - workDir := archive.NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) - t.Cleanup(workDir.Remove) - store := archive.NewEntryStore(workDir, "test.tar", nil) - t.Cleanup(func() { require.NoError(t, store.Close()) }) - charge := archive.NewLimiter(archive.Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}).Charge() - + store := storeFor(t, memCharge(1<<20)) for _, e := range entries { - hdr := tar.Header{Name: e.name, Mode: 0o644, Typeflag: tar.TypeReg, Size: int64(len(e.body))} + hdr := regularHeader(e.name, int64(len(e.body))) switch { case e.linkname != "" && e.hard: hdr = tar.Header{Name: e.name, Mode: 0o644, Typeflag: tar.TypeLink, Linkname: e.linkname} case e.linkname != "": hdr = tar.Header{Name: e.name, Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: e.linkname} } - _, err := store.Add(hdr, bytes.NewReader([]byte(e.body)), charge) - require.NoError(t, err) + require.NoError(t, store.Add(hdr, bytes.NewReader([]byte(e.body)))) } - - r, err := NewFromArchiveEntries("", "outer.tar", store, nil) - require.NoError(t, err) - return r + return NewIndex(store, "", "outer.tar") } -func globPaths(t *testing.T, r *ArchiveIndex, pattern string) []string { +func globPaths(t *testing.T, r *Index, pattern string) []string { t.Helper() locations, err := r.FilesByGlob(pattern) require.NoError(t, err) @@ -60,14 +47,9 @@ func globPaths(t *testing.T, r *ArchiveIndex, pattern string) []string { return out } -// TestArchiveIndex_linksCollapseToOnePath covers the resolver contract that several paths to one file -// answer as one location, and which path that is. -// -// The rule is a property of the candidates, not of the order they were found in: the path that is the -// file itself wins, and otherwise the lowest-sorting one. That matters because the search accumulates -// into a map and the answer is sorted afterwards - neither of which the choice may depend on, or the -// same archive would name a different path from run to run. -func TestArchiveIndex_linksCollapseToOnePath(t *testing.T) { +// several paths to one file answer as one location: the file's own path when it matched, otherwise +// the lowest-sorting link +func TestIndex_linksCollapseToOnePath(t *testing.T) { tests := []struct { name string entries []linkEntry @@ -171,11 +153,8 @@ func TestArchiveIndex_linksCollapseToOnePath(t *testing.T) { } } -// TestArchiveIndex_collapseIgnoresTheOrderItFinds is what holds the rule to being a rule. -// -// Sorting the answer would hide a choice made by iteration order, so this exercises the choice -// directly: the same candidates in shuffled order must always yield the same path. -func TestArchiveIndex_collapseIgnoresTheOrderItFinds(t *testing.T) { +// the choice must not depend on the order the search found the candidates in +func TestIndex_collapseIgnoresTheOrderItFinds(t *testing.T) { r := indexWithLinks(t, linkEntry{name: "opt/real.jar", body: "PK\x03\x04"}, linkEntry{name: "opt/a-sym.jar", linkname: "real.jar"}, @@ -183,12 +162,12 @@ func TestArchiveIndex_collapseIgnoresTheOrderItFinds(t *testing.T) { linkEntry{name: "opt/z-sym.jar", linkname: "real.jar"}, ) - candidates := []*indexNode{ + candidates := []*node{ r.byPath["/opt/real.jar"], r.byPath["/opt/a-sym.jar"], r.byPath["/opt/m-sym.jar"], r.byPath["/opt/z-sym.jar"], } - for _, node := range candidates { - require.NotNil(t, node) + for _, n := range candidates { + require.NotNil(t, n) } rng := rand.New(rand.NewSource(1)) @@ -197,43 +176,38 @@ func TestArchiveIndex_collapseIgnoresTheOrderItFinds(t *testing.T) { candidates[i], candidates[j] = candidates[j], candidates[i] }) - found := map[*indexNode]struct{}{} - for _, node := range candidates { - found[node] = struct{}{} + found := map[*node]struct{}{} + for _, n := range candidates { + found[n] = struct{}{} } - best := collapseToOnePathPerFile(found) + best := onePathPerFile(found) require.Len(t, best, 1, "four paths, one file") - for target, access := range best { - assert.Equal(t, "/opt/real.jar", target.path) - assert.Equal(t, "/opt/real.jar", access.path, "the file itself must win from any order") - } + assert.Equal(t, "/opt/real.jar", best[0].path, "the file itself must win from any order") } // and with the file itself not among the candidates, the lowest-sorting link wins from any order links := candidates[:0:0] - for _, node := range candidates { - if node.path != "/opt/real.jar" { - links = append(links, node) + for _, n := range candidates { + if n.path != "/opt/real.jar" { + links = append(links, n) } } for range 64 { rng.Shuffle(len(links), func(i, j int) { links[i], links[j] = links[j], links[i] }) - found := map[*indexNode]struct{}{} - for _, node := range links { - found[node] = struct{}{} + found := map[*node]struct{}{} + for _, n := range links { + found[n] = struct{}{} } - for _, access := range collapseToOnePathPerFile(found) { - assert.Equal(t, "/opt/a-sym.jar", access.path, "the lowest-sorting link must win from any order") - } + best := onePathPerFile(found) + require.Len(t, best, 1) + assert.Equal(t, "/opt/a-sym.jar", best[0].path, "the lowest-sorting link must win from any order") } } -// TestArchiveIndex_collapsedLocationReadsTheTarget covers what the collapsed answer is for: the -// location has to name the link a reader recognizes while reading the bytes the file actually holds. -func TestArchiveIndex_collapsedLocationReadsTheTarget(t *testing.T) { +func TestIndex_collapsedLocationReadsTheTarget(t *testing.T) { r := indexWithLinks(t, linkEntry{name: "opt/real.bin", body: "PK\x03\x04 real contents"}, linkEntry{name: "opt/a-sym.jar", linkname: "real.bin"}, @@ -257,9 +231,7 @@ func TestArchiveIndex_collapsedLocationReadsTheTarget(t *testing.T) { "a location standing for a link must still read the file's bytes, not the link's none") } -// TestArchiveIndex_dangingLinkHasNoContent covers the other half of keeping a dangling link: it is -// answered as a path, and reading it fails rather than returning something invented. -func TestArchiveIndex_dangingLinkHasNoContent(t *testing.T) { +func TestIndex_danglingLinkHasNoContent(t *testing.T) { r := indexWithLinks(t, linkEntry{name: "opt/gone.jar", linkname: "missing.jar"}) locations, err := r.FilesByGlob("**/*.jar") @@ -268,12 +240,10 @@ func TestArchiveIndex_dangingLinkHasNoContent(t *testing.T) { assert.Equal(t, "opt/gone.jar", locations[0].RealPath) reader, err := r.FileContentsByLocation(locations[0]) - if err == nil { - t.Cleanup(func() { _ = reader.Close() }) - body, readErr := io.ReadAll(reader) - require.NoError(t, readErr) - assert.Empty(t, string(body), "a link to nothing holds nothing") - } + require.NoError(t, err) + body, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Empty(t, string(body), "a link to nothing holds nothing") metadata, err := r.FileMetadataByLocation(locations[0]) require.NoError(t, err) @@ -281,12 +251,8 @@ func TestArchiveIndex_dangingLinkHasNoContent(t *testing.T) { "where it pointed is still reported, resolved against the link's own directory") } -// TestArchiveIndex_mimeTypeAnswersOnlyWithFiles covers why a link is never a MIME answer: the type is -// sniffed from content, a link has none, and the file it points at answers under its own path. -// -// Without the rule this passes by accident - a link's empty entry sniffs to no type, so no real query -// reaches one - and the accident shows through on a query for the empty type. -func TestArchiveIndex_mimeTypeAnswersOnlyWithFiles(t *testing.T) { +// a link has no content to sniff a type from; the file it points at answers under its own path +func TestIndex_mimeTypeAnswersOnlyWithFiles(t *testing.T) { r := indexWithLinks(t, linkEntry{name: "opt/real.jar", body: "PK\x03\x04 zip contents here"}, linkEntry{name: "opt/sym.jar", linkname: "real.jar"}, @@ -314,13 +280,9 @@ func TestArchiveIndex_mimeTypeAnswersOnlyWithFiles(t *testing.T) { assert.Equal(t, []string{"opt/notes.txt", "opt/real.jar"}, byMIME("application/zip", "text/plain")) } -// TestArchiveIndex_allLocationsKeepsEveryPath covers the opposite rule, and why it is opposite. -// -// AllLocations is an enumeration of what the archive holds, not a search for a file. The file metadata -// cataloger reads it to record a row per path - a symlink's row being its type and where it points - -// so collapsing links here would delete the very records it exists to produce. Callers wanting only -// real files filter by type, as file.cataloger/internal.AllRegularFiles does. -func TestArchiveIndex_allLocationsKeepsEveryPath(t *testing.T) { +// AllLocations enumerates what the archive holds rather than searching for files: the file metadata +// cataloger records a row per path, links included +func TestIndex_allLocationsKeepsEveryPath(t *testing.T) { r := indexWithLinks(t, linkEntry{name: "opt/real.jar", body: "PK\x03\x04"}, linkEntry{name: "opt/sym.jar", linkname: "real.jar"}, diff --git a/syft/internal/fileresolver/archive_index_test.go b/internal/archive/index_test.go similarity index 50% rename from syft/internal/fileresolver/archive_index_test.go rename to internal/archive/index_test.go index 2da403825..e02f13e0f 100644 --- a/syft/internal/fileresolver/archive_index_test.go +++ b/internal/archive/index_test.go @@ -1,24 +1,20 @@ -package fileresolver +package archive import ( "archive/tar" "bytes" "context" "io" - "os" - "path/filepath" "sort" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/anchore/syft/internal/archive" - "github.com/anchore/syft/internal/tmpdir" "github.com/anchore/syft/syft/file" ) -func TestArchiveIndex_globByExtensionUsesTheNameIndex(t *testing.T) { +func TestIndex_globByExtensionUsesTheNameIndex(t *testing.T) { r := indexOver(t, 1<<20, jarEntriesFixture) locations, err := r.FilesByGlob("**/*.jar") @@ -26,7 +22,7 @@ func TestArchiveIndex_globByExtensionUsesTheNameIndex(t *testing.T) { assert.Equal(t, []string{"/BOOT-INF/lib/inner.jar"}, relPaths(t, r, locations)) } -func TestArchiveIndex_globByExactNameAtAnyDepth(t *testing.T) { +func TestIndex_globByExactNameAtAnyDepth(t *testing.T) { r := indexOver(t, 1<<20, jarEntriesFixture) locations, err := r.FilesByGlob("**/pom.properties") @@ -34,9 +30,7 @@ func TestArchiveIndex_globByExactNameAtAnyDepth(t *testing.T) { assert.Equal(t, []string{"/META-INF/maven/com.example/lib/pom.properties"}, relPaths(t, r, locations)) } -func TestArchiveIndex_globFindsRootLevelFiles(t *testing.T) { - // `/*` names the entries at the top of the archive, where a jar such as kafka-clients carries its - // license +func TestIndex_globFindsRootLevelFiles(t *testing.T) { r := indexOver(t, 1<<20, jarEntriesFixture) locations, err := r.FilesByGlob("/*") @@ -44,7 +38,7 @@ func TestArchiveIndex_globFindsRootLevelFiles(t *testing.T) { assert.Equal(t, []string{"/LICENSE"}, relPaths(t, r, locations)) } -func TestArchiveIndex_globOneDirectoryDeep(t *testing.T) { +func TestIndex_globOneDirectoryDeep(t *testing.T) { r := indexOver(t, 1<<20, jarEntriesFixture) locations, err := r.FilesByGlob("/META-INF/*") @@ -53,9 +47,8 @@ func TestArchiveIndex_globOneDirectoryDeep(t *testing.T) { "one level only: the maven files are deeper") } -func TestArchiveIndex_globAcrossSyntheticDirectories(t *testing.T) { - // the archive never names META-INF/ or META-INF/maven/, so the directories a middle segment needs are - // synthesized from the paths of the entries that were named +func TestIndex_globAcrossSyntheticDirectories(t *testing.T) { + // the archive never lists META-INF/ or META-INF/maven/ r := indexOver(t, 1<<20, jarEntriesFixture) locations, err := r.FilesByGlob("**/maven/*/*/pom.xml") @@ -63,11 +56,9 @@ func TestArchiveIndex_globAcrossSyntheticDirectories(t *testing.T) { assert.Equal(t, []string{"/META-INF/maven/com.example/lib/pom.xml"}, relPaths(t, r, locations)) } -func TestArchiveIndex_globHonorsNonWildcardMetacharacters(t *testing.T) { - // the name-index shortcuts read a segment as literal text around at most one `*`, so a segment - // carrying any other metacharacter must reach doublestar instead. Taking the shortcut anyway looks - // the pattern up as a literal name and returns nothing, losing every `**/{go,go.exe}` and - // `**/libstd-????????????????.so` glob the binary classifier cataloger has. +func TestIndex_globHonorsNonWildcardMetacharacters(t *testing.T) { + // a segment with any metacharacter other than one `*` must reach doublestar rather than be looked + // up as a literal name r := indexOver(t, 1<<20, map[string]string{ "usr/local/go/bin/go": "binary", "usr/lib/libstd-0123456789abcdef.so": "binary", @@ -84,7 +75,6 @@ func TestArchiveIndex_globHonorsNonWildcardMetacharacters(t *testing.T) { {"**/libstd-????????????????.so", []string{"/usr/lib/libstd-0123456789abcdef.so"}}, {"**/*.[jw]ar", []string{"/opt/a.jar", "/opt/b.war"}}, {"**/{a,b}.{jar,war}", []string{"/opt/a.jar", "/opt/b.war"}}, - // the shapes the shortcuts do answer must keep working {"**/*.jar", []string{"/opt/a.jar"}}, {"**/go", []string{"/usr/local/go/bin/go"}}, {"**/notes*", []string{"/opt/notes.txt"}}, @@ -98,9 +88,7 @@ func TestArchiveIndex_globHonorsNonWildcardMetacharacters(t *testing.T) { } } -func TestArchiveIndex_globMetacharactersInAMiddleSegment(t *testing.T) { - // the same case one level down, where childrenMatching could take the directory index's exact-name - // and prefix shortcuts on any segment holding no `*` +func TestIndex_globMetacharactersInAMiddleSegment(t *testing.T) { r := indexOver(t, 1<<20, map[string]string{ "META-INF/maven/com.example/lib/pom.xml": "", "WEB-INF/lib/dep.jar": "PK\x03\x04", @@ -111,7 +99,7 @@ func TestArchiveIndex_globMetacharactersInAMiddleSegment(t *testing.T) { assert.Equal(t, []string{"/META-INF/maven/com.example/lib/pom.xml"}, relPaths(t, r, locations)) } -func TestArchiveIndex_pathsAndContents(t *testing.T) { +func TestIndex_pathsAndContents(t *testing.T) { r := indexOver(t, 1<<20, jarEntriesFixture) locations, err := r.FilesByPath("/META-INF/MANIFEST.MF") @@ -130,9 +118,7 @@ func TestArchiveIndex_pathsAndContents(t *testing.T) { assert.Equal(t, "Manifest-Version: 1.0\n", string(body)) } -func TestArchiveIndex_contentsSurviveTheMoveToDisk(t *testing.T) { - // the index is built over entries, not offsets, so content moving out to the overflow blob must be - // invisible to every lookup +func TestIndex_contentsSurviveTheMoveToDisk(t *testing.T) { r := indexOver(t, 8, jarEntriesFixture) locations, err := r.FilesByGlob("**/*.jar") @@ -147,7 +133,7 @@ func TestArchiveIndex_contentsSurviveTheMoveToDisk(t *testing.T) { assert.Equal(t, "PK\x03\x04 not really", string(body)) } -func TestArchiveIndex_mimeTypesAreSniffedFromContent(t *testing.T) { +func TestIndex_mimeTypesAreSniffedFromContent(t *testing.T) { r := indexOver(t, 1<<20, map[string]string{ "notes.txt": "just some text", "run.sh": "#!/bin/bash\necho hello\n", @@ -158,35 +144,7 @@ func TestArchiveIndex_mimeTypesAreSniffedFromContent(t *testing.T) { assert.Equal(t, []string{"/run.sh"}, relPaths(t, r, locations)) } -func TestArchiveIndex_exclusionsPruneWholeSubtrees(t *testing.T) { - // a filter answering SkipDir removes the directory and everything under it, as in a directory walk. - // An archive is a sequence rather than a walk, so this is checked per entry against what was pruned. - // the filter sees each entry's archive-relative path, the only frame an archive has - skipMetaInf := func(_, entryPath string, _ os.FileInfo, _ error) error { - if entryPath == "META-INF" { - return filepath.SkipDir - } - return nil - } - - // the archive names the directory itself, so the filter has something to prune on - entries := map[string]string{} - for k, v := range jarEntriesFixture { - entries[k] = v - } - entries["META-INF/"] = "" - - r := indexOver(t, 1<<20, entries, skipMetaInf) - - locations, err := r.FilesByGlob("**/*") - require.NoError(t, err) - got := relPaths(t, r, locations) - assert.NotContains(t, got, "/META-INF/MANIFEST.MF") - assert.NotContains(t, got, "/META-INF/maven/com.example/lib/pom.xml") - assert.Contains(t, got, "/LICENSE", "everything outside the pruned subtree is still there") -} - -func TestArchiveIndex_allLocationsCoversEveryFile(t *testing.T) { +func TestIndex_allLocationsCoversEveryFile(t *testing.T) { r := indexOver(t, 1<<20, jarEntriesFixture) var got []string @@ -205,16 +163,8 @@ func TestArchiveIndex_allLocationsCoversEveryFile(t *testing.T) { }, got) } -func TestArchiveIndex_recordsCountsWhatTheFilesystemHolds(t *testing.T) { - r := indexOver(t, 1<<20, jarEntriesFixture) - assert.Equal(t, len(jarEntriesFixture), r.Records()) -} - -func TestArchiveIndex_answersAreOrdered(t *testing.T) { - // every answer must come back in the same order every time: results merge into one shared SBOM, so - // order decides which locations group into which package, and an index built over a Go map has none. - // Unordered, an image whose jars are hard links to one another split one package of 24 locations into - // three holding 6, 9 and 9 of the same 24. +func TestIndex_answersAreOrdered(t *testing.T) { + // order decides which locations group into which package when results merge into the SBOM entries := map[string]string{} for _, name := range []string{ "b/z.jar", "a/y.jar", "c/x.jar", "a/b/c/w.jar", "z.jar", "a.jar", @@ -233,61 +183,31 @@ func TestArchiveIndex_answersAreOrdered(t *testing.T) { for range 8 { again, err := r.FilesByGlob("**/*.jar") require.NoError(t, err) - assert.Equal(t, first, again, "the same glob must return the same order every time") + assert.Equal(t, first, again) mimeAgain, err := r.FilesByMIMEType("application/octet-stream") require.NoError(t, err) - assert.Equal(t, byMIME, mimeAgain, "and so must a MIME lookup") + assert.Equal(t, byMIME, mimeAgain) var all []string for loc := range r.AllLocations(context.Background()) { all = append(all, loc.RealPath) } - assert.True(t, sort.StringsAreSorted(all), "AllLocations must be ordered, not merely repeatable") + assert.True(t, sort.StringsAreSorted(all)) } var paths []string for _, loc := range first { paths = append(paths, loc.RealPath) } - assert.True(t, sort.StringsAreSorted(paths), "and the order is by path, so it is predictable rather than merely stable") + assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable") } -func TestArchiveIndex_CloseReleasesStore(t *testing.T) { - // forcing content to disk (memory 0) so the store holds an open overflow blob: Close must release it, - // reads after Close no longer resolve, and a second Close is a no-op - r := indexOver(t, 0, map[string]string{"a.txt": "hello"}) - - locs, err := r.FilesByPath("a.txt") - require.NoError(t, err) - require.Len(t, locs, 1) - - rc, err := r.FileContentsByLocation(locs[0]) - require.NoError(t, err) - b, err := io.ReadAll(rc) - require.NoError(t, err) - require.NoError(t, rc.Close()) - assert.Equal(t, "hello", string(b)) - - require.NoError(t, r.Close()) - - _, err = r.FileContentsByLocation(locs[0]) - assert.Error(t, err, "the overflow blob is closed, so its content can no longer be read") - - assert.NoError(t, r.Close(), "Close is idempotent") -} - -// storeWith builds an entry store holding the given entries, in name order so a test that depends on -// order does not depend on a map. -func storeWith(t testing.TB, maxInMemory int64, entries map[string]string) *archive.EntryStore { +// storeWith builds an entry store holding the given entries in name order. A name ending in "/" with +// an empty body is a directory. +func storeWith(t testing.TB, maxInMemory int64, entries map[string]string) *EntryStore { t.Helper() - workDir := archive.NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) - t.Cleanup(workDir.Remove) - store := archive.NewEntryStore(workDir, "test.jar", nil) - // the limiter decides how much stays in memory, so a test wanting content on disk sets a memory bound - // and leaves disk unbounded - charge := archive.NewLimiter(archive.Limits{MaxMemoryBytes: maxInMemory, MaxDiskBytes: -1}).Charge() - t.Cleanup(func() { require.NoError(t, store.Close()) }) + store := storeFor(t, memCharge(maxInMemory)) names := make([]string, 0, len(entries)) for name := range entries { @@ -297,29 +217,21 @@ func storeWith(t testing.TB, maxInMemory int64, entries map[string]string) *arch for _, name := range names { body := entries[name] - hdr := tar.Header{Name: name, Size: int64(len(body)), Mode: 0o600, Typeflag: tar.TypeReg} if body == "" && name[len(name)-1] == '/' { - hdr = tar.Header{Name: name, Mode: 0o755, Typeflag: tar.TypeDir} - _, err := store.Add(hdr, nil, charge) - require.NoError(t, err) + require.NoError(t, store.Add(tar.Header{Name: name, Mode: 0o755, Typeflag: tar.TypeDir}, nil)) continue } - _, err := store.Add(hdr, bytes.NewReader([]byte(body)), charge) - require.NoError(t, err) + require.NoError(t, store.Add(regularHeader(name, int64(len(body))), bytes.NewReader([]byte(body)))) } return store } -// indexOver builds an index with no budget of its own, so these tests see every entry they put in. -// What it charges, and what it does when a charge is refused, is in archive_index_charge_test.go. -func indexOver(t testing.TB, maxInMemory int64, entries map[string]string, filters ...PathIndexVisitor) *ArchiveIndex { +func indexOver(t testing.TB, maxInMemory int64, entries map[string]string) *Index { t.Helper() - r, err := NewFromArchiveEntries("", "outer.jar", storeWith(t, maxInMemory, entries), nil, filters...) - require.NoError(t, err) - return r + return NewIndex(storeWith(t, maxInMemory, entries), "", "outer.jar") } -func relPaths(t *testing.T, r *ArchiveIndex, locations []file.Location) []string { +func relPaths(t *testing.T, r *Index, locations []file.Location) []string { t.Helper() var out []string for _, loc := range locations { @@ -337,3 +249,23 @@ var jarEntriesFixture = map[string]string{ "BOOT-INF/lib/inner.jar": "PK\x03\x04 not really", "com/example/Thing.class": "bytecode", } + +func TestIndex_impliedDirectoriesArePathsButNotFiles(t *testing.T) { + r := indexOver(t, 1<<20, jarEntriesFixture) + + assert.True(t, r.HasPath("META-INF")) + assert.True(t, r.HasPath("/META-INF/maven")) + + locations, err := r.FilesByPath("META-INF") + require.NoError(t, err) + assert.Empty(t, locations) + locations, err = r.FilesByGlob("**/maven") + require.NoError(t, err) + assert.Empty(t, locations) +} + +func TestIndex_invalidGlobIsAnError(t *testing.T) { + r := indexOver(t, 1<<20, jarEntriesFixture) + _, err := r.FilesByGlob("**/[") + assert.Error(t, err) +} diff --git a/internal/archive/limits.go b/internal/archive/limits.go index 21733e468..11223953f 100644 --- a/internal/archive/limits.go +++ b/internal/archive/limits.go @@ -5,52 +5,41 @@ import ( "sync" ) -// ErrDiskLimitReached reports that placing an archive's content would exceed the disk limit. Not a -// scan failure: the archive is skipped and the walk continues. +// ErrDiskLimitReached reports that placing an archive's content would exceed the disk limit. var ErrDiskLimitReached = errors.New("archive content would exceed the disk limit") -// Limits bounds how much archive content one scan may hold at once, and so where content is held. +// Limits bound how much archive content one scan holds at once. Each limit falls as archives are +// released, so it bounds peak usage rather than the total ever written. // -// These are in-use limits, not counters: each measures what is held right now and falls when an -// archive is released, bounding peak concurrent usage. Work directories are removed on release, so a -// bound on bytes ever written would measure state that never exists at once. -// -// Both fields read three states the same way: positive is the limit, zero forbids that resource, and -// negative is unbounded. Unbounded is a real hazard, so it must be asked for explicitly. +// For both limits: positive is the bound, zero forbids the resource, negative is unbounded. type Limits struct { - // MaxMemoryBytes bounds archive content held in memory at once; content spills to disk when it does - // not admit more, so there is no separate spill threshold. Zero sends all content to disk; negative - // is unbounded. + // MaxMemoryBytes bounds archive content held in memory. Content that does not fit is written to + // disk instead. MaxMemoryBytes int64 - // MaxDiskBytes bounds archive content on disk at once: content spilled there plus the entry bytes - // extracted from it. Zero writes nothing, so content that does not fit in memory has nowhere to go - // and its archive is skipped; negative is unbounded. + // MaxDiskBytes bounds archive content written to disk. Content that does not fit is dropped and + // the archive is skipped or truncated. MaxDiskBytes int64 } -// Limiter measures how much archive content a scan is holding right now. One instance per task run -// (one scan), the only scope under which "held at once" means anything. A nil *Limiter enforces -// nothing. Mutex-guarded because the archive task merges into a shared builder while top-level -// catalogers run. +// Limiter tracks how much archive content a scan holds right now against its Limits. A nil *Limiter +// enforces nothing. type Limiter struct { limits Limits + mu sync.Mutex memory int64 disk int64 - // peaks are high-water marks of the two gauges above: the most held at any one moment, not the sum - // across archives. They only rise. peakMemory int64 peakDisk int64 } -// NewLimiter returns a limiter enforcing the given limits. func NewLimiter(limits Limits) *Limiter { return &Limiter{limits: limits} } -// Peak reports the most archive content held at any one moment, the quantities the limits bound. +// Peak reports the most memory and disk held at any one moment. func (l *Limiter) Peak() (memory, disk int64) { if l == nil { return 0, 0 @@ -60,7 +49,7 @@ func (l *Limiter) Peak() (memory, disk int64) { return l.peakMemory, l.peakDisk } -// InUse reports the archive content held right now, in memory and on disk. +// InUse reports the memory and disk held right now. func (l *Limiter) InUse() (memory, disk int64) { if l == nil { return 0, 0 @@ -70,7 +59,7 @@ func (l *Limiter) InUse() (memory, disk int64) { return l.memory, l.disk } -// Charge starts one archive's accounting. Releasing the returned charge gives back what it took. +// Charge starts accounting for one archive. Releasing the charge gives back everything it took. func (l *Limiter) Charge() *Charge { if l == nil { return nil @@ -78,19 +67,16 @@ func (l *Limiter) Charge() *Charge { return &Charge{limiter: l} } -// Charge is one archive's accumulated draw on the limiter. A nil *Charge charges and refuses nothing. +// Charge is one archive's draw on a Limiter. A nil *Charge admits everything. // -// Bytes are charged as they land, never from a declared size: tar and zip headers both carry a size a -// crafted archive can lie about, so reserving against it would bound the claim, not the usage. +// Bytes are charged as they land, never from the sizes an archive declares, since those can lie. type Charge struct { limiter *Limiter memory int64 disk int64 } -// admitsThreeState reports whether n more bytes may be charged against a limit already holding held. -// Shared by Memory and Disk so the two readings of Limits cannot drift. -func admitsThreeState(limit, held, n int64) bool { +func admits(limit, held, n int64) bool { switch { case limit < 0: return true @@ -101,100 +87,78 @@ func admitsThreeState(limit, held, n int64) bool { } } -// Memory charges n bytes held in memory, reporting false when the memory limit does not admit it (for -// a zero limit, always). Nothing is charged when it reports false. +// Memory charges n bytes held in memory, or reports false (charging nothing) when the memory limit +// does not admit them. func (c *Charge) Memory(n int64) bool { - if c == nil || c.limiter == nil || n <= 0 { + if c == nil || n <= 0 { return true } l := c.limiter l.mu.Lock() defer l.mu.Unlock() - if !admitsThreeState(l.limits.MaxMemoryBytes, l.memory, n) { + if !admits(l.limits.MaxMemoryBytes, l.memory, n) { return false } l.memory += n c.memory += n - if l.memory > l.peakMemory { - l.peakMemory = l.memory - } + l.peakMemory = max(l.peakMemory, l.memory) return true } -// Disk charges n bytes placed on disk, reporting false when the disk limit does not admit it (for a -// zero limit, always, which skips the archive). Nothing is charged when it reports false. +// Disk charges n bytes written to disk, or reports false (charging nothing) when the disk limit does +// not admit them. func (c *Charge) Disk(n int64) bool { - if c == nil || c.limiter == nil || n <= 0 { + if c == nil || n <= 0 { return true } l := c.limiter l.mu.Lock() defer l.mu.Unlock() - if !admitsThreeState(l.limits.MaxDiskBytes, l.disk, n) { + if !admits(l.limits.MaxDiskBytes, l.disk, n) { return false } l.disk += n c.disk += n - if l.disk > l.peakDisk { - l.peakDisk = l.disk - } + l.peakDisk = max(l.peakDisk, l.disk) return true } -// IndexRecord charges the memory an archive's index records keep regardless of content weight, -// preferring the memory budget and falling back to disk. It reports false only when neither budget -// admits the record, truncating the archive. -// -// It bounds the dimension the byte budgets miss: an archive of many empty entries, or one entry named -// a thousand components deep, weighs almost nothing as content yet would grow the index without limit. -// -// The disk fallback exists because a resolver needs an entry's record to reach its bytes at all, and -// MaxMemoryBytes may legitimately be zero (the configuration that spills every archive). So records -// are bounded by the disk budget once memory is full, though they are only ever held in memory. -func (c *Charge) IndexRecord(n int64) bool { - if c == nil || c.limiter == nil || n <= 0 { - return true - } - if c.Memory(n) { - return true - } - return c.Disk(n) +// Index charges n bytes of index bookkeeping. The index always lives in memory, but a zero memory +// limit is a valid configuration that must still index archives, so it falls back to the disk +// budget when memory does not admit it. Reports false only when neither does. +func (c *Charge) Index(n int64) bool { + return c.Memory(n) || c.Disk(n) } -// RefundDisk gives back n bytes charged for content since removed, so a partial file dropped at a -// limit does not leave the limiter holding bytes no longer on disk. -func (c *Charge) RefundDisk(n int64) { - if c == nil || c.limiter == nil || n <= 0 { - return - } - l := c.limiter - l.mu.Lock() - defer l.mu.Unlock() - if n > c.disk { - n = c.disk - } - l.disk -= n - c.disk -= n -} - -// RefundMemory gives back n bytes charged for content no longer held in memory. +// RefundMemory gives back n bytes no longer held in memory. func (c *Charge) RefundMemory(n int64) { - if c == nil || c.limiter == nil || n <= 0 { + if c == nil || n <= 0 { return } l := c.limiter l.mu.Lock() defer l.mu.Unlock() - if n > c.memory { - n = c.memory - } + n = min(n, c.memory) l.memory -= n c.memory -= n } -// Release returns everything this charge took. Safe to call more than once. +// RefundDisk gives back n bytes no longer on disk. +func (c *Charge) RefundDisk(n int64) { + if c == nil || n <= 0 { + return + } + l := c.limiter + l.mu.Lock() + defer l.mu.Unlock() + n = min(n, c.disk) + l.disk -= n + c.disk -= n +} + +// Release gives back everything this charge took. Safe to call more than once. func (c *Charge) Release() { - if c == nil || c.limiter == nil { + if c == nil { return } l := c.limiter diff --git a/internal/archive/limits_test.go b/internal/archive/limits_test.go index 4c7f2d94b..d26fa2f51 100644 --- a/internal/archive/limits_test.go +++ b/internal/archive/limits_test.go @@ -38,7 +38,6 @@ func TestLimiter_chargeAndRelease(t *testing.T) { } func TestLimiter_refund(t *testing.T) { - // a partial entry dropped at a bound is no longer on disk, so the limiter must not still hold it limiter := NewLimiter(Limits{MaxDiskBytes: 100}) charge := limiter.Charge() @@ -55,8 +54,6 @@ func TestLimiter_refund(t *testing.T) { } func TestLimiter_boundsAreIndependent(t *testing.T) { - // a caller may bound one limit without the other: a limit on one side does not change how the other - // reads its own value memoryBounded := NewLimiter(Limits{MaxMemoryBytes: 10, MaxDiskBytes: -1}).Charge() assert.False(t, memoryBounded.Memory(50)) assert.True(t, memoryBounded.Disk(1_000_000), "a negative disk limit admits anything") @@ -67,8 +64,6 @@ func TestLimiter_boundsAreIndependent(t *testing.T) { } func TestLimiter_threeStateReadingIsUniform(t *testing.T) { - // Memory and Disk read their limit the same way: positive is the limit, zero refuses, negative - // admits t.Run("positive is the limit", func(t *testing.T) { c := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 100}).Charge() assert.True(t, c.Memory(100)) @@ -91,8 +86,6 @@ func TestLimiter_threeStateReadingIsUniform(t *testing.T) { } func TestLimiter_nilIsUnbounded(t *testing.T) { - // what an extraction with no configured bounds gets: charges succeed and nothing is measured. - // Distinct from a configured but non-positive MaxMemoryBytes, which refuses instead. var limiter *Limiter charge := limiter.Charge() assert.Nil(t, charge) @@ -112,9 +105,6 @@ func TestLimiter_nilIsUnbounded(t *testing.T) { } func TestLimiter_concurrentChargesAgree(t *testing.T) { - // the walk is sequential today, but the archive task merges into a shared builder while top-level - // catalogers run, so the accounting is guarded. Both limits need an explicit value, since zero - // refuses every charge: memory is set above what this test charges and disk is negative. limiter := NewLimiter(Limits{MaxMemoryBytes: 1_000_000, MaxDiskBytes: -1}) var wg sync.WaitGroup @@ -145,8 +135,6 @@ func TestLimiter_concurrentChargesAgree(t *testing.T) { } func Test_Limiter_peakIsAHighWaterMark(t *testing.T) { - // the peak is what the scan held at its worst moment, so it must survive the release that takes the - // in-use gauge back down l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) first := l.Charge() @@ -163,12 +151,11 @@ func Test_Limiter_peakIsAHighWaterMark(t *testing.T) { assert.Equal(t, int64(50), disk) peakMemory, peakDisk := l.Peak() - assert.Equal(t, int64(100), peakMemory, "peak memory is the most held at once, not the most recently held") - assert.Equal(t, int64(300), peakDisk, "and the same for disk") + assert.Equal(t, int64(100), peakMemory, "the peak is the most held at once, not the most recently held") + assert.Equal(t, int64(300), peakDisk) } func Test_Limiter_peakCountsConcurrentHoldersTogether(t *testing.T) { - // two archives held at once peak at their sum, the number the limits are enforced against l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) outer := l.Charge() @@ -190,8 +177,6 @@ func Test_Limiter_peakOfANilLimiterIsZero(t *testing.T) { } func Test_Limiter_refusedChargeDoesNotMoveThePeak(t *testing.T) { - // nothing is charged when a limit refuses, so nothing may be recorded: a peak that rose on refusal - // would report bytes the scan never held l := NewLimiter(Limits{MaxMemoryBytes: 50, MaxDiskBytes: 50}) c := l.Charge() @@ -202,3 +187,15 @@ func Test_Limiter_refusedChargeDoesNotMoveThePeak(t *testing.T) { assert.Zero(t, peakMemory) assert.Zero(t, peakDisk) } + +func TestCharge_indexFallsBackToDisk(t *testing.T) { + memoryOnly := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 0}).Charge() + assert.True(t, memoryOnly.Index(100)) + assert.False(t, memoryOnly.Index(1), "neither budget admits it") + + diskOnly := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 100}).Charge() + assert.True(t, diskOnly.Index(100)) + mem, disk := diskOnly.held() + assert.Zero(t, mem) + assert.Equal(t, int64(100), disk) +} diff --git a/internal/archive/notify.go b/internal/archive/notify.go deleted file mode 100644 index 4ce5df59a..000000000 --- a/internal/archive/notify.go +++ /dev/null @@ -1,76 +0,0 @@ -package archive - -import ( - "github.com/anchore/syft/internal/log" -) - -// Notify receives structured events from archive extraction. A nil Notify is the no-op. -// -// func(any) rather than typed callbacks so adding an event changes no signature: a handler asserts -// for the events it wants and ignores the rest. Boxing into an any allocates, so on this hot path -// guard the call rather than relying on a nil check inside the callee: -// -// if notify != nil { -// notify(EntriesOverflowed{Archive: name, Bytes: n}) -// } -type Notify func(msg any) - -// OverflowReason says why content was written to disk rather than held. -type OverflowReason string - -const ( - // MemoryLimitReached means the scan was already holding as much as the memory limit admits. - MemoryLimitReached OverflowReason = "memory limit reached" - - // MemoryHoldsNothing means the memory limit is zero, so content goes to disk by policy rather than - // under pressure. - MemoryHoldsNothing OverflowReason = "memory limit is zero" -) - -// ContentOverflowed reports that one archive's own bytes were written to disk rather than held in -// memory. Bytes is the size of the archive. -type ContentOverflowed struct { - Archive string - Bytes int64 - Reason OverflowReason -} - -// EntriesOverflowed reports that the entries a store was holding were written to its overflow blob. -// Entries and Bytes are what moved in this spill, not the archive's totals: a store spills everything -// it holds at once, and may spill again as it refills. -type EntriesOverflowed struct { - Archive string - Entries int - Bytes int64 - Reason OverflowReason -} - -// Skipped reports that an archive contributed nothing, and why. The scan continues, but any packages -// inside that archive are absent from the SBOM. -type Skipped struct { - Archive string - Reason string -} - -// Truncated reports that an archive was cataloged from part of its contents, naming the bound that -// stopped it. -type Truncated struct { - Archive string - Reason TruncationReason -} - -// LogNotify is the default handler: it writes each known event to the debug log and ignores the rest. -func LogNotify(msg any) { - switch e := msg.(type) { - case ContentOverflowed: - log.WithFields("archive", e.Archive, "bytes", e.Bytes, "reason", string(e.Reason)). - Debug("archive content written to disk rather than held in memory") - case EntriesOverflowed: - log.WithFields("archive", e.Archive, "entries", e.Entries, "bytes", e.Bytes, "reason", string(e.Reason)). - Debug("archive entries written to disk rather than held in memory") - case Skipped: - log.WithFields("archive", e.Archive, "reason", e.Reason).Debug("archive skipped") - case Truncated: - log.WithFields("archive", e.Archive, "reason", string(e.Reason)).Debug("archive cataloged from part of its contents") - } -} diff --git a/internal/archive/notify_test.go b/internal/archive/notify_test.go deleted file mode 100644 index fa598f938..000000000 --- a/internal/archive/notify_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package archive - -import ( - "archive/tar" - "bytes" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// recorder is what a consumer of these events looks like: one func, a type assertion for what it -// cares about, silence for everything else. -type recorder struct { - overflows []EntriesOverflowed - other int -} - -func TestNotify_reportsWhichArchiveSpilledAndWhy(t *testing.T) { - rec := &recorder{} - s := NewEntryStore(WorkDirAt(t.TempDir()), "lib/app.jar", rec.notify) - t.Cleanup(func() { require.NoError(t, s.Close()) }) - - charge := memCharge(8) - for _, body := range []string{"aaaa", "bbbbbb"} { - _, err := s.Add(regularHeader(body+".txt", int64(len(body))), bytes.NewReader([]byte(body)), charge) - require.NoError(t, err) - } - - require.Len(t, rec.overflows, 1) - assert.Equal(t, "lib/app.jar", rec.overflows[0].Archive, "the event has to name the archive, or it says nothing actionable") - assert.Equal(t, MemoryLimitReached, rec.overflows[0].Reason) - assert.Equal(t, 2, rec.overflows[0].Entries, "a store moves everything it is holding, not just the entry that did not fit") - assert.Equal(t, int64(10), rec.overflows[0].Bytes) -} - -func TestNotify_distinguishesPressureFromPolicy(t *testing.T) { - // a scan at its memory bound and a scan whose bound is zero are different findings: the first says - // the limit is biting, the second that nothing was ever going to be held - rec := &recorder{} - s := NewEntryStore(WorkDirAt(t.TempDir()), "app.jar", rec.notify) - t.Cleanup(func() { require.NoError(t, s.Close()) }) - - _, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), memCharge(0)) - require.NoError(t, err) - - require.Len(t, rec.overflows, 1) - assert.Equal(t, MemoryHoldsNothing, rec.overflows[0].Reason) -} - -func TestNotify_nilHandlerIsTheNoOp(t *testing.T) { - s := NewEntryStore(WorkDirAt(t.TempDir()), "app.jar", nil) - t.Cleanup(func() { require.NoError(t, s.Close()) }) - - _, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), memCharge(0)) - assert.NoError(t, err, "no handler must not mean no extraction") -} - -func TestNotify_directoryEntriesDoNotSpill(t *testing.T) { - // a directory carries no content, so it can never be the thing that pushes a store to disk - rec := &recorder{} - s := NewEntryStore(WorkDirAt(t.TempDir()), "app.jar", rec.notify) - t.Cleanup(func() { require.NoError(t, s.Close()) }) - - _, err := s.Add(tar.Header{Name: "lib/", Typeflag: tar.TypeDir, Mode: 0o755}, nil, memCharge(0)) - require.NoError(t, err) - assert.Empty(t, rec.overflows) -} - -func (r *recorder) notify(msg any) { - if e, ok := msg.(EntriesOverflowed); ok { - r.overflows = append(r.overflows, e) - return - } - r.other++ -} diff --git a/internal/archive/path_guards_test.go b/internal/archive/path_guards_test.go deleted file mode 100644 index dfffee94e..000000000 --- a/internal/archive/path_guards_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package archive - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// The path-safety guards are covered here directly, since extraction no longer writes a file or link -// per archive entry: -// -// resolvesInsideRoot, resolveLink and isWithin guard the one filesystem write left in this package - -// an archive's own bytes spilling into its work directory, under a name derived from where the -// archive was found, which inside another archive is attacker-supplied text. -// -// A symlink entry creates nothing on disk: it is stored as a header, and where the link points is -// decided inside the archive's own filetree. - -func TestResolvesInsideRoot_multiStepSymlinkChainIsRefused(t *testing.T) { - // the escape the guard exists for: two cooperating links each pass a lexical check and together - // resolve above the root. "d" -> "." lands on the root, so "d/up" -> ".." becomes "/up" -> - // ".." pointing above it, and a write through "d/up/x" lands outside. A lexical check sees - // /d/../x and allows it; only resolving against the real filesystem catches it. - base := t.TempDir() - root := filepath.Join(base, "contents") - require.NoError(t, os.MkdirAll(root, 0o755)) - - require.NoError(t, os.Symlink(".", filepath.Join(root, "d"))) - require.NoError(t, os.Symlink("..", filepath.Join(root, "up"))) - - inside, err := resolvesInsideRoot(root, filepath.Join(root, "d", "up", "PWNED")) - require.NoError(t, err) - assert.False(t, inside, "a write through a link chain that leaves the root must be refused") -} - -func TestResolvesInsideRoot_legitimatePathsAreAllowed(t *testing.T) { - base := t.TempDir() - root := filepath.Join(base, "contents") - require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755)) - - for _, path := range []string{ - filepath.Join(root, "top.txt"), - filepath.Join(root, "lib", "deep", "file.txt"), - } { - inside, err := resolvesInsideRoot(root, path) - require.NoError(t, err) - assert.True(t, inside, "%s is inside the root and must be allowed", path) - } -} - -func TestResolvesInsideRoot_siblingSharingThePrefixIsRefused(t *testing.T) { - // the string-prefix hole: "contents-evil" shares "contents" as a prefix and is not inside it - base := t.TempDir() - root := filepath.Join(base, "contents") - require.NoError(t, os.MkdirAll(root, 0o755)) - require.NoError(t, os.MkdirAll(filepath.Join(base, "contents-evil"), 0o755)) - - inside, err := resolvesInsideRoot(root, filepath.Join(base, "contents-evil", "x.txt")) - require.NoError(t, err) - assert.False(t, inside) -} - -func TestResolvesInsideRoot_KNOWN_LIMIT_aLinkWhoseOwnTargetPathLeavesTheRoot(t *testing.T) { - // KNOWN GAP, recorded as a finding rather than a passing assertion of safety. - // - // Two links then a write: "up" -> "..", then "bin/link.txt" -> "../up/PWNED", then a write to - // "bin/link.txt/through". resolvesInsideRoot walks the write path, meets bin/link.txt, and resolves - // its target lexically without walking the target's own path, never seeing that /up is itself - // a link out of the root. On the real filesystem the write lands above the root. - // - // Not a live exposure: resolvesInsideRoot's only caller writes one flat name into a work directory - // it just created, where no link can be planted. Anything that gives it a caller over - // archive-supplied paths must close this first. - base := t.TempDir() - root := filepath.Join(base, "contents") - require.NoError(t, os.MkdirAll(filepath.Join(root, "bin"), 0o755)) - require.NoError(t, os.Symlink("..", filepath.Join(root, "up"))) - require.NoError(t, os.Symlink("../up/PWNED", filepath.Join(root, "bin", "link.txt"))) - - inside, err := resolvesInsideRoot(root, filepath.Join(root, "bin", "link.txt", "through")) - require.NoError(t, err) - assert.True(t, inside, - "the guard allows this write, and the real filesystem puts it above the root - the hole this "+ - "case exists to record") -} - -func TestArchiveFileName_refusesNamesThatWouldLeaveTheWorkDirectory(t *testing.T) { - // an archive spills under the basename of where it was found, and Base returns ".." for an access - // path ending in it, which joined to the work directory would name the directory above. - // overflowContent checks its destination too, so this is the earlier of two refusals. - for _, accessPath := range []string{"", ".", "..", "a/..", string(filepath.Separator)} { - assert.Equal(t, "archive", archiveFileName(accessPath), - "%q must not become a name joined to the work directory", accessPath) - } - - // and a real name is preserved, compound extension included, so format detection has the best - // chance of identifying the archive - assert.Equal(t, "bundle.tar.gz", archiveFileName("some/path/bundle.tar.gz")) -} diff --git a/internal/archive/resolver.go b/internal/archive/resolver.go deleted file mode 100644 index 0f3bc6226..000000000 --- a/internal/archive/resolver.go +++ /dev/null @@ -1,250 +0,0 @@ -package archive - -import ( - "context" - "crypto" - "errors" - "fmt" - "io" - "path/filepath" - - intFile "github.com/anchore/syft/internal/file" - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/file" -) - -// StoreResolverFactory builds an indexed resolver over one archive's entries. Injected so this -// package does not depend on the internal fileresolver package. -type StoreResolverFactory func(*EntryStore, Overflow) (file.Resolver, IndexResult, error) - -// Overflow names what the resolver built over one archive's entries must know. -type Overflow struct { - // FileSystemID is the identifier of the filesystem the archive file itself lives on (a layer - // digest, or blank), inherited unchanged down the nesting chain, stamped onto every Location. - FileSystemID string - - // ArchivePath is the colon-delimited chain of archive paths from the scan root to this archive, - // stamped onto every Location. It keeps identically-named files in different archives from - // colliding in the coordinate-keyed SBOM tables. - ArchivePath string - - // Charge is this archive's draw on the resource limiter; nil enforces nothing. The index keeps more - // per entry than the store does (one node per path component), so it charges for what it keeps - // rather than letting the store estimate for it. - Charge *Charge -} - -// IndexResult reports what indexing one archive's entries produced. -type IndexResult struct { - // Truncated reports that the budget refused an index record and the remaining entries were never - // indexed. What was indexed is complete and usable. - Truncated bool -} - -// archiveDigestHashes are the digests taken of an archive as a whole: SHA-1 alone, matching what the -// java archive cataloger reports for a jar. -var archiveDigestHashes = []crypto.Hash{crypto.SHA1} - -// digestsOf takes the archive's own digests in one pass over the acquired content, which the java -// cataloger reads when it can no longer open the archive itself. A failure is logged and returns -// nothing: an archive that cataloged fine is not discarded for a missing hash. -func digestsOf(ctx context.Context, content Content, accessPath string) []file.Digest { - if _, err := content.Reader.Seek(0, io.SeekStart); err != nil { - log.WithFields("archive", accessPath, "error", err).Trace("unable to rewind archive content for digests") - return nil - } - digests, err := intFile.NewDigestsFromFile(ctx, io.NopCloser(content.Reader), archiveDigestHashes) - if err != nil { - log.WithFields("archive", accessPath, "error", err).Trace("unable to take archive digests") - return nil - } - return digests -} - -// ExtractedArchive is a standalone, fully-indexed filesystem built from an extracted archive. -type ExtractedArchive struct { - // Resolver is an indexed resolver over the extracted contents, its Locations stamped with - // FileSystemID. - Resolver file.Resolver - - // Digests are the digests of the archive file itself, taken while its bytes were in hand. - Digests []file.Digest - // FileSystemID is the filesystem the archive file itself lives on, inherited unchanged down the - // nesting chain. - FileSystemID string - // Result reports how much was extracted and whether a bound truncated it. A truncated archive - // still yields a usable Resolver over what was written before the bound; see ExtractionResult. - Result ExtractionResult - - // content is this archive's own bytes, in memory or spilled to disk. Retained until Cleanup so the - // bytes the limiter counts as held stay reachable and lazy readers keep their backing. - content Content - - cleanup func() -} - -// Cleanup removes the temp directory backing this archive's filesystem and releases what it held -// against the limiter. Safe to call more than once. -func (e *ExtractedArchive) Cleanup() { - if e == nil || e.cleanup == nil { - return - } - // close before removing the work directory, so the overflow blob handle is released while the file - // still exists. A resolver holding nothing open is not an io.Closer. - if closer, ok := e.Resolver.(io.Closer); ok { - if err := closer.Close(); err != nil { - log.WithFields("fsid", e.FileSystemID, "error", err).Trace("unable to close archive resolver") - } - } - e.cleanup() - e.cleanup = nil - // drop the last reference so the limit falls as the bytes become collectable - e.content = Content{} -} - -// ExtractToResolver holds the given archive content (in memory while the memory limit admits it, on -// disk once it does not), extracts it, and returns a standalone indexed resolver over its contents -// whose Locations are stamped with fileSystemID and archivePath. Returns (nil, nil) when the content -// is not an extractable archive. The caller owns the result and must call Cleanup on it. Everything -// the archive holds is charged to one limiter handle and released by Cleanup. -// -// Reaching a limit never waits for capacity: the walk descends before it unwinds, so a containing -// archive still holds its content while its children are cataloged, and nothing would be released -// while this one waited. Content that cannot be admitted yields ErrDiskLimitReached. -// -// Reaching a bound part way through is not an error: the ExtractedArchive is indexed over the partial -// contents and Result.Truncated() reports it. Any other non-nil error means nothing usable was -// produced. -func ExtractToResolver(ctx context.Context, content io.Reader, archiveAccessPath, fileSystemID, archivePath string, extractors []Extractor, limiter *Limiter, newStoreResolver StoreResolverFactory, notify Notify) (*ExtractedArchive, error) { - // created only if this archive writes something; most never do - workDir := NewWorkDir(ctx) - - charge := limiter.Charge() - - var held Content - cleanup := func() { - if closeErr := held.Close(); closeErr != nil { - log.WithFields("archive", archiveAccessPath, "error", closeErr).Trace("unable to close archive content") - } - held = Content{} - workDir.Remove() - charge.Release() - } - - held, err := acquireContent(content, workDir, archiveFileName(archiveAccessPath), charge, notify) - if err != nil { - cleanup() - if notify != nil && errors.Is(err, ErrDiskLimitReached) { - notify(Skipped{Archive: archiveAccessPath, Reason: "content would exceed the disk limit"}) - } - // unwrapped so the caller can attribute an ErrDiskLimitReached skip to this archive - return nil, err - } - - extractor := FindExtractor(ctx, extractors, held) - if extractor == nil { - // not extractable (e.g. an executable that matched a broad mime filter) - cleanup() - return nil, nil - } - - resolver, result, err := storeAndIndex(ctx, extractor, held, workDir, fileSystemID, archivePath, archiveAccessPath, charge, newStoreResolver, notify) - if err != nil { - cleanup() - return nil, err - } - - return &ExtractedArchive{ - Resolver: resolver, - Digests: digestsOf(ctx, held, archiveAccessPath), - FileSystemID: fileSystemID, - Result: result, - content: held, - cleanup: cleanup, - }, nil -} - -// inPlaceContent is content already random-access and already charged for by the archive holding it. -// A nested archive arrives as a reader over one entry of its parent's EntryStore supporting Read, -// Seek and ReadAt, so it is read where it lies rather than copied or spilled again. -// -// A marker method rather than a type assertion, because the concrete type belongs to the fileresolver -// package, which this one does not import. Nothing else satisfies it. -type inPlaceContent interface { - io.Reader - io.ReaderAt - io.Seeker - - // OverflowArchiveEntry marks a reader over one entry of an archive's EntryStore. - OverflowArchiveEntry() -} - -// acquireContent gets one archive's bytes ready for random access: content already held by a parent -// archive is used in place, anything else goes through the limiter. In-place content is charged -// nothing, since the parent already charged for those bytes and releases them on its own Cleanup. -func acquireContent(r io.Reader, workDir *WorkDir, name string, charge *Charge, notify Notify) (Content, error) { - if entry, ok := r.(inPlaceContent); ok { - return Content{Name: name, Reader: entry}, nil - } - return holdContent(r, workDir, name, charge, notify) -} - -// archiveFileName derives a safe basename for the saved archive, preserving compound extensions -// (e.g. ".tar.gz") so format detection can identify it. "..", empty and root are refused, since -// joining them to the work directory would name the directory above; the destination is rechecked -// before the write. -func archiveFileName(accessPath string) string { - name := filepath.Base(accessPath) - if name == "" || name == "." || name == ".." || name == string(filepath.Separator) { - return "archive" - } - return name -} - -// storeAndIndex puts the archive's entries into an EntryStore and builds the resolver over them. -// -// ErrDiskLimitReached travels unwrapped so the caller can attribute the skip to this archive. -func storeAndIndex(ctx context.Context, extractor Extractor, held Content, workDir *WorkDir, fsID, archivePath, accessPath string, charge *Charge, newStoreResolver StoreResolverFactory, notify Notify) (file.Resolver, ExtractionResult, error) { - var result ExtractionResult - - store := NewEntryStore(workDir, accessPath, notify) - - result, err := extractor.Extract(ctx, held, store, charge) - if err != nil { - if closeErr := store.Close(); closeErr != nil { - log.WithFields("archive", fsID, "error", closeErr).Trace("unable to close archive entry store") - } - return nil, result, fmt.Errorf("unable to extract archive %q: %w", fsID, err) - } - - // the disk limit refused the first byte, so every entry is content-less: skip the archive rather - // than catalog it as empty. An archive that stored some entries is a truncation instead. - if result.Truncation == TruncatedByDiskLimit && store.OnDisk() == 0 { - if closeErr := store.Close(); closeErr != nil { - log.WithFields("archive", fsID, "error", closeErr).Trace("unable to close archive entry store") - } - return nil, result, ErrDiskLimitReached - } - - resolver, indexed, err := newStoreResolver(store, Overflow{ - FileSystemID: fsID, - ArchivePath: archivePath, - Charge: charge, - }) - if err != nil { - if closeErr := store.Close(); closeErr != nil { - log.WithFields("archive", fsID, "error", closeErr).Trace("unable to close archive entry store") - } - return nil, result, fmt.Errorf("unable to index extracted archive %q: %w", fsID, err) - } - - // an extraction already stopped at a bound keeps that reason: it names where the archive was cut - // short, and the index only ran out of budget behind it - if indexed.Truncated && !result.Truncated() { - result.Truncation = TruncatedByIndexLimit - } - if notify != nil && result.Truncated() { - notify(Truncated{Archive: accessPath, Reason: result.Truncation}) - } - return resolver, result, nil -} diff --git a/internal/archive/resolver_test.go b/internal/archive/resolver_test.go deleted file mode 100644 index 9838ac531..000000000 --- a/internal/archive/resolver_test.go +++ /dev/null @@ -1,213 +0,0 @@ -package archive - -import ( - "archive/zip" - "bytes" - "context" - "io" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/anchore/syft/internal/tmpdir" - "github.com/anchore/syft/syft/file" -) - -func TestExtractToResolver_writesUnderTheScansTempRoot(t *testing.T) { - // where an archive's scratch space lands is the scan's decision: the root on the context is what the - // caller configured, and what gets cleaned up if an archive's own cleanup is missed - root := t.TempDir() - ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)) - - factory := func(_ *EntryStore, _ Overflow) (file.Resolver, IndexResult, error) { - return nil, IndexResult{}, nil - } - - // a zero memory bound, so this archive really does write: a work directory exists only once - // something has spilled into it - extracted, err := ExtractToResolver( - ctx, newTestZip(t, map[string]string{"hello.txt": "hi"}), "app.zip", "", "app.zip", - DefaultExtractors(), NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}), factory, nil, - ) - require.NoError(t, err) - require.NotNil(t, extracted) - t.Cleanup(extracted.Cleanup) - - // the work directory is not named by anything the resolver is handed, so it is observed where it - // lands: under the temp root the scan configured - workDirs := workDirsUnder(t, root) - require.Len(t, workDirs, 1, "expected one archive work directory under the scan's temp root") - - // and the archive still cleans up after itself, so the root is a safety net rather than the only - // thing reclaiming the space - extracted.Cleanup() - _, statErr := os.Stat(workDirs[0]) - assert.True(t, os.IsNotExist(statErr), "expected the archive work directory to be removed by Cleanup") -} - -func TestExtractToResolver_anArchiveThatNeverWritesCreatesNoWorkDir(t *testing.T) { - // the scratch directory is created at the point of writing, so an archive held entirely in memory - - // which is most of them - costs no directory on the filesystem at all - root := t.TempDir() - ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)) - - factory := func(_ *EntryStore, _ Overflow) (file.Resolver, IndexResult, error) { - return nil, IndexResult{}, nil - } - - extracted, err := ExtractToResolver( - ctx, newTestZip(t, map[string]string{"hello.txt": "hi"}), "app.zip", "", "app.zip", - DefaultExtractors(), NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), factory, nil, - ) - require.NoError(t, err) - require.NotNil(t, extracted) - t.Cleanup(extracted.Cleanup) - - assert.Empty(t, workDirsUnder(t, root), "nothing was written, so no work directory was created") - - // and cleanup of an archive that created nothing is still safe - extracted.Cleanup() - assert.Empty(t, workDirsUnder(t, root)) -} - -func TestExtractToResolver(t *testing.T) { - content := newTestZip(t, map[string]string{"dir/hello.txt": "hello world"}) - - var got Overflow - var gotStore *EntryStore - factory := func(store *EntryStore, overflow Overflow) (file.Resolver, IndexResult, error) { - got, gotStore = overflow, store - return nil, IndexResult{}, nil - } - - extracted, err := ExtractToResolver( - context.Background(), content, "some/path/app.zip", "parentFS", "app.war:some/path/app.zip", - DefaultExtractors(), nil, factory, nil, - ) - require.NoError(t, err) - require.NotNil(t, extracted) - t.Cleanup(extracted.Cleanup) - - // the filesystem the archive lives on is passed through unchanged; the nesting chain is carried - // separately as the archive path, and both are handed to the resolver factory - assert.Equal(t, "parentFS", extracted.FileSystemID) - assert.Equal(t, "parentFS", got.FileSystemID) - assert.Equal(t, "app.war:some/path/app.zip", got.ArchivePath) - - // the archive's entries are in the store handed to the factory, keyed by their archive-relative - // path: nothing was unpacked to a directory for the resolver to be pointed at - assert.Equal(t, "hello world", readStoreEntry(t, gotStore, "dir/hello.txt")) - - extracted.Cleanup() -} - -func TestExtractToResolver_notAnArchive(t *testing.T) { - factory := func(*EntryStore, Overflow) (file.Resolver, IndexResult, error) { return nil, IndexResult{}, nil } - - extracted, err := ExtractToResolver( - context.Background(), strings.NewReader("this is not an archive"), "notes.txt", "", "notes.txt", - DefaultExtractors(), nil, factory, nil, - ) - require.NoError(t, err) - assert.Nil(t, extracted, "non-archive content should yield a nil ExtractedArchive") -} - -func TestExtractToResolver_passesTheArchivesChargeToTheIndex(t *testing.T) { - // the index keeps more per entry than the store can measure - one node per path component - so it - // charges the scan's limiter itself, against this archive's handle - limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: 1 << 20}) - - var got Overflow - factory := func(_ *EntryStore, overflow Overflow) (file.Resolver, IndexResult, error) { - got = overflow - return nil, IndexResult{}, nil - } - - extracted, err := ExtractToResolver( - context.Background(), newTestZip(t, map[string]string{"hello.txt": "hi"}), "app.zip", "", "app.zip", - DefaultExtractors(), limiter, factory, nil, - ) - require.NoError(t, err) - require.NotNil(t, extracted) - t.Cleanup(extracted.Cleanup) - - require.NotNil(t, got.Charge, "the index must be handed this archive's charge, not left unbounded") - - // and it is this archive's handle, so Cleanup gives back whatever the index charged through it - require.True(t, got.Charge.Memory(4096)) - before, _ := limiter.InUse() - require.Positive(t, before) - - extracted.Cleanup() - after, _ := limiter.InUse() - assert.Zero(t, after, "everything the archive charged, the index included, is released with it") -} - -func TestExtractToResolver_reportsAnIndexThatRanOutOfBudget(t *testing.T) { - // an index that stops part way is a truncation, not a failure: the archive is cataloged from what was - // indexed, and the reason names the index rather than the disk, which nothing was written to - factory := func(_ *EntryStore, _ Overflow) (file.Resolver, IndexResult, error) { - return nil, IndexResult{Truncated: true}, nil - } - - extracted, err := ExtractToResolver( - context.Background(), newTestZip(t, map[string]string{"a.txt": "a", "b.txt": "b"}), "app.zip", "", "app.zip", - DefaultExtractors(), nil, factory, nil, - ) - require.NoError(t, err) - require.NotNil(t, extracted) - t.Cleanup(extracted.Cleanup) - - assert.True(t, extracted.Result.Truncated()) - assert.Equal(t, TruncatedByIndexLimit, extracted.Result.Truncation) -} - -// workDirsUnder returns the archive work directories directly under the scan's temp root. -func workDirsUnder(t *testing.T, root string) []string { - t.Helper() - children, err := os.ReadDir(root) - require.NoError(t, err) - - var out []string - for _, child := range children { - if child.IsDir() && strings.HasPrefix(child.Name(), workDirName) { - out = append(out, filepath.Join(root, child.Name())) - } - } - return out -} - -func readStoreEntry(t *testing.T, store *EntryStore, name string) string { - t.Helper() - require.NotNil(t, store) - for _, entry := range store.Entries() { - if entry.Header.Name != name { - continue - } - reader, err := store.Open(entry) - require.NoError(t, err) - body, err := io.ReadAll(reader) - require.NoError(t, err) - return string(body) - } - t.Fatalf("no entry %q in the store", name) - return "" -} - -func newTestZip(t *testing.T, files map[string]string) *bytes.Buffer { - t.Helper() - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - for name, body := range files { - w, err := zw.Create(name) - require.NoError(t, err) - _, err = w.Write([]byte(body)) - require.NoError(t, err) - } - require.NoError(t, zw.Close()) - return &buf -} diff --git a/internal/archive/storage_test.go b/internal/archive/storage_test.go index c7eb50858..c1a85f9ea 100644 --- a/internal/archive/storage_test.go +++ b/internal/archive/storage_test.go @@ -3,7 +3,6 @@ package archive import ( "archive/zip" "bytes" - "context" "fmt" "io" "io/fs" @@ -14,80 +13,52 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/anchore/syft/internal/tmpdir" - "github.com/anchore/syft/syft/file" ) -// unboundedLimits hold everything in memory, as an archive well inside the default bounds does. -var unboundedLimits = Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1} +var ( + unboundedLimits = Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1} -// spillingLimits hold nothing in memory, so every entry lands in the store's overflow blob. The -// filesystem-cost requirement must be measured in this shape: an archive that never touches disk -// satisfies it trivially. -var spillingLimits = Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1} + // spillingLimits hold nothing in memory, so every entry lands in the overflow file + spillingLimits = Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1} +) func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) { - // an archive of a hundred thousand entries must not put a hundred thousand files on the filesystem. - // Asserted as an exact set of paths for two archives three orders of magnitude apart in entry count, - // since a count that merely grows slowly would pass a bound and still be proportional to content. _, smallStore, smallRoot := extractedStore(t, manyEntryZip(t, 2), spillingLimits) _, largeStore, largeRoot := extractedStore(t, manyEntryZip(t, 5000), spillingLimits) - smallWork, largeWork := workDirIn(t, smallRoot), workDirIn(t, largeRoot) - // the archive's own bytes spill too under a zero memory bound, so the set is the spilled archive and - // the blob its entries went into - two, whatever it holds - want := []string{"app.zip", "contents.blob"} - assert.Equal(t, want, filesystemEntries(t, smallWork)) - assert.Equal(t, want, filesystemEntries(t, largeWork), - "5000 entries must cost the same filesystem entries as 2") + want := []string{entriesFileName} + assert.Equal(t, want, filesystemEntries(t, workDirIn(t, smallRoot))) + assert.Equal(t, want, filesystemEntries(t, workDirIn(t, largeRoot)), "5000 entries cost the same filesystem entries as 2") - // and the entries really are all there, so this is not passing because nothing was extracted assert.Len(t, smallStore.Entries(), 2) assert.Len(t, largeStore.Entries(), 5000) - // an archive costs its entry content and nothing else on disk: no headers, no padding, so a one-byte - // entry costs one byte - assert.Equal(t, int64(5000), largeStore.OnDisk()) + // entries cost their content and nothing else on disk: no headers, no padding assert.Equal(t, int64(2), smallStore.OnDisk()) + assert.Equal(t, int64(5000), largeStore.OnDisk()) } func TestStorage_anArchiveWithinTheMemoryBoundTouchesNoFilesystemAtAll(t *testing.T) { - // an archive that fits in memory pays for no write at all - not even the directory it would have - // written into, which is created only when something actually spills _, store, root := extractedStore(t, manyEntryZip(t, 5000), unboundedLimits) assert.Len(t, store.Entries(), 5000) assert.Zero(t, store.OnDisk()) - assert.Empty(t, filesystemEntries(t, root), - "an archive held entirely in memory creates no work directory at all") + assert.Empty(t, filesystemEntries(t, root)) } -func TestStorage_cleanupRemovesOneFileAndDoesNotWalkPerEntry(t *testing.T) { - // cleanup is bounded by the same constant as the extraction, so the count of things removed is - // asserted rather than only that the directory is gone +func TestStorage_cleanupRemovesTheWorkDirectory(t *testing.T) { extracted, _, root := extractedStore(t, manyEntryZip(t, 5000), spillingLimits) workDir := workDirIn(t, root) - - before := filesystemEntries(t, workDir) - require.Len(t, before, 2, "the release has two things to remove whatever the archive held") + require.Len(t, filesystemEntries(t, workDir), 1) extracted.Cleanup() - _, err := os.Stat(filepath.Join(workDir, overflowBlobName)) - assert.True(t, os.IsNotExist(err), "the one file holding every entry must be gone") - _, err = os.Stat(workDir) - assert.True(t, os.IsNotExist(err), "and so must the work directory") - - // safe to call more than once, since the cataloger defers it past the sub-pipeline and recursion + assert.NoDirExists(t, workDir) extracted.Cleanup() } func TestStorage_anEntryAtTheEndIsReadBySeeking(t *testing.T) { - // "overflow content MUST remain directly readable without re-reading or re-decompressing what - // precedes it". Asserted through the mechanism: the store records where the last entry's content - // begins, near the end of a blob whose earlier entries are large, and hands back a bounded window - // onto the file at that offset. + // the reader over an entry is a bounded window at its offset, not a re-read of what precedes it const filler = 64 * 1024 var buf bytes.Buffer @@ -105,95 +76,59 @@ func TestStorage_anEntryAtTheEndIsReadBySeeking(t *testing.T) { require.NoError(t, err) require.NoError(t, zw.Close()) - _, store, root := extractedStore(t, &buf, spillingLimits) + _, store, root := extractedStore(t, buf.Bytes(), spillingLimits) - blobPath := filepath.Join(workDirIn(t, root), overflowBlobName) - info, err := os.Stat(blobPath) - require.NoError(t, err) - total := info.Size() - require.Greater(t, total, int64(8*filler), "the fixture must put real bulk ahead of the last entry") - - // the last entry's content sits near the end of the blob, so everything before it is what a - // re-streaming design would have had to read - body, err := os.ReadFile(blobPath) + body, err := os.ReadFile(filepath.Join(workDirIn(t, root), entriesFileName)) require.NoError(t, err) offset := int64(bytes.Index(body, []byte(wanted))) - require.Positive(t, offset) - assert.Greater(t, offset, total*9/10, "the last entry must really be at the end of the blob") + require.Greater(t, offset, int64(len(body))*9/10, "the last entry must really be at the end of the file") entries := store.Entries() entry := entries[len(entries)-1] require.Equal(t, "zz-last.txt", entry.Header.Name) - // the reader is bounded to this entry rather than the whole blob: its end is the entry's own size - random, err := store.Open(entry) - require.NoError(t, err) - - end, err := random.Seek(0, io.SeekEnd) + reader := store.Open(entry) + end, err := reader.Seek(0, io.SeekEnd) require.NoError(t, err) assert.Equal(t, int64(len(wanted)), end) - // offset zero of the reader is the last entry's first byte, not the blob's; a reader over the whole - // file would answer with the first filler entry's content head := make([]byte, 8) - _, err = random.ReadAt(head, 0) + _, err = reader.ReadAt(head, 0) require.NoError(t, err) assert.Equal(t, wanted[:8], string(head)) } -func manyEntryZip(t *testing.T, count int) *bytes.Buffer { +func manyEntryZip(t *testing.T, count int) []byte { t.Helper() - var buf bytes.Buffer - zw := zip.NewWriter(&buf) + files := make(map[string]string, count) for i := range count { - w, err := zw.Create(fmt.Sprintf("e%06d.txt", i)) - require.NoError(t, err) - _, err = w.Write([]byte("x")) - require.NoError(t, err) + files[fmt.Sprintf("e%06d.txt", i)] = "x" } - require.NoError(t, zw.Close()) - return &buf + return zipBytes(t, files) } -// extractedStore extracts one archive and returns the store its entries went into, with the scan temp -// root the extraction was pointed at. -// -// The root is where a work directory would appear: nothing the resolver is handed names a path on the -// host, and an archive that never writes creates no directory to name. -func extractedStore(t *testing.T, content io.Reader, limits Limits) (*ExtractedArchive, *EntryStore, string) { +// extractedStore extracts one archive and returns the store its entries went into, along with the +// scan temp root its work directory would appear under. +func extractedStore(t *testing.T, data []byte, limits Limits) (*Extracted, *EntryStore, string) { t.Helper() + ctx, root := scanContext(t) - root := t.TempDir() - ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)) - - var got *EntryStore - factory := func(store *EntryStore, _ Overflow) (file.Resolver, IndexResult, error) { - got = store - return nil, IndexResult{}, nil - } - - extracted, err := ExtractToResolver( - ctx, content, "app.zip", "", "app.zip", - DefaultExtractors(), NewLimiter(limits), factory, nil, - ) + extracted, err := Extract(ctx, bytes.NewReader(data), "", "app.zip", NewLimiter(limits), nil) require.NoError(t, err) require.NotNil(t, extracted) t.Cleanup(extracted.Cleanup) - return extracted, got, root + return extracted, extracted.Resolver.(*Index).store, root } -// workDirIn returns the one work directory an archive that spilled created under the scan temp root. func workDirIn(t *testing.T, root string) string { t.Helper() - children, err := os.ReadDir(root) - require.NoError(t, err) - require.Len(t, children, 1, "expected exactly one archive work directory under the temp root") - return filepath.Join(root, children[0].Name()) + dirs := workDirsUnder(t, root) + require.Len(t, dirs, 1) + return dirs[0] } -// filesystemEntries is every directory entry under dir: an archive must not put one of these on the -// filesystem per entry it holds. +// filesystemEntries is every directory entry under dir, recursively. func filesystemEntries(t *testing.T, dir string) []string { t.Helper() var out []string @@ -201,14 +136,9 @@ func filesystemEntries(t *testing.T, dir string) []string { if err != nil { return err } - rel, relErr := filepath.Rel(dir, p) - if relErr != nil { - return relErr + if rel, _ := filepath.Rel(dir, p); rel != "." { + out = append(out, filepath.ToSlash(rel)) } - if rel == "." { - return nil - } - out = append(out, filepath.ToSlash(rel)) return nil })) return out diff --git a/internal/archive/traversal.go b/internal/archive/traversal.go index 3a6467908..d4bc81d44 100644 --- a/internal/archive/traversal.go +++ b/internal/archive/traversal.go @@ -7,55 +7,34 @@ import ( "github.com/anchore/syft/syft/file" ) -// Traversal describes one level of archive nesting. The archive cataloger task places one on the -// context before running the sub-pipeline against an archive's contents, so ecosystem catalogers -// (e.g. java) can reconstruct nesting-aware identity like "app.war:WEB-INF/lib/dep.jar" without -// owning the recursion. +// Traversal describes the archive whose contents a cataloger is running against. The archive +// cataloger task puts one on the context before running catalogers inside an archive, so a cataloger +// such as java can describe the archive it is inside without owning the recursion. type Traversal struct { - // Location is the archive file's location within its parent filesystem. + // Location is the archive file's location within its parent filesystem. Its ArchivePath names the + // archives above it, so VirtualPath(Location) is the full chain from the scan root. Location file.Location - // VirtualPath is the colon-delimited chain of archive paths from the scan root to this archive, - // e.g. "app.war:WEB-INF/lib/dep.jar". - VirtualPath string - - // Digests are of the archive file itself, taken once during extraction. A cataloger running against - // the extracted filesystem cannot compute them, since the archive is not inside itself. + // Digests are of the archive file itself, taken during extraction; a cataloger running inside the + // archive cannot compute them since the archive is not inside itself. Digests []file.Digest } -// VirtualPathOf returns the colon-delimited virtual path for an entry within this archive, or the -// entry path unchanged on a nil traversal. The leading slash is trimmed to match the java cataloger -// format, which joins slash-less zip entry names onto the containing archive's path. -func (t *Traversal) VirtualPathOf(entryPath string) string { - if t == nil { - return entryPath +// VirtualPath returns the colon-delimited chain of archives from the scan root to loc, ending in loc +// itself: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path unchanged. A +// colon in an entry name is escaped so splitting on ':' recovers the archive boundaries; the leading +// slash is dropped to match the java cataloger, which joins slash-less zip entry names onto the +// containing archive's path. +func VirtualPath(loc file.Location) string { + if loc.ArchivePath == "" { + return loc.Path() } - entryPath = strings.ReplaceAll(entryPath, ":", "%3A") - if t.VirtualPath == "" { - return entryPath - } - return t.VirtualPath + ":" + strings.TrimPrefix(entryPath, "/") -} - -type limiterCtxKey struct{} - -// WithLimiter returns a context carrying the scan's archive limiter, so a cataloger inside the walk -// can read the live draw on the budgets (Limiter.InUse). It rides the context rather than each -// Traversal so every level reads one shared total. -func WithLimiter(ctx context.Context, l *Limiter) context.Context { - return context.WithValue(ctx, limiterCtxKey{}, l) -} - -// LimiterFromContext returns the scan's archive limiter, or nil outside the recursive archive walk. -func LimiterFromContext(ctx context.Context) *Limiter { - l, _ := ctx.Value(limiterCtxKey{}).(*Limiter) - return l + entryPath := strings.ReplaceAll(loc.Path(), ":", "%3A") + return loc.ArchivePath + ":" + strings.TrimPrefix(entryPath, "/") } type traversalCtxKey struct{} -// WithTraversal returns a context carrying the given archive traversal. func WithTraversal(ctx context.Context, t *Traversal) context.Context { return context.WithValue(ctx, traversalCtxKey{}, t) } @@ -65,3 +44,18 @@ func TraversalFromContext(ctx context.Context) *Traversal { t, _ := ctx.Value(traversalCtxKey{}).(*Traversal) return t } + +type nestedCatalogingCtxKey struct{} + +// WithNestedCataloging marks the context of a scan whose archive cataloger task will extract every +// archive and run the catalogers inside it. syft.CreateSBOM sets it before any task runs, so a +// cataloger that would otherwise open archives itself can leave them to the task instead. +func WithNestedCataloging(ctx context.Context) context.Context { + return context.WithValue(ctx, nestedCatalogingCtxKey{}, true) +} + +// NestedCatalogingEnabled reports whether the archive cataloger task is extracting archives in this scan. +func NestedCatalogingEnabled(ctx context.Context) bool { + enabled, _ := ctx.Value(nestedCatalogingCtxKey{}).(bool) + return enabled +} diff --git a/internal/archive/traversal_test.go b/internal/archive/traversal_test.go index 772bc5f27..f53f8ae26 100644 --- a/internal/archive/traversal_test.go +++ b/internal/archive/traversal_test.go @@ -14,165 +14,92 @@ import ( func TestTraversalContextRoundTrip(t *testing.T) { assert.Nil(t, TraversalFromContext(context.Background())) - trav := &Traversal{ - Location: file.NewLocation("app.war"), - VirtualPath: "app.war", - } + trav := &Traversal{Location: file.NewLocation("app.war")} ctx := WithTraversal(context.Background(), trav) require.Same(t, trav, TraversalFromContext(ctx)) - child := &Traversal{ - Location: file.NewLocation("/WEB-INF/lib/dep.jar"), - VirtualPath: trav.VirtualPathOf("/WEB-INF/lib/dep.jar"), - } + child := &Traversal{Location: locationIn("app.war", "/WEB-INF/lib/dep.jar")} ctx = WithTraversal(ctx, child) require.Same(t, child, TraversalFromContext(ctx)) } -func TestTraversal_VirtualPathOf(t *testing.T) { +// locationIn is a location as the Index returns it: an entry path stamped with the containing chain. +func locationIn(archivePath, entryPath string) file.Location { + loc := file.NewLocation(entryPath) + loc.ArchivePath = archivePath + return loc +} + +func TestVirtualPath(t *testing.T) { tests := []struct { - name string - traversal *Traversal - entryPath string - expected string + name string + location file.Location + expected string }{ { - name: "nil traversal returns entry path unchanged", - traversal: nil, - entryPath: "/WEB-INF/lib/dep.jar", - expected: "/WEB-INF/lib/dep.jar", + name: "outside an archive returns the path unchanged", + location: file.NewLocation("/WEB-INF/lib/dep.jar"), + expected: "/WEB-INF/lib/dep.jar", }, { - name: "nil traversal preserves relative path", - traversal: nil, - entryPath: "some/dir/example.jar", - expected: "some/dir/example.jar", + name: "outside an archive preserves a relative path", + location: file.NewLocation("some/dir/example.jar"), + expected: "some/dir/example.jar", }, { - name: "single level trims leading slash", - traversal: &Traversal{VirtualPath: "app.war"}, - entryPath: "/WEB-INF/lib/dep.jar", - expected: "app.war:WEB-INF/lib/dep.jar", + name: "single level trims leading slash", + location: locationIn("app.war", "/WEB-INF/lib/dep.jar"), + expected: "app.war:WEB-INF/lib/dep.jar", }, { - name: "single level with slash-less entry", - traversal: &Traversal{VirtualPath: "app.war"}, - entryPath: "WEB-INF/lib/dep.jar", - expected: "app.war:WEB-INF/lib/dep.jar", + name: "single level with slash-less entry", + location: locationIn("app.war", "WEB-INF/lib/dep.jar"), + expected: "app.war:WEB-INF/lib/dep.jar", }, { - name: "multi-level chain", - traversal: &Traversal{VirtualPath: "app.war:WEB-INF/lib/dep.jar"}, - entryPath: "/inner/most.jar", - expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar", + name: "multi-level chain", + location: locationIn("app.war:WEB-INF/lib/dep.jar", "/inner/most.jar"), + expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar", }, { - // a colon in the entry name must not be mistaken for a chain delimiter - see - // decisions.md#colon-in-an-entry-path-is-escaped - name: "colon in entry path is escaped", - traversal: &Traversal{VirtualPath: "app.war"}, - entryPath: "WEB-INF/lib/dep:special.jar", - expected: "app.war:WEB-INF/lib/dep%3Aspecial.jar", + // a colon in the entry name must not be mistaken for a chain delimiter + name: "colon in entry path is escaped", + location: locationIn("app.war", "WEB-INF/lib/dep:special.jar"), + expected: "app.war:WEB-INF/lib/dep%3Aspecial.jar", }, { - name: "every colon in the entry path is escaped", - traversal: &Traversal{VirtualPath: "app.war"}, - entryPath: "lib/a:b:c.jar", - expected: "app.war:lib/a%3Ab%3Ac.jar", + name: "every colon in the entry path is escaped", + location: locationIn("app.war", "lib/a:b:c.jar"), + expected: "app.war:lib/a%3Ab%3Ac.jar", }, { - // a traversal carrying no virtual path (as distinct from a nil traversal) still yields - // the entry path alone, never a bare leading colon - name: "empty virtual path yields the entry path alone, no leading colon", - traversal: &Traversal{VirtualPath: ""}, - entryPath: "WEB-INF/lib/dep.jar", - expected: "WEB-INF/lib/dep.jar", + name: "access path is preferred over real path", + location: file.NewVirtualLocation("/real/dep.jar", "/link/dep.jar"), + expected: "/link/dep.jar", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, tt.traversal.VirtualPathOf(tt.entryPath)) + assert.Equal(t, tt.expected, VirtualPath(tt.location)) }) } } -// TestTraversal_VirtualPathOf_noColonIsByteIdenticalToPreEscaping carries the compatibility guarantee -// behind java-virtual-path-identity-preserved (decisions.md#colon-in-an-entry-path-is-escaped): -// escaping a colon must not alter the result for any entry path without one. Every case below is -// colon-free, and every expected value is what VirtualPathOf produced before escaping existed. Do not -// fix a failure here by updating the expected string - that is the regression this test catches. -func TestTraversal_VirtualPathOf_noColonIsByteIdenticalToPreEscaping(t *testing.T) { - tests := []struct { - name string - traversal *Traversal - entryPath string - expected string - }{ - { - name: "nil traversal, absolute entry path", - traversal: nil, - entryPath: "/WEB-INF/lib/dep.jar", - expected: "/WEB-INF/lib/dep.jar", - }, - { - name: "nil traversal, relative entry path", - traversal: nil, - entryPath: "some/dir/example.jar", - expected: "some/dir/example.jar", - }, - { - name: "single level, leading slash trimmed", - traversal: &Traversal{VirtualPath: "app.war"}, - entryPath: "/WEB-INF/lib/dep.jar", - expected: "app.war:WEB-INF/lib/dep.jar", - }, - { - name: "single level, no leading slash", - traversal: &Traversal{VirtualPath: "app.war"}, - entryPath: "WEB-INF/lib/dep.jar", - expected: "app.war:WEB-INF/lib/dep.jar", - }, - { - name: "multi-level chain", - traversal: &Traversal{VirtualPath: "app.war:WEB-INF/lib/dep.jar"}, - entryPath: "/inner/most.jar", - expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, tt.traversal.VirtualPathOf(tt.entryPath)) - }) - } -} - -// TestTraversal_VirtualPathOf_escapingComposesAcrossNestingLevels asserts escaping applies at every -// level of a chain, not only the first: splitting the composed result on ':' yields exactly one -// segment per archive boundary, since no escaped colon survives as a literal ':' byte. -func TestTraversal_VirtualPathOf_theEscapeCharacterIsNotItselfEscaped(t *testing.T) { - // a known and accepted limit: an entry path holding the literal text %3A is indistinguishable from - // one holding a colon. Escaping % as %25 would close it and is deliberately not done, since % is far - // more common in real paths than : and doing so would move many well-defined identities. - t.Parallel() - - outer := &Traversal{VirtualPath: "outer.zip"} - - fromColon := outer.VirtualPathOf("a:b.jar") - fromLiteral := outer.VirtualPathOf("a%3Ab.jar") +func TestVirtualPath_theEscapeCharacterIsNotItselfEscaped(t *testing.T) { + // an accepted limit: an entry path holding the literal text %3A is indistinguishable from one holding + // a colon. Escaping % too would move many more real identities than it protects. + fromColon := VirtualPath(locationIn("outer.zip", "a:b.jar")) + fromLiteral := VirtualPath(locationIn("outer.zip", "a%3Ab.jar")) assert.Equal(t, "outer.zip:a%3Ab.jar", fromColon) - assert.Equal(t, "outer.zip:a%3Ab.jar", fromLiteral) - assert.Equal(t, fromColon, fromLiteral, - "the collision is accepted; if this ever differs, the escape character started being escaped") + assert.Equal(t, fromColon, fromLiteral) } -func TestTraversal_VirtualPathOf_escapingComposesAcrossNestingLevels(t *testing.T) { - level1 := &Traversal{VirtualPath: "outer.zip"} - level2VirtualPath := level1.VirtualPathOf("lib/mid:dle.war") - require.Equal(t, "outer.zip:lib/mid%3Adle.war", level2VirtualPath) +func TestVirtualPath_escapingComposesAcrossNestingLevels(t *testing.T) { + level2 := VirtualPath(locationIn("outer.zip", "lib/mid:dle.war")) + require.Equal(t, "outer.zip:lib/mid%3Adle.war", level2) - level2 := &Traversal{VirtualPath: level2VirtualPath} - final := level2.VirtualPathOf("WEB-INF/lib/inn:er.jar") + final := VirtualPath(locationIn(level2, "WEB-INF/lib/inn:er.jar")) require.Equal(t, "outer.zip:lib/mid%3Adle.war:WEB-INF/lib/inn%3Aer.jar", final) assert.Equal(t, diff --git a/internal/archive/workdir.go b/internal/archive/workdir.go index 21e93acbb..8a7ffd6cf 100644 --- a/internal/archive/workdir.go +++ b/internal/archive/workdir.go @@ -4,85 +4,66 @@ import ( "context" "fmt" "os" - "sync" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/tmpdir" ) -// workDirName prefixes the directory holding one archive's content and entries, so a scan's leftovers -// are recognizable. Tests that sample what is on disk match on it. +// workDirName prefixes the directory holding one archive's overflow files. const workDirName = "syft-archive" -// WorkDir is the scratch space one archive writes into, created lazily the first time something needs -// to write - an archive's own bytes spilling out of memory, or its entries spilling into the overflow -// blob. -// -// Eager creation would put one directory on disk per archive opened, whether or not it ever wrote a -// byte; most do not, and a scan walks far more archives than it spills. So the path is asked for at -// the point of writing. A nil *WorkDir has no path and fails the write. +// WorkDir is the scratch directory one archive writes overflow into. It is created on first use, so +// an archive that stays in memory never touches the filesystem. type WorkDir struct { - mu sync.Mutex + tempDir *tmpdir.TempDir - // create makes the directory and returns it with the func that removes it. Called at most once. - create func() (string, func(), error) - - path string - remove func() - err error - done bool + created bool + path string + remove func() + err error } -// NewWorkDir returns the lazily-created work directory for one archive, rooted under the scan's temp -// root (internal/tmpdir) when the context carries one - so files land where the caller configured and -// are removed together if an archive's own cleanup is missed - otherwise the system temp dir. +// NewWorkDir returns a work directory under the scan's temp root when ctx carries one (see +// internal/tmpdir), otherwise under the system temp directory. func NewWorkDir(ctx context.Context) *WorkDir { - return &WorkDir{create: func() (string, func(), error) { - if td := tmpdir.FromContext(ctx); td != nil { - dir, cleanup, err := td.NewChild(workDirName) //nolint:gocritic // cleanup is returned, not deferred here - if err != nil { - return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err) - } - return dir, cleanup, nil - } - - dir, err := os.MkdirTemp("", workDirName+"-") - if err != nil { - return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err) - } - return dir, func() { - if rmErr := os.RemoveAll(dir); rmErr != nil { - log.WithFields("dir", dir, "error", rmErr).Trace("unable to remove archive temp dir") - } - }, nil - }} + return &WorkDir{tempDir: tmpdir.FromContext(ctx)} } -// Path returns the directory, creating it on the first call. Later calls return the same result, -// including the same failure. +// Path returns the directory, creating it on the first call. func (w *WorkDir) Path() (string, error) { if w == nil { return "", fmt.Errorf("no work directory for archive content") } - - w.mu.Lock() - defer w.mu.Unlock() - if !w.done { + if !w.created { + w.created = true w.path, w.remove, w.err = w.create() - w.done = true } return w.path, w.err } -// Remove deletes the directory and its contents, and does nothing if it was never created. Safe to -// call more than once. -func (w *WorkDir) Remove() { - if w == nil { - return +func (w *WorkDir) create() (string, func(), error) { + if w.tempDir != nil { + dir, remove, err := w.tempDir.NewChild(workDirName) //nolint:gocritic // the cleanup is returned, not deferred + if err != nil { + return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err) + } + return dir, remove, nil } - w.mu.Lock() - defer w.mu.Unlock() - if w.remove == nil { + + dir, err := os.MkdirTemp("", workDirName+"-") + if err != nil { + return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err) + } + return dir, func() { + if err := os.RemoveAll(dir); err != nil { + log.WithFields("dir", dir, "error", err).Trace("unable to remove archive temp dir") + } + }, nil +} + +// Remove deletes the directory if it was created. Safe to call more than once. +func (w *WorkDir) Remove() { + if w == nil || w.remove == nil { return } w.remove() diff --git a/internal/archive/workdir_test.go b/internal/archive/workdir_test.go index 8052427a5..10453d154 100644 --- a/internal/archive/workdir_test.go +++ b/internal/archive/workdir_test.go @@ -1,84 +1,52 @@ package archive import ( - "context" "os" "path/filepath" - "sync" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/anchore/syft/internal/tmpdir" ) func TestWorkDir_isNotCreatedUntilAskedFor(t *testing.T) { - root := t.TempDir() - w := NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))) + ctx, root := scanContext(t) + w := NewWorkDir(ctx) - assert.False(t, w.created(), "constructing a work directory must touch nothing") - entries, err := os.ReadDir(root) - require.NoError(t, err) - assert.Empty(t, entries) + assert.False(t, w.wasCreated()) + assert.Empty(t, workDirsUnder(t, root)) dir, err := w.Path() require.NoError(t, err) assert.DirExists(t, dir) - assert.True(t, w.created()) + assert.True(t, w.wasCreated()) + assert.Equal(t, []string{dir}, workDirsUnder(t, root)) } func TestWorkDir_pathIsStableAcrossCalls(t *testing.T) { - w := NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) + ctx, _ := scanContext(t) + w := NewWorkDir(ctx) first, err := w.Path() require.NoError(t, err) second, err := w.Path() require.NoError(t, err) - assert.Equal(t, first, second, "the directory is created once, not once per writer") -} - -func TestWorkDir_concurrentCallersShareOneDirectory(t *testing.T) { - // the store spills under its own lock while the archive's content may be spilling too, so the - // creation has to be safe to race - root := t.TempDir() - w := NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))) - - var wg sync.WaitGroup - paths := make([]string, 8) - for i := range paths { - wg.Add(1) - go func() { - defer wg.Done() - dir, err := w.Path() - assert.NoError(t, err) - paths[i] = dir - }() - } - wg.Wait() - - for _, p := range paths { - assert.Equal(t, paths[0], p) - } - entries, err := os.ReadDir(root) - require.NoError(t, err) - assert.Len(t, entries, 1, "racing callers must not each create a directory") + assert.Equal(t, first, second) } func TestWorkDir_removeIsANoOpWhenNothingWasCreated(t *testing.T) { - root := t.TempDir() - w := NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))) + ctx, root := scanContext(t) + w := NewWorkDir(ctx) w.Remove() w.Remove() - entries, err := os.ReadDir(root) - require.NoError(t, err) - assert.Empty(t, entries) + assert.Empty(t, workDirsUnder(t, root)) } func TestWorkDir_removeDeletesWhatWasCreated(t *testing.T) { - w := NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) + ctx, _ := scanContext(t) + w := NewWorkDir(ctx) dir, err := w.Path() require.NoError(t, err) @@ -86,8 +54,6 @@ func TestWorkDir_removeDeletesWhatWasCreated(t *testing.T) { w.Remove() assert.NoDirExists(t, dir) - - // safe to call more than once, as the extraction's cleanup is w.Remove() } @@ -95,7 +61,7 @@ func TestWorkDir_nilHasNowhereToWrite(t *testing.T) { var w *WorkDir _, err := w.Path() - assert.Error(t, err, "a caller with nowhere to put content must fail rather than pick a directory") - assert.False(t, w.created()) + assert.Error(t, err) + assert.False(t, w.wasCreated()) w.Remove() } diff --git a/internal/bus/helpers.go b/internal/bus/helpers.go index 1e4b45b5d..2867aeff2 100644 --- a/internal/bus/helpers.go +++ b/internal/bus/helpers.go @@ -41,24 +41,17 @@ func Notify(message string) { type catalogerTaskRegistryKey struct{} -// catalogerTaskRegistry remembers the row each cataloger publishes, keyed by the publishing task's -// identity. -// -// A cataloger runs more than once per scan: once over the scan root, then again inside every archive -// the archive cataloger walks. A row per run floods the display, and reusing a published ID replaces -// the live row - which is then never completed, hanging a UI (like the syft CLI) that waits on every -// row it started. So a re-run reports into the row it already owns, and its count covers the whole -// scan rather than the root alone. +// catalogerTaskRegistry remembers the progress row each cataloger publishes. A cataloger runs once +// over the scan root and again inside every archive the archive cataloger walks; a re-run reports +// into the row it already owns rather than publishing another, and republishing an ID would replace +// the live row with one the UI never sees completed. type catalogerTaskRegistry struct { mu sync.Mutex rows map[string]*monitor.TaskProgress } -// WithCatalogerTaskRegistry returns a context under which each cataloger publishes one row no matter -// how many times it runs; without it every run publishes, which suits a single-pass caller. -// -// Install once per scan before any task runs, so a re-run finds the first run's row. Idempotent: a -// second registry would hold none of the existing rows, so every re-run would publish again. +// WithCatalogerTaskRegistry returns a context under which each cataloger publishes one progress row +// however many times it runs. A context that already carries a registry is returned unchanged. func WithCatalogerTaskRegistry(ctx context.Context) context.Context { if catalogerTaskRegistryFromContext(ctx) != nil { return ctx @@ -73,8 +66,8 @@ func catalogerTaskRegistryFromContext(ctx context.Context) *catalogerTaskRegistr return reg } -// catalogerTaskKey identifies the row a task owns: its ID where it has one (a package cataloger's ID -// is its name), otherwise its parent and title (as every file cataloger, which publishes no ID). +// catalogerTaskKey identifies the row a task owns: its ID when it has one, otherwise its parent and +// title (file catalogers publish no ID). func catalogerTaskKey(info monitor.GenericTask) string { if info.ID != "" { return "id:" + info.ID @@ -104,14 +97,7 @@ func StartCatalogerTask(ctx context.Context, info monitor.GenericTask, size int6 } // resumeCatalogerTask reopens a row an earlier run completed and grows its total by what this run -// adds. -// -// Growing the total matters: a row completes once its count reaches its total, so incrementing a row -// whose total covers only the first run makes it read as finished mid-walk. An indeterminate total on -// either side stays indeterminate. -// -// Only a completed row is reopened; one carrying a real failure keeps it, since a later clean run -// does not mean the earlier one succeeded. +// adds, so the row does not read as finished mid-walk. A row carrying a real failure keeps it. func resumeCatalogerTask(row *monitor.TaskProgress, size int64, initialStage string) { if size < 0 || row.Size() < 0 { row.SetTotal(-1) diff --git a/internal/file/zip_file_manifest.go b/internal/file/zip_file_manifest.go index 34a2cb66d..4ab1d73a6 100644 --- a/internal/file/zip_file_manifest.go +++ b/internal/file/zip_file_manifest.go @@ -46,7 +46,7 @@ func (z ZipFileManifest) Add(entry string, info os.FileInfo) { } // GlobMatch returns the path keys to files (not directories) that match the given value(s). -func (z ZipFileManifest) GlobMatch(caseInsensitive bool, patterns ...string) []string { +func (z ZipFileManifest) GlobMatch(patterns ...string) []string { uniqueMatches := strset.New() for _, pattern := range patterns { @@ -58,11 +58,7 @@ func (z ZipFileManifest) GlobMatch(caseInsensitive bool, patterns ...string) []s // We want to match globs as if entries begin with a leading slash (akin to an absolute path) // so that glob logic is consistent inside and outside of ZIP archives - normalizedEntry := normalizeZipEntryName(caseInsensitive, entry) - - if caseInsensitive { - pattern = strings.ToLower(pattern) - } + normalizedEntry := normalizeZipEntryName(entry) matches, err := doublestar.Match(pattern, normalizedEntry) if err != nil { @@ -82,10 +78,7 @@ func (z ZipFileManifest) GlobMatch(caseInsensitive bool, patterns ...string) []s } // normalizeZipEntryName takes the given path entry and ensures it is prefixed with "/". -func normalizeZipEntryName(caseInsensitive bool, entry string) string { - if caseInsensitive { - entry = strings.ToLower(entry) - } +func normalizeZipEntryName(entry string) string { if !strings.HasPrefix(entry, "/") { return "/" + entry } diff --git a/internal/file/zip_file_manifest_test.go b/internal/file/zip_file_manifest_test.go index 50b798882..e0c0e5ee6 100644 --- a/internal/file/zip_file_manifest_test.go +++ b/internal/file/zip_file_manifest_test.go @@ -122,10 +122,6 @@ func TestZipFileManifest_GlobMatch(t *testing.T) { "**/a-file.txt", []string{"some-dir/a-file.txt"}, }, - { - "**/A-file.txt", - []string{"some-dir/a-file.txt"}, - }, { "**/*.zip", []string{"nested.zip"}, @@ -136,7 +132,7 @@ func TestZipFileManifest_GlobMatch(t *testing.T) { t.Run(tc.glob, func(t *testing.T) { glob := tc.glob - results := z.GlobMatch(true, glob) + results := z.GlobMatch(glob) require.ElementsMatch(t, tc.expected, results) }) diff --git a/internal/index/README.md b/internal/index/README.md new file mode 100644 index 000000000..c2f378fce --- /dev/null +++ b/internal/index/README.md @@ -0,0 +1,39 @@ +# internal/index + +A key-split (radix) index over strings, and a forward/reverse pair of them that answers prefix and +suffix lookups. Ported from a prototype directory-scan package and kept close to the original so the +two can be diffed. + +## Types + +- `KeySplitIndex[T]` / `Node[T]`: a radix tree. Each node holds at most one value and splits its + children on the longest common prefix of their keys, so `pom.xml` and `pom.properties` share a + `pom.` node. Reads take a read lock per node; writes upgrade to a write lock only where the tree + changes. `KeySplitIndex` also marshals to and from a flat JSON object of key → value. +- `PrefixSuffix[T]`: two `KeySplitIndex` values, one keyed by the string and one by its reverse, so + `ByPrefix("pom")` and `BySuffix(".jar")` are both a single descent followed by a collect. + +## Operations + +| call | answers | +|---|---| +| `Get(key)` | the value at exactly `key` | +| `Set(key, v)` / `Update(key, fn)` | store, or read-modify-write, the value at `key` | +| `ByPrefix(p)` | every value whose key starts with `p` | +| `BySuffix(s)` (PrefixSuffix only) | every value whose key ends with `s` | + +Values are whatever the caller stores; `internal/archive` stores `[]*node` per base name because a +name repeats across directories. + +## Where it is used + +`internal/archive.Index` keeps one `PrefixSuffix[[]*node]` of files by base name. A cataloger glob +whose last segment is a literal, `*suffix` or `prefix*` (nearly all of them: `**/*.jar`, +`**/pom.properties`) is answered from it instead of by walking every path. + +## Notes + +- Lookups are byte-wise; `reverse` works on runes so a multi-byte name reverses correctly. +- `lock.go` returns unlock functions rather than exposing `Unlock`, which is how a node tells a read + lock from a write lock when deciding whether to upgrade. +- `_find`, `_collect` and `_makeNodeP` are the prototype's internals and are left with its naming. diff --git a/internal/index/key_split.go b/internal/index/key_split.go index 27c9e6a24..a74378364 100644 --- a/internal/index/key_split.go +++ b/internal/index/key_split.go @@ -1,10 +1,5 @@ -// Package index provides a concurrent key-split (radix) index and a forward/reverse pair over it. -// -// It answers path queries like "which entries end in .jar" or "live under this directory" as keyed -// lookups rather than tree walks, so a cataloger's `**/*.ext` glob need not scan every path. -// -// Ported from a prototype dir-scan package and kept close to the original for diffing. The lock type -// is local (see lock.go) rather than the prototype's, which carries unrelated helpers. +// Package index provides a concurrent key-split (radix) index, and a forward/reverse pair over it +// that answers prefix and suffix lookups such as "which names end in .jar". package index import ( @@ -101,41 +96,6 @@ func (n *Node[T]) ByPrefix(s string) []T { return v.Collect() } -// ByPrefixUpTo returns the values under s, or false once there are more than limit of them. It lets a -// caller choosing between two lookups take whichever is selective without paying for the other: a -// complete result costs at most limit. The values returned alongside false are partial; discard them. -func (n *Node[T]) ByPrefixUpTo(s string, limit int) (values []T, complete bool) { - v, _ := n._find(s) - if v == nil { - return nil, true - } - return v.CollectUpTo(limit) -} - -// CollectUpTo gathers this node's values, giving up as soon as there are more than limit. -func (n *Node[T]) CollectUpTo(limit int) (values []T, complete bool) { - complete = n._collectUpTo(&values, limit) - return values, complete -} - -func (n *Node[T]) _collectUpTo(values *[]T, limit int) bool { - defer n.RLock()() - - if n.set { - if len(*values) >= limit { - return false - } - *values = append(*values, n.value) - } - - for _, v := range n.keyMap { - if !v._collectUpTo(values, limit) { - return false - } - } - return true -} - func (n *Node[T]) Update(name string, f NodeUpdateFunc[T]) { unlock := n.Lock() node := n._makeNodeP(&unlock, name, nil) diff --git a/internal/index/prefix_suffix.go b/internal/index/prefix_suffix.go index fada252dd..7d4c453eb 100644 --- a/internal/index/prefix_suffix.go +++ b/internal/index/prefix_suffix.go @@ -28,16 +28,6 @@ func (f *PrefixSuffix[T]) BySuffix(suffix string) []T { return f.reverseIndex.ByPrefix(reverse(suffix)) } -// ByPrefixUpTo and BySuffixUpTo are the bounded forms, reporting false rather than returning more than -// limit values, so a caller can take whichever side is selective without paying for the other. -func (f *PrefixSuffix[T]) ByPrefixUpTo(prefix string, limit int) ([]T, bool) { - return f.forwardIndex.ByPrefixUpTo(prefix, limit) -} - -func (f *PrefixSuffix[T]) BySuffixUpTo(suffix string, limit int) ([]T, bool) { - return f.reverseIndex.ByPrefixUpTo(reverse(suffix), limit) -} - func reverse(s string) string { runes := []rune(s) for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { diff --git a/syft/archive_nesting_test.go b/internal/task/archive_nesting_test.go similarity index 53% rename from syft/archive_nesting_test.go rename to internal/task/archive_nesting_test.go index 676ccdb2e..a9afab352 100644 --- a/syft/archive_nesting_test.go +++ b/internal/task/archive_nesting_test.go @@ -1,4 +1,4 @@ -package syft +package task import ( "context" @@ -6,7 +6,6 @@ import ( "io/fs" "os" "path/filepath" - "reflect" "sort" "strings" "sync" @@ -17,18 +16,14 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/anchore/go-logger" - "github.com/anchore/go-logger/adapter/discard" "github.com/anchore/syft/internal/archive" - "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" - "github.com/anchore/syft/syft/cataloging/filecataloging" - "github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/java" "github.com/anchore/syft/syft/sbom" - "github.com/anchore/syft/syft/source/directorysource" ) // This file covers nesting across archive families. The other nested-archive fixtures are all zips, @@ -73,7 +68,8 @@ type nestedArchive struct { // sizes is each archive's size in bytes, outermost first. A container is never smaller than what it // holds, so these decrease, and the limit cases place a bound between two of them. sizes []int - // paths is each archive's path within its parent, outermost first; paths[0] is name. + // paths is each archive's path within its parent, outermost first; paths[0] is the outermost archive + // as the scan's resolver reports it. paths []string // fileSystemIDs is the composed chain each archive's own contents carry, outermost first. fileSystemIDs []string @@ -92,31 +88,22 @@ type nestingRow struct { type nestingVisit struct { virtualPath string depth int - // overflowed names the archives whose content was on disk at this moment, sampled only when the - // probe was given a temp dir to look in. - overflowed []string - // diskBytes is the scan's draw on the disk budget at this moment, read from the limiter on the - // context (Limiter.InUse): spilled archive content, entry blobs, and index records charged to disk. - // It exceeds the bytes in the work directories, since index records are charged too. - diskBytes int64 + // heldFiles are the basenames of the files in the live extraction work directories at this moment, + // sampled only when the probe was given a temp dir to look in. An archive's own bytes are read where + // they lie, so only "entries" files appear, one per archive whose entries did not fit in memory. + heldFiles []string } -// nestingProbe is a stub cataloger recording one visit per filesystem the archive cataloger runs its -// sub-pipeline against, reading the chain off the traversal on the context. It lets a case assert -// which archives were cataloged rather than inferring it from the SBOM. +// nestingProbe records one visit per filesystem the archive cataloger runs its sub-pipeline against, +// reading the chain off the traversal on the context. type nestingProbe struct { - // tempDir, when set, is where extraction work directories live, so each visit can record which - // archives were overflowed to disk while they were all still held. + // tempDir, when set, is where extraction work directories live tempDir string mu sync.Mutex visits []nestingVisit } -// entryStorageFiles are the basenames internal/archive gives the file one archive's entries are stored -// in. Which appears depends on the resolver under test, not on the routing this probe reports. -var entryStorageFiles = []string{"contents.tar", "contents.blob"} - func Test_mixedFamilyNesting_leafIsCatalogedOnceWithTheFullChain(t *testing.T) { // family order must not change the result: the same package, addressed by a chain naming its own // containers in its own order @@ -126,7 +113,7 @@ func Test_mixedFamilyNesting_leafIsCatalogedOnceWithTheFullChain(t *testing.T) { nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) probe := &nestingProbe{} - s := scanDirWith(t, scanDir, nestingScanConfig(3, markerCataloger{}, probe)) + s := runNesting(t, scanDir, 3, cataloging.DefaultArchiveSearchConfig(), markerTask(), probe.task()) assert.Equal(t, 1, packageCount(s, "marker-pkg"), "%s: the leaf package must be cataloged exactly once", row.why) @@ -152,8 +139,7 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) { scanDir := t.TempDir() nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) - cfg := withFileCataloging(nestingScanConfig(3, markerCataloger{})) - s := scanDirWith(t, scanDir, cfg) + s := runNesting(t, scanDir, 3, cataloging.DefaultArchiveSearchConfig(), markerTask(), fileMetadataTask()) // each archive's coordinates as seen in its parent; the outermost sits in the scanned // directory, so it carries no filesystem id of its own @@ -211,7 +197,7 @@ func Test_mixedFamilyNesting_javaLeafVirtualPathIsTheColonJoinedChain(t *testing scanDir := t.TempDir() nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) - s := scanDirWith(t, scanDir, nestingScanConfig(3, markerCataloger{})) + s := runNesting(t, scanDir, 3, cataloging.DefaultArchiveSearchConfig(), markerTask(), javaTask()) // the innermost archive is the jar, so its own package is the java leaf and the chain of // all three archives is exactly its virtual path @@ -242,7 +228,7 @@ func Test_mixedFamilyNesting_jarIsExercisedAsAContainer(t *testing.T) { nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) probe := &nestingProbe{} - s := scanDirWith(t, scanDir, nestingScanConfig(3, markerCataloger{}, probe)) + s := runNesting(t, scanDir, 3, cataloging.DefaultArchiveSearchConfig(), markerTask(), javaTask(), probe.task()) for _, level := range containerLevels { name := fmt.Sprintf("level%d", level) @@ -270,8 +256,7 @@ func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) { nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) probe := &nestingProbe{} - cfg := withFileCataloging(nestingScanConfig(2, markerCataloger{}, probe)) - s := scanDirWith(t, scanDir, cfg) + s := runNesting(t, scanDir, 2, cataloging.DefaultArchiveSearchConfig(), markerTask(), fileMetadataTask(), probe.task()) assert.Equal(t, nested.fileSystemIDs[:2], probe.archiveFileSystemIDs(), "at depth 2 the outer and middle archives are cataloged and the innermost is not") @@ -302,7 +287,7 @@ func Test_mixedFamilyNesting_defaultBoundsCatalogEveryRow(t *testing.T) { nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) probe := &nestingProbe{} - s := scanDirWith(t, scanDir, nestingScanConfig(3, markerCataloger{}, probe)) + s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(), "every level of the chain must be cataloged at the default bounds") @@ -314,47 +299,34 @@ func Test_mixedFamilyNesting_defaultBoundsCatalogEveryRow(t *testing.T) { } func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing.T) { - // memory pressure degrades to disk: an archive that will not fit spills rather than being refused, - // and the memory limit alone decides where that boundary falls + // entries that will not fit in memory are written to disk rather than refused scanDir := t.TempDir() tempDir := isolatedTempDir(t) nested := writeNestedArchive(t, scanDir, gradedChain()) - t.Run("with a generous memory limit nothing overflows", func(t *testing.T) { + t.Run("with a generous memory limit nothing is written to disk", func(t *testing.T) { probe := &nestingProbe{tempDir: tempDir} - cfg := nestingScanConfig(3, markerCataloger{}, probe) - // all three levels are held at once at the deepest point, so the limit must admit their sum - generous := int64(nested.sizes[0]+nested.sizes[1]+nested.sizes[2]) + 1 - cfg = cfg.WithArchiveConfig(cfg.Archive.WithMaxMemoryBytes(generous)) - scanDirWith(t, scanDir, cfg) + bounds := cataloging.DefaultArchiveSearchConfig(). + WithMaxMemoryBytes(int64(2 * (nested.sizes[0] + nested.sizes[1] + nested.sizes[2]))) + runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) deepest := probe.deepestVisit() require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held") - assert.Empty(t, deepest.overflowed, "with room in memory for every level, nothing is written to disk") + assert.Empty(t, deepest.heldFiles) }) - t.Run("a memory limit the outermost archive does not fit in", func(t *testing.T) { - // only the outermost archive is routed by the memory limit, being the one whose bytes are copied - // out of the scanned filesystem. Every archive below it is already an entry of its parent's store - // and is read where it lies, taking neither memory nor a second copy. This pins the remaining - // half: too large for memory means spilled, not failed, with the same outcome as a run with room. + t.Run("with a memory limit the outermost archive's entries do not fit in", func(t *testing.T) { probe := &nestingProbe{tempDir: tempDir} - cfg := nestingScanConfig(3, markerCataloger{}, probe) - cfg = cfg.WithArchiveConfig(cfg.Archive. + bounds := cataloging.DefaultArchiveSearchConfig(). WithMaxMemoryBytes(int64(nested.sizes[2])). - WithMaxDiskBytes(-1)) // unbounded: the disk limit must not be what decides anything here - s := scanDirWith(t, scanDir, cfg) + WithMaxDiskBytes(-1) + s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) deepest := probe.deepestVisit() require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held") - require.Less(t, int64(nested.sizes[2]), int64(nested.sizes[0]), - "the outermost archive must be the one that does not fit, or this case asserts nothing") - assert.Equal(t, []string{"level0.tar.gz"}, deepest.overflowed, - "the archive that does not fit in memory spills rather than failing, and the levels inside "+ - "it are read where they lie rather than being copied anywhere") + assert.Contains(t, deepest.heldFiles, "entries", "entries that do not fit in memory are written to disk") - // and the outcome is identical to the default-bounds run - assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs()) + assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(), "and every level is still cataloged") locs := leafLocations(s) require.Len(t, locs, 1) assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath) @@ -362,53 +334,41 @@ func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing } func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) { - // a nested archive is an entry of its parent's store, already charged to that parent, and its reader - // supports Read, Seek and ReadAt - everything a zip's central directory needs - so it is handed to - // the archive format where it lies. Nothing is copied, so nothing competes for the memory limit. - // - // Asserted at both ends of that limit: generous or zero, the nested levels neither hold memory nor - // put a second copy of themselves on disk, and the whole chain is cataloged either way. + // an archive's own bytes are read where they lie: a scanned file from disk, a nested archive from + // its parent's entries. Nothing copies them, so no "archive" file ever appears in a work directory + // whatever the memory limit; only entries are held, and only they can be written out. scanDir := t.TempDir() tempDir := isolatedTempDir(t) nested := writeNestedArchive(t, scanDir, gradedChain()) for _, tc := range []struct { - name string - memoryBytes int64 - outerAlsoOverflows bool + name string + memoryBytes int64 + wantHeldFiles []string }{ { - name: "with room in memory for the outermost archive", - memoryBytes: int64(nested.sizes[0]), + name: "with room in memory for every level's entries", + memoryBytes: int64(2 * (nested.sizes[0] + nested.sizes[1] + nested.sizes[2])), }, { - name: "with a memory limit of zero, so even the outermost overflows", - memoryBytes: 0, - outerAlsoOverflows: true, + name: "with a memory limit of zero, so every level's entries are written out", + memoryBytes: 0, + wantHeldFiles: []string{"entries", "entries", "entries"}, }, } { t.Run(tc.name, func(t *testing.T) { probe := &nestingProbe{tempDir: tempDir} - cfg := nestingScanConfig(3, markerCataloger{}, probe) - cfg = cfg.WithArchiveConfig(cfg.Archive. + bounds := cataloging.DefaultArchiveSearchConfig(). WithMaxMemoryBytes(tc.memoryBytes). - WithMaxDiskBytes(-1)) // unbounded: only the memory limit is under test + WithMaxDiskBytes(-1) - s := scanDirWith(t, scanDir, cfg) + s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) deepest := probe.deepestVisit() require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held") + assert.Equal(t, tc.wantHeldFiles, deepest.heldFiles) - var want []string - if tc.outerAlsoOverflows { - want = []string{"level0.tar.gz"} - } - assert.Equal(t, want, deepest.overflowed, - "no copy of a nested archive is written anywhere: level1.zip and level2.jar are read "+ - "out of the tar of the archive holding them") - - assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(), - "and every level is still cataloged") + assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs()) locs := leafLocations(s) require.Len(t, locs, 1) assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath) @@ -416,65 +376,53 @@ func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) { } } -func Test_mixedFamilyNesting_diskLimitSkipsTheLevelItCannotAdmit(t *testing.T) { - // the disk limit is terminal and the walk descends before it unwinds, so when the innermost level - // cannot be admitted the outer and middle archives still hold everything they took: that level is - // skipped, the shallower ones keep what they found, and the scan finishes. An implementation that - // waited for capacity would deadlock, hence the timeout. +func Test_mixedFamilyNesting_diskLimitTruncatesTheLevelItCannotAdmit(t *testing.T) { + // the walk descends before it unwinds, so the outer and middle archives still hold their entries + // when the innermost level's do not fit in memory. With nothing admitted on disk that level is + // cataloged from nothing, the shallower ones keep what they found, and the scan finishes. An + // implementation that waited for capacity would deadlock, hence the timeout. scanDir := t.TempDir() - tempDir := isolatedTempDir(t) nested := writeNestedArchive(t, scanDir, deepChain()) - // a sibling of a different family, to show the bound is per archive: it is processed once the chain - // has been released, so it is cataloged in full + // a sibling of a different family, processed once the chain has been released, so cataloged in full require.NoError(t, os.WriteFile( filepath.Join(scanDir, "sibling.tgz"), - buildTarGzBytes(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}), + makeTarGz(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}), 0o644, )) - // the limit is measured rather than computed: what the chain holds on disk at a given level is a - // fact about this corpus, so the chain runs once with no disk ceiling and the probe reports what was - // on disk while two levels were held. - // - // The real run gets that figure with no slack, making this a skip rather than a truncation. Entries - // are stored without framing, so any slack would admit some of the innermost level's content, and an - // archive that stored some entries is truncated instead - covered elsewhere. - measuring := &nestingProbe{tempDir: tempDir} - measureCfg := nestingScanConfig(3, markerCataloger{}, measuring) - measureCfg = measureCfg.WithArchiveConfig(measureCfg.Archive.WithMaxMemoryBytes(0).WithMaxDiskBytes(-1)) - scanDirWith(t, scanDir, measureCfg) - - twoLevelsHeld := measuring.visitAtDepth(2).diskBytes - require.Positive(t, twoLevelsHeld, "the measuring run must have reached the second level") - diskLimit := twoLevelsHeld + // room for the outer two levels' entries (each about the size of the archive it holds) but not the + // innermost's payload, which is half an archive larger than the slack + memoryLimit := int64(nested.sizes[0] + nested.sizes[1] + deepChainPayloadBytes/2) probe := &nestingProbe{} - cfg := nestingScanConfig(3, markerCataloger{}, probe) - cfg = cfg.WithArchiveConfig(cfg.Archive. - WithMaxMemoryBytes(0). // every archive spills, so its own bytes are charged to disk - WithMaxDiskBytes(diskLimit)) + bounds := cataloging.DefaultArchiveSearchConfig(). + WithMaxMemoryBytes(memoryLimit). + WithMaxDiskBytes(0) - logs := captureLogs(t) - s := scanWithinTimeout(t, scanDir, cfg, 60*time.Second) + s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) - want := append(append([]string{}, nested.fileSystemIDs[:2]...), "sibling.tgz") + want := append(append([]string{}, nested.fileSystemIDs...), "/sibling.tgz") assert.ElementsMatch(t, want, probe.archiveFileSystemIDs(), - "the level that would exceed the limit is skipped, the shallower ones are still cataloged, "+ - "and a sibling of another family is unaffected") - - // the skip names the archive that triggered it, not just the bound - assert.True(t, logs.sawFields(map[string]any{ - "archive": nested.paths[2], - "limit": diskLimit, - }), "the skip must be attributed to the archive that reached the limit; saw %v", logs.eventsWith("archive")) + "every level is visited, the truncated one with nothing inside it") fsIDs := strset.New() for _, loc := range leafLocations(s) { fsIDs.Add(loc.ArchivePath) } - assert.False(t, fsIDs.Has(nested.leafFileSystemID()), "the skipped level's contents must be absent") - assert.True(t, fsIDs.Has("sibling.tgz"), "the sibling of a different family must be cataloged fully") + assert.False(t, fsIDs.Has(nested.leafFileSystemID()), "the truncated level's contents must be absent") + assert.True(t, fsIDs.Has("/sibling.tgz"), "the sibling of a different family must be cataloged fully") + + var truncated []string + for coords, reasons := range s.Artifacts.Unknowns { + for _, reason := range reasons { + if strings.Contains(reason, "disk limit") { + truncated = append(truncated, coords.ArchivePath+":"+coords.RealPath) + } + } + } + assert.Equal(t, []string{nested.fileSystemIDs[1] + ":" + nested.paths[2]}, truncated, + "the truncation is recorded against the archive that reached the limit") } func Test_mixedFamilyNesting_unboundedLimitsEnforceNothing(t *testing.T) { @@ -486,12 +434,11 @@ func Test_mixedFamilyNesting_unboundedLimitsEnforceNothing(t *testing.T) { nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) probe := &nestingProbe{} - cfg := nestingScanConfig(3, markerCataloger{}, probe) - cfg = cfg.WithArchiveConfig(cfg.Archive. + bounds := cataloging.DefaultArchiveSearchConfig(). WithMaxMemoryBytes(-1). - WithMaxDiskBytes(-1)) + WithMaxDiskBytes(-1) - s := scanWithinTimeout(t, scanDir, cfg, 60*time.Second) + s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs()) locs := leafLocations(s) @@ -501,161 +448,90 @@ func Test_mixedFamilyNesting_unboundedLimitsEnforceNothing(t *testing.T) { } } -func Test_mixedFamilyNesting_bothLimitsZeroSkipsEveryArchive(t *testing.T) { - // the degenerate but well-defined configuration where nothing is admitted anywhere: unlike negative - // (unbounded), zero on both limits skips every archive, and the scan still succeeds rather than - // failing or finding anything inside the archives +func Test_mixedFamilyNesting_bothLimitsZeroCatalogsNothingInsideAnyArchive(t *testing.T) { + // unlike negative (unbounded), zero on both limits admits no entry anywhere: the outermost archive is + // cataloged from nothing, so the level inside it is never found, and the scan still succeeds for _, row := range nestingRows() { t.Run(row.name, func(t *testing.T) { scanDir := t.TempDir() - writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) + nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) probe := &nestingProbe{} - cfg := nestingScanConfig(3, markerCataloger{}, probe) - cfg = cfg.WithArchiveConfig(cfg.Archive. + bounds := cataloging.DefaultArchiveSearchConfig(). WithMaxMemoryBytes(0). - WithMaxDiskBytes(0)) + WithMaxDiskBytes(0) - s := scanWithinTimeout(t, scanDir, cfg, 60*time.Second) + s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) - assert.Empty(t, probe.archiveFileSystemIDs(), "no archive content can be admitted anywhere") - assert.Empty(t, leafLocations(s), "the leaf is never reached since every archive is skipped") + assert.Equal(t, nested.fileSystemIDs[:1], probe.archiveFileSystemIDs()) + assert.Empty(t, leafLocations(s)) }) } } -// A skip is reported by a log line alone - reaching a bound must not fail the scan - so asserting the -// attribution means capturing what was logged. cmd/syft/internal/ui/log_writer_test.go uses the same -// log.Set/log.Get pair; this adds the field-carrying half, where the attribution lives. - -// logCapture collects the field-carrying log events emitted while it is installed. -type logCapture struct { - mu sync.Mutex - collected []map[string]any -} - -// captureLogs installs a capturing logger for the test and restores the previous one afterwards. -func captureLogs(t *testing.T) *logCapture { +// runNesting runs the archive cataloger task over scanDir with the given sub-pipeline and returns the +// SBOM it built. It runs on its own goroutine and fails if the task has not finished in time. +// +// The depth-3 limit cases need the timeout: the walk descends before it unwinds, so the outer and +// middle archives still hold their content when the inner one is refused. An implementation that +// waited for capacity would deadlock, and a test asserting only the result would hang rather than fail. +func runNesting(t *testing.T, scanDir string, depth int, bounds cataloging.ArchiveSearchConfig, subPipeline ...Task) *sbom.SBOM { t.Helper() - capture := &logCapture{} - previous := log.Get() - t.Cleanup(func() { log.Set(previous) }) - log.Set(&capturingLogger{Logger: discard.New(), capture: capture}) - return capture -} + tsk := newTestTask(t, bounds.WithMaxDepth(depth), subPipeline...) + s := &sbom.SBOM{Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(), + FileMetadata: map[file.Coordinates]file.Metadata{}, + }} -func (c *logCapture) record(event map[string]any) { - c.mu.Lock() - defer c.mu.Unlock() - c.collected = append(c.collected, event) -} + done := make(chan error, 1) + go func() { + done <- tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(s)) + }() -func (c *logCapture) events() []map[string]any { - c.mu.Lock() - defer c.mu.Unlock() - return append([]map[string]any(nil), c.collected...) -} - -// eventsWith is the captured events carrying the given field. A scan logs thousands of lines, so a -// failure message shows this subset rather than all of them. -func (c *logCapture) eventsWith(field string) []map[string]any { - var out []map[string]any - for _, event := range c.events() { - if _, ok := event[field]; ok { - out = append(out, event) - } + select { + case err := <-done: + require.NoError(t, err, "reaching a bound must not fail the scan") + return s + case <-time.After(60 * time.Second): + t.Fatal("the scan did not finish in time: reaching a limit must skip the archive that " + + "reached it, never wait for capacity that nothing in the chain will release") + return nil } - return out } -// sawFields reports whether any captured event carried every given field with exactly the given -// value. -func (c *logCapture) sawFields(want map[string]any) bool { - for _, got := range c.events() { - matched := true - for key, value := range want { - if !reflect.DeepEqual(got[key], value) { - matched = false - break +// markerTask emits one package per "marker.txt" found, so the matrix asserts behavior without +// depending on any real cataloger's fixture format. +func markerTask() Task { + return NewTask("marker-cataloger", func(_ context.Context, resolver file.Resolver, builder sbomsync.Builder) error { + locations, err := resolver.FilesByGlob("**/marker.txt") + if err != nil { + return err + } + for _, loc := range locations { + p := pkg.Package{ + Name: "marker-pkg", + Version: "1.0.0", + Type: pkg.BinaryPkg, + Locations: file.NewLocationSet(loc), } + p.SetID() + builder.AddPackages(p) } - if matched { - return true - } - } - return false + return nil + }) } -// capturingLogger discards every message but records the fields attached to those that carry any. -// Nested is left to the embedded discard logger, since nothing under test uses it. -type capturingLogger struct { - logger.Logger - capture *logCapture +// javaTask is the java archive cataloger as the cataloging factory builds it, so a jar is met as a jar +// wherever it sits in a chain. +func javaTask() Task { + return NewPackageTask(CatalogingFactoryConfig{}, java.NewArchiveCataloger(java.DefaultArchiveCatalogerConfig())) } -func (l *capturingLogger) WithFields(fields ...any) logger.MessageLogger { - return &capturingMessageLogger{ - MessageLogger: l.Logger.WithFields(fields...), - capture: l.capture, - fields: fieldMap(fields), - } -} - -type capturingMessageLogger struct { - logger.MessageLogger - capture *logCapture - fields map[string]any -} - -func (l *capturingMessageLogger) emit(message string) { - event := map[string]any{"message": message} - for key, value := range l.fields { - event[key] = value - } - l.capture.record(event) -} - -func (l *capturingMessageLogger) Error(args ...any) { l.emit(fmt.Sprint(args...)) } -func (l *capturingMessageLogger) Warn(args ...any) { l.emit(fmt.Sprint(args...)) } -func (l *capturingMessageLogger) Info(args ...any) { l.emit(fmt.Sprint(args...)) } -func (l *capturingMessageLogger) Debug(args ...any) { l.emit(fmt.Sprint(args...)) } -func (l *capturingMessageLogger) Trace(args ...any) { l.emit(fmt.Sprint(args...)) } - -func (l *capturingMessageLogger) Errorf(format string, args ...any) { - l.emit(fmt.Sprintf(format, args...)) -} -func (l *capturingMessageLogger) Warnf(format string, args ...any) { - l.emit(fmt.Sprintf(format, args...)) -} -func (l *capturingMessageLogger) Infof(format string, args ...any) { - l.emit(fmt.Sprintf(format, args...)) -} -func (l *capturingMessageLogger) Debugf(format string, args ...any) { - l.emit(fmt.Sprintf(format, args...)) -} -func (l *capturingMessageLogger) Tracef(format string, args ...any) { - l.emit(fmt.Sprintf(format, args...)) -} - -// fieldMap turns the alternating key/value form log.WithFields takes into a map, folding in any -// logger.Fields passed as a whole. -func fieldMap(fields []any) map[string]any { - out := map[string]any{} - for i := 0; i < len(fields); i++ { - if asMap, ok := fields[i].(logger.Fields); ok { - for key, value := range asMap { - out[key] = value - } - continue - } - key, ok := fields[i].(string) - if !ok || i+1 >= len(fields) { - continue - } - out[key] = fields[i+1] - i++ - } - return out +// fileMetadataTask records every file, which the CONTAINS chain needs: archive-to-file edges key on +// coordinates the file catalogers recorded, so without them an inner archive has no coordinate to +// hang an edge on. +func fileMetadataTask() Task { + return newFileMetadataCatalogerTask(file.AllFilesSelection) } // pack builds one archive of this family holding the given entries. @@ -663,20 +539,33 @@ func (f archiveFamily) pack(t *testing.T, base string, entries map[string][]byte t.Helper() switch f { case familyZip: - return buildZipBytesRaw(t, entries) + return makeZip(t, entries) case familyJar: // a manifest makes it a real jar to the java cataloger at any level; the package java reports is // named after the file, not the manifest title - return jarBytes(t, base, "1.0", entries) + return makeJar(t, base, "1.0", entries) case familyTar: - return buildTarBytes(t, entries) + return makeTar(t, entries) case familyTarGz, familyTgz: - return buildTarGzBytes(t, entries) + return makeTarGz(t, entries) } t.Fatalf("unknown archive family %q", f) return nil } +func makeJar(t *testing.T, title, version string, extra map[string][]byte) []byte { + t.Helper() + entries := map[string][]byte{ + "META-INF/MANIFEST.MF": []byte( + "Manifest-Version: 1.0\nImplementation-Title: " + title + "\nImplementation-Version: " + version + "\n", + ), + } + for k, v := range extra { + entries[k] = v + } + return makeZip(t, entries) +} + // leafFileSystemID is the chain the innermost archive's contents carry: every archive, outer to // inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check. func (n nestedArchive) leafFileSystemID() string { @@ -701,13 +590,14 @@ func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive { for i, family := range plan.families { base := fmt.Sprintf("level%d", i) if i == 0 { - out.paths[i] = base + string(family) + // the outermost archive is a file in the scanned directory, which the resolver reports rooted + out.name = base + string(family) + out.paths[i] = "/" + out.name continue } out.paths[i] = fmt.Sprintf("nest%d/%s%s", i, base, family) } - out.name = out.paths[0] for i := range plan.families { if i == 0 { out.fileSystemIDs[i] = out.paths[0] @@ -749,24 +639,12 @@ func writeNestedArchive(t *testing.T, dir string, plan nestPlan) nestedArchive { } // markerLeaf is the family-agnostic leaf payload: one file no archive format has an opinion about, -// found by markerCataloger. It keeps the rows comparable - the same package must come out whichever +// found by markerTask. It keeps the rows comparable - the same package must come out whichever // families the chain is made of. func markerLeaf() map[string][]byte { return map[string][]byte{"nested/marker.txt": []byte("leaf")} } -// incompressibleBytes returns n bytes deflate and gzip cannot shrink, so per-level sizes are -// predictable enough to place a bound between two of them. -func incompressibleBytes(n int) []byte { - b := make([]byte, n) - x := uint32(0x9e3779b9) - for i := range b { - x = x*1664525 + 1013904223 - b[i] = byte(x >> 24) - } - return b -} - // nestingRows is the depth-3 matrix. Each row covers a case the others do not. func nestingRows() []nestingRow { return []nestingRow{ @@ -798,30 +676,26 @@ func nestingRows() []nestingRow { } } -func (p *nestingProbe) Name() string { return "nesting-probe" } +func (p *nestingProbe) task() Task { + return NewTask("nesting-probe", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { + trav := archive.TraversalFromContext(ctx) + visit := nestingVisit{} + if trav != nil { + visit.virtualPath = archive.VirtualPath(trav.Location) + visit.depth = strings.Count(archive.VirtualPath(trav.Location), ":") + 1 + } + if p.tempDir != "" { + visit.heldFiles = heldFiles(p.tempDir) + } -func (p *nestingProbe) Catalog(ctx context.Context, _ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { - trav := archive.TraversalFromContext(ctx) - visit := nestingVisit{} - if trav != nil { - visit.virtualPath = trav.VirtualPath - visit.depth = strings.Count(trav.VirtualPath, ":") + 1 - } - if lim := archive.LimiterFromContext(ctx); lim != nil { - _, visit.diskBytes = lim.InUse() - } - if p.tempDir != "" { - visit.overflowed = overflowArchives(p.tempDir) - } - - p.mu.Lock() - defer p.mu.Unlock() - p.visits = append(p.visits, visit) - return nil, nil, nil + p.mu.Lock() + defer p.mu.Unlock() + p.visits = append(p.visits, visit) + return nil + }) } -// archiveFileSystemIDs is the chain of every archive whose contents were cataloged, in visit order, -// with the scan root dropped. +// archiveFileSystemIDs is the chain of every archive whose contents were cataloged, in visit order. func (p *nestingProbe) archiveFileSystemIDs() []string { p.mu.Lock() defer p.mu.Unlock() @@ -835,19 +709,6 @@ func (p *nestingProbe) archiveFileSystemIDs() []string { return out } -// visitAtDepth is the visit at the given nesting depth, for a case measuring what a run held while a -// particular level was being cataloged. -func (p *nestingProbe) visitAtDepth(depth int) nestingVisit { - p.mu.Lock() - defer p.mu.Unlock() - for _, v := range p.visits { - if v.depth == depth { - return v - } - } - return nestingVisit{} -} - // deepestVisit is the visit at the innermost level reached, the only moment at which every level of a // chain is still held and its routing observable. It returns the whole visit so a case can also assert // it got as deep as it meant to: an empty overflow sample from a visit that never left the scan root @@ -864,40 +725,10 @@ func (p *nestingProbe) deepestVisit() nestingVisit { return best } -// overflowArchives names the archive files in the live extraction work directories under tempDir. -// Content held in memory writes nothing there; content spilled to disk is written beside the archive's -// logical root under the archive's own name, which makes the routing observable without a hook into -// the limiter. Meaningful only while the archives are still held, so it is sampled from inside the -// sub-pipeline rather than after the scan. -// -// The file an archive's entries are stored in is skipped: it says nothing about where the archive's -// own bytes were routed. -func overflowArchives(tempDir string) []string { - var out []string - forEachHeldFile(tempDir, func(name string, _ os.FileInfo) { - if isEntryStorageFile(name) { - return - } - out = append(out, name) - }) - sort.Strings(out) - return out -} - -func isEntryStorageFile(name string) bool { - for _, storage := range entryStorageFiles { - if name == storage { - return true - } - } - return false -} - -// forEachHeldFile visits every file an archive work directory is holding, wherever under tempDir those -// directories were created. It walks the tree rather than reading one level, since an archive's work -// directory sits under the scan's own temp root (internal/tmpdir), whose name this probe does not -// choose. -func forEachHeldFile(tempDir string, visit func(name string, info os.FileInfo)) { +// heldFiles names the files in the live extraction work directories under tempDir. Meaningful only +// while the archives are still held, so it is sampled from inside the sub-pipeline rather than after +// the scan. +func heldFiles(tempDir string) (names []string) { _ = filepath.WalkDir(tempDir, func(path string, d fs.DirEntry, err error) error { if err != nil || !d.IsDir() || !strings.HasPrefix(d.Name(), "syft-archive-") { return nil //nolint:nilerr // an unreadable directory holds nothing this probe can report @@ -907,17 +738,14 @@ func forEachHeldFile(tempDir string, visit func(name string, info os.FileInfo)) return filepath.SkipDir } for _, entry := range held { - if entry.IsDir() { - continue + if !entry.IsDir() { + names = append(names, entry.Name()) } - info, err := entry.Info() - if err != nil { - continue - } - visit(entry.Name(), info) } return filepath.SkipDir }) + sort.Strings(names) + return names } // isolatedTempDir points archive extraction at a directory of this test's own, so an overflow sample @@ -930,25 +758,6 @@ func isolatedTempDir(t *testing.T) string { return dir } -// nestingScanConfig selects the java catalogers - so a jar is met as a jar wherever it sits - and -// always runs the given catalogers. Selected by tag rather than the whole default set, which includes -// the RPM cataloger and its sqlite driver this test binary does not register. -func nestingScanConfig(depth int, catalogers ...pkg.Cataloger) *CreateSBOMConfig { - cfg := DefaultCreateSBOMConfig(). - WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")) - for _, c := range catalogers { - cfg = cfg.WithCatalogers(pkgcataloging.NewAlwaysEnabledCatalogerReference(c)) - } - return cfg.WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth)) -} - -// withFileCataloging turns on file cataloging, which the CONTAINS chain needs: archive-to-file edges -// key on coordinates the file catalogers recorded, so without them an inner archive has no coordinate -// to hang an edge on. -func withFileCataloging(cfg *CreateSBOMConfig) *CreateSBOMConfig { - return cfg.WithFilesConfig(filecataloging.Config{Selection: file.AllFilesSelection}) -} - func leafLocations(s *sbom.SBOM) []file.Location { var out []file.Location for _, p := range s.Artifacts.Packages.Sorted() { @@ -980,38 +789,6 @@ func packageCount(s *sbom.SBOM, name string) int { return n } -// scanWithinTimeout runs the scan on its own goroutine and fails if it has not finished in time. -// -// The depth-3 limit cases need it: the walk descends before it unwinds, so the outer and middle -// archives still hold their content when the inner one is refused. An implementation that waited for -// capacity would deadlock, and a test asserting only the result would hang rather than fail. -func scanWithinTimeout(t *testing.T, dir string, cfg *CreateSBOMConfig, timeout time.Duration) *sbom.SBOM { - t.Helper() - src, err := directorysource.New(directorysource.Config{Path: dir}) - require.NoError(t, err) - t.Cleanup(func() { _ = src.Close() }) - - type outcome struct { - s *sbom.SBOM - err error - } - done := make(chan outcome, 1) - go func() { - s, err := cfg.Create(context.Background(), src) - done <- outcome{s: s, err: err} - }() - - select { - case got := <-done: - require.NoError(t, got.err, "reaching a bound must not fail the scan") - return got.s - case <-time.After(timeout): - t.Fatalf("the scan did not finish within %s: reaching a limit must skip the archive that "+ - "reached it, never wait for capacity that nothing in the chain will release", timeout) - return nil - } -} - func coordKey(c file.Coordinates) string { return c.ArchivePath + "|" + c.RealPath } @@ -1024,22 +801,25 @@ func gradedChain() nestPlan { families: []archiveFamily{familyTarGz, familyZip, familyJar}, leaf: markerLeaf(), extra: map[int]map[string][]byte{ - 0: {"pad0.bin": incompressibleBytes(24 * 1024)}, - 1: {"pad1.bin": incompressibleBytes(24 * 1024)}, - 2: {"pad2.bin": incompressibleBytes(8 * 1024)}, + 0: {"pad0.bin": incompressible(24 * 1024)}, + 1: {"pad1.bin": incompressible(24 * 1024)}, + 2: {"pad2.bin": incompressible(8 * 1024)}, }, } } +// deepChainPayloadBytes is the size of deepChain's leaf payload. +const deepChainPayloadBytes = 64 * 1024 + // deepChain is the headline row with one large, incompressible payload at the leaf and no other -// padding. The payload dominates every level, so the three archives are about the same size, giving -// the disk limit arithmetic below slack of half an archive rather than a few hundred bytes. +// padding. The payload dominates every level, so the three archives are about the same size. It sorts +// before the marker so it is the first entry refused when the leaf archive's entries do not fit. func deepChain() nestPlan { return nestPlan{ families: []archiveFamily{familyTarGz, familyZip, familyJar}, leaf: map[string][]byte{ + "0-payload.bin": incompressible(deepChainPayloadBytes), "nested/marker.txt": []byte("leaf"), - "payload.bin": incompressibleBytes(64 * 1024), }, } } diff --git a/internal/task/archive_tasks.go b/internal/task/archive_tasks.go index 439d082b8..f7d59196e 100644 --- a/internal/task/archive_tasks.go +++ b/internal/task/archive_tasks.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" "io" - "sync" + "maps" "time" "github.com/dustin/go-humanize" @@ -28,48 +28,37 @@ import ( const ArchiveCatalogerTaskName = "archive-cataloger" -// NewArchiveCatalogerTask returns a task that recursively extracts archives (including JAR-family -// ones, whose cataloger skips its own unarchiving when this task is enabled), treats each as its own -// standalone indexed filesystem, runs subPipeline against each, and records file-level CONTAINS -// relationships from the archive to what was found inside it. +// NewArchiveCatalogerTask returns a task that extracts every archive it finds, runs subPipeline +// against the archive's contents as a standalone filesystem, and records CONTAINS relationships +// from the archive to what was found inside it. It recurses into the archives it extracts up to +// cfg.MaxDepth; the task drops itself from subPipeline so that is the only recursion. // -// Returns nil when archive cataloging is disabled (MaxDepth == 0; negative means unbounded). -// -// subPipeline must not include this task, so recursion is driven only by the depth-limited walk here. -// newStoreResolver is injected so this package does not depend on the internal fileresolver package. -func NewArchiveCatalogerTask(cfg cataloging.ArchiveSearchConfig, subPipeline []Task, newStoreResolver archive.StoreResolverFactory, notify archive.Notify) Task { - if cfg.MaxDepth == 0 || newStoreResolver == nil { +// exclusions are the scan's exclusion patterns, applied inside each archive. Returns nil when archive +// cataloging is disabled (MaxDepth == 0; negative means unbounded). +func NewArchiveCatalogerTask(cfg cataloging.ArchiveSearchConfig, subPipeline []Task, exclusions []string) Task { + if cfg.MaxDepth == 0 { return nil } - - // this task drives recursion itself: in its own sub-pipeline, each nesting level would be processed - // once by the walk and again by the nested copy subPipeline = withoutArchiveCataloger(subPipeline) if len(subPipeline) == 0 { return nil } + fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error { - // this task re-runs catalogers, so it must ensure each keeps to one row; a scan installs the - // registry up front, this covers a pipeline assembled without one + // catalogers run once per archive, and each must report into the one progress row it owns ctx = bus.WithCatalogerTaskRegistry(ctx) - // one progress row for the whole walk; the sub-pipeline's catalogers each report into the row - // they already own (see bus.WithCatalogerTaskRegistry), not this one - prog := bus.StartCatalogerTask(ctx, archiveCatalogerProgressInfo(), -1, "") c := &archiveCataloger{ - cfg: cfg, - subPipeline: subPipeline, - extractors: archive.DefaultExtractors(), - limiter: archive.NewLimiter(archive.DefaultLimits(cfg)), - newStoreResolver: newStoreResolver, - notify: notify, - prog: prog, + maxDepth: cfg.MaxDepth, + subPipeline: subPipeline, + limiter: archive.NewLimiter(archive.Limits{MaxMemoryBytes: cfg.MaxMemoryBytes, MaxDiskBytes: cfg.MaxDiskBytes}), + exclusions: archive.NewExclusions(exclusions), + progress: bus.StartCatalogerTask(ctx, archiveCatalogerProgressInfo(), -1, ""), } - err := c.catalog(archive.WithLimiter(ctx, c.limiter), resolver, nil, 0, builder) - c.left(nil) + err := c.catalog(ctx, resolver, 0, builder) + c.progress.AtomicStage.Set(fmt.Sprintf("%s archives", humanize.Comma(c.progress.Current()))) + c.progress.SetCompleted() c.logStats() - prog.SetCompleted() - return err } return NewTask(ArchiveCatalogerTaskName, fn, pkgcataloging.PackageTag, "archive") @@ -88,14 +77,9 @@ func archiveCatalogerProgressInfo() monitor.GenericTask { } func withoutArchiveCataloger(tasks []Task) []Task { - out := make([]Task, 0, len(tasks)) + var out []Task for _, t := range tasks { - if t == nil { - continue - } - if t.Name() == ArchiveCatalogerTaskName { - log.WithFields("task", t.Name()). - Debug("dropping archive cataloger task from its own sub-pipeline") + if t == nil || t.Name() == ArchiveCatalogerTaskName { continue } out = append(out, t) @@ -104,213 +88,120 @@ func withoutArchiveCataloger(tasks []Task) []Task { } type archiveCataloger struct { - cfg cataloging.ArchiveSearchConfig - subPipeline []Task - extractors []archive.Extractor - newStoreResolver archive.StoreResolverFactory + maxDepth int + subPipeline []Task + limiter *archive.Limiter + exclusions archive.Exclusions - // notify carries structured extraction events - which archive spilled to disk and why, which was - // skipped. Nil is the no-op. - notify archive.Notify + // progress is the one row for the whole walk: its count is how many archives have been entered + progress *monitor.TaskProgress - // limiter measures the archive content held right now, falling as each archive is released. One - // instance per task run, since one task run is one scan. - limiter *archive.Limiter - - // prog is the single progress row for the whole walk: its count and stage are how many archives have - // been entered, its context names the one being cataloged right now. - prog *monitor.TaskProgress - - // slowest is the archive that spent the most time on its own extraction and cataloging, by virtual - // path. Guarded because the walk runs alongside the top-level catalogers. - statsMu sync.Mutex - slowest time.Duration - slowestPath string - archivesSeen int64 + // the archive that spent the longest on its own extraction and cataloging, excluding nested archives + slowest time.Duration + slowestPath string } -// recordArchiveTime keeps the longest self time seen and the archive that spent it. Self time -// excludes nested archives; inclusive time would name the outermost archive every run. -func (c *archiveCataloger) recordArchiveTime(virtualPath string, took time.Duration) { - c.statsMu.Lock() - defer c.statsMu.Unlock() - c.archivesSeen++ - if took > c.slowest { - c.slowest = took - c.slowestPath = virtualPath - } -} - -// logStats reports what the walk cost, once, when it is done. The peaks are high-water marks of the -// gauges the limits are enforced against - comparable to the configured bounds, not process memory or -// filesystem usage. One line per scan, so it logs at info; per-archive lines stay at debug. +// logStats reports what the walk cost, once. The peaks are the most held against each limit at any +// one moment, comparable to the configured bounds. func (c *archiveCataloger) logStats() { - c.statsMu.Lock() - slowest, slowestPath, seen := c.slowest, c.slowestPath, c.archivesSeen - c.statsMu.Unlock() - peakMemory, peakDisk := c.limiter.Peak() fields := []any{ - "archives", seen, + "archives", c.progress.Current(), "peak-memory", humanize.Bytes(uint64(peakMemory)), "peak-disk", humanize.Bytes(uint64(peakDisk)), } - if slowestPath != "" { - fields = append(fields, "slowest", slowestPath, "slowest-took", slowest.Round(time.Millisecond)) + if c.slowestPath != "" { + fields = append(fields, "slowest", c.slowestPath, "slowest-took", c.slowest.Round(time.Millisecond)) } log.WithFields(fields...).Info("nested archive cataloging complete") } -// catalog discovers archives in the given resolver and processes each, recursing up to MaxDepth. -// parent is the traversal for the archive being searched (nil at the scan root). -func (c *archiveCataloger) catalog(ctx context.Context, resolver file.Resolver, parent *archive.Traversal, depth int, builder sbomsync.Builder) error { - if c.cfg.MaxDepth >= 0 && depth >= c.cfg.MaxDepth { +// catalog processes every archive in the resolver, recursing into each up to maxDepth. +func (c *archiveCataloger) catalog(ctx context.Context, resolver file.Resolver, depth int, builder sbomsync.Builder) error { + if c.maxDepth >= 0 && depth >= c.maxDepth { return nil } - var errs error for _, candidate := range c.discoverArchives(resolver) { - // no between-archives check: a limit falls when an archive is released, so reaching a bound skips - // only the archive that reached it - if err := c.processArchive(ctx, resolver, candidate, parent, depth, builder); err != nil { + if err := c.processArchive(ctx, resolver, candidate, depth, builder); err != nil { errs = unknown.Append(errs, candidate.location, err) } } return errs } -func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver file.Resolver, candidate archiveCandidate, parent *archive.Traversal, depth int, builder sbomsync.Builder) error { - archiveLoc := candidate.location - content, err := parentResolver.FileContentsByLocation(archiveLoc) +func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver file.Resolver, candidate archiveCandidate, depth int, builder sbomsync.Builder) error { + location := candidate.location + content, err := parentResolver.FileContentsByLocation(location) if err != nil { return err } - defer internal.CloseAndLogError(content, archiveLoc.AccessPath) + defer internal.CloseAndLogError(content, location.AccessPath) - // check a non-sniffing candidate cheaply before reading it in full: an appended archive carries an - // entry signature near the front, and almost nothing else does var archiveContent io.Reader = content if !candidate.sniffedAsArchive { var mayHide bool - archiveContent, mayHide = archive.MayHideAnAppendedArchive(content) - if !mayHide { + if archiveContent, mayHide = archive.MayHideAnAppendedArchive(content); !mayHide { return nil } } - // self time: extraction, sub-pipeline and merge, stopped before the recursive descent so a - // containing archive is not charged for what it contains started := time.Now() - // extracted files inherit the filesystem id of where the archive was found (a layer digest for an - // image source, empty for a directory source). The nesting chain rides separately as the archive - // path, so same-named files in different archives - and the same archive path in different layers - - // stay distinct in coordinate-keyed tables. - archivePath := parent.VirtualPathOf(accessPath(archiveLoc)) - extracted, err := archive.ExtractToResolver(ctx, archiveContent, archiveLoc.AccessPath, archiveLoc.FileSystemID, archivePath, c.extractors, c.limiter, c.newStoreResolver, c.notify) + // extracted files keep the filesystem ID of where the archive was found; the nesting chain is + // carried separately as the archive path + archivePath := archive.VirtualPath(location) + extracted, err := archive.Extract(ctx, archiveContent, location.FileSystemID, archivePath, c.limiter, c.exclusions) if errors.Is(err, archive.ErrDiskLimitReached) { - // terminal: nothing is released while this archive waits, so skip it and continue the scan - log.WithFields("archive", accessPath(archiveLoc), "limit", c.cfg.MaxDiskBytes). - Debug("skipping archive whose content would exceed the disk limit") + // nothing is released while this archive waits, so skip it rather than block the scan + appendUnknowns(builder, ArchiveCatalogerTaskName, []unknown.CoordinateError{{ + Coordinates: location.Coordinates, + Reason: fmt.Errorf("archive skipped: its content would exceed the disk limit"), + }}) return nil } if err != nil { return err } if extracted == nil { - // nothing could open it: a format with no extractor (ar, a bare non-tar gzip), or a candidate - // whose head only looked like it hid an archive. Logged rather than returned: an image can hold - // thousands, and accumulating an error each into a chain later walked for coordinate errors is - // quadratic. A corrupt archive does return an error. - log.WithFields("archive", accessPath(archiveLoc), "sniffed-as-archive", candidate.sniffedAsArchive). + // a format with no extractor, or a candidate whose head only looked like an appended archive. + // Logged rather than returned: an image can hold thousands of these + log.WithFields("archive", archivePath, "sniffed-as-archive", candidate.sniffedAsArchive). Trace("no extractor could read this candidate; skipping") return nil } - // releases the extraction directory and this archive's charges, after the sub-pipeline and recursion defer extracted.Cleanup() - if extracted.Result.Truncated() { - // catalog what was stored rather than discarding the archive, and record it in the SBOM's - // unknowns as well as the log, since the packages found here are not the full set - log.WithFields("archive", accessPath(archiveLoc), "limit", string(extracted.Result.Truncation)). - Debug("archive extraction truncated by a configured limit; cataloging partial contents") - + if extracted.Truncated { appendUnknowns(builder, ArchiveCatalogerTaskName, []unknown.CoordinateError{{ - Coordinates: archiveLoc.Coordinates, - Reason: fmt.Errorf("archive cataloged from part of its contents: extraction stopped at the %s", - string(extracted.Result.Truncation)), + Coordinates: location.Coordinates, + Reason: fmt.Errorf("archive cataloged from part of its contents: extraction stopped at the disk limit"), }}) } - // expose the nesting chain to the sub-pipeline and deeper recursion, so catalogers can reconstruct - // nesting-aware identity such as the java cataloger's colon-delimited virtual paths - trav := &archive.Traversal{ - Location: archiveLoc, - VirtualPath: archivePath, - Digests: extracted.Digests, + traversal := &archive.Traversal{Location: location, Digests: extracted.Digests} + ctx = archive.WithTraversal(ctx, traversal) + + c.progress.Increment() + scratch := c.runSubPipeline(ctx, extracted.Resolver, location.Coordinates) + mergeArchiveResults(location.Coordinates, scratch, builder) + c.progress.AtomicStage.Set(fmt.Sprintf("%s archives (%s)", humanize.Comma(c.progress.Current()), archivePath)) + + // self time stops before descending, so a containing archive is not charged for what it contains + if took := time.Since(started); took > c.slowest { + c.slowest, c.slowestPath = took, archivePath } - subCtx := archive.WithTraversal(ctx, trav) - c.entering() - - scratch := c.runSubPipeline(subCtx, extracted.Resolver, archiveLoc.Coordinates) - - mergeArchiveResults(archiveLoc.Coordinates, scratch, builder) - - c.left(trav) - c.recordArchiveTime(trav.VirtualPath, time.Since(started)) - - return c.catalog(subCtx, extracted.Resolver, trav, depth+1, builder) + return c.catalog(ctx, extracted.Resolver, depth+1, builder) } -// entering counts this archive onto the walk's single progress row; left takes the row's stage back -// once the archive is done, naming it unless the whole walk has finished. Both are nil-safe. +// runSubPipeline runs every task against the resolver into a throwaway SBOM, so results can be +// attributed to the containing archive before merging into the shared SBOM. // -// The stage is reset because each sub-pipeline signs off with its own summary, which would otherwise -// stand while the next archive is extracted. The count of archives entered is always true. -func (c *archiveCataloger) entering() { - if c.prog == nil { - return - } - c.prog.Increment() -} - -func (c *archiveCataloger) left(traversal *archive.Traversal) { - if c.prog == nil { - return - } - if traversal == nil { - c.prog.AtomicStage.Set(fmt.Sprintf("%s archives", humanize.Comma(c.prog.Current()))) - } else { - c.prog.AtomicStage.Set(fmt.Sprintf("%s archives (%s)", humanize.Comma(c.prog.Current()), traversal.VirtualPath)) - } -} - -func accessPath(loc file.Location) string { - if loc.AccessPath != "" { - return loc.AccessPath - } - return loc.RealPath -} - -// runSubPipeline runs every cataloger task against the given resolver into a throwaway SBOM, so -// results can be attributed to the containing archive before merging into the shared SBOM. -// -// Failures are recorded as RunTask records them at the top level (executor.go), so they land in the -// SBOM's unknowns rather than only the logs: errors carrying coordinates keep them, anything left is -// attributed to archiveCoord. A failure here never fails the archive or the scan. -func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Resolver, archiveCoord file.Coordinates) *sbom.SBOM { - scratch := &sbom.SBOM{ - Artifacts: sbom.Artifacts{ - Packages: pkg.NewCollection(), - FileMetadata: map[file.Coordinates]file.Metadata{}, - FileDigests: map[file.Coordinates][]file.Digest{}, - FileContents: map[file.Coordinates]string{}, - FileLicenses: map[file.Coordinates][]file.License{}, - Executables: map[file.Coordinates]file.Executable{}, - Unknowns: map[file.Coordinates][]string{}, - }, - } +// Failures are recorded as RunTask records them at the top level: errors carrying coordinates keep +// them, anything else is attributed to the archive. A failure never fails the archive or the scan. +func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Resolver, archiveCoordinates file.Coordinates) *sbom.SBOM { + scratch := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} scratchBuilder := sbomsync.NewBuilder(scratch) for _, t := range c.subPipeline { err := runTaskSafely(ctx, t, resolver, scratchBuilder) @@ -321,47 +212,37 @@ func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Res unknowns, remaining := unknown.ExtractCoordinateErrors(err) if remaining != nil { - unknowns = append(unknowns, unknown.CoordinateError{Coordinates: archiveCoord, Reason: remaining}) + unknowns = append(unknowns, unknown.CoordinateError{Coordinates: archiveCoordinates, Reason: remaining}) } appendUnknowns(scratchBuilder, t.Name(), unknowns) } return scratch } -// archiveCandidate is one file the walk will try to open, and how it came to be a candidate. +// archiveCandidate is one file the walk will try to open. type archiveCandidate struct { location file.Location - // sniffedAsArchive is true when the file's own content types it as an archive. Such a file that - // cannot be extracted is a broken archive and is reported; a candidate here only because its head - // might hide one is passed over silently. + // sniffedAsArchive is true when the file's content types it as an archive. Such a file that cannot + // be extracted is a broken archive and is reported; a file that is a candidate only by name is + // passed over silently. sniffedAsArchive bool } -// zipArchiveGlobs find files worth a second look: those named like zip archives whose content did not -// sniff as one, such as a Spring Boot executable jar - a launcher stub with a zip appended, which -// sniffs as a shell script. +// zipArchiveGlobs find files named like zip archives whose content did not sniff as one, such as a +// Spring Boot executable jar: a launcher script with a zip appended. The name only decides what gets +// opened; the end-of-central-directory record decides whether it is an archive. // -// Not a detection rule: the end-of-central-directory record settles whether a candidate is an -// archive, so a `.jar` holding prose is still refused. The name decides only what gets opened, which -// is why a self-extracting archive with no telling extension is not found. -// -// A glob rather than a MIME lookup because asking for every file sniffed as executable, script or -// unrecognized returns thousands of locations that are all discarded, which tripled the scan on a -// stock ruby image. One pattern per extension because brace alternation matched nothing. +// Globs rather than a MIME lookup, because asking for every executable, script and unrecognized file +// returned thousands of locations that were all discarded. var zipArchiveGlobs = []string{ "**/*.jar", "**/*.war", "**/*.ear", "**/*.par", "**/*.sar", "**/*.nar", "**/*.kar", "**/*.hpi", "**/*.jpi", "**/*.far", "**/*.rar", "**/*.zip", "**/*.apk", "**/*.aar", "**/*.egg", "**/*.whl", "**/*.lpkg", "**/*.zap", "**/*.exe", } -// discoverArchives returns every location worth opening: files that are archives by their own -// content, plus files named like zip archives whose head says script or executable but whose tail may -// still be a whole zip. The second group is only opened; the extractor refuses anything with no -// end-of-central-directory record. -// -// No exclusion filter here: exclusion patterns are applied where each filesystem is indexed, so an -// excluded archive is not in the resolver to begin with. +// discoverArchives returns every file worth opening: those whose content is an archive, plus those +// named like a zip whose tail may still hold one. func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCandidate { archives, err := resolver.FilesByMIMEType(mimetype.ArchiveMIMETypeSet.List()...) if err != nil { @@ -369,23 +250,19 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan return nil } - candidates := make([]archiveCandidate, 0, len(archives)) + var candidates []archiveCandidate + seen := map[file.Coordinates]struct{}{} for _, loc := range archives { + seen[loc.Coordinates] = struct{}{} candidates = append(candidates, archiveCandidate{location: loc, sniffedAsArchive: true}) } - appended, err := resolver.FilesByGlob(zipArchiveGlobs...) + namedLikeZips, err := resolver.FilesByGlob(zipArchiveGlobs...) if err != nil { - // the archives found above are still worth walking log.WithFields("error", err).Debug("unable to list files that may carry an appended archive") return candidates } - seen := make(map[file.Coordinates]struct{}, len(candidates)) - for _, c := range candidates { - seen[c.location.Coordinates] = struct{}{} - } - for _, loc := range appended { - // one file reached by both lookups must be opened once, or every package it holds is doubled + for _, loc := range namedLikeZips { if _, dup := seen[loc.Coordinates]; dup { continue } @@ -396,78 +273,52 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan } // mergeArchiveResults copies the throwaway SBOM's packages, files and relationships into the shared -// SBOM, adding a file-level CONTAINS relationship from the archive to everything found inside it. -func mergeArchiveResults(archiveCoord file.Coordinates, scratch *sbom.SBOM, builder sbomsync.Builder) { +// SBOM, adding a CONTAINS relationship from the archive to everything found inside it. +func mergeArchiveResults(archiveCoordinates file.Coordinates, scratch *sbom.SBOM, builder sbomsync.Builder) { pkgs := scratch.Artifacts.Packages.Sorted() if len(pkgs) > 0 { builder.AddPackages(pkgs...) } - accessor, _ := builder.(sbomsync.Accessor) - if accessor != nil { + if accessor, ok := builder.(sbomsync.Accessor); ok { accessor.WriteToSBOM(func(s *sbom.SBOM) { - mergeFileArtifacts(s, scratch) + mergeFileArtifacts(&s.Artifacts, &scratch.Artifacts) }) } var rels []artifact.Relationship for _, p := range pkgs { - rels = append(rels, artifact.Relationship{ - From: archiveCoord, - To: p, - Type: artifact.ContainsRelationship, - }) + rels = append(rels, artifact.Relationship{From: archiveCoordinates, To: p, Type: artifact.ContainsRelationship}) } - for _, coord := range scratch.AllCoordinates() { - rels = append(rels, artifact.Relationship{ - From: archiveCoord, - To: coord, - Type: artifact.ContainsRelationship, - }) + for _, coordinates := range scratch.AllCoordinates() { + rels = append(rels, artifact.Relationship{From: archiveCoordinates, To: coordinates, Type: artifact.ContainsRelationship}) } - // preserve relationships discovered within the archive (e.g. package evident-by file) rels = append(rels, scratch.Relationships...) - if len(rels) > 0 { builder.AddRelationships(rels...) } } -func mergeFileArtifacts(dst, src *sbom.SBOM) { - if dst.Artifacts.FileMetadata == nil { - dst.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} +func mergeFileArtifacts(dst, src *sbom.Artifacts) { + mergeInto(&dst.FileMetadata, src.FileMetadata) + mergeInto(&dst.FileDigests, src.FileDigests) + mergeInto(&dst.FileContents, src.FileContents) + mergeInto(&dst.FileLicenses, src.FileLicenses) + mergeInto(&dst.Executables, src.Executables) + if len(src.Unknowns) > 0 && dst.Unknowns == nil { + dst.Unknowns = map[file.Coordinates][]string{} } - for k, v := range src.Artifacts.FileMetadata { - dst.Artifacts.FileMetadata[k] = v - } - if dst.Artifacts.FileDigests == nil { - dst.Artifacts.FileDigests = map[file.Coordinates][]file.Digest{} - } - for k, v := range src.Artifacts.FileDigests { - dst.Artifacts.FileDigests[k] = v - } - if dst.Artifacts.FileContents == nil { - dst.Artifacts.FileContents = map[file.Coordinates]string{} - } - for k, v := range src.Artifacts.FileContents { - dst.Artifacts.FileContents[k] = v - } - if dst.Artifacts.FileLicenses == nil { - dst.Artifacts.FileLicenses = map[file.Coordinates][]file.License{} - } - for k, v := range src.Artifacts.FileLicenses { - dst.Artifacts.FileLicenses[k] = v - } - if dst.Artifacts.Executables == nil { - dst.Artifacts.Executables = map[file.Coordinates]file.Executable{} - } - for k, v := range src.Artifacts.Executables { - dst.Artifacts.Executables[k] = v - } - if dst.Artifacts.Unknowns == nil { - dst.Artifacts.Unknowns = map[file.Coordinates][]string{} - } - for k, v := range src.Artifacts.Unknowns { - dst.Artifacts.Unknowns[k] = append(dst.Artifacts.Unknowns[k], v...) + for coordinates, reasons := range src.Unknowns { + dst.Unknowns[coordinates] = append(dst.Unknowns[coordinates], reasons...) } } + +func mergeInto[K comparable, V any](dst *map[K]V, src map[K]V) { + if len(src) == 0 { + return + } + if *dst == nil { + *dst = make(map[K]V, len(src)) + } + maps.Copy(*dst, src) +} diff --git a/internal/task/archive_tasks_test.go b/internal/task/archive_tasks_test.go index 39a46927d..83de81351 100644 --- a/internal/task/archive_tasks_test.go +++ b/internal/task/archive_tasks_test.go @@ -11,7 +11,6 @@ import ( "io" "io/fs" "os" - "path" "path/filepath" "sort" "strings" @@ -39,45 +38,63 @@ import ( "github.com/anchore/syft/syft/sbom" ) -// dirTestResolver is a minimal file.Resolver over a directory: FilesByMIMEType reports any zip-family -// file (ignoring the MIME arguments) and FileContentsByLocation reads from disk. Other methods are -// unimplemented. +// dirTestResolver is a minimal file.Resolver over a directory. FilesByMIMEType sniffs each file's +// content as a real resolver does; FileContentsByLocation opens the file, which is random-access. type dirTestResolver struct { file.Resolver dir string fsid string } -// storeTestResolver is a minimal file.Resolver over one archive's entry store, standing in for the -// real one so these tests need not import the fileresolver package. -// -// FilesByMIMEType reports any zip-family entry (ignoring the MIME arguments) and -// FileContentsByLocation opens it from the store. The reader carries the marker the extraction path -// looks for, so a nested archive is exercised through the read-in-place route rather than a copy. -type storeTestResolver struct { - file.Resolver - store *archive.EntryStore - entries map[string]*archive.Entry - fsid string - archivePath string +// streamingDirResolver is dirTestResolver handing out plain streams, so an archive's own bytes must be +// held in memory or written to disk rather than read in place. +type streamingDirResolver struct { + dirTestResolver } -// nopCloserAt gives a store reader the Close the resolver interface asks for; the store owns the -// bytes, so closing a reader over them must close nothing. -type nopCloserAt struct { - archive.ReaderAtSeeker +func (d streamingDirResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { + f, err := d.dirTestResolver.FileContentsByLocation(loc) + if err != nil { + return nil, err + } + return struct { + io.Reader + io.Closer + }{f, f}, nil } -type randomAccessReadCloser interface { - io.ReadCloser - io.ReaderAt - io.Seeker +func newTestTask(t *testing.T, cfg cataloging.ArchiveSearchConfig, subPipeline ...Task) Task { + t.Helper() + tsk := NewArchiveCatalogerTask(cfg, subPipeline, nil) + require.NotNil(t, tsk) + return tsk } -// testOverflowEntry carries the marker method saying "already random access on a file this scan -// wrote", which is how a nested archive gets read where it lies. -type testOverflowEntry struct { - randomAccessReadCloser +func newTestSBOM() *sbom.SBOM { + return &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} +} + +func countingTask(ran *int) Task { + return NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { + *ran++ + return nil + }) +} + +// capturingTask records the virtual path of every archive the sub-pipeline is run against and how many +// files its resolver holds. +func capturingTask(t *testing.T, seen *[]string, fileCounts *map[string]int) Task { + return NewTask("capture", func(ctx context.Context, r file.Resolver, _ sbomsync.Builder) error { + trav := archive.TraversalFromContext(ctx) + require.NotNil(t, trav) + *seen = append(*seen, archive.VirtualPath(trav.Location)) + if fileCounts != nil { + locs, err := r.FilesByGlob("**") + require.NoError(t, err) + (*fileCounts)[archive.VirtualPath(trav.Location)] = len(locs) + } + return nil + }) } func Test_archiveCataloger_traversalThreading(t *testing.T) { @@ -92,40 +109,21 @@ func Test_archiveCataloger_traversalThreading(t *testing.T) { rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) - var captured []*archive.Traversal - captureTask := NewTask("capture-traversal", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { - captured = append(captured, archive.TraversalFromContext(ctx)) - return nil - }) + var seen []string + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3), capturingTask(t, &seen, nil)) - cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))) - newResolver := newStoreTestResolver - - tsk := NewArchiveCatalogerTask(cfg, []Task{captureTask}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) - require.NoError(t, err) - - var chains []string - for _, trav := range captured { - require.NotNil(t, trav) - chains = append(chains, trav.VirtualPath) - } // java archives are traversed like any other archive when the task is enabled assert.ElementsMatch(t, []string{ "/outer.zip", "/outer.zip:lib/inner.jar", "/outer.zip:nested/inner.zip", - }, chains) + }, seen) } func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) { - // a limit is a truncation, not a failure: the sub-pipeline still runs over what was written, so - // packages found before the limit survive. Discarding the whole archive silently lost every package - // in an oversized one. + // reaching a limit is a truncation, not a failure: the sub-pipeline still runs over what was stored big := bytes.Repeat([]byte("x"), 4096) outerZip := makeZip(t, map[string][]byte{ "a/small.txt": []byte("small"), @@ -135,32 +133,23 @@ func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) { rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) - var ran int - countTask := NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { - ran++ - return nil - }) - - // nothing is held in memory, so every entry is charged to disk as it lands: the archive's own bytes - // spill first, then small.txt, and big.txt is refused part way through + // nothing is held in memory, so each entry is charged to disk: small.txt lands, big.txt is refused cfg := cataloging.DefaultArchiveSearchConfig(). WithMaxDepth(1). WithMaxMemoryBytes(0). - WithMaxDiskBytes(int64(len(outerZip)) + 1500) + WithMaxDiskBytes(6000) - newResolver := newStoreTestResolver + var seen []string + fileCounts := map[string]int{} + tsk := newTestTask(t, cfg, capturingTask(t, &seen, &fileCounts)) - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + s := newTestSBOM() err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) require.NoError(t, err, "a truncated extraction must not fail the scan") - assert.Equal(t, 1, ran, "the sub-pipeline must run over the partial contents") + assert.Equal(t, []string{"/outer.zip"}, seen, "the sub-pipeline must run over the partial contents") + assert.Equal(t, 1, fileCounts["/outer.zip"], "only the entry stored before the limit is visible") - // and the SBOM records, at the archive's coordinates, that it was read in part: a scan that saw only - // some of an archive is not the same SBOM as one that saw all of it reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] require.True(t, ok, "the truncation must be recorded against the archive; got %v", s.Artifacts.Unknowns) require.Len(t, reasons, 1) @@ -169,13 +158,9 @@ func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) { } func Test_archiveCataloger_failureIsRecordedAndSkipsSubPipeline(t *testing.T) { - // a genuine failure is not a truncation: nothing usable was produced, so the sub-pipeline must not - // run over an empty tree, and the failure must reach the SBOM rather than only the logs rootDir := t.TempDir() - // a zip whose entry data has been overwritten: the header and end-of-central-directory record are - // intact, so it is detected as an archive and fails when the entry is read. Prose named `.zip` is a - // different case - not detected as an archive at all - see - // Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError. + // entry data overwritten but header and end-of-central-directory intact: detected as an archive, + // fails when the entry is read corrupt := makeZip(t, map[string][]byte{"data.bin": incompressible(4096)}) require.Greater(t, len(corrupt), 1024) for i := 100; i < 600; i++ { @@ -183,86 +168,61 @@ func Test_archiveCataloger_failureIsRecordedAndSkipsSubPipeline(t *testing.T) { } require.NoError(t, os.WriteFile(filepath.Join(rootDir, "corrupt.zip"), corrupt, 0o600)) - // a second, healthy archive proves the failure does not stop the walk goodZip := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")}) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "good.zip"), goodZip, 0o600)) var ran int - countTask := NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { - ran++ - return nil - }) + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), countingTask(&ran)) - cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1) - newResolver := newStoreTestResolver + err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())) - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) - - // the failure must be a CoordinateError: that is what executor.go's - // unknown.ExtractCoordinateErrors pulls out into sbom.Artifacts.Unknowns. A plain error would - // propagate as a scan failure instead of being reported against the archive + // a coordinate error is what executor.go pulls out into sbom.Artifacts.Unknowns; a plain error would + // fail the scan instead require.Error(t, err) coordErrs, remaining := unknown.ExtractCoordinateErrors(err) assert.NoError(t, remaining, "nothing may escape as a non-coordinate error") require.Len(t, coordErrs, 1) assert.Equal(t, "/corrupt.zip", coordErrs[0].Coordinates.RealPath) - // and the healthy archive was still cataloged: one bad archive must not stop the walk assert.Equal(t, 1, ran, "the sub-pipeline must still run for the archive that extracted cleanly") } func Test_NewArchiveCatalogerTask_gating(t *testing.T) { - newResolver := newStoreTestResolver someTask := NewTask("noop", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { return nil }) tests := []struct { name string cfg cataloging.ArchiveSearchConfig subPipeline []Task - resolver archive.StoreResolverFactory wantTask bool }{ { name: "disabled at depth 0", cfg: cataloging.DefaultArchiveSearchConfig(), subPipeline: []Task{someTask}, - resolver: newResolver, }, { name: "no sub-pipeline", cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), subPipeline: nil, - resolver: newResolver, - }, - { - name: "no resolver factory", - cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), - subPipeline: []Task{someTask}, - resolver: nil, }, { name: "enabled", cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), subPipeline: []Task{someTask}, - resolver: newResolver, wantTask: true, }, { name: "enabled at negative depth", cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(-1), subPipeline: []Task{someTask}, - resolver: newResolver, wantTask: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := NewArchiveCatalogerTask(tt.cfg, tt.subPipeline, tt.resolver, nil) + got := NewArchiveCatalogerTask(tt.cfg, tt.subPipeline, nil) if tt.wantTask { require.NotNil(t, got) assert.Equal(t, ArchiveCatalogerTaskName, got.Name()) @@ -274,30 +234,17 @@ func Test_NewArchiveCatalogerTask_gating(t *testing.T) { } func Test_archiveCataloger_depthBoundIsExact(t *testing.T) { - // an archive in the scan source is depth 1, so MaxDepth 1 must catalog it and not descend innerZip := makeZip(t, map[string][]byte{"deep.txt": []byte("deep")}) outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) - newResolver := newStoreTestResolver - run := func(t *testing.T, depth int) []string { t.Helper() var seen []string - captureTask := NewTask("capture", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { - trav := archive.TraversalFromContext(ctx) - require.NotNil(t, trav) - seen = append(seen, trav.VirtualPath) - return nil - }) - - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth), []Task{captureTask}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth), capturingTask(t, &seen, nil)) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())), "reaching the depth bound must not be an error") return seen } @@ -307,10 +254,7 @@ func Test_archiveCataloger_depthBoundIsExact(t *testing.T) { } func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) { - // the task drives recursion with its own depth-bounded walk, so a copy of itself in the sub-pipeline - // would process every nesting level twice. The depth bound still terminates, so the symptom is - // duplicated packages rather than a hang - the kind of failure nobody notices in a large SBOM. - newResolver := newStoreTestResolver + // a copy of itself in the sub-pipeline would process every nesting level twice cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2) innerZip := makeZip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) @@ -319,36 +263,25 @@ func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) var ran int - countTask := NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { - ran++ - return nil - }) + countTask := countingTask(&ran) - // a sub-pipeline that (incorrectly) carries an archive cataloger task alongside real work - poisoned := []Task{countTask, NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil)} + poisoned := []Task{countTask, NewArchiveCatalogerTask(cfg, []Task{countTask}, nil)} require.NotNil(t, poisoned[1]) - tsk := NewArchiveCatalogerTask(cfg, poisoned, newResolver, nil) - require.NotNil(t, tsk) + tsk := newTestTask(t, cfg, poisoned...) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))) - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) - - // two archives, one sub-pipeline task each: the nested archive task was dropped, so no level is - // processed twice - assert.Equal(t, 2, ran) + assert.Equal(t, 2, ran, "two archives, one sub-pipeline task each") t.Run("a sub-pipeline of nothing but itself yields no task", func(t *testing.T) { - only := []Task{NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil)} - assert.Nil(t, NewArchiveCatalogerTask(cfg, only, newResolver, nil)) + only := []Task{NewArchiveCatalogerTask(cfg, []Task{countTask}, nil)} + assert.Nil(t, NewArchiveCatalogerTask(cfg, only, nil)) }) } func Test_archiveCataloger_chainStartsAtTheArchivesOwnFileSystemID(t *testing.T) { - // covers syft/archive-content-identity#nested-filesystem-id-chain, scenario "image source chain - // extends the layer digest": extracted files keep the layer digest as their FileSystemID at every - // level, while the nesting chain rides on the traversal's VirtualPath and the coordinate's - // ArchivePath + // extracted files keep the layer digest as their FileSystemID at every level; the nesting chain + // rides on the traversal's VirtualPath and the coordinate's ArchivePath innerZip := makeZip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip}) @@ -357,37 +290,24 @@ func Test_archiveCataloger_chainStartsAtTheArchivesOwnFileSystemID(t *testing.T) const layerDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111" - var chains []string - var fsids []string + var chains, fsids []string captureTask := NewTask("capture-fsid", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { trav := archive.TraversalFromContext(ctx) require.NotNil(t, trav) - chains = append(chains, trav.VirtualPath) + chains = append(chains, archive.VirtualPath(trav.Location)) fsids = append(fsids, trav.Location.FileSystemID) return nil }) - newResolver := newStoreTestResolver + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), captureTask) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir, fsid: layerDigest}, sbomsync.NewBuilder(newTestSBOM()))) - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), []Task{captureTask}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir, fsid: layerDigest}, sbomsync.NewBuilder(s))) - - // the layer digest is preserved unchanged at every level as the filesystem the archive lives on assert.Equal(t, []string{layerDigest, layerDigest}, fsids) - - // exact strings: the archive path is serialized into the SBOM, so its shape is output format - assert.ElementsMatch(t, []string{ - "/outer.zip", - "/outer.zip:nested/inner.zip", - }, chains) + assert.ElementsMatch(t, []string{"/outer.zip", "/outer.zip:nested/inner.zip"}, chains) } -// twoLayerResolver reports the same archive path twice under two different filesystem ids, each with -// different content - what an all-layers image scan yields for an archive rewritten in a later layer. -// FilesByMIMEType dedupes on the file reference rather than the path, so both survive. +// twoLayerResolver reports the same archive path under two filesystem ids with different content, as an +// all-layers image scan does for an archive rewritten in a later layer. type twoLayerResolver struct { file.Resolver path string @@ -395,8 +315,6 @@ type twoLayerResolver struct { } func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T) { - // the collision the chain exists to prevent: two layers holding /app/bundle.zip would otherwise - // compose the same identifier and overwrite each other in the coordinate-keyed file tables resolver := twoLayerResolver{ path: "/app/bundle.zip", contentBy: map[string][]byte{ @@ -406,8 +324,6 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T } var chains []string - // record a file artifact per location the way a file cataloger does, so the merge into the shared - // SBOM exercises the coordinate-keyed tables rather than only the traversal fileTask := NewTask("record-files", func(ctx context.Context, r file.Resolver, builder sbomsync.Builder) error { trav := archive.TraversalFromContext(ctx) require.NotNil(t, trav) @@ -415,19 +331,17 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T locs, err := r.FilesByGlob("**") require.NoError(t, err) - accessor := builder.(sbomsync.Accessor) - accessor.WriteToSBOM(func(s *sbom.SBOM) { - for _, loc := range locs { - s.Artifacts.FileMetadata[loc.Coordinates] = file.Metadata{Path: loc.RealPath} - } + metadata := map[file.Coordinates]file.Metadata{} + for _, loc := range locs { + metadata[loc.Coordinates] = file.Metadata{Path: loc.RealPath} + } + builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) { + s.Artifacts.FileMetadata = metadata }) return nil }) - newResolver := newStoreTestResolver - - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{fileTask}, newResolver, nil) - require.NotNil(t, tsk) + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), fileTask) s := &sbom.SBOM{Artifacts: sbom.Artifacts{ Packages: pkg.NewCollection(), @@ -435,29 +349,20 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T }} require.NoError(t, tsk.Execute(context.Background(), resolver, sbomsync.NewBuilder(s))) - assert.ElementsMatch(t, []string{"layer-one", "layer-two"}, chains, - "the two archives keep the distinct filesystem ids of the layers they were found in") - - // asserted as a count: the tables are keyed by Coordinates, so a collision is a silently missing - // entry rather than an error. The two stay distinct by FileSystemID (the layer); their ArchivePath - // is identical here. + assert.ElementsMatch(t, []string{"layer-one", "layer-two"}, chains) + // the tables are keyed by Coordinates, so a collision is a silently missing entry assert.Len(t, s.Artifacts.FileMetadata, 2) } func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) { - // syft/nested-archive-cataloging#archive-failure-is-not-fatal wants a catalog failure inside an - // archive "visible in output rather than only in logs". runSubPipeline bypasses RunTask, where the - // coordinate-error to unknown conversion lives, so it does that conversion itself. + // runSubPipeline bypasses RunTask, where the coordinate-error to unknown conversion lives outerZip := makeZip(t, map[string][]byte{"lib/broken.json": []byte("{")}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) - newResolver := newStoreTestResolver cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1) t.Run("a coordinate error keeps the coordinates the cataloger named", func(t *testing.T) { - // a failure attributed to a file the cataloger was reading is more precise than the archive, so - // it must not be flattened onto the archive's coordinates failing := NewTask("failing-cataloger", func(_ context.Context, r file.Resolver, _ sbomsync.Builder) error { locs, err := r.FilesByGlob("**") require.NoError(t, err) @@ -469,16 +374,14 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) return errs }) - tsk := NewArchiveCatalogerTask(cfg, []Task{failing}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + tsk := newTestTask(t, cfg, failing) + s := newTestSBOM() require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), "a cataloger failing inside an archive must not fail the archive or the scan") require.Len(t, s.Artifacts.Unknowns, 1) for coords, reasons := range s.Artifacts.Unknowns { - assert.Equal(t, "/lib/broken.json", coords.RealPath, "the inner coordinates must survive the merge") + assert.Equal(t, "lib/broken.json", coords.RealPath, "the inner coordinates must survive the merge") assert.Equal(t, "/outer.zip", coords.ArchivePath, "and must be addressable per archive") assert.Equal(t, []string{"failing-cataloger: unable to parse"}, reasons) } @@ -489,10 +392,8 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) return errors.New("cataloger blew up") }) - tsk := NewArchiveCatalogerTask(cfg, []Task{failing}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + tsk := newTestTask(t, cfg, failing) + s := newTestSBOM() require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) require.Len(t, s.Artifacts.Unknowns, 1) @@ -502,15 +403,12 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) }) t.Run("a panicking cataloger is recovered and recorded", func(t *testing.T) { - // runTaskSafely converts the panic to an error, which now has somewhere to go panicking := NewTask("panicking-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { panic("nope") }) - tsk := NewArchiveCatalogerTask(cfg, []Task{panicking}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + tsk := newTestTask(t, cfg, panicking) + s := newTestSBOM() require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), "a panic inside an archive must not fail the scan") @@ -522,15 +420,16 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) } func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) { - // mergeFileArtifacts appends to the shared SBOM's unknowns, so the ones a sub-cataloger wrote into - // the scratch SBOM must survive alongside the one the failure produced outerZip := makeZip(t, map[string][]byte{"lib/thing.json": []byte("{}")}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) - preexisting := file.Coordinates{RealPath: "/lib/thing.json", ArchivePath: "outer.zip"} + preexisting := file.Coordinates{RealPath: "lib/thing.json", ArchivePath: "/outer.zip"} writer := NewTask("scratch-writer", func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error { builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) { + if s.Artifacts.Unknowns == nil { + s.Artifacts.Unknowns = map[file.Coordinates][]string{} + } s.Artifacts.Unknowns[preexisting] = append(s.Artifacts.Unknowns[preexisting], "recorded by the cataloger itself") }) return nil @@ -539,11 +438,8 @@ func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) return errors.New("boom") }) - newResolver := newStoreTestResolver - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{writer, failing}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), writer, failing) + s := newTestSBOM() require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) assert.Equal(t, []string{"recorded by the cataloger itself"}, s.Artifacts.Unknowns[preexisting]) @@ -551,8 +447,8 @@ func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) } func Test_archiveCataloger_manySmallArchivesAreEachCataloged(t *testing.T) { - // the case a monotonic counter wrongly rejected: every archive is released before the next begins, - // so a scan whose archives sum well past the disk limit never holds more than one at a time + // every archive is released before the next begins, so a scan whose archives sum well past a limit + // never holds more than one at a time entry := bytes.Repeat([]byte("z"), 400) entries := map[string][]byte{} for i := range 5 { @@ -565,32 +461,27 @@ func Test_archiveCataloger_manySmallArchivesAreEachCataloged(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, entries), 0o600)) } - var ran int - countTask := NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { - ran++ - return nil - }) - - // one archive extracts 2000 bytes, so the limit holds one comfortably and the five together would be - // four times over a counter that never fell + // room for one archive's entries and their index cost, not for two cfg := cataloging.DefaultArchiveSearchConfig(). WithMaxDepth(1). - WithMaxDiskBytes(4000) + WithMaxMemoryBytes(0). + WithMaxDiskBytes(20000) - newResolver := newStoreTestResolver + var seen []string + fileCounts := map[string]int{} + tsk := newTestTask(t, cfg, capturingTask(t, &seen, &fileCounts)) - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + s := newTestSBOM() require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) - assert.Equal(t, len(names), ran, "every archive must be cataloged: no two were ever held at once") + assert.Len(t, seen, len(names), "every archive must be cataloged: no two were ever held at once") + for _, archivePath := range seen { + assert.Equal(t, 5, fileCounts[archivePath], "%s must extract in full", archivePath) + } + assert.Empty(t, s.Artifacts.Unknowns) } func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) { - // a second archive the same size as the first is cataloged in full, which a counter would have - // refused, and both limits are back at zero once the walk unwinds entry := bytes.Repeat([]byte("z"), 400) entries := map[string][]byte{} for i := range 5 { @@ -602,96 +493,46 @@ func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, entries), 0o600)) } - // each entry costs a header block plus a padded data block, plus the tar's end-of-archive marker: - // five entries of 400 bytes is 11 blocks. Room for one archive's tar, not two at once. - cfg := cataloging.DefaultArchiveSearchConfig(). - WithMaxDepth(1). - WithMaxDiskBytes(15 * 512) + // room in memory for one archive's entries and their index cost, not for two at once + limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: 20000, MaxDiskBytes: 0}) - limiter := archive.NewLimiter(archive.DefaultLimits(cfg)) - - // measured while the archive is still held: the resolver is built straight after extraction, before - // anything is cleaned up + // sampled while the archive is still held var extracted []int var peakMemory int64 - newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - resolver, indexed, err := newStoreTestResolver(store, overflow) - extracted = append(extracted, len(store.Entries())) + sample := NewTask("sample", func(_ context.Context, r file.Resolver, _ sbomsync.Builder) error { + locs, err := r.FilesByGlob("**") + require.NoError(t, err) + extracted = append(extracted, len(locs)) if mem, _ := limiter.InUse(); mem > peakMemory { peakMemory = mem } - return resolver, indexed, err - } + return nil + }) - c := &archiveCataloger{ - cfg: cfg, - subPipeline: []Task{NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil })}, - extractors: archive.DefaultExtractors(), - limiter: limiter, - newStoreResolver: newResolver, - } - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, nil, 0, sbomsync.NewBuilder(s))) + c := newTestCataloger(1, limiter, sample) + require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM()))) assert.Equal(t, []int{5, 5}, extracted, "both archives must extract in full: the first is released before the second is admitted") - assert.Positive(t, peakMemory, - "an archive within the memory limit is held in memory, so memory in use must rise") + assert.Positive(t, peakMemory, "entries within the memory limit are held in memory") mem, disk := limiter.InUse() assert.Zero(t, mem, "the memory limit must fall back to nothing once the walk unwinds") assert.Zero(t, disk, "and so must the disk limit") } -func Test_archiveCataloger_archiveExceedingTheDiskLimitIsSkipped(t *testing.T) { - // the disk limit is terminal, so an archive whose content will not fit is skipped rather than waited - // for, and a sibling that does fit is cataloged in full - oversized := makeZip(t, map[string][]byte{"payload.bin": incompressible(8000)}) - small := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")}) - - rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "small.zip"), small, 0o600)) - - // room for the small archive's own bytes and the tar its one entry is written into - a header block, - // a padded data block and an end-of-archive marker - and nowhere near the oversized one - diskLimit := int64(len(small)) + 8*512 - require.Greater(t, int64(len(oversized)), diskLimit, "the fixture must actually exceed the limit") - - var sawArchivePaths []string - newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - sawArchivePaths = append(sawArchivePaths, overflow.ArchivePath) - return newStoreTestResolver(store, overflow) +func newTestCataloger(maxDepth int, limiter *archive.Limiter, subPipeline ...Task) *archiveCataloger { + return &archiveCataloger{ + maxDepth: maxDepth, + subPipeline: subPipeline, + limiter: limiter, + progress: bus.StartCatalogerTask(context.Background(), archiveCatalogerProgressInfo(), -1, ""), } - - cfg := cataloging.DefaultArchiveSearchConfig(). - WithMaxDepth(1). - WithMaxMemoryBytes(0). // every archive overflows, so its own bytes are charged to disk - WithMaxDiskBytes(diskLimit) - - var ran int - countTask := NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { - ran++ - return nil - }) - - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), - "reaching a limit must not fail the scan") - - assert.Equal(t, []string{"/small.zip"}, sawArchivePaths, - "the oversized archive is skipped and the sibling is still cataloged") - assert.Equal(t, 1, ran) } -func Test_archiveCataloger_zeroDiskLimitSkipsWhatWillNotFitInMemory(t *testing.T) { - // with a zero disk limit there is nowhere to overflow, so an archive is cataloged exactly when it - // and its entries fit in memory. The memory limit below admits the small archive and not the - // oversized one; the scan still succeeds and the skip names the archive that could not be placed. +func Test_archiveCataloger_archiveWhoseOwnBytesExceedTheDiskLimitIsSkipped(t *testing.T) { + // a streamed archive's own bytes must be placed before it can be opened; when they do not fit on + // disk the archive is skipped rather than waited for, and a sibling that fits is cataloged in full oversized := makeZip(t, map[string][]byte{"payload.bin": incompressible(8000)}) small := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")}) @@ -699,121 +540,129 @@ func Test_archiveCataloger_zeroDiskLimitSkipsWhatWillNotFitInMemory(t *testing.T require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "small.zip"), small, 0o600)) - // room for the small archive and its entries, and not for the oversized one - memoryLimit := int64(len(small) + 1000) - - var sawArchivePaths []string - newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - sawArchivePaths = append(sawArchivePaths, overflow.ArchivePath) - return newStoreTestResolver(store, overflow) - } + // room for the small archive's bytes, its one entry and that entry's index cost + diskLimit := int64(len(small)) + 3000 + require.Greater(t, int64(len(oversized)), diskLimit, "the fixture must actually exceed the limit") cfg := cataloging.DefaultArchiveSearchConfig(). WithMaxDepth(1). - WithMaxMemoryBytes(memoryLimit). - WithMaxDiskBytes(0) // nowhere to overflow what memory will not hold + WithMaxMemoryBytes(0). + WithMaxDiskBytes(diskLimit) - var ran int - countTask := NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { - ran++ - return nil - }) + var seen []string + fileCounts := map[string]int{} + tsk := newTestTask(t, cfg, capturingTask(t, &seen, &fileCounts)) - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) - require.NotNil(t, tsk) + s := newTestSBOM() + require.NoError(t, tsk.Execute(context.Background(), streamingDirResolver{dirTestResolver{dir: rootDir}}, sbomsync.NewBuilder(s)), + "reaching a limit must not fail the scan") - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + assert.Equal(t, []string{"/small.zip"}, seen, "the oversized archive is skipped and the sibling is still cataloged") + assert.Equal(t, 1, fileCounts["/small.zip"]) + + reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/oversized.zip"}] + require.Len(t, reasons, 1, "the skip must be recorded against the archive; got %v", s.Artifacts.Unknowns) + assert.Contains(t, reasons[0], "archive skipped") +} + +func Test_archiveCataloger_zeroDiskLimitTruncatesWhatWillNotFitInMemory(t *testing.T) { + // with a zero disk limit there is nowhere to overflow, so an archive's entries are stored exactly + // while they fit in memory + oversized := makeZip(t, map[string][]byte{"payload.bin": incompressible(8000)}) + small := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")}) + + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "small.zip"), small, 0o600)) + + // room for the small archive's entry and its index cost, not for the oversized one's + cfg := cataloging.DefaultArchiveSearchConfig(). + WithMaxDepth(1). + WithMaxMemoryBytes(3000). + WithMaxDiskBytes(0) + + var seen []string + fileCounts := map[string]int{} + tsk := newTestTask(t, cfg, capturingTask(t, &seen, &fileCounts)) + + s := newTestSBOM() require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), "reaching the disk limit must not fail the scan") - assert.Equal(t, []string{"/small.zip"}, sawArchivePaths, - "the archive that fits in memory needs no disk at all and is still cataloged") - assert.Equal(t, 1, ran) + assert.ElementsMatch(t, []string{"/oversized.zip", "/small.zip"}, seen) + assert.Equal(t, 1, fileCounts["/small.zip"], "the archive that fits in memory needs no disk at all") + assert.Equal(t, 0, fileCounts["/oversized.zip"], "the entry that does not fit is refused") + + assert.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "/oversized.zip"}) + assert.NotContains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "/small.zip"}) } -func Test_archiveCataloger_bothLimitsZeroSkipsEveryArchiveAndSucceeds(t *testing.T) { - // the degenerate configuration where nothing is admitted anywhere: every archive is skipped, which - // is a well-defined success rather than a failure +func Test_archiveCataloger_bothLimitsZeroTruncatesEveryArchiveAndSucceeds(t *testing.T) { entries := map[string][]byte{"ok.txt": []byte("ok")} rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "one.zip"), makeZip(t, entries), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "two.zip"), makeZip(t, entries), 0o600)) - var sawArchivePaths []string - newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - sawArchivePaths = append(sawArchivePaths, overflow.ArchivePath) - return newStoreTestResolver(store, overflow) - } - cfg := cataloging.DefaultArchiveSearchConfig(). WithMaxDepth(1). WithMaxMemoryBytes(0). WithMaxDiskBytes(0) - var ran int - countTask := NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { - ran++ - return nil - }) + var seen []string + fileCounts := map[string]int{} + tsk := newTestTask(t, cfg, capturingTask(t, &seen, &fileCounts)) - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + s := newTestSBOM() require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), "both limits at zero is a real configuration, not a failure") - assert.Empty(t, sawArchivePaths, "no archive content can be admitted anywhere, so every archive is skipped") - assert.Zero(t, ran, "the sub-pipeline never runs since nothing was extracted") + assert.ElementsMatch(t, []string{"/one.zip", "/two.zip"}, seen) + for _, archivePath := range seen { + assert.Equal(t, 0, fileCounts[archivePath], "no entry can be admitted anywhere") + assert.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: archivePath}) + } + assert.Empty(t, s.Artifacts.Packages.Sorted()) } -func Test_archiveCataloger_nestedArchiveExceedingALimitIsSkippedNotBlocked(t *testing.T) { - // the walk descends before it unwinds, so a parent still holds its content while its children are - // cataloged and nothing is released while a child waits. Waiting for capacity would deadlock, so a - // child that does not fit is skipped and the scan finishes. +func Test_archiveCataloger_nestedArchiveExceedingALimitIsTruncatedNotBlocked(t *testing.T) { + // the walk descends before it unwinds, so nothing is released while a child waits; a child that does + // not fit is truncated and the scan finishes inner := makeZip(t, map[string][]byte{"payload.bin": incompressible(3000)}) outer := makeZip(t, map[string][]byte{"inner.zip": inner}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outer, 0o600)) - // enough for the outer archive's own bytes plus the inner.zip it extracts, and not enough for the - // inner archive's bytes on top of them - diskLimit := int64(len(outer)+len(inner)) + 500 - require.Greater(t, int64(len(inner)), int64(500), "the inner archive must not fit in the slack") - - var sawArchivePaths []string - newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - sawArchivePaths = append(sawArchivePaths, overflow.ArchivePath) - return newStoreTestResolver(store, overflow) - } + // enough for the outer archive's one entry and its index cost, plus the inner's index cost, and not + // the inner's payload on top + diskLimit := int64(len(inner)) + 2*2100 + 500 + require.Greater(t, 3000, 500, "the inner payload must not fit in the slack") cfg := cataloging.DefaultArchiveSearchConfig(). WithMaxDepth(2). - WithMaxMemoryBytes(0). // every archive overflows, so its own bytes are charged to disk + WithMaxMemoryBytes(0). WithMaxDiskBytes(diskLimit) - noop := NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil }) - tsk := NewArchiveCatalogerTask(cfg, []Task{noop}, newResolver, nil) - require.NotNil(t, tsk) + var seen []string + fileCounts := map[string]int{} + tsk := newTestTask(t, cfg, capturingTask(t, &seen, &fileCounts)) - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + s := newTestSBOM() require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) - assert.Equal(t, []string{"/outer.zip"}, sawArchivePaths, - "the parent is cataloged and the child it cannot afford is skipped rather than blocking") + assert.Equal(t, []string{"/outer.zip", "/outer.zip:inner.zip"}, seen) + assert.Equal(t, 1, fileCounts["/outer.zip"], "the parent is cataloged in full") + assert.Equal(t, 0, fileCounts["/outer.zip:inner.zip"], "the child it cannot afford is truncated") + assert.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "inner.zip", ArchivePath: "/outer.zip"}) } -// allFilesResolver is dirTestResolver without a filesystem id. Both sniff content the way a real -// resolver does, so which files reach the task is decided by what they are. +// allFilesResolver is dirTestResolver without a filesystem id. type allFilesResolver struct { file.Resolver dir string } func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) { - // every other task-level fixture is a zip, so the tar branch of FindExtractor had never been reached - // through the task's own discovery marker := map[string][]byte{"nested/marker.txt": []byte("hello")} scanDir := t.TempDir() @@ -828,34 +677,17 @@ func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) { } var seen []string - captureTask := NewTask("capture", func(ctx context.Context, r file.Resolver, _ sbomsync.Builder) error { - trav := archive.TraversalFromContext(ctx) - require.NotNil(t, trav) - - // the contents are there, so extraction really happened rather than the traversal being built - // over an empty tree - locs, err := r.FilesByGlob("**") - require.NoError(t, err) - assert.NotEmpty(t, locs, "expected contents inside %s", trav.VirtualPath) - - seen = append(seen, trav.VirtualPath) - return nil - }) - - newResolver := newStoreTestResolver - - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{captureTask}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, tsk.Execute(context.Background(), allFilesResolver{dir: scanDir}, sbomsync.NewBuilder(s))) + fileCounts := map[string]int{} + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, &fileCounts)) + require.NoError(t, tsk.Execute(context.Background(), allFilesResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM()))) assert.ElementsMatch(t, []string{"/bundle.tar.gz", "/bundle.tgz", "/bundle.tar", "/bundle.zip"}, seen, "every tar-family archive must be reached, and the plain text file must not be") + for _, archivePath := range seen { + assert.Equal(t, 1, fileCounts[archivePath], "%s must have been extracted", archivePath) + } t.Run("the MIME types the tar family is sniffed as are in the set discovery asks for", func(t *testing.T) { - // discoverArchives passes mimetype.ArchiveMIMETypeSet to FilesByMIMEType, so a tar type missing - // from it makes the whole tar path unreachable whatever the extractor can do for _, mt := range []string{"application/x-tar", "application/gzip", "application/x-gtar", "application/zip"} { assert.True(t, mimetype.ArchiveMIMETypeSet.Has(mt), "%s must be a candidate archive type", mt) } @@ -867,11 +699,8 @@ type recordingPublisher struct { } func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) { - // the sub-pipeline re-runs catalogers that start progress rows keyed by cataloger name. Publishing - // those again per archive floods the UI, and a consumer that keys rows by ID (as the syft CLI does) - // replaces the live row, never renders the replaced one again, and so never observes its - // completion - hanging teardown. Each cataloger keeps to the one row it already owns instead, and - // what it finds inside archives is counted onto that row. + // a consumer keying rows by ID (as the syft CLI does) replaces a republished row and never sees it + // complete, so each cataloger keeps to the one row it already owns inner := makeZip(t, map[string][]byte{"inner/file.txt": []byte("hello")}) scanDir := t.TempDir() for _, name := range []string{"one.zip", "two.zip"} { @@ -882,9 +711,12 @@ func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) bus.Set(publisher) t.Cleanup(func() { bus.Set(nil) }) - // a task that starts a progress row and counts what it found, the way a file cataloger does + // the archive row's stage as each archive's sub-pipeline starts: the previous archive is done by then, + // so the row must read the walk's own count rather than anything a nested cataloger left behind var rows []*monitor.TaskProgress + var stageAtEntry []string noisy := NewTask("noisy", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { + stageAtEntry = append(stageAtEntry, publisher.events[0].Value.(progress.StagedProgressable).Stage()) p := bus.StartCatalogerTask(ctx, monitor.GenericTask{ID: "noisy"}, 3, "") rows = append(rows, p) p.AtomicStage.Set("working on something") @@ -893,21 +725,8 @@ func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) return nil }) - // the archive row's stage as each archive is about to be extracted: the previous archive's - // sub-pipeline has finished by then, so the row must read the walk's own count and the archive it - // last finished, rather than anything a nested cataloger left behind - var stageAtExtraction []string - newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - row := publisher.events[0].Value.(progress.StagedProgressable) - stageAtExtraction = append(stageAtExtraction, row.Stage()) - return newStoreTestResolver(store, overflow) - } - - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{noisy}, newResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(s))) + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), noisy) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM()))) var started []string for _, e := range publisher.events { @@ -934,14 +753,12 @@ func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) assert.Equal(t, "2 archives", prog.Stage(), "and says so once the walk is done") // one.zip sorts before two.zip, and the resolver walks the scan directory in lexical order - assert.Equal(t, []string{"", "1 archives (/one.zip)"}, stageAtExtraction, + assert.Equal(t, []string{"", "1 archives (/one.zip)"}, stageAtEntry, "the row must take its stage back from the sub-pipeline once an archive is done") } func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) { - // the slowest archive is the one that spent the time itself. Inclusive time would name the outer - // archive every scan, so here the outer's own sub-pipeline is fast and the inner's slow: the report - // must name the inner. + // self time excludes nested archives, or the outer archive would be named every scan innerJar := makeZip(t, map[string][]byte{"README.txt": []byte("inner")}) outerZip := makeZip(t, map[string][]byte{"lib/inner.jar": innerJar}) @@ -950,89 +767,59 @@ func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) { const innerWork = 40 * time.Millisecond slow := NewTask("slow-inside-the-inner-archive", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { - if trav := archive.TraversalFromContext(ctx); trav != nil && strings.HasSuffix(trav.VirtualPath, "inner.jar") { + if trav := archive.TraversalFromContext(ctx); trav != nil && strings.HasSuffix(archive.VirtualPath(trav.Location), "inner.jar") { time.Sleep(innerWork) } return nil }) - cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3) - limiter := archive.NewLimiter(archive.DefaultLimits(cfg)) - c := &archiveCataloger{ - cfg: cfg, - subPipeline: []Task{slow}, - extractors: archive.DefaultExtractors(), - limiter: limiter, - newStoreResolver: newStoreTestResolver, - } + limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) + c := newTestCataloger(3, limiter, slow) + require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM()))) - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, nil, 0, sbomsync.NewBuilder(s))) - - assert.Equal(t, "/outer.zip:lib/inner.jar", c.slowestPath, - "the slowest archive is named by its full chain, not by its base name, and self time must exclude the archives nested inside") + assert.Equal(t, "/outer.zip:lib/inner.jar", c.slowestPath, "the slowest archive is named by its full chain") assert.GreaterOrEqual(t, c.slowest, innerWork) - assert.Equal(t, int64(2), c.archivesSeen) + assert.Equal(t, int64(2), c.progress.Current()) - // the peaks outlive the walk that set them, so they are still readable once everything is released + // the peaks outlive the walk that set them peakMemory, _ := limiter.Peak() - assert.Positive(t, peakMemory, "content held in memory must show on the peak") + assert.Positive(t, peakMemory, "entries held in memory must show on the peak") memory, disk := limiter.InUse() assert.Zero(t, memory) assert.Zero(t, disk) } func Test_archiveCataloger_slowestIsUnsetWhenNoArchiveIsCataloged(t *testing.T) { - // a directory with nothing extractable in it: there is no archive to name rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.txt"), []byte("not an archive"), 0o600)) - cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2) - c := &archiveCataloger{ - cfg: cfg, - subPipeline: []Task{NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil })}, - extractors: archive.DefaultExtractors(), - limiter: archive.NewLimiter(archive.DefaultLimits(cfg)), - newStoreResolver: newStoreTestResolver, - } - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, nil, 0, sbomsync.NewBuilder(s))) + noop := NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil }) + c := newTestCataloger(2, archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), noop) + require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM()))) assert.Empty(t, c.slowestPath) - assert.Zero(t, c.archivesSeen) + assert.Zero(t, c.progress.Current()) c.logStats() // must not panic with nothing to report } func Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError(t *testing.T) { - // `syft/nested-archive-cataloging#content-based-archive-detection`, scenario "misnamed non-archive - // is not extracted": the name says archive, the content does not, and it never becomes a candidate - // because its content types it as text + // the name says archive, the content does not: it is never opened rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.zip"), []byte("this is prose, not an archive, whatever the extension claims\n"), 0o600)) var ran int - countTask := NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { - ran++ - return nil - }) + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), countingTask(&ran)) - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), - []Task{countTask}, newStoreTestResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) + err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())) assert.NoError(t, err, "a file that is not an archive is not a failure to report") assert.Zero(t, ran, "and nothing is cataloged inside it") } func Test_archiveCataloger_catalogsAZipBehindALauncherScript(t *testing.T) { - // end to end for the Spring Boot executable jar shape: a launcher script with a zip appended. - // Content sniffing types it text/x-shellscript, so it reaches the walk only through the - // appended-archive candidate set, and qualifies only by its end-of-central-directory record. + // the Spring Boot executable jar shape: content sniffing types it as a shell script, so it reaches + // the walk only by name and qualifies only by its end-of-central-directory record inner := makeZip(t, map[string][]byte{"nested/marker.txt": []byte("found me")}) prefixed := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"), makeZip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...) @@ -1041,19 +828,8 @@ func Test_archiveCataloger_catalogsAZipBehindALauncherScript(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(rootDir, "app.jar"), prefixed, 0o600)) var seen []string - capture := NewTask("capture", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { - if trav := archive.TraversalFromContext(ctx); trav != nil { - seen = append(seen, trav.VirtualPath) - } - return nil - }) - - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3), - []Task{capture}, newStoreTestResolver, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) + tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3), capturingTask(t, &seen, nil)) + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))) assert.Equal(t, []string{"/app.jar", "/app.jar:BOOT-INF/lib/inner.jar"}, seen, "the stub must not hide the archive, and the walk must carry on into what it holds") @@ -1063,9 +839,6 @@ func (d dirTestResolver) FilesByMIMEType(types ...string) ([]file.Location, erro return filesByMIMEType(d.dir, d.fsid, types...) } -// filesByMIMEType sniffs each file's content and returns the ones whose type was asked for, as a real -// resolver does. Returning everything would make every caller look like it was handed archives, -// hiding that the task asks two different questions with two different consequences. func filesByMIMEType(dir, fsid string, types ...string) ([]file.Location, error) { wanted := strset.New(types...) var locs []file.Location @@ -1099,86 +872,8 @@ func (d dirTestResolver) FileContentsByLocation(loc file.Location) (io.ReadClose return os.Open(filepath.Join(d.dir, filepath.FromSlash(loc.RealPath))) } -// newStoreTestResolver is an archive.StoreResolverFactory over the archive's entries. -func newStoreTestResolver(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - r := &storeTestResolver{ - store: store, - entries: map[string]*archive.Entry{}, - fsid: overflow.FileSystemID, - archivePath: overflow.ArchivePath, - } - for _, entry := range store.Entries() { - r.entries[path.Clean("/"+entry.Header.Name)] = entry - } - return r, archive.IndexResult{}, nil -} - -func (r *storeTestResolver) sortedNames() []string { - names := make([]string, 0, len(r.entries)) - for name := range r.entries { - names = append(names, name) - } - sort.Strings(names) - return names -} - -func (r *storeTestResolver) location(name string) file.Location { - return file.NewLocationFromCoordinates(file.Coordinates{RealPath: name, FileSystemID: r.fsid, ArchivePath: r.archivePath}) -} - -func (r *storeTestResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) { - var locs []file.Location - for _, name := range r.sortedNames() { - switch strings.ToLower(filepath.Ext(name)) { - case ".zip", ".jar", ".war": - locs = append(locs, r.location(name)) - } - } - return locs, nil -} - -func (r *storeTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { - var locs []file.Location - for _, name := range r.sortedNames() { - for _, pattern := range patterns { - if matched, err := doublestar.Match(pattern, strings.TrimPrefix(name, "/")); err == nil && matched { - locs = append(locs, r.location(name)) - break - } - } - } - return locs, nil -} - -func (r *storeTestResolver) FilesByPath(paths ...string) ([]file.Location, error) { - var locs []file.Location - for _, p := range paths { - name := path.Clean("/" + p) - if _, ok := r.entries[name]; ok { - locs = append(locs, r.location(name)) - } - } - return locs, nil -} - -func (r *storeTestResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { - entry, ok := r.entries[loc.RealPath] - if !ok { - return nil, fmt.Errorf("no entry %q", loc.RealPath) - } - contents, err := r.store.Open(entry) - if err != nil { - return nil, err - } - return testOverflowEntry{nopCloserAt{contents}}, nil -} - -func (nopCloserAt) Close() error { return nil } - -func (testOverflowEntry) OverflowArchiveEntry() {} - -// makeZip builds a zip in sorted entry-name order rather than map order: limits are enforced as the -// walk proceeds, so entry order decides which entries land before a truncation. +// makeZip builds a zip in sorted entry-name order: limits are enforced as the walk proceeds, so entry +// order decides which entries land before a truncation. func makeZip(t *testing.T, entries map[string][]byte) []byte { t.Helper() names := make([]string, 0, len(entries)) @@ -1199,8 +894,8 @@ func makeZip(t *testing.T, entries map[string][]byte) []byte { return buf.Bytes() } -// matchesAnyGlob reports whether the path matches one of the patterns, against the resolver-relative -// path with a leading slash, the form a real resolver globs over. No patterns matches everything. +// matchesAnyGlob matches the resolver-relative path with a leading slash, the form a real resolver +// globs over. No patterns matches everything. func matchesAnyGlob(root, path string, patterns []string) bool { if len(patterns) == 0 { return true @@ -1218,8 +913,6 @@ func matchesAnyGlob(root, path string, patterns []string) bool { return false } -// FilesByGlob for the resolvers that otherwise only stand in for a MIME lookup: the archive walk also -// asks them for archive-named files, and an unimplemented method on an embedded nil resolver panics. func (a allFilesResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { return dirTestResolver{dir: a.dir}.FilesByGlob(patterns...) } @@ -1228,9 +921,6 @@ func (r twoLayerResolver) FilesByGlob(_ ...string) ([]file.Location, error) { return nil, nil } -// FilesByGlob lists the matching files under the resolver's directory, stamped with its filesystem id, -// so a sub-pipeline task can record file artifacts the way a real file cataloger does. No patterns -// lists everything, which is what the sub-pipeline tasks here want. func (d dirTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { var locs []file.Location err := filepath.WalkDir(d.dir, func(path string, entry fs.DirEntry, err error) error { @@ -1267,8 +957,8 @@ func (r twoLayerResolver) FileContentsByLocation(loc file.Location) (io.ReadClos return io.NopCloser(bytes.NewReader(body)), nil } -// incompressible returns n bytes deflate cannot shrink, so an archive built from them has a size on -// disk predictable enough to size a limit against. +// incompressible returns n bytes deflate cannot shrink, so an archive built from them has a +// predictable size to set a limit against. func incompressible(n int) []byte { b := make([]byte, n) x := uint32(12345) @@ -1279,7 +969,6 @@ func incompressible(n int) []byte { return b } -// makeTarGz builds a gzipped tar in memory, in sorted entry order for the same reason makeZip does. func makeTarGz(t *testing.T, entries map[string][]byte) []byte { t.Helper() var buf bytes.Buffer diff --git a/syft/archive_cataloger_test.go b/syft/archive_cataloger_test.go index cdff1ac9e..973002760 100644 --- a/syft/archive_cataloger_test.go +++ b/syft/archive_cataloger_test.go @@ -7,7 +7,6 @@ import ( "compress/gzip" "context" "io" - "io/fs" "os" "path/filepath" "sort" @@ -16,7 +15,6 @@ import ( "testing" "unicode/utf8" - "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/wagoodman/go-partybus" @@ -201,137 +199,8 @@ func TestArchiveCataloger_archiveContentsDoNotLeakIntoParentTree(t *testing.T) { } } -func TestArchiveCataloger_siblingArchivesDoNotCollide(t *testing.T) { - // two archives holding the same internal path: the file tables are keyed by Coordinates, so without - // a distinct FileSystemID the second write overwrites the first and a package disappears silently - scanDir := t.TempDir() - writeTestZip(t, filepath.Join(scanDir, "one.zip"), map[string]string{"nested/marker.txt": "from one"}) - writeTestZip(t, filepath.Join(scanDir, "two.zip"), map[string]string{"nested/marker.txt": "from two"}) - - s := scanDirWith(t, scanDir, archiveScanConfig(1, markerCataloger{})) - - // the two markers are the same package by syft's identity rules (same name, version and type), so - // they merge into one carrying two locations; what matters is that both locations survive as - // distinct coordinates - fsIDs := strset.New() - var locCount int - for _, p := range s.Artifacts.Packages.Sorted() { - if p.Name != "marker-pkg" { - continue - } - for _, loc := range p.Locations.ToSlice() { - locCount++ - fsIDs.Add(loc.ArchivePath) - assert.Equal(t, "nested/marker.txt", loc.RealPath, - "the paths are identical, which is exactly why the FileSystemID has to differ") - } - } - - assert.Equal(t, 2, locCount, "both archives' markers must survive as distinct locations") - assert.ElementsMatch(t, []string{"one.zip", "two.zip"}, fsIDs.List(), - "each archive must produce a distinct FileSystemID") -} - -func TestArchiveCataloger_containsEdgesChainAcrossLevels(t *testing.T) { - innerZip := buildZipBytes(t, map[string]string{"nested/marker.txt": "hello"}) - scanDir := t.TempDir() - writeTestZipRaw(t, filepath.Join(scanDir, "outer.zip"), map[string][]byte{"deeper/inner.zip": innerZip}) - - // file cataloging is on because archive-to-file edges are recorded for the files the file catalogers - // found inside an archive; without them inner.zip has no coordinate to hang an edge on - cfg := archiveScanConfig(2, markerCataloger{}). - WithFilesConfig(filecataloging.Config{Selection: file.AllFilesSelection}) - s := scanDirWith(t, scanDir, cfg) - - // an edge from outer.zip to inner.zip (file-to-file), and one from inner.zip to the package - var outerToInner, innerToPkg bool - for _, rel := range s.Relationships { - if rel.Type != artifact.ContainsRelationship { - continue - } - from, ok := rel.From.(file.Coordinates) - if !ok { - continue - } - if to, ok := rel.To.(file.Coordinates); ok { - if strings.HasSuffix(from.RealPath, "outer.zip") && strings.HasSuffix(to.RealPath, "inner.zip") { - outerToInner = true - } - continue - } - if p, ok := rel.To.(pkg.Package); ok && p.Name == "marker-pkg" { - if strings.HasSuffix(from.RealPath, "inner.zip") { - innerToPkg = true - } - } - } - - assert.True(t, outerToInner, "expected a CONTAINS edge from outer.zip to inner.zip") - assert.True(t, innerToPkg, "expected a CONTAINS edge from inner.zip to the nested package") -} - -func TestArchiveCataloger_truncationRetainsWhatWasFound(t *testing.T) { - // a bound truncates rather than discards: the marker extracted before the disk limit filled stays in - // the SBOM, and a small sibling archive is cataloged in full, the first having been released - big := bytes.Repeat([]byte("x"), 8192) - scanDir := t.TempDir() - writeTestZipRaw(t, filepath.Join(scanDir, "oversized.zip"), map[string][]byte{ - "a/marker.txt": []byte("small enough"), - "b/payload.bin": big, - }) - writeTestZip(t, filepath.Join(scanDir, "small.zip"), map[string]string{"nested/marker.txt": "fine"}) - - cfg := archiveScanConfig(1, markerCataloger{}) - cfg.Archive = cfg.Archive.WithMaxDiskBytes(1024) - - s := scanDirWith(t, scanDir, cfg) - - fsIDs := strset.New() - for _, p := range s.Artifacts.Packages.Sorted() { - for _, loc := range p.Locations.ToSlice() { - fsIDs.Add(loc.ArchivePath) - } - } - - assert.True(t, fsIDs.Has("oversized.zip"), - "the marker extracted before the limit must survive rather than the archive being discarded") - assert.True(t, fsIDs.Has("small.zip"), - "one archive hitting its limit must not affect another") -} - -func TestArchiveCataloger_detectionIsContentBased(t *testing.T) { - marker := map[string]string{"nested/marker.txt": "hello"} - - t.Run("extensionless and renamed archives are extracted", func(t *testing.T) { - scanDir := t.TempDir() - // no extension at all, and an extension that says something else entirely - require.NoError(t, os.WriteFile(filepath.Join(scanDir, "bundle"), buildZipBytes(t, marker), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(scanDir, "data.bin"), buildZipBytes(t, marker), 0o644)) - - s := scanDirWith(t, scanDir, archiveScanConfig(1, markerCataloger{})) - - fsIDs := strset.New() - for _, p := range s.Artifacts.Packages.Sorted() { - for _, loc := range p.Locations.ToSlice() { - fsIDs.Add(loc.ArchivePath) - } - } - assert.True(t, fsIDs.Has("bundle"), "an extensionless zip must be detected by content") - assert.True(t, fsIDs.Has("data.bin"), "a renamed zip must be detected by content") - }) - - t.Run("a misnamed non-archive is not extracted and is not an error", func(t *testing.T) { - scanDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(scanDir, "notes.zip"), []byte("just some text"), 0o644)) - - s := scanDirWith(t, scanDir, archiveScanConfig(1, markerCataloger{})) - assert.Empty(t, s.Artifacts.Packages.Sorted()) - }) -} - -// TestArchiveCataloger_exclusionsAreTheScansOwn covers exclusions-apply-by-pattern-scope from the -// outside: there is no archive-specific setting, and a pattern's shape decides whether it reaches -// inside an archive. +// there is no archive-specific exclusion setting: a scan pattern's shape decides whether it reaches +// inside an archive func TestArchiveCataloger_exclusionsAreTheScansOwn(t *testing.T) { marker := map[string]string{"nested/marker.txt": "hello"} @@ -460,53 +329,6 @@ func TestArchiveCataloger_exclusionsAreTheScansOwn(t *testing.T) { }) } -// TestArchiveCataloger_exclusionPatternPrecedence covers every-boundary-populates-the-patterns. -// cataloging.ArchiveSearchConfig.ExclusionPatterns is a derived field with no yaml/json/mapstructure -// tag, written by two boundaries: cmd/syft/internal/options.Catalog.ToArchiveConfig from --exclude, -// and CreateSBOM from the source. Precedence is defined rather than left to ordering: a value already -// present is never overwritten, and CreateSBOM only fills an empty field. -func TestArchiveCataloger_exclusionPatternPrecedence(t *testing.T) { - marker := map[string]string{"nested/marker.txt": "hello"} - - t.Run("a value already present is not overwritten by the source", func(t *testing.T) { - scanDir := t.TempDir() - writeTestZip(t, filepath.Join(scanDir, "app.zip"), marker) - - cfg := archiveScanConfig(1, markerCataloger{}) - cfg.Archive.ExclusionPatterns = []string{"**/marker.txt"} - - // the source publishes a different, non-matching pattern: if it won, the caller-supplied value - // would be replaced - s := scanDirWithExclusions(t, scanDir, cfg, "**/does-not-exist.txt") - assert.Empty(t, s.Artifacts.Packages.Sorted(), - "a value already present on the config must be what is applied, not overwritten by what the source publishes") - }) - - t.Run("an empty field is filled from the source", func(t *testing.T) { - // the configure-only-your-source path: a consumer who never touches ExclusionPatterns still - // gets the source's own patterns applied inside archives - scanDir := t.TempDir() - writeTestZip(t, filepath.Join(scanDir, "app.zip"), marker) - - cfg := archiveScanConfig(1, markerCataloger{}) - - s := scanDirWithExclusions(t, scanDir, cfg, "**/marker.txt") - assert.Empty(t, s.Artifacts.Packages.Sorted(), - "an empty field must be filled from the source's own published patterns") - }) - - t.Run("an empty field with no source exclusions excludes nothing", func(t *testing.T) { - scanDir := t.TempDir() - writeTestZip(t, filepath.Join(scanDir, "app.zip"), marker) - - cfg := archiveScanConfig(1, markerCataloger{}) - - s := scanDirWith(t, scanDir, cfg) - assert.NotEmpty(t, s.Artifacts.Packages.Sorted(), - "with nothing configured on either boundary, no exclusion should apply") - }) -} - // resolverProbeCataloger reports what the resolver it was handed can see, which is how "absent from // the index" is told from "present and skipped" within a scan. type resolverProbeCataloger struct { @@ -514,39 +336,6 @@ type resolverProbeCataloger struct { record func([]string) } -func TestArchiveCataloger_leavesNothingBehind(t *testing.T) { - // extracted content is written outside the scanned source and must not survive the scan, so a - // long-running process scanning many archives does not accumulate temp trees - innerZip := buildZipBytes(t, map[string]string{"nested/marker.txt": "hello"}) - scanDir := t.TempDir() - writeTestZipRaw(t, filepath.Join(scanDir, "outer.zip"), map[string][]byte{"deeper/inner.zip": innerZip}) - // a corrupt archive so the failure path's cleanup is exercised too - require.NoError(t, os.WriteFile(filepath.Join(scanDir, "corrupt.zip"), []byte("PK\x03\x04 truncated"), 0o644)) - - before := archiveTempDirs(t) - - scanDirWith(t, scanDir, archiveScanConfig(2, markerCataloger{})) - - assert.ElementsMatch(t, before, archiveTempDirs(t), - "no syft-archive-* temp dir may survive the scan, including for the archive that failed") - - // and nothing may be written into the directory under scan - var found []string - require.NoError(t, filepath.WalkDir(scanDir, func(path string, entry fs.DirEntry, err error) error { - if err != nil || entry.IsDir() { - return err - } - rel, relErr := filepath.Rel(scanDir, path) - if relErr != nil { - return relErr - } - found = append(found, rel) - return nil - })) - assert.ElementsMatch(t, []string{"outer.zip", "corrupt.zip"}, found, - "nothing new may appear under the scanned directory") -} - func TestArchiveCataloger_unicodeEntryNamesSurviveToOutput(t *testing.T) { // entry names reach the FileSystemID and then the encoded SBOM; a mangled one is a corrupt // identifier rather than a visible failure @@ -646,9 +435,7 @@ func TestArchiveCataloger_noExtractionPathReachesTheSBOM(t *testing.T) { for _, p := range paths { assert.NotContains(t, p, "syft-archive-", "an extraction directory reached the SBOM at %q", p) - // the storage shape is an implementation detail, so the file an archive's entries were - // overflowed into must not name itself either - in a path, or as a segment of one - assert.NotContains(t, p, "contents.tar", + assert.NotEqual(t, "entries", filepath.Base(p), "the file holding an archive's entries reached the SBOM at %q", p) if filepath.IsAbs(p) { // the directory resolver has always reported the scan root absolute: it is the path the @@ -685,42 +472,6 @@ func TestArchiveCataloger_repeatedScansAgreeOnNestedPaths(t *testing.T) { assert.Equal(t, first, scan(t)) } -func TestArchiveCataloger_tarFamilyIsCatalogedEndToEnd(t *testing.T) { - // nothing asserted the tar path end to end: the extensionless-archive case builds a zip named - // "bundle", so TarExtractor was only ever reached by its own unit tests. The compound extension is - // the interesting one - the archive is saved to a temp file before detection, and the basename is - // preserved specifically so ".tar.gz" survives that round trip. - dep := jarBytes(t, "tar-nested-lib", "3.1", nil) - entry := map[string][]byte{"lib/tar-nested-lib-3.1.jar": dep} - - for name, body := range map[string][]byte{ - "bundle.tar.gz": buildTarGzBytes(t, entry), - "bundle.tgz": buildTarGzBytes(t, entry), - "bundle.tar": buildTarBytes(t, entry), - } { - t.Run(name, func(t *testing.T) { - scanDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), body, 0o644)) - - s := scanDirWith(t, scanDir, javaScanConfig(2)) - - var count int - var virtualPath string - for _, p := range s.Artifacts.Packages.Sorted() { - if p.Name != "tar-nested-lib" { - continue - } - count++ - if metadata, ok := p.Metadata.(pkg.JavaArchive); ok { - virtualPath = metadata.VirtualPath - } - } - require.Equal(t, 1, count, "the jar inside the tar must be cataloged exactly once") - assert.Equal(t, name+":lib/tar-nested-lib-3.1.jar", virtualPath) - }) - } -} - func TestArchiveCataloger_legacySearchSettingsAreSuperseded(t *testing.T) { // syft/nested-archive-cataloging#legacy-archive-search-settings-are-superseded. The row that matters // is nested-on with both settings false: it looks like a bug and is not, so a future change that @@ -933,20 +684,6 @@ func (c resolverProbeCataloger) Catalog(_ context.Context, resolver file.Resolve return nil, nil, nil } -func archiveTempDirs(t *testing.T) []string { - t.Helper() - entries, err := os.ReadDir(os.TempDir()) - require.NoError(t, err) - - var out []string - for _, entry := range entries { - if entry.IsDir() && strings.HasPrefix(entry.Name(), "syft-archive-") { - out = append(out, entry.Name()) - } - } - return out -} - func buildTarGzBytes(t *testing.T, files map[string][]byte) []byte { t.Helper() var buf bytes.Buffer @@ -956,13 +693,6 @@ func buildTarGzBytes(t *testing.T, files map[string][]byte) []byte { return buf.Bytes() } -func buildTarBytes(t *testing.T, files map[string][]byte) []byte { - t.Helper() - var buf bytes.Buffer - writeTarEntries(t, &buf, files) - return buf.Bytes() -} - func writeTarEntries(t *testing.T, w io.Writer, files map[string][]byte) { t.Helper() names := make([]string, 0, len(files)) diff --git a/syft/archive_exclusions.go b/syft/archive_exclusions.go deleted file mode 100644 index f575fb08f..000000000 --- a/syft/archive_exclusions.go +++ /dev/null @@ -1,58 +0,0 @@ -package syft - -import ( - "os" - "path/filepath" - - "github.com/bmatcuk/doublestar/v4" - - "github.com/anchore/syft/syft/internal/fileresolver" - "github.com/anchore/syft/syft/source" -) - -// newArchiveExclusionVisitor builds an index visitor that drops the given patterns from one archive's -// contents, or nil when there is nothing to exclude. -// -// Patterns match against each entry's archive-relative path, unlike the directory source, which -// rewrites patterns absolute against the scan root. An archive is indexed in memory with no host -// location to rewrite against, and matching absolute would let a segment of the scan's temp directory -// satisfy a pattern the contents never do. Relative matching also keeps patterns intact, so `**/x` -// matches x at the archive root as well as below it. -// -// Only `**/`-prefixed patterns reach here (see cataloging.ArchiveExclusionPatterns): those mean the -// same thing at any depth, which is what makes an archive-relative match well defined. -func newArchiveExclusionVisitor(patterns []string) fileresolver.PathIndexVisitor { - if len(patterns) == 0 { - return nil - } - - return func(_, entryPath string, info os.FileInfo, _ error) error { - for _, pattern := range patterns { - matches, err := doublestar.Match(pattern, entryPath) - if err != nil { - // a malformed pattern excludes nothing rather than failing the archive; the source - // already validated its patterns - continue - } - if !matches { - continue - } - if info != nil && info.IsDir() { - return filepath.SkipDir - } - return fileresolver.ErrSkipPath - } - return nil - } -} - -// sourceExclusions returns the exclusion patterns the source was configured with, or nil. Exclusions -// belong to the source and appear nowhere in CreateSBOMConfig, so this is how the cataloging side -// learns them; a source that is not a source.PathExcluder excludes nothing. -func sourceExclusions(src source.Source) []string { - excluder, ok := src.(source.PathExcluder) - if !ok { - return nil - } - return excluder.ExcludedPaths() -} diff --git a/syft/archive_exclusions_test.go b/syft/archive_exclusions_test.go deleted file mode 100644 index 6403ec7b5..000000000 --- a/syft/archive_exclusions_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package syft - -import ( - "archive/tar" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/anchore/syft/syft/internal/fileresolver" - "github.com/anchore/syft/syft/source" - "github.com/anchore/syft/syft/source/directorysource" -) - -func Test_newArchiveExclusionVisitor(t *testing.T) { - // entry paths are archive-relative, the frame the index hands every filter; nothing here is a path - // on the host, so the fixtures are plain strings rather than files in a temp directory - dir := entryInfo(tar.TypeDir) - regular := entryInfo(tar.TypeReg) - - t.Run("no patterns means no visitor", func(t *testing.T) { - // so the archive resolver is built exactly as it was before this existed - assert.Nil(t, newArchiveExclusionVisitor(nil)) - }) - - t.Run("matching is relative to the archive, not to any host path", func(t *testing.T) { - // an archive is indexed in memory, so a pattern can only be satisfied by what the archive holds; - // there is no scratch directory whose own segments could match - visitor := newArchiveExclusionVisitor([]string{"**/vendor"}) - require.NotNil(t, visitor) - - assert.ErrorIs(t, visitor("/", "vendor", dir, nil), filepath.SkipDir) - assert.NoError(t, visitor("/", "keep/vendored.txt", regular, nil), - "a path that merely contains the word must not match") - assert.NoError(t, visitor("/", "vendor-ish", regular, nil)) - }) - - t.Run("a matching file is skipped and a matching directory is pruned", func(t *testing.T) { - visitor := newArchiveExclusionVisitor([]string{"**/vendor", "**/*.rpm"}) - require.NotNil(t, visitor) - - assert.ErrorIs(t, visitor("/", "a/pkg.rpm", regular, nil), fileresolver.ErrSkipPath) - assert.ErrorIs(t, visitor("/", "a/vendor", dir, nil), filepath.SkipDir) - }) - - t.Run("an any-depth pattern reaches the archive's own root", func(t *testing.T) { - // `**/x` matches x at zero depth as well as below it, so "exclude .rpm everywhere" stops an .rpm - // sitting directly inside another archive - visitor := newArchiveExclusionVisitor([]string{"**/*.rpm"}) - require.NotNil(t, visitor) - - assert.ErrorIs(t, visitor("/", "pkg.rpm", regular, nil), fileresolver.ErrSkipPath) - assert.ErrorIs(t, visitor("/", "deep/down/pkg.rpm", regular, nil), fileresolver.ErrSkipPath) - }) - - t.Run("a non-matching path is kept", func(t *testing.T) { - visitor := newArchiveExclusionVisitor([]string{"**/*.rpm"}) - require.NotNil(t, visitor) - - assert.NoError(t, visitor("/", "keep.txt", regular, nil)) - }) - - t.Run("a malformed pattern excludes nothing rather than failing the archive", func(t *testing.T) { - visitor := newArchiveExclusionVisitor([]string{"[", "**/*.rpm"}) - require.NotNil(t, visitor) - - assert.NoError(t, visitor("/", "keep.txt", regular, nil)) - assert.ErrorIs(t, visitor("/", "pkg.rpm", regular, nil), fileresolver.ErrSkipPath) - }) -} - -func Test_sourceExclusions(t *testing.T) { - t.Run("a source publishes what it was configured with", func(t *testing.T) { - src, err := directorysource.New(directorysource.Config{ - Path: t.TempDir(), - Exclude: source.ExcludeConfig{Paths: []string{"**/*.rpm", "./x"}}, - }) - require.NoError(t, err) - t.Cleanup(func() { _ = src.Close() }) - - assert.Equal(t, []string{"**/*.rpm", "./x"}, sourceExclusions(src)) - }) - - t.Run("building the source's own resolver does not rewrite them", func(t *testing.T) { - // the source rewrites exclusions absolute against the scan root while building its resolver. If that - // reached the configured patterns, none would begin "**/" any more and an archive's exclusions would - // silently degrade to none. - src, err := directorysource.New(directorysource.Config{ - Path: t.TempDir(), - Exclude: source.ExcludeConfig{Paths: []string{"**/*.rpm"}}, - }) - require.NoError(t, err) - t.Cleanup(func() { _ = src.Close() }) - - _, err = src.FileResolver(source.SquashedScope) - require.NoError(t, err) - - assert.Equal(t, []string{"**/*.rpm"}, sourceExclusions(src)) - }) - - t.Run("a source that publishes nothing excludes nothing", func(t *testing.T) { - assert.Nil(t, sourceExclusions(source.FromDescription(source.Description{}))) - }) -} - -// entryInfo is the FileInfo an archive entry carries, which is what the index hands every filter - -// derived from the entry's header, not from anything on the filesystem. -func entryInfo(typeflag byte) os.FileInfo { - mode := int64(0o600) - if typeflag == tar.TypeDir { - mode = 0o755 - } - return (&tar.Header{Typeflag: typeflag, Mode: mode}).FileInfo() -} diff --git a/syft/cataloging/archive_exclusions.go b/syft/cataloging/archive_exclusions.go deleted file mode 100644 index 205d98900..000000000 --- a/syft/cataloging/archive_exclusions.go +++ /dev/null @@ -1,32 +0,0 @@ -package cataloging - -import "strings" - -// anyDepthPrefix marks the one pattern shape that reaches inside an archive. -// -// A directory source accepts three shapes (directorysource.GetDirectoryExclusionFunctions): -// -// ./x a path at the scan root -// */x a path exactly one level below the scan root - `*` matches one segment and no separator, -// so it matches a/x but neither x nor a/b/x -// **/x x at any depth, the scan root included -// -// Only the third is depth-independent, so only it means the same thing re-rooted at an archive's -// extraction directory. The rule holds for image sources too, which do not validate these shapes. -const anyDepthPrefix = "**/" - -// ArchiveExclusionPatterns returns the subset of the scan's exclusion patterns that reach inside an -// archive, normalized for matching. A pattern's own shape decides: see anyDepthPrefix. -// -// The trailing slash is trimmed as the directory source trims it: it reads as "a directory" but -// doublestar.Match discards it, so a pattern keeping it matches nothing (issue #4839). -func ArchiveExclusionPatterns(exclusions []string) []string { - var out []string - for _, exclusion := range exclusions { - if !strings.HasPrefix(exclusion, anyDepthPrefix) { - continue - } - out = append(out, strings.TrimSuffix(exclusion, "/")) - } - return out -} diff --git a/syft/cataloging/archive_exclusions_test.go b/syft/cataloging/archive_exclusions_test.go deleted file mode 100644 index cb1ad44f4..000000000 --- a/syft/cataloging/archive_exclusions_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package cataloging - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_ArchiveExclusionPatterns(t *testing.T) { - // the pattern's shape is the whole rule, so every accepted shape is listed. `*/x` names a definite - // depth from the scan root, anchoring it there exactly as `./x` is; only `**/x` means the same thing - // wherever it is rooted. - tests := []struct { - name string - exclusions []string - want []string - }{ - { - name: "nothing configured", - exclusions: nil, - want: nil, - }, - { - name: "root-anchored does not reach inside", - exclusions: []string{"./nested/marker.txt"}, - want: nil, - }, - { - name: "one-level-down is anchored too and does not reach inside", - exclusions: []string{"*/vendor"}, - want: nil, - }, - { - name: "any-depth reaches inside", - exclusions: []string{"**/*.rpm"}, - want: []string{"**/*.rpm"}, - }, - { - name: "mixed keeps only the any-depth ones, in order", - exclusions: []string{"./a", "**/b", "*/c", "**/d"}, - want: []string{"**/b", "**/d"}, - }, - { - name: "a trailing slash is trimmed", - // it reads as "a directory" but doublestar.Match discards it, so a pattern keeping it - // matches nothing (issue #4839) - exclusions: []string{"**/vendor/"}, - want: []string{"**/vendor"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, ArchiveExclusionPatterns(tt.exclusions)) - }) - } -} diff --git a/syft/cataloging/archive_search.go b/syft/cataloging/archive_search.go index 4dc274ba6..08dd0e493 100644 --- a/syft/cataloging/archive_search.go +++ b/syft/cataloging/archive_search.go @@ -3,19 +3,16 @@ package cataloging import "github.com/anchore/stereoscope/pkg/file" const ( - DefaultArchiveMaxDepth = 0 // off by default, so old scans work the same way + // DefaultArchiveMaxDepth is 0: nested archives are not cataloged unless asked for. + DefaultArchiveMaxDepth = 0 - // DefaultArchiveMaxMemoryBytes bounds the archive content held in memory at once. Both bounds are - // in-use gauges: each measures what is held at a moment and falls as the scan releases an archive, - // so the peak they cap is the deepest point of a nesting chain, not the sum of everything opened. - // Set generously: it decides only whether an archive costs a write. Once refused, an archive's - // entries spill to the overflow blob and syft catalogs it either way. + // DefaultArchiveMaxMemoryBytes bounds the archive content held in memory at once. It decides only + // whether an archive costs a write: content that does not fit goes to disk and is cataloged anyway. DefaultArchiveMaxMemoryBytes = 2 * file.GB - // DefaultArchiveMaxDiskBytes bounds the archive content on disk at once. This bound is terminal: - // spilled content has nowhere further to go, so syft skips an archive whose first byte it refuses, - // leaving that archive's packages out of the SBOM. Set high, as a backstop against a pathological - // archive filling the disk; it is a poor working budget. + // DefaultArchiveMaxDiskBytes bounds the archive content on disk at once. Content that does not fit + // has nowhere to go, so the archive holding it is skipped or truncated. Set high as a backstop + // against a pathological archive filling the disk. DefaultArchiveMaxDiskBytes = 100 * file.GB ) @@ -26,30 +23,16 @@ type ArchiveSearchConfig struct { // IncludeUnindexedArchives indicates whether to search within unindexed archive files (e.g., .tar*). IncludeUnindexedArchives bool `yaml:"include-unindexed-archives" json:"include-unindexed-archives" mapstructure:"include-unindexed-archives"` - // MaxDepth is the maximum depth of recursive archive extraction (0 disables it). + // MaxDepth is how many levels of nested archives to catalog: 0 disables it, negative is unbounded. MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"` - // MaxMemoryBytes is the max archive content held in memory at once, an in-use gauge that falls as - // the scan releases an archive, so it bounds the peak. Content spills to disk once this stops - // admitting it, the only spill threshold. Zero spills everything; negative is unbounded. + // MaxMemoryBytes bounds the archive content held in memory at once; content that does not fit is + // written to disk. Zero writes everything to disk, negative is unbounded. MaxMemoryBytes int64 `yaml:"max-memory-bytes" json:"max-memory-bytes" mapstructure:"max-memory-bytes"` - // MaxDiskBytes is the max archive content on disk at once: spilled content plus the entries - // extracted from it. Also an in-use gauge, and terminal, since spilled content has nowhere further - // to go; syft skips an archive that would exceed it. Zero keeps every archive out of the work - // directory; negative is unbounded. + // MaxDiskBytes bounds the archive content written to disk at once; an archive whose content does + // not fit is skipped or truncated. Zero writes nothing to disk, negative is unbounded. MaxDiskBytes int64 `yaml:"max-disk-bytes" json:"max-disk-bytes" mapstructure:"max-disk-bytes"` - - // ExclusionPatterns carries the scan's file exclusion patterns so the code indexing an archive's - // contents need not interrogate the source for them. It has no config key, so no config file, flag - // or environment variable reaches it and `syft config` never names it. - // - // Each boundary holding exclusion patterns populates it: the CLI from `--exclude` - // (options.Catalog.ToArchiveConfig), and syft.CreateSBOM from the source's published exclusions - // (source.PathExcluder). CreateSBOM fills only an empty field, so the boundary closest to the user - // wins, and a consumer who configures only their source still gets those patterns applied inside - // archives. Callers building this config directly should leave it alone. - ExclusionPatterns []string `yaml:"-" json:"-" mapstructure:"-"` } func DefaultArchiveSearchConfig() ArchiveSearchConfig { @@ -86,11 +69,3 @@ func (c ArchiveSearchConfig) WithMaxDiskBytes(size int64) ArchiveSearchConfig { c.MaxDiskBytes = size return c } - -// WithExclusionPatterns sets the scan's exclusion patterns for the code that indexes an archive's -// contents. Only patterns whose shape reaches inside an archive belong here; ArchiveExclusionPatterns -// selects them. syft.CreateSBOM will not overwrite a value already set here. -func (c ArchiveSearchConfig) WithExclusionPatterns(patterns []string) ArchiveSearchConfig { - c.ExclusionPatterns = patterns - return c -} diff --git a/syft/cataloging/archive_search_test.go b/syft/cataloging/archive_search_test.go index d5c1e88db..46d88feb3 100644 --- a/syft/cataloging/archive_search_test.go +++ b/syft/cataloging/archive_search_test.go @@ -1,7 +1,6 @@ package cataloging import ( - "reflect" "testing" "github.com/stretchr/testify/assert" @@ -17,36 +16,6 @@ func TestDefaultArchiveSearchConfig(t *testing.T) { assert.Equal(t, int64(DefaultArchiveMaxDiskBytes), cfg.MaxDiskBytes) } -func TestArchiveSearchConfig_namesNoSeparateOverflowSize(t *testing.T) { - // one list, two assertions. The memory limit alone decides the boundary between memory and disk, so - // the surface must never grow a field naming a separate overflow size. And nested archive cataloging - // honors the scan's own exclusion patterns, so the surface must name no exclusion setting of its own - // - an archive extension list here would be a second mechanism for a question the scan answers. - // ExclusionPatterns is on the struct but is not config surface: derived, written only by CreateSBOM - // from what the source published, reachable from no file, flag or variable. It is listed here and - // separately asserted to carry no tags, which keeps this test catching a settable addition rather - // than any addition. - var settable, derived []string - typ := reflect.TypeOf(ArchiveSearchConfig{}) - for i := 0; i < typ.NumField(); i++ { - f := typ.Field(i) - if f.Tag.Get("yaml") == "-" && f.Tag.Get("json") == "-" && f.Tag.Get("mapstructure") == "-" { - derived = append(derived, f.Name) - continue - } - settable = append(settable, f.Name) - } - assert.Equal(t, []string{ - "IncludeIndexedArchives", - "IncludeUnindexedArchives", - "MaxDepth", - "MaxMemoryBytes", - "MaxDiskBytes", - }, settable, "a new SETTABLE field means a new way to configure this capability") - assert.Equal(t, []string{"ExclusionPatterns"}, derived, - "a derived field must stay untagged, or it becomes config surface without anyone deciding it should") -} - func TestArchiveSearchConfig_WithMethods(t *testing.T) { cfg := DefaultArchiveSearchConfig() diff --git a/syft/create_sbom.go b/syft/create_sbom.go index ff65162b5..89755c86a 100644 --- a/syft/create_sbom.go +++ b/syft/create_sbom.go @@ -10,6 +10,7 @@ import ( "github.com/scylladb/go-set/strset" "github.com/anchore/go-sync" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/log" @@ -38,16 +39,7 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( srcMetadata := src.Describe() - // the archive cataloger needs the source's exclusions to index each archive the way the scan - // indexes itself. Copied locally so the caller's config is not mutated, and only when empty: - // options.ToArchiveConfig fills the same field from --exclude and wins when set, while a consumer - // who configures only their source still gets theirs. - archiveCfg := cfg.Archive - if len(archiveCfg.ExclusionPatterns) == 0 { - archiveCfg = archiveCfg.WithExclusionPatterns(sourceExclusions(src)) - } - - taskGroups, audit, err := cfg.makeTaskGroups(srcMetadata, archiveCfg) + taskGroups, audit, err := cfg.makeTaskGroups(srcMetadata, sourceExclusions(src)) if err != nil { return nil, err } @@ -121,11 +113,14 @@ func setupContext(ctx context.Context, cfg *CreateSBOMConfig) (context.Context, // configure parallel executors ctx = setContextExecutors(ctx, cfg) - // one progress row per cataloger for the whole scan, however many times it runs: the archive - // cataloger re-runs the pipeline against every archive it walks, and those findings belong on the - // row that cataloger already owns + // one progress row per cataloger for the whole scan, however many archives it is run against ctx = bus.WithCatalogerTaskRegistry(ctx) + // the archive cataloger task recurses into jars too, so the java cataloger must not unarchive them itself + if cfg.Archive.MaxDepth != 0 { + ctx = archive.WithNestedCataloging(ctx) + } + // configure temp dir factory for catalogers (if not already set) if tmpdir.FromContext(ctx) == nil { ctx, _ = tmpdir.Root(ctx, "syft-cataloger") @@ -232,3 +227,12 @@ func formatTaskNames(tasks []task.Task) []string { sort.Strings(list) return list } + +// sourceExclusions returns the exclusion patterns the source was configured with, so the same +// exclusions apply inside the archives found in it. +func sourceExclusions(src source.Source) []string { + if excluder, ok := src.(source.PathExcluder); ok { + return excluder.ExcludedPaths() + } + return nil +} diff --git a/syft/create_sbom_config.go b/syft/create_sbom_config.go index 5579da2ae..96fa0605d 100644 --- a/syft/create_sbom_config.go +++ b/syft/create_sbom_config.go @@ -5,45 +5,19 @@ import ( "errors" "fmt" "runtime/debug" + "slices" "strings" - "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/task" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/cataloging/filecataloging" "github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/file" - "github.com/anchore/syft/syft/internal/fileresolver" "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/source" ) -// archiveEntryResolverFactory returns a factory that builds an indexed resolver over an extracted -// archive's entries. Passed to the archive cataloger task so internal/task and internal/archive need -// not import the internal fileresolver package directly. -// -// exclusions reach here through ArchiveSearchConfig.ExclusionPatterns rather than from the source. -// Those whose shape reaches inside an archive become index visitors, so an excluded file is absent -// from the archive's filesystem rather than present and skipped, and an excluded archive is never -// found to extract. -func archiveEntryResolverFactory(exclusions []string) archive.StoreResolverFactory { - // built once for the scan, not per archive: the visitor matches against an entry's archive-relative - // path, so it carries nothing specific to the archive it is asked about - var filters []fileresolver.PathIndexVisitor - if visitor := newArchiveExclusionVisitor(cataloging.ArchiveExclusionPatterns(exclusions)); visitor != nil { - filters = append(filters, visitor) - } - - return func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - resolver, err := fileresolver.NewFromArchiveEntries(overflow.FileSystemID, overflow.ArchivePath, store, overflow.Charge, filters...) - if err != nil { - return nil, archive.IndexResult{}, err - } - return resolver, archive.IndexResult{Truncated: resolver.Truncated()}, nil - } -} - // CreateSBOMConfig specifies all parameters needed for creating an SBOM. type CreateSBOMConfig struct { // required configuration input to specify how cataloging should be performed @@ -173,8 +147,8 @@ func (c *CreateSBOMConfig) WithFilesConfig(cfg filecataloging.Config) *CreateSBO return c } -// WithArchiveConfig allows for defining recursive archive cataloging parameters (e.g. max depth and -// extraction limits). MaxDepth 0, the default, disables recursive archive cataloging. +// WithArchiveConfig sets how archives are cataloged: how deep to recurse into nested archives (MaxDepth +// 0, the default, does not recurse) and how much extracted content the scan may hold at once. func (c *CreateSBOMConfig) WithArchiveConfig(cfg cataloging.ArchiveSearchConfig) *CreateSBOMConfig { c.Archive = cfg return c @@ -220,10 +194,8 @@ func (c *CreateSBOMConfig) WithCatalogers(catalogerRefs ...pkgcataloging.Catalog // The final set of task groups is returned along with a cataloger manifest that describes the catalogers that were // selected and the tokens that were sensitive to this selection (both for adding and removing from the final set). // -// archiveCfg is c.Archive with ExclusionPatterns populated by the caller (CreateSBOM reads -// source.PathExcluder off the source). Passed in rather than read off c so deriving the field never -// mutates the consumer's CreateSBOMConfig. -func (c *CreateSBOMConfig) makeTaskGroups(src source.Description, archiveCfg cataloging.ArchiveSearchConfig) ([][]task.Task, *catalogerManifest, error) { +// exclusions are the source's exclusion patterns, which the archive cataloger applies inside each archive. +func (c *CreateSBOMConfig) makeTaskGroups(src source.Description, exclusions []string) ([][]task.Task, *catalogerManifest, error) { var taskGroups [][]task.Task // generate package and file tasks based on the configuration @@ -246,15 +218,9 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description, archiveCfg cat taskGroups = append(taskGroups, append(pkgTasks, fileTasks...)) } - // recursively catalog archives as standalone indexed filesystems (opt-in via Archive.MaxDepth). the - // sub-pipeline is the package/file cataloger set, so the same catalogers run against archive - // contents; it runs after the base scan group and before relationship/unknowns post-processing - - var subPipeline []task.Task - subPipeline = append(subPipeline, pkgTasks...) - subPipeline = append(subPipeline, fileTasks...) - - if archiveTask := task.NewArchiveCatalogerTask(archiveCfg, subPipeline, archiveEntryResolverFactory(archiveCfg.ExclusionPatterns), archive.LogNotify); archiveTask != nil { + // the same package and file catalogers run against the contents of every archive, after the scan + // root and before relationship and unknowns post-processing + if archiveTask := task.NewArchiveCatalogerTask(c.Archive, append(slices.Clone(pkgTasks), fileTasks...), exclusions); archiveTask != nil { taskGroups = append(taskGroups, []task.Task{archiveTask}) } @@ -318,11 +284,6 @@ func (c *CreateSBOMConfig) selectTasks(src source.Description) ([]task.Task, []t FilesConfig: c.Files, } - // when enabled, the archive cataloger task recurses into all archives (JAR-family included), so the - // java cataloger must not unarchive them itself. This field exists only for that; derive it here so - // no value supplied through WithPackagesConfig is lost. - cfg.PackagesConfig.JavaArchive.NestedArchivesHandledExternally = c.Archive.MaxDepth != 0 - persistentPkgTasks, selectablePkgTasks, err := c.allPackageTasks(cfg) if err != nil { return nil, nil, nil, fmt.Errorf("unable to create package cataloger tasks: %w", err) diff --git a/syft/create_sbom_config_test.go b/syft/create_sbom_config_test.go index f26672263..08290d1fe 100644 --- a/syft/create_sbom_config_test.go +++ b/syft/create_sbom_config_test.go @@ -330,7 +330,7 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { require.NotEmpty(t, tt.wantTaskNames) // test the subject - gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src, tt.cfg.Archive) + gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src, nil) tt.wantErr(t, err) if err != nil { return @@ -583,7 +583,7 @@ func TestCreateSBOMConfig_archiveTaskGroup(t *testing.T) { t.Run("no archive task by default", func(t *testing.T) { // the feature is off by default, and off must mean no task at all rather than a registered // task that does nothing: a registered task is reported as a selected cataloger - groups, _, err := DefaultCreateSBOMConfig().makeTaskGroups(src, DefaultCreateSBOMConfig().Archive) + groups, _, err := DefaultCreateSBOMConfig().makeTaskGroups(src, nil) require.NoError(t, err) assert.False(t, archiveTaskIn(groups)) }) @@ -591,7 +591,7 @@ func TestCreateSBOMConfig_archiveTaskGroup(t *testing.T) { t.Run("archive task is present at negative depth", func(t *testing.T) { cfg := DefaultCreateSBOMConfig(). WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(-1)) - groups, _, err := cfg.makeTaskGroups(src, cfg.Archive) + groups, _, err := cfg.makeTaskGroups(src, nil) require.NoError(t, err) assert.True(t, archiveTaskIn(groups)) }) @@ -599,23 +599,21 @@ func TestCreateSBOMConfig_archiveTaskGroup(t *testing.T) { t.Run("archive task is present when enabled", func(t *testing.T) { cfg := DefaultCreateSBOMConfig(). WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)) - groups, _, err := cfg.makeTaskGroups(src, cfg.Archive) + groups, _, err := cfg.makeTaskGroups(src, nil) require.NoError(t, err) assert.True(t, archiveTaskIn(groups)) }) t.Run("enabling the feature does not mutate the caller's package config", func(t *testing.T) { - // the java side's yield flag is derived from the archive depth, into the cataloging factory's - // own copy: a consumer who handed us a package config gets it back exactly as supplied. - // Nothing user-settable is overwritten, and nothing the framework sets leaks back out. + // the java side learns the archive task is running from the context, not from its config: a + // consumer who handed us a package config gets it back exactly as supplied. supplied := java.DefaultArchiveCatalogerConfig().WithUseNetwork(true) - require.False(t, supplied.NestedArchivesHandledExternally) cfg := DefaultCreateSBOMConfig(). WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)). WithPackagesConfig(pkgcataloging.Config{JavaArchive: supplied}) - _, _, err := cfg.makeTaskGroups(src, cfg.Archive) + _, _, err := cfg.makeTaskGroups(src, nil) require.NoError(t, err) assert.Equal(t, supplied, cfg.Packages.JavaArchive, diff --git a/syft/file/coordinates.go b/syft/file/coordinates.go index 985b4c30f..bf1bde3cd 100644 --- a/syft/file/coordinates.go +++ b/syft/file/coordinates.go @@ -12,7 +12,7 @@ type Coordinates struct { // RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). RealPath string `json:"path" cyclonedx:"path"` - // FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in; the nesting chain itself is carried by ArchivePath. + // FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in. FileSystemID string `json:"layerID,omitempty" cyclonedx:"layerID"` // ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. "app.war:WEB-INF/lib/dep.jar"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. @@ -53,10 +53,8 @@ func (c Coordinates) GetCoordinates() Coordinates { return c } -// HashInclude controls which fields participate in the artifact ID hash (see artifact.IDByHash). An -// empty ArchivePath is excluded, so non-archive coordinates keep the identity they had before the -// field existed; a non-empty one is included, keeping identically-named files in different archives -// from sharing an ID. +// HashInclude keeps an empty ArchivePath out of the artifact ID hash (see artifact.IDByHash), so +// coordinates outside archives keep the IDs they had before the field existed. func (c Coordinates) HashInclude(field string, _ any) (bool, error) { if field == "ArchivePath" && c.ArchivePath == "" { return false, nil diff --git a/syft/internal/fileresolver/archive_entry.go b/syft/internal/fileresolver/archive_entry.go deleted file mode 100644 index 549849a2e..000000000 --- a/syft/internal/fileresolver/archive_entry.go +++ /dev/null @@ -1,110 +0,0 @@ -package fileresolver - -import ( - "io" - "os" - "path" - "path/filepath" - "strings" - - "github.com/anchore/syft/internal/log" -) - -// overflowEntryReader is a reader over one entry of an archive's store. Its marker method lets the -// archive cataloger recognize a nested archive as already-random-access, already-charged content, -// without either package importing the other. -type overflowEntryReader struct { - entryReadCloser -} - -// OverflowArchiveEntry marks this reader as content that needs no copy to be read at random. -func (overflowEntryReader) OverflowArchiveEntry() {} - -// entryReadCloser is a bounded view onto the bytes the store holds for one entry. -type entryReadCloser interface { - io.ReadCloser - io.ReaderAt - io.Seeker -} - -// archiveRoot is the root handed to a path filter. Archive entries are relative to the archive, which -// has no host location, so the root is the archive itself. -const archiveRoot = "/" - -type filterOutcome int - -const ( - keepEntry filterOutcome = iota - skipEntry - skipTree -) - -// runPathFilters asks each filter about one entry, mapping a walk's answers (skip path, skip dir) -// onto an unordered pass over archive entries. -// -// entryPath is archive-relative and the root is the archive's own root, not a host path: the archive -// is indexed in memory, and matching against the host temp path could match segments the archive's -// contents never carry. -func runPathFilters(filters []PathIndexVisitor, entryPath string, info os.FileInfo) filterOutcome { - for _, filter := range filters { - if filter == nil { - continue - } - switch err := filter(archiveRoot, entryPath, info, nil); err { - case nil: - continue - case filepath.SkipDir, filepath.SkipAll: - return skipTree - default: - return skipEntry - } - } - return keepEntry -} - -// isPruned reports whether an entry sits inside an already-pruned directory. An archive is a -// sequence, not a walk, so SkipDir cannot end a subtree; each later entry is checked instead. -func isPruned(entryPath string, pruned []string) bool { - for _, dir := range pruned { - if entryPath == dir || strings.HasPrefix(entryPath, dir+"/") { - return true - } - } - return false -} - -// maxEntryPathBytes is the longest entry path this package will index: PATH_MAX. -// -// Each path component becomes a tree node, so a long name like "a/a/.../a/f" expands a few hundred -// compressed bytes into a huge directory chain (a 352-byte tar.gz produced 200,000 nodes and 44 MB of -// index). This caps the per-entry cost; the index's per-node charge caps how many such entries a scan -// admits. -// -// A longer path is one no filesystem could hold, so refusing it drops nothing a real extraction or -// directory scan would have found. Go's tar reader alone allows names far longer. -const maxEntryPathBytes = 4096 - -// sanitizeEntryName converts an archive entry's name into a path inside that archive, reporting false -// when the name is empty or too long to be a path. -// -// Nothing is written through the name; this guards the path the SBOM reports. Cleaning against the -// root clamps "../" climbs, so an attacker-supplied name yields at worst a wrong path inside the right -// archive, never a claim about the host filesystem. -func sanitizeEntryName(name string) (string, bool) { - if strings.ContainsRune(name, 0) { - return "", false - } - if len(name) > maxEntryPathBytes { - // tested before cleaning: cleaning only shortens, and building the cleaned copy of a - // multi-megabyte name is itself part of what this refuses - log.WithFields("entry-name-bytes", len(name)). - Trace("skipping archive entry whose name is longer than any path a filesystem could hold") - return "", false - } - cleaned := strings.TrimPrefix(path.Clean("/"+name), "/") - if cleaned == "" || cleaned == "." { - // the archive's own root, which is not an entry in it - return "", false - } - return cleaned, true -} diff --git a/syft/internal/fileresolver/archive_entry_test.go b/syft/internal/fileresolver/archive_entry_test.go deleted file mode 100644 index 1614f1960..000000000 --- a/syft/internal/fileresolver/archive_entry_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package fileresolver - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -// Test_resolveLinkWithinArchive pins the frame a link entry is resolved in: the archive's own root, -// never the host filesystem. An archive's link targets are attacker-supplied text, so a target -// naming the host must land inside this archive or nowhere. -func Test_resolveLinkWithinArchive(t *testing.T) { - tests := []struct { - name string - linkPath string - linkname string - hardLink bool - want string - }{ - { - name: "a relative target resolves against the link's own directory", - linkPath: "/a/b/link", - linkname: "sibling.txt", - want: "/a/b/sibling.txt", - }, - { - name: "a relative target may climb within the archive", - linkPath: "/a/b/link", - linkname: "../c/target.txt", - want: "/a/c/target.txt", - }, - { - name: "a climb past the archive root is clamped at it", - linkPath: "/a/b/link", - linkname: "../../../../etc/passwd", - want: "/etc/passwd", - }, - { - name: "an absolute target is archive-absolute, not host-absolute", - linkPath: "/a/b/link", - linkname: "/etc/passwd", - want: "/etc/passwd", - }, - { - name: "a link at the archive root resolves against the root", - linkPath: "/link", - linkname: "target.txt", - want: "/target.txt", - }, - { - name: "a target that is only a climb lands on the root itself", - linkPath: "/a/link", - linkname: "..", - want: "/", - }, - { - name: "a hard link names its target from the archive root, not from its own directory", - linkPath: "/opt/hard.jar", - linkname: "opt/real.jar", - hardLink: true, - want: "/opt/real.jar", - }, - { - name: "a symlink with the same target text is relative to its own directory", - linkPath: "/opt/sym.jar", - linkname: "opt/real.jar", - want: "/opt/opt/real.jar", - }, - { - name: "a hard link target is clamped at the root like any other", - linkPath: "/opt/hard.jar", - linkname: "../../etc/passwd", - hardLink: true, - want: "/etc/passwd", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, resolveLinkWithinArchive(tt.linkPath, tt.linkname, tt.hardLink)) - }) - } -} diff --git a/syft/internal/fileresolver/archive_glob_bounds.go b/syft/internal/fileresolver/archive_glob_bounds.go deleted file mode 100644 index 8aa91457b..000000000 --- a/syft/internal/fileresolver/archive_glob_bounds.go +++ /dev/null @@ -1,401 +0,0 @@ -package fileresolver - -import "strings" - -// globMeta are the characters that end a run of literal text in a glob segment. A lone `]` or `}` is -// literal to doublestar but is treated as a stop here anyway, which only shortens a bound - a loss of -// selectivity, never of correctness. See literalBounds. -const globMeta = `*?[]{}\` - -// minLiteralBound is the shortest bound worth a lookup: any non-empty fixed text, since a lookup is a -// binary search plus a walk of one contiguous run. -const minLiteralBound = 1 - -// maxAlternatives bounds how far brace groups and character classes are enumerated. Past it, a -// segment reports nothing to look up rather than producing exponentially many lookups. -const maxAlternatives = 32 - -// maxClassRange is the widest `a-z` span enumerated into members. Wider is more lookups than the scan -// it replaces. -const maxClassRange = 64 - -// literalBounds returns the leading and trailing literal runs any matching name must carry: -// `libstd-????????????????.so` gives "libstd-" and ".so"; `*.[jw]ar` gives "" and "ar". A side -// starting with a metacharacter fixes nothing. -// -// Bounds only filter the index; doublestar still decides. A short bound merely widens the candidate -// set, so every globMeta character stops a run to avoid dropping real matches. -func literalBounds(segment string) (prefix, suffix string) { - first := strings.IndexAny(segment, globMeta) - if first < 0 { - return segment, segment - } - last := strings.LastIndexAny(segment, globMeta) - return segment[:first], segment[last+1:] -} - -// segmentBounds returns literal prefixes and suffixes covering every name this segment can match; a -// list is nil when the segment fixes too little on that side. -// -// Braces are expanded first, since an alternative can fix text the raw segment does not (`{go,go.exe}` -// prefixes "go" only once expanded). A side is usable only when every alternative carries a bound, -// else that alternative's matches would be lost. -func segmentBounds(segment string) (prefixes, suffixes []string) { - alternatives, ok := expandBraces(segment) - if !ok { - return nil, nil - } - - prefixes = make([]string, 0, len(alternatives)) - suffixes = make([]string, 0, len(alternatives)) - for _, alternative := range alternatives { - prefix, suffix := literalBounds(alternative) - if len(prefix) < minLiteralBound { - prefixes = nil - } - if len(suffix) < minLiteralBound { - suffixes = nil - } - if prefixes != nil { - prefixes = append(prefixes, prefix) - } - if suffixes != nil { - suffixes = append(suffixes, suffix) - } - } - return dedupe(prefixes), dedupe(suffixes) -} - -func dedupe(values []string) []string { - if len(values) < 2 { - return values - } - seen := make(map[string]struct{}, len(values)) - out := values[:0] - for _, v := range values { - if _, dup := seen[v]; dup { - continue - } - seen[v] = struct{}{} - out = append(out, v) - } - return out -} - -// expandBraces enumerates the alternatives a segment's `{a,b}` groups stand for, reporting false past -// maxAlternatives. -// -// It errs toward expanding: over-expanding costs extra lookups, under-expanding costs matches. An -// unmatched `{` is left for literalBounds to stop on. -func expandBraces(segment string) ([]string, bool) { - open, closing := braceGroup(segment) - if open < 0 { - return []string{segment}, true - } - - var out []string - for _, choice := range splitAlternatives(segment[open+1 : closing]) { - expanded, ok := expandBraces(segment[:open] + choice + segment[closing+1:]) - if !ok || len(out)+len(expanded) > maxAlternatives { - return nil, false - } - out = append(out, expanded...) - } - return out, true -} - -// braceGroup returns the bounds of the first top-level `{...}` group, or (-1, -1) when there is none. -// Escapes and character classes are skipped, so `\{` and `[{]` are not groups. -func braceGroup(segment string) (open, closing int) { - depth := 0 - open = -1 - for i := 0; i < len(segment); i++ { - switch segment[i] { - case '\\': - i++ - case '[': - if end := classEnd(segment, i); end >= 0 { - i = end - } - case '{': - if depth == 0 { - open = i - } - depth++ - case '}': - if depth > 0 { - depth-- - if depth == 0 { - return open, i - } - } - } - } - return -1, -1 -} - -// splitAlternatives splits the inside of a brace group on its top-level commas, leaving the commas -// of any nested group or character class alone. -func splitAlternatives(inner string) []string { - var out []string - depth, start := 0, 0 - for i := 0; i < len(inner); i++ { - switch inner[i] { - case '\\': - i++ - case '[': - if end := classEnd(inner, i); end >= 0 { - i = end - } - case '{': - depth++ - case '}': - depth-- - case ',': - if depth == 0 { - out = append(out, inner[start:i]) - start = i + 1 - } - } - } - return append(out, inner[start:]) -} - -// classEnd returns the index of the `]` closing the class opened at open, or -1 when it is never -// closed. A `]` first in the class, or directly after a leading negation, is a literal member. -func classEnd(segment string, open int) int { - i := open + 1 - if i < len(segment) && (segment[i] == '!' || segment[i] == '^') { - i++ - } - if i < len(segment) && segment[i] == ']' { - i++ - } - for ; i < len(segment); i++ { - switch segment[i] { - case '\\': - i++ - case ']': - return i - } - } - return -1 -} - -// segmentKind is what a name index can be asked for on behalf of one glob segment. -type segmentKind int - -const ( - // segExact is a literal name. - segExact segmentKind = iota - // segPrefix is a name with a fixed head, what the forward index answers. - segPrefix - // segSuffix is a name with a fixed tail, what the reverse index answers. - segSuffix -) - -// nameLookup is one index lookup: an exact name, or a name with a fixed head or tail. -type nameLookup struct { - kind segmentKind - literal string -} - -// enumerateLookups returns index lookups covering every name this segment can match, or nil when it -// cannot be reduced to a finite set. -// -// A segment whose only metacharacters are alternations and classes denotes a finite set: `{go,go.exe}` -// is two names, `*.[jw]ar` two suffixes, `[lm]*` two prefixes - a few lookups rather than a scan. -// Bounds serve these poorly: `*.[jw]ar` bounds to suffix "ar" (catches `calendar`), `[lm]*` bounds to -// nothing. -// -// Over-broad enumeration only costs filtering; under-broad loses files, so anything not exactly -// enumerable - a negated class, a POSIX class, a metacharacter member - abandons the attempt. -func enumerateLookups(segment string) []nameLookup { - alternatives, ok := expandBraces(segment) - if !ok { - return nil - } - - var expanded []string - for _, alternative := range alternatives { - members, ok := expandClasses(alternative) - if !ok || len(expanded)+len(members) > maxAlternatives { - return nil - } - expanded = append(expanded, members...) - } - - lookups := make([]nameLookup, 0, len(expanded)) - seen := make(map[nameLookup]struct{}, len(expanded)) - for _, alternative := range dedupe(expanded) { - lookup, ok := lookupFor(alternative) - if !ok { - // one alternative without a lookup would leave its matches out - return nil - } - if _, dup := seen[lookup]; dup { - continue - } - seen[lookup] = struct{}{} - lookups = append(lookups, lookup) - } - return lookups -} - -// lookupFor classifies one fully-expanded alternative (no alternation, no class) as an index lookup, -// or reports false when the index cannot answer it usefully. The answerable shapes are an exact name, -// `*tail`, `head*`; a `?`, an escape, or a second `*` leaves text the index cannot key on. -func lookupFor(alternative string) (nameLookup, bool) { - if strings.ContainsAny(alternative, nonWildcardMeta) { - return nameLookup{}, false - } - first := strings.IndexByte(alternative, '*') - last := strings.LastIndexByte(alternative, '*') - switch { - case first < 0: - return nameLookup{kind: segExact, literal: alternative}, true - case first != last || len(alternative) == 1: - // `*a*` needs every name anyway, and a bare `*` takes them all - return nameLookup{}, false - case first == 0: - return worthLooking(nameLookup{kind: segSuffix, literal: alternative[1:]}) - case last == len(alternative)-1: - return worthLooking(nameLookup{kind: segPrefix, literal: alternative[:len(alternative)-1]}) - } - // `head*tail`, which neither index answers on its own - return nameLookup{}, false -} - -func worthLooking(lookup nameLookup) (nameLookup, bool) { - if len(lookup.literal) < minLiteralBound { - return nameLookup{}, false - } - return lookup, true -} - -// expandClasses enumerates the names a segment's `[...]` classes stand for, reporting false when a -// class cannot be enumerated exactly. -// -// Members are substituted back as themselves, so a metacharacter member would change the pattern's -// meaning (`[*]` matches one asterisk, not anything). Such a class is refused rather than escaped, -// since the caller's fallback is correct. -func expandClasses(segment string) ([]string, bool) { - open := firstClass(segment) - if open < 0 { - return []string{segment}, true - } - closing := classEnd(segment, open) - if closing < 0 { - // unterminated, so doublestar reads the `[` as literal; lookupFor will refuse it - return []string{segment}, true - } - - members, ok := classMembers(segment[open : closing+1]) - if !ok { - return nil, false - } - - var out []string - for _, member := range members { - expanded, ok := expandClasses(segment[:open] + member + segment[closing+1:]) - if !ok || len(out)+len(expanded) > maxAlternatives { - return nil, false - } - out = append(out, expanded...) - } - return out, true -} - -// firstClass returns the index of the first unescaped `[`, or -1. -func firstClass(segment string) int { - for i := 0; i < len(segment); i++ { - switch segment[i] { - case '\\': - i++ - case '[': - return i - } - } - return -1 -} - -// classMembers returns the characters the given `[...]` matches, or false when the class is not -// enumerable. -func classMembers(class string) ([]string, bool) { - runes := []rune(class[1 : len(class)-1]) - if !enumerableClass(runes) { - return nil, false - } - - var out []string - for i := 0; i < len(runes); i++ { - members, width, ok := classItemAt(runes, i) - if !ok { - return nil, false - } - out = append(out, members...) - if len(out) > maxAlternatives { - return nil, false - } - i += width - } - if len(out) == 0 { - return nil, false - } - return out, true -} - -// enumerableClass reports whether the inside of a `[...]` describes a listable finite set. A negated -// class stands for all but a few characters, and a POSIX class for a set this does not model. -func enumerableClass(inner []rune) bool { - switch { - case len(inner) == 0: - return false - case inner[0] == '!' || inner[0] == '^': - return false - case strings.Contains(string(inner), "[:"): - return false - } - return true -} - -// classItemAt returns the members of the class item starting at i and how many runes past i it -// consumed. An item is an escape, an `a-z` range, or a single character. -func classItemAt(runes []rune, i int) (members []string, width int, ok bool) { - switch { - case runes[i] == '\\': - if i+1 >= len(runes) || !enumerableMember(runes[i+1]) { - return nil, 0, false - } - return []string{string(runes[i+1])}, 1, true - - case i+2 < len(runes) && runes[i+1] == '-': - return classRange(runes[i], runes[i+2]) - - default: - if !enumerableMember(runes[i]) { - return nil, 0, false - } - return []string{string(runes[i])}, 0, true - } -} - -// classRange expands an `a-z` range into its members, refusing one wider than maxClassRange. -func classRange(low, high rune) (members []string, width int, ok bool) { - if high < low || high-low > maxClassRange { - return nil, 0, false - } - for r := low; r <= high; r++ { - if !enumerableMember(r) { - return nil, 0, false - } - members = append(members, string(r)) - } - return members, 2, true -} - -// enumerableMember reports whether a class member can stand in the pattern as itself, which a -// metacharacter cannot: see expandClasses. -func enumerableMember(r rune) bool { - return !strings.ContainsRune(globMeta, r) -} diff --git a/syft/internal/fileresolver/archive_glob_bounds_test.go b/syft/internal/fileresolver/archive_glob_bounds_test.go deleted file mode 100644 index 0a22767f9..000000000 --- a/syft/internal/fileresolver/archive_glob_bounds_test.go +++ /dev/null @@ -1,462 +0,0 @@ -package fileresolver - -import ( - "fmt" - "strings" - "testing" - - "github.com/bmatcuk/doublestar/v4" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - syftindex "github.com/anchore/syft/internal/index" -) - -func Test_literalBounds(t *testing.T) { - tests := []struct { - segment string - prefix string - suffix string - }{ - {"pom.properties", "pom.properties", "pom.properties"}, - {"libstd-????????????????.so", "libstd-", ".so"}, - {"*.[jw]ar", "", "ar"}, - {"*.jar", "", ".jar"}, - {"pom*", "pom", ""}, - {"*", "", ""}, - {"{go,go.exe}", "", ""}, - {"lib{a,b}.so", "lib", ".so"}, - {"a?b", "a", "b"}, - {`esc\*ape`, "esc", "ape"}, - } - for _, tt := range tests { - t.Run(tt.segment, func(t *testing.T) { - prefix, suffix := literalBounds(tt.segment) - assert.Equal(t, tt.prefix, prefix, "prefix") - assert.Equal(t, tt.suffix, suffix, "suffix") - }) - } -} - -func Test_expandBraces(t *testing.T) { - tests := []struct { - segment string - want []string - wantOK bool - }{ - {segment: "plain.txt", want: []string{"plain.txt"}, wantOK: true}, - {segment: "{go,go.exe}", want: []string{"go", "go.exe"}, wantOK: true}, - {segment: "lib{a,b}.so", want: []string{"liba.so", "libb.so"}, wantOK: true}, - {segment: "{a,{b,c}}x", want: []string{"ax", "bx", "cx"}, wantOK: true}, - {segment: "{a,b}{c,d}", want: []string{"ac", "ad", "bc", "bd"}, wantOK: true}, - // a comma inside a character class is not a separator - {segment: "{x[a,b]y,z}", want: []string{"x[a,b]y", "z"}, wantOK: true}, - // an escaped brace is not a group - {segment: `\{a,b\}`, want: []string{`\{a,b\}`}, wantOK: true}, - // unmatched, so left alone for literalBounds to stop at - {segment: "{unclosed", want: []string{"{unclosed"}, wantOK: true}, - } - for _, tt := range tests { - t.Run(tt.segment, func(t *testing.T) { - got, ok := expandBraces(tt.segment) - require.Equal(t, tt.wantOK, ok) - assert.Equal(t, tt.want, got) - }) - } - - t.Run("an exponential expansion is refused rather than enumerated", func(t *testing.T) { - _, ok := expandBraces(strings.Repeat("{a,b}", 10)) - assert.False(t, ok, "2^10 alternatives is not less work than a scan") - }) -} - -func Test_segmentBounds(t *testing.T) { - tests := []struct { - name string - segment string - wantPrefixes []string - wantSuffixes []string - }{ - { - name: "both sides fixed", - segment: "libstd-????????????????.so", - wantPrefixes: []string{"libstd-"}, - wantSuffixes: []string{".so"}, - }, - { - name: "only the tail is fixed", - segment: "*.[jw]ar", - wantSuffixes: []string{"ar"}, - }, - { - name: "only the head is fixed", - segment: "pom?.xml*", - wantPrefixes: []string{"pom"}, - }, - { - name: "alternation fixes each branch", - segment: "{go,go.exe}", - wantPrefixes: []string{"go", "go.exe"}, - wantSuffixes: []string{"go", "go.exe"}, - }, - { - name: "alternation around fixed text", - segment: "{mariadb,mysql}", - wantPrefixes: []string{"mariadb", "mysql"}, - wantSuffixes: []string{"mariadb", "mysql"}, - }, - { - name: "a branch with nothing fixed disqualifies the whole side", - segment: "{go,*}", - }, - { - name: "a one-character bound is still a lookup", - segment: "a?", - wantPrefixes: []string{"a"}, - }, - { - name: "nothing fixed at all", - segment: "*", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - prefixes, suffixes := segmentBounds(tt.segment) - assert.Equal(t, tt.wantPrefixes, prefixes, "prefixes") - assert.Equal(t, tt.wantSuffixes, suffixes, "suffixes") - }) - } -} - -// Test_segmentBounds_neverExcludesAMatch is the invariant the optimization rests on: the bounds pick -// which names doublestar is shown, so a name the pattern matches must carry one. A bound that is too -// short only widens the candidate set; one that is too long silently loses files. -func Test_segmentBounds_neverExcludesAMatch(t *testing.T) { - segments := []string{ - "libstd-????????????????.so", "*.[jw]ar", "{go,go.exe}", "{mariadb,mysql}", - "lib{a,b}.so", "pom?.xml", "*.jar", "pom*", "*", "a?b", "{a,{b,c}}x", - `esc\*ape`, "[!x]yz", "{firefox,firefox.exe}", "**", - } - names := []string{ - "go", "go.exe", "gopher", "a.jar", "b.war", "b.rar", "libstd-0123456789abcdef.so", - "libstd-short.so", "liba.so", "libb.so", "libc.so", "pom1.xml", "pom.xml", "pom", - "mariadb", "mysql", "postgres", "axb", "ax", "bx", "cx", "esc*ape", "escape", - "xyz", "ayz", "firefox", "firefox.exe", "", "{go,go.exe}", - } - - for _, segment := range segments { - t.Run(segment, func(t *testing.T) { - prefixes, suffixes := segmentBounds(segment) - for _, name := range names { - matched, err := doublestar.Match(segment, name) - if err != nil || !matched { - continue - } - if len(prefixes) > 0 { - assert.True(t, anyHasPrefix(name, prefixes), - "%q matches %q but carries none of the prefixes %q", name, segment, prefixes) - } - if len(suffixes) > 0 { - assert.True(t, anyHasSuffix(name, suffixes), - "%q matches %q but carries none of the suffixes %q", name, segment, suffixes) - } - } - }) - } -} - -func anyHasPrefix(name string, prefixes []string) bool { - for _, p := range prefixes { - if strings.HasPrefix(name, p) { - return true - } - } - return false -} - -func anyHasSuffix(name string, suffixes []string) bool { - for _, s := range suffixes { - if strings.HasSuffix(name, s) { - return true - } - } - return false -} - -func Test_classMembers(t *testing.T) { - tests := []struct { - class string - want []string - wantOK bool - }{ - {class: "[jw]", want: []string{"j", "w"}, wantOK: true}, - {class: "[a-c]", want: []string{"a", "b", "c"}, wantOK: true}, - {class: "[a-cx]", want: []string{"a", "b", "c", "x"}, wantOK: true}, - {class: "[-a]", want: []string{"-", "a"}, wantOK: true}, - {class: "[a-]", want: []string{"a", "-"}, wantOK: true}, - // a negated class stands for every character but a few, which is not a finite lookup - {class: "[!x]"}, - {class: "[^x]"}, - // a POSIX class is a set this does not model - {class: "[[:alpha:]]"}, - // a member that is itself a metacharacter would change meaning on substitution - {class: "[*]"}, - {class: "[?]"}, - // wider than maxClassRange - {class: "[\x20-\x7e]"}, - {class: "[]"}, - } - for _, tt := range tests { - t.Run(tt.class, func(t *testing.T) { - got, ok := classMembers(tt.class) - require.Equal(t, tt.wantOK, ok) - assert.Equal(t, tt.want, got) - }) - } -} - -func Test_enumerateLookups(t *testing.T) { - tests := []struct { - name string - segment string - want []nameLookup - }{ - { - name: "alternation of exact names", - segment: "{go,go.exe}", - want: []nameLookup{{segExact, "go"}, {segExact, "go.exe"}}, - }, - { - name: "class inside an extension becomes exact suffixes", - segment: "*.[jw]ar", - want: []nameLookup{{segSuffix, ".jar"}, {segSuffix, ".war"}}, - }, - { - name: "leading class becomes prefix lookups, where bounds fix nothing at all", - segment: "[lm]*", - want: []nameLookup{{segPrefix, "l"}, {segPrefix, "m"}}, - }, - { - name: "multi-character alternation likewise", - segment: "{lib,mod}*", - want: []nameLookup{{segPrefix, "lib"}, {segPrefix, "mod"}}, - }, - { - name: "alternation and class together", - segment: "{lib,bin}[0-2].so", - want: []nameLookup{ - {segExact, "lib0.so"}, {segExact, "lib1.so"}, {segExact, "lib2.so"}, - {segExact, "bin0.so"}, {segExact, "bin1.so"}, {segExact, "bin2.so"}, - }, - }, - { - name: "duplicate alternatives collapse to one lookup", - segment: "{go,go}", - want: []nameLookup{{segExact, "go"}}, - }, - // a `?` is not a finite set of names - {name: "question mark", segment: "libstd-?.so"}, - // a negated class is not either - {name: "negated class", segment: "[!x]yz"}, - // nor is a wildcard in the middle - {name: "interior wildcard", segment: "{a,b}*.{c,d}"}, - // nor a bare wildcard - {name: "bare wildcard", segment: "{a,*}"}, - // and an exponential expansion is refused rather than enumerated - {name: "too many alternatives", segment: "[a-z][a-z]"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, enumerateLookups(tt.segment)) - }) - } -} - -// Test_enumerateLookups_neverExcludesAMatch is the same invariant for the stronger claim this pass -// makes: the lookups are what the index is asked for, so a name the pattern matches must be found by -// at least one of them. Over-broad only costs filtering. -func Test_enumerateLookups_neverExcludesAMatch(t *testing.T) { - segments := []string{ - "{go,go.exe}", "*.[jw]ar", "[lm]*", "{lib,mod}*", "{mariadb,mysql}", "{lib,bin}[0-2].so", - "lib{a,b}.so", "[abc]", "*.[jw]ar", "{firefox,firefox.exe}", "x[!y]z", "[a-c]*.so", - "pom.{xml,properties}", "*.{jar,war}", "{a,b}{c,d}", - // only some branches reduce to a lookup - "{go,*}", "{lib,l}*", "{lo,m}*", "{*.jar,pom.xml}", - } - names := []string{ - "go", "go.exe", "gopher", "a.jar", "b.war", "c.rar", "calendar", "lib0.so", "lib1.so", - "lib3.so", "bin2.so", "liba.so", "libb.so", "libc.so", "mariadb", "mysql", "a", "b", "c", "d", - "lmn", "mno", "nop", "xyz", "xaz", "ayz", "firefox", "firefox.exe", "pom.xml", - "log.txt", "mn.txt", "lib.so", - "pom.properties", "pom.txt", "ac", "ad", "bc", "bd", "ab", "", "az.so", - } - - for _, segment := range segments { - t.Run(segment, func(t *testing.T) { - lookups := enumerateLookups(segment) - if len(lookups) == 0 { - return - } - for _, name := range names { - matched, err := doublestar.Match(segment, name) - if err != nil || !matched { - continue - } - assert.True(t, anyLookupFinds(name, lookups), - "%q matches %q but no lookup in %v would find it", name, segment, lookups) - } - }) - } -} - -func anyLookupFinds(name string, lookups []nameLookup) bool { - for _, l := range lookups { - switch l.kind { - case segExact: - if name == l.literal { - return true - } - case segPrefix: - if strings.HasPrefix(name, l.literal) { - return true - } - case segSuffix: - if strings.HasSuffix(name, l.literal) { - return true - } - } - } - return false -} - -// Test_lookupsAreExactlyTheirAlternative is what lets byLookups return the index's answer without -// asking doublestar to confirm it. -// -// The bounds path only narrows, so an imprecise bound there costs filtering. A lookup is the answer -// itself, so anything byPrefix("l") selected that `l*` does not match would be reported as a match. -// This pins the three shapes to doublestar's verdict. -func Test_lookupsAreExactlyTheirAlternative(t *testing.T) { - alternatives := []string{ - "go", "go.exe", "l", "lib", "*.jar", "*ar", "*", "l*", "lib*", "libstd-*", - "*.so", "a", "pom.xml", ".hidden", "*.", ".*", - } - names := []string{ - "go", "go.exe", "gopher", "l", "lib", "lib.so", "libstd-0.a", "libz.so", "a.jar", - "calendar", "ar", ".jar", "", "a", "pom.xml", ".hidden", "x.", ".x", "日本.jar", "日本", - } - - for _, alternative := range alternatives { - t.Run(alternative, func(t *testing.T) { - lookup, ok := lookupFor(alternative) - if !ok { - return - } - for _, name := range names { - want, err := doublestar.Match(alternative, name) - require.NoError(t, err) - assert.Equal(t, want, anyLookupFinds(name, []nameLookup{lookup}), - "pattern %q, name %q: the lookup and doublestar must agree exactly", alternative, name) - } - }) - } -} - -// Test_boundedLookupMatchesTheFullOne holds the probe to the lookup it stands in for: a bounded -// lookup that completes has to return exactly what the unbounded one would, or the side it picks is -// answering a different question than the side it rejects. -func Test_boundedLookupMatchesTheFullOne(t *testing.T) { - var x syftindex.PrefixSuffix[[]*indexNode] - names := []string{ - "libstd-a.so", "libstd-b.a", "libstd-c.a", "lib.so", "libz.so", - "go", "go.exe", "gopher", "a.jar", "b.war", "\xff\xfe", - } - for _, name := range names { - node := &indexNode{name: name} - x.Update(name, appendNode(node)) - } - - for _, probe := range []string{"lib", "libstd-", "go", "z", "", "a", "\xff", "libstd-a.so"} { - t.Run("prefix "+probe, func(t *testing.T) { - full := flattenNodes(x.ByPrefix(probe)) - got, complete := x.ByPrefixUpTo(probe, len(names)) - require.True(t, complete, "the budget covers every name, so nothing may give up") - assert.ElementsMatch(t, full, flattenNodes(got)) - }) - t.Run("suffix "+probe, func(t *testing.T) { - full := flattenNodes(x.BySuffix(probe)) - got, complete := x.BySuffixUpTo(probe, len(names)) - require.True(t, complete) - assert.ElementsMatch(t, full, flattenNodes(got)) - }) - } -} - -// Test_boundedLookupGivesUpPastTheBudget covers the other half: past the budget the lookup must report -// failure rather than a short answer, which a caller would read as a narrow side. -func Test_boundedLookupGivesUpPastTheBudget(t *testing.T) { - var x syftindex.PrefixSuffix[[]*indexNode] - for i := range 100 { - name := fmt.Sprintf("lib%03d.so", i) - x.Update(name, appendNode(&indexNode{name: name})) - } - - _, complete := x.ByPrefixUpTo("lib", 10) - assert.False(t, complete, "100 names under a budget of 10 must give up") - - _, complete = x.ByPrefixUpTo("lib000", 10) - assert.True(t, complete, "one name under a budget of 10 must not") -} - -// Test_narrowByBounds_takesTheNarrowerSide covers what probing the index buys. Both ends of -// `libstd-????????????????.so` can answer it, and which should is a property of the archive: here the -// seven-character head is the broad side and the three-character tail the narrow one, so a rule -// preferring the longer literal would take the worse lookup. -func Test_narrowByBounds_takesTheNarrowerSide(t *testing.T) { - entries := map[string]string{"usr/lib/libstd-0123456789abcdef.so": "real"} - for i := range 200 { - entries[fmt.Sprintf("usr/lib/libstd-%04d.a", i)] = "decoy" - } - r := indexOver(t, 1<<20, entries) - - const segment = "libstd-????????????????.so" - prefixes, suffixes := segmentBounds(segment) - require.Equal(t, []string{"libstd-"}, prefixes) - require.Equal(t, []string{".so"}, suffixes) - - // the head reaches every decoy and so must never be the side that runs; the tail reaches one - _, headComplete := r.fileNames.ByPrefixUpTo("libstd-", boundProbe) - require.False(t, headComplete, "the longer bound reaches past the probe") - tail, tailComplete := r.fileNames.BySuffixUpTo(".so", boundProbe) - require.True(t, tailComplete) - require.Len(t, flattenNodes(tail), 1, "the shorter bound reaches only the real file") - - candidates, ok := r.narrowByBounds(&r.fileNames, segment) - require.True(t, ok) - assert.Len(t, candidates, 1, "the narrower side must be the one that ran") - - // and the answer is the same either way, so this is only a question of cost - locations, err := r.FilesByGlob("**/" + segment) - require.NoError(t, err) - require.Len(t, locations, 1) - assert.Equal(t, "usr/lib/libstd-0123456789abcdef.so", locations[0].RealPath) -} - -// Test_narrowByBounds_fallsBackWhenNeitherSideIsNarrow covers the case the probe cannot decide: both -// ends reach past it, so the answer still has to be complete. -func Test_narrowByBounds_fallsBackWhenNeitherSideIsNarrow(t *testing.T) { - entries := map[string]string{} - for i := range boundProbe * 3 { - entries[fmt.Sprintf("usr/lib/libstd-%04d.so", i)] = "decoy" - } - entries["usr/lib/libstd-0123456789abcdef.so"] = "real" - r := indexOver(t, 1<<20, entries) - - candidates, ok := r.narrowByBounds(&r.fileNames, "libstd-????????????????.so") - require.True(t, ok) - assert.Len(t, candidates, boundProbe*3+1, "a probe that decides nothing must still gather everything") - - locations, err := r.FilesByGlob("**/libstd-????????????????.so") - require.NoError(t, err) - require.Len(t, locations, 1) - assert.Equal(t, "usr/lib/libstd-0123456789abcdef.so", locations[0].RealPath) -} diff --git a/syft/internal/fileresolver/archive_index.go b/syft/internal/fileresolver/archive_index.go deleted file mode 100644 index 555deebd2..000000000 --- a/syft/internal/fileresolver/archive_index.go +++ /dev/null @@ -1,546 +0,0 @@ -package fileresolver - -import ( - "archive/tar" - "context" - "fmt" - "io" - "path" - "reflect" - "sort" - "strings" - - stereoscopeFile "github.com/anchore/stereoscope/pkg/file" - "github.com/anchore/syft/internal/archive" - syftindex "github.com/anchore/syft/internal/index" - "github.com/anchore/syft/syft/file" -) - -// The index is the largest per-archive structure a scan keeps, and the only archive cost the entry -// store cannot measure: one entry is one store record but one node per path component here. So the -// index charges the scan's budget node by node and stops when refused - see ArchiveIndex.Truncated. -// -// The constants below are what one node and one file keep, deliberately over-estimated: an under-count -// lets the budget admit several times its setting. -// Test_indexChargeCoversWhatTheIndexKeeps holds them above the heap an index actually retains. -const ( - // mapSlotCost is what one string-keyed, pointer-valued pair costs a Go map beyond the pair itself: - // control bytes, load-factor headroom, and the doubling it grows by. - mapSlotCost = 96 - - // sliceSlotCost is one pointer held in a slice grown by appending, including the headroom append - // leaves behind it. - sliceSlotCost = 24 - - // keySplitNodeCost is one node of a key-split index - its lock, its value, and the two map headers - // it carries whether or not it has children - rounded up. The rounding is the headroom for a key - // that splits an existing one and so allocates a second node. - keySplitNodeCost = 112 - - // keySplitEntryCost is what one key costs the index that holds it - a node of its own, plus the - // slot it takes in its parent's two maps. A key that splits an existing one costs a second node, - // which is why this is charged per key rather than per distinct path through the trie. - keySplitEntryCost = keySplitNodeCost + 2*mapSlotCost - - // nameIndexCost is what one name keeps in a base-name index on top of its node: an entry in each of - // the forward and reverse tries, and a slot in the node slice each of those holds. The reverse - // trie's reversed name copy is charged by length alongside this, as is the ordered view - // AllLocations and FilesByMIMEType read. - nameIndexCost = 2*(keySplitEntryCost+sliceSlotCost) + sliceSlotCost -) - -// indexNodeCost is what one tree node keeps: the node itself - which carries a key-split index of its -// children by value - its slot in the by-path map, its slot in its parent's child list, and the slot -// it takes in its parent's child index. Name and path are substrings of the entry path, which is -// charged once per entry, so neither is counted again here. -var indexNodeCost = int64(reflect.TypeFor[indexNode]().Size()) + mapSlotCost + sliceSlotCost + keySplitEntryCost - -// ArchiveIndex is a resolver over the entries of one extracted archive. It indexes archive.Entry -// values rather than file offsets, so an entry's bytes can move between memory and the overflow blob -// without the index changing. -// -// Glob lookups go through a base-name index rather than a walk: cataloger globs are overwhelmingly -// `**/*.ext` and `**/name`, answered by suffix and exact lookup respectively. -type ArchiveIndex struct { - fileSystemID string - archivePath string - root *indexNode - - // byPath is every node by its full archive-relative path; path lookups and HasPath answer from it. - byPath map[string]*indexNode - - // fileNames and dirNames are the global base-name indexes the search answers from: files for a - // pattern's last segment, directories for the segment after a `**`. Values are slices because one - // name repeats across directories. - fileNames syftindex.PrefixSuffix[[]*indexNode] - dirNames syftindex.PrefixSuffix[[]*indexNode] - - store *archive.EntryStore - records int - - // truncated reports that the budget refused a node and the remaining entries were never indexed. - // What was indexed before that is complete and usable. - truncated bool - - // sorted is every file in a stable order, built once. See ordered. - sorted []*indexNode -} - -// indexNode is one file or directory in an archive's filesystem. -// -// A directory may be synthesized: archives routinely name `a/b/c.txt` without naming `a/` or `a/b/`, -// and a glob whose middle segment is a directory needs those nodes. A synthesized node has no entry. -type indexNode struct { - name string - path string - isDir bool - entry *archive.Entry - metadata file.Metadata - - // target is the node whose entry holds this node's content: itself for an ordinary file, what it - // points at for a link, and nil when the link dangles. Resolved once by resolveLinks. - target *indexNode - - // children indexes this directory's children by name, which is what an exact or `name*` segment - // looks up. childList is the same set in sorted order, for the segments that have to be matched - // one by one and for a stable result order. - children syftindex.KeySplitIndex[*indexNode] - childList []*indexNode -} - -// NewFromArchiveEntries builds a resolver over the entries of one archive. -// -// An archive is its own root: entries are keyed, reported and filtered by their archive-relative -// path, and nothing here names a directory on the host. pathFilters are applied per entry as in a -// directory walk, including the prune-a-subtree answer, and are handed that same archive-relative -// path - see runPathFilters. -// -// charge is this archive's draw on the scan's resource limiter, charged per node kept. A refused -// charge truncates rather than fails the archive, as a refused entry does during extraction; a nil -// charge enforces nothing. -func NewFromArchiveEntries(fileSystemID, archivePath string, store *archive.EntryStore, charge *archive.Charge, pathFilters ...PathIndexVisitor) (*ArchiveIndex, error) { - if store == nil { - return nil, fmt.Errorf("no entry store for archive %q", fileSystemID) - } - - r := &ArchiveIndex{ - fileSystemID: fileSystemID, - archivePath: archivePath, - root: &indexNode{name: "", path: "/", isDir: true}, - byPath: map[string]*indexNode{}, - store: store, - } - r.byPath["/"] = r.root - - var pruned []string - for _, entry := range store.Entries() { - name, ok := sanitizeEntryName(entry.Header.Name) - if !ok { - continue - } - if isPruned(name, pruned) { - continue - } - switch runPathFilters(pathFilters, name, entry.Header.FileInfo()) { - case skipEntry: - continue - case skipTree: - pruned = append(pruned, name) - continue - case keepEntry: - } - - if !r.add(entry, "/"+name, charge) { - // stop rather than carrying on and admitting later entries that need fewer nodes - r.truncated = true - break - } - } - - r.finalize() - return r, nil -} - -// add puts one entry into the tree and the indexes, creating the directories its path implies. It -// reports false when the budget refused a node, which truncates the index. -func (r *ArchiveIndex) add(entry *archive.Entry, entryPath string, charge *archive.Charge) bool { - isDir := entry.Header.FileInfo().IsDir() - - // every node and by-path key is a substring of entryPath, so the one string backs the whole chain - // and is charged here rather than once per node - if !charge.IndexRecord(int64(len(entryPath))) { - return false - } - - node, ok := r.node(entryPath, isDir, charge) - if !ok { - return false - } - if node == nil { - // names nothing inside this archive; not a budget refusal, so the walk carries on - return true - } - - if node.entry != nil { - // an archive naming the same path twice: the first entry wins - return true - } - - // charged before committing the entry, so a refusal never leaves a file counted but unsearchable. - // The reverse index's key is the one string the name index allocates per name. - if !isDir && !charge.IndexRecord(nameIndexCost+int64(len(node.name))) { - return false - } - - node.entry = entry - node.metadata = r.metadataOf(entry, entryPath) - r.records++ - - if isDir { - // already placed in dirNames when the node was created; an archive naming a directory it also - // implies must not index it twice - return true - } - - r.fileNames.Update(node.name, appendNode(node)) - return true -} - -// appendNode adds one node to whatever a name already holds. The index stores a slice per key because -// base names repeat across directories. -func appendNode(node *indexNode) syftindex.NodeUpdateFunc[[]*indexNode] { - return func(current *syftindex.Node[[]*indexNode]) []*indexNode { - return append(current.Value(), node) - } -} - -// node returns the node at the given path, creating any missing parents and charging the budget for -// each. It reports false when the budget refused a node, and a nil node for a path that names nothing -// inside the archive. -// -// Two passes: climb to the deepest ancestor already in the tree, then build down from there. The climb -// keeps the common case cheap - an entry whose parent is already placed costs one lookup however deep. -// -// Neither pass recurses: every key and name is a substring of entryPath, so a name 2,000 components -// deep costs 2,000 iterations, not 2,000 stack frames. Placing a node is still proportional to path -// length, which is why sanitizeEntryName bounds it. -func (r *ArchiveIndex) node(entryPath string, isDir bool, charge *archive.Charge) (*indexNode, bool) { - if existing, found := r.byPath[entryPath]; found { - return existing, true - } - - // climb: find the deepest ancestor already in the tree, and where in entryPath it ends - anchor, anchorEnd := r.root, 0 - for end := len(entryPath); ; { - slash := strings.LastIndexByte(entryPath[:end], '/') - if slash <= 0 { - // the parent is the archive root, which is always in the tree - break - } - if existing, found := r.byPath[entryPath[:slash]]; found { - anchor, anchorEnd = existing, slash - break - } - end = slash - } - - // descend: create what is missing below the anchor. entryPath is rooted, cleaned and free of - // trailing separators (see sanitizeEntryName), so every stretch between separators is a component. - for start := anchorEnd + 1; start < len(entryPath); { - end := len(entryPath) - if i := strings.IndexByte(entryPath[start:], '/'); i >= 0 { - end = start + i - } - - if !charge.IndexRecord(indexNodeCost) { - return nil, false - } - child := &indexNode{ - name: entryPath[start:end], - path: entryPath[:end], - // anything with more path below it is a directory whatever the entry itself is - isDir: end < len(entryPath) || isDir, - } - anchor.children.Set(child.name, child) - anchor.childList = append(anchor.childList, child) - r.byPath[child.path] = child - - if child.isDir { - // a directory the archive never named still has to be findable by name, since the segment - // after a `**` is matched against directories - see searchFromRoot - if !charge.IndexRecord(nameIndexCost + int64(len(child.name))) { - return nil, false - } - r.dirNames.Update(child.name, appendNode(child)) - } - - anchor = child - start = end + 1 - } - - if anchor == r.root { - return nil, true - } - return anchor, true -} - -// finalize sorts each directory's child list by name. The key-split indexes need no finalizing - they -// are searchable as they are built - but childList is what the segments that cannot be looked up are -// matched against, and a map's iteration order there would make results unstable. -func (r *ArchiveIndex) finalize() { - for _, node := range r.byPath { - sort.Slice(node.childList, func(i, j int) bool { return node.childList[i].name < node.childList[j].name }) - node.target = r.linkTarget(node) - } -} - -// maxLinkHops bounds a chain of links pointing at links. No real archive is shaped that way, and the -// bound is also what stops a cycle from being followed forever. -const maxLinkHops = 8 - -// linkTarget returns the node whose entry holds this node's content: the node itself when it is not a -// link, and what it points at when it is, following a chain. -// -// nil means there is nothing to read: the chain leaves the archive, ends at a directory, or does not -// end. Such a link still exists as a path, and is answered under that path with no content - which is -// all the archive says about it. -func (r *ArchiveIndex) linkTarget(node *indexNode) *indexNode { - for hops := 0; ; hops++ { - if node.entry == nil || node.isDir { - return nil - } - header := node.entry.Header - if header.Linkname == "" { - return node - } - if hops == maxLinkHops { - return nil - } - next, ok := r.byPath[resolveLinkWithinArchive(node.path, header.Linkname, header.Typeflag == tar.TypeLink)] - if !ok { - return nil - } - node = next - } -} - -// metadataOf describes one entry as a file, including the MIME type sniffed from its content, which -// FilesByMIMEType and nested archive detection both read. -func (r *ArchiveIndex) metadataOf(entry *archive.Entry, entryPath string) file.Metadata { - var contents io.Reader - if opened, err := r.store.Open(entry); err == nil { - contents = opened - } - - metadata := stereoscopeFile.NewMetadata(entry.Header, contents) - // NewMetadata copies the header by value and its fs.FileInfo points into that copy, pinning a second - // tar.Header per entry. Point it at the header the store already holds and charged for. - metadata.FileInfo = entry.Header.FileInfo() - metadata.Path = reportedPath(entryPath) - - if entry.Header.Linkname != "" { - metadata.LinkDestination = reportedPath( - resolveLinkWithinArchive(entryPath, entry.Header.Linkname, entry.Header.Typeflag == tar.TypeLink)) - } - - return metadata -} - -// resolveLinkWithinArchive returns where a link entry points, resolved against the archive's own root -// rather than the host filesystem. linkPath is the archive-rooted path of the link itself. -// -// A hard link's target is named from the archive root (per the tar format), and an absolute symlink -// target is read as archive-absolute, so "/etc/passwd" names an entry of this archive, not the host's. -// Only a relative symlink target is joined onto the link's own directory. The leading "/" makes -// path.Join clamp climbs at the archive root, so "../../../etc/passwd" lands at "/etc/passwd" inside -// the archive. -// -// Nothing here follows the link: the result is a path in this archive's frame, resolving to an entry -// of this archive or to nothing. -func resolveLinkWithinArchive(linkPath, linkname string, hardLink bool) string { - if hardLink || path.IsAbs(linkname) { - return path.Join("/", linkname) - } - return path.Join("/", path.Dir(linkPath), linkname) -} - -func (r *ArchiveIndex) Records() int { return r.records } - -// Truncated reports whether the budget stopped indexing before the archive's last entry. What is -// held is usable either way, just not all of the archive. -func (r *ArchiveIndex) Truncated() bool { return r.truncated } - -// Close releases the entry store, and with it the overflow blob's file handle. Readers handed out by -// this resolver borrow the store, so Close must wait until the extracted archive is torn down; -// ExtractedArchive.Cleanup calls it there. Safe to call more than once. -func (r *ArchiveIndex) Close() error { - if r == nil || r.store == nil { - return nil - } - // the reference is kept, not cleared: EntryStore.Close is idempotent, and a read arriving after - // Close fails through the store rather than dereferencing nil - return r.store.Close() -} - -func (r *ArchiveIndex) location(node *indexNode) file.Location { - return file.NewLocationFromCoordinates(r.coordinates(node)) -} - -// accessedLocation names the file at target, reached by the path of access. The two differ when a glob -// matched a link: the content is the target's, while the path the archive was searched by - and the -// one a reader will recognize - is the link's. -func (r *ArchiveIndex) accessedLocation(target, access *indexNode) file.Location { - if access == target { - return r.location(target) - } - return file.NewVirtualLocationFromCoordinates(r.coordinates(target), reportedPath(access.path)) -} - -func (r *ArchiveIndex) coordinates(node *indexNode) file.Coordinates { - return file.Coordinates{ - RealPath: reportedPath(node.path), - FileSystemID: r.fileSystemID, - ArchivePath: r.archivePath, - } -} - -// reportedPath is how a path leaves this resolver: archive-relative with no leading slash, the form -// SBOMs and archive-path chains use. Nodes are keyed with the slash, which path.Dir and path.Join -// want. -func reportedPath(nodePath string) string { - return strings.TrimPrefix(nodePath, "/") -} - -// FileContentsByLocation returns a reader over the entry's content, from memory or the overflow blob. -// Either way it supports Read, Seek and ReadAt, so a nested archive is read where it lies. -func (r *ArchiveIndex) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { - node, ok := r.byPath[r.requestPath(location.RealPath)] - if !ok || node.entry == nil { - return nil, fmt.Errorf("no archive entry for path=%q", location.RealPath) - } - reader, err := r.store.Open(node.entry) - if err != nil { - return nil, err - } - return overflowEntryReader{nopCloserAt{reader}}, nil -} - -// FileMetadataByLocation returns what was recorded for the entry when it was indexed. -func (r *ArchiveIndex) FileMetadataByLocation(location file.Location) (file.Metadata, error) { - node, ok := r.byPath[r.requestPath(location.RealPath)] - if !ok { - return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.RealPath) - } - return node.metadata, nil -} - -// HasPath reports whether the archive holds anything at that path, file or directory. -func (r *ArchiveIndex) HasPath(userPath string) bool { - _, ok := r.byPath[r.requestPath(userPath)] - return ok -} - -// FilesByPath returns the files at the given paths. Directories are not files, as in every other -// resolver. -func (r *ArchiveIndex) FilesByPath(paths ...string) ([]file.Location, error) { - var out []file.Location - seen := map[string]struct{}{} - for _, userPath := range paths { - node, ok := r.byPath[r.requestPath(userPath)] - if !ok || node.isDir || node.entry == nil { - continue - } - if _, dup := seen[node.path]; dup { - continue - } - seen[node.path] = struct{}{} - out = append(out, r.location(node)) - } - return out, nil -} - -func (r *ArchiveIndex) FilesByMIMEType(types ...string) ([]file.Location, error) { - wanted := map[string]struct{}{} - for _, t := range types { - wanted[t] = struct{}{} - } - - var out []file.Location - for _, node := range r.ordered() { - if node.target != node { - // only a file that holds bytes has a sniffed type; a link has none of its own. The target - // answers for itself under its own path. - continue - } - if _, ok := wanted[node.metadata.MIMEType]; !ok { - continue - } - out = append(out, r.location(node)) - } - return out, nil -} - -// ordered returns the archive's files (not directories) in a stable path order. -// -// Order decides which locations group into which package when results merge into the SBOM, so an -// unstable order splits one package into several across runs. -func (r *ArchiveIndex) ordered() []*indexNode { - if r.sorted != nil { - return r.sorted - } - out := make([]*indexNode, 0, len(r.byPath)) - for _, node := range r.byPath { - if node.isDir { - continue - } - out = append(out, node) - } - sort.Slice(out, func(i, j int) bool { return out[i].path < out[j].path }) - r.sorted = out - return out -} - -// RelativeFileByPath resolves against the archive's root, the only frame an archive has. -func (r *ArchiveIndex) RelativeFileByPath(_ file.Location, userPath string) *file.Location { - locations, err := r.FilesByPath(userPath) - if err != nil || len(locations) == 0 { - return nil - } - return &locations[0] -} - -func (r *ArchiveIndex) AllLocations(ctx context.Context) <-chan file.Location { - out := make(chan file.Location) - go func() { - defer close(out) - for _, node := range r.ordered() { - if node.entry == nil { - continue - } - select { - case out <- r.location(node): - case <-ctx.Done(): - return - } - } - }() - return out -} - -// requestPath puts a caller's path into the archive's frame; a leading slash is optional. -func (r *ArchiveIndex) requestPath(userPath string) string { - if userPath == "" { - return "/" - } - return path.Clean("/" + strings.TrimPrefix(userPath, "/")) -} - -var _ file.Resolver = (*ArchiveIndex)(nil) - -// nopCloserAt gives an entry reader the Close the resolver interface requires. The store owns the -// bytes and the overflow file, so closing a reader over them must close nothing. -type nopCloserAt struct { - archive.ReaderAtSeeker -} - -func (nopCloserAt) Close() error { return nil } diff --git a/syft/internal/fileresolver/archive_index_charge_test.go b/syft/internal/fileresolver/archive_index_charge_test.go deleted file mode 100644 index 5ad90b8b0..000000000 --- a/syft/internal/fileresolver/archive_index_charge_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package fileresolver - -import ( - "archive/tar" - "context" - "fmt" - "runtime" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/anchore/syft/internal/archive" - "github.com/anchore/syft/internal/tmpdir" -) - -// Test_indexChargeCoversWhatTheIndexKeeps is what the cost constants in archive_index.go rest on. -// -// The charge exists to stop an archive before its index is built, so it must over-estimate what the -// index really retains. An under-count is worse than no limit at all: it reads as a configured bound -// while admitting some multiple of it. A previous accounting charged 216 bytes plus the name per -// entry where the index kept 627, and charged nothing for the directory nodes an entry's path -// implies - which a single entry can name 200,000 of. -func Test_indexChargeCoversWhatTheIndexKeeps(t *testing.T) { - tests := []struct { - name string - names []string - }{ - { - name: "flat, short names", - names: func() []string { - out := make([]string, 60000) - for i := range out { - out[i] = fmt.Sprintf("d/%06d", i) - } - return out - }(), - }, - { - name: "nested, jar-shaped names", - names: func() []string { - out := make([]string, 60000) - for i := range out { - out[i] = fmt.Sprintf("com/example/pkg%03d/sub%02d/Thing%06d.class", i%500, i%50, i) - } - return out - }(), - }, - { - name: "one entry per directory, so every entry synthesizes a chain", - names: func() []string { - out := make([]string, 20000) - for i := range out { - out[i] = fmt.Sprintf("a%05d/b%05d/c%05d/d%05d/file.txt", i, i, i, i) - } - return out - }(), - }, - { - name: "deep single entry, at the path cap", - names: []string{strings.Repeat("a/", (maxEntryPathBytes-2)/2) + "f"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - store := storeOf(t, tt.names) - limiter, charge := unboundedCharge() - - var index *ArchiveIndex - kept := heapGrowth(t, func() { - var err error - index, err = NewFromArchiveEntries("", "outer.jar", store, charge) - require.NoError(t, err) - }) - require.False(t, index.Truncated(), "an unbounded charge must never truncate") - - charged, _ := limiter.InUse() - t.Logf("entries=%d records=%d charged=%d kept=%d (charged/kept=%.2fx)", - len(tt.names), index.Records(), charged, kept, float64(charged)/float64(kept)) - - assert.GreaterOrEqual(t, charged, kept, - "the index charged less than it kept, so the configured bound admits more than it says") - - // an over-estimate is the point, but one wildly over wastes the budget on archives doing - // nothing wrong - assert.Less(t, charged, kept*4, - "the index charged several times what it kept, which spends a scan's budget on nothing") - - runtime.KeepAlive(index) - }) - } -} - -// Test_deepEntryPathsAreRefused covers the bound on an entry's path, which keeps one name from -// becoming an arbitrarily long chain of nodes. -func Test_deepEntryPathsAreRefused(t *testing.T) { - atCap := strings.Repeat("a/", (maxEntryPathBytes-2)/2) + "f" - overCap := strings.Repeat("a/", maxEntryPathBytes) + "f" - - require.LessOrEqual(t, len(atCap), maxEntryPathBytes) - require.Greater(t, len(overCap), maxEntryPathBytes) - - index, err := NewFromArchiveEntries("", "outer.jar", - storeOf(t, []string{"kept.txt", atCap, overCap}), nil) - require.NoError(t, err) - - assert.Equal(t, 2, index.Records(), "the entry over the cap is dropped and the rest are kept") - assert.True(t, index.HasPath("/kept.txt")) - assert.True(t, index.HasPath("/"+atCap)) - assert.False(t, index.HasPath("/"+overCap)) - assert.False(t, index.Truncated(), "dropping an unusable name is not a budget truncation") -} - -// Test_deepEntryPathsStayAffordable guards the two-pass placement in ArchiveIndex.node. -// -// Placing a node used to recurse from the leaf up, rebuilding the path with path.Dir at every level, -// making a chain cost work quadratic in its depth. Against the replacement: 400 entries sharing one -// deep prefix went from 9.8ms to 0.77ms, 400 entries each with their own chain from 2.87s to 0.33s, -// and a single 200,000-component name from 59.3s to 1.3s. That last shape no longer reaches here - -// sanitizeEntryName refuses it - which bounds the residual per-node cost neither pass can remove: -// placing a node means keying it by its full path. -// -// The budget below is enormous next to the ~0.1s this takes: it guards against an order-of-magnitude -// regression, not a few percent. -func Test_deepEntryPathsStayAffordable(t *testing.T) { - if testing.Short() { - t.Skip("timing guard") - } - - names := make([]string, 400) - for i := range names { - names[i] = fmt.Sprintf("%s%04d.txt", strings.Repeat("a/", (maxEntryPathBytes-16)/2), i) - } - - store := storeOf(t, names) - started := time.Now() - index, err := NewFromArchiveEntries("", "outer.jar", store, nil) - require.NoError(t, err) - took := time.Since(started) - - require.Equal(t, len(names), index.Records()) - assert.Less(t, took, 30*time.Second, "placing deep paths has regressed by an order of magnitude") - t.Logf("indexed %d entries of %d bytes of path in %s", len(names), len(names[0]), took) -} - -// Test_indexTruncatesWhenTheBudgetRefuses covers what an index does when the budget runs out part way -// through: it keeps what it has, says so, and stays usable. -func Test_indexTruncatesWhenTheBudgetRefuses(t *testing.T) { - names := make([]string, 500) - for i := range names { - names[i] = fmt.Sprintf("dir/file%04d.txt", i) - } - - // room for a handful of entries, nowhere near all 500 - limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: 10 * indexNodeCost, MaxDiskBytes: 0}) - index, err := NewFromArchiveEntries("", "outer.jar", storeOf(t, names), limiter.Charge()) - require.NoError(t, err) - - assert.True(t, index.Truncated(), "the budget refused a record, so the index is truncated") - assert.Positive(t, index.Records(), "what was indexed before the refusal is kept") - assert.Less(t, index.Records(), len(names)) - - // and what it did index is a working filesystem, not a half-built one - locations, err := index.FilesByGlob("**/*.txt") - require.NoError(t, err) - assert.Len(t, locations, index.Records()) - for _, loc := range locations { - assert.True(t, index.HasPath("/"+loc.RealPath)) - } -} - -// Test_zeroMemoryBudgetStillIndexes covers why index records fall back to the disk budget: -// MaxMemoryBytes of zero is a supported configuration meaning "spill everything", and an archive must -// still be reachable under it. -func Test_zeroMemoryBudgetStillIndexes(t *testing.T) { - limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) - index, err := NewFromArchiveEntries("", "outer.jar", - storeOf(t, []string{"a/b/c.txt", "a/d.txt"}), limiter.Charge()) - require.NoError(t, err) - - assert.False(t, index.Truncated()) - assert.Equal(t, 2, index.Records()) - - memory, disk := limiter.InUse() - assert.Zero(t, memory, "a zero memory budget admits nothing to memory") - assert.Positive(t, disk, "so the index records are charged against the disk budget instead") -} - -// storeOf puts the given entry names into a store with no budget of its own, so what a test measures -// afterwards is the index's charge and nothing else. -func storeOf(t testing.TB, names []string) *archive.EntryStore { - t.Helper() - workDir := archive.NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) - t.Cleanup(workDir.Remove) - store := archive.NewEntryStore(workDir, "test.jar", nil) - t.Cleanup(func() { require.NoError(t, store.Close()) }) - for _, name := range names { - hdr := tar.Header{Name: name, Mode: 0o600, Typeflag: tar.TypeReg} - _, err := store.Add(hdr, strings.NewReader(""), nil) - require.NoError(t, err) - } - return store -} - -func unboundedCharge() (*archive.Limiter, *archive.Charge) { - limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) - return limiter, limiter.Charge() -} - -// heapGrowth reports how much live heap build left behind, which for an index kept alive is what it -// retains. -func heapGrowth(t testing.TB, build func()) int64 { - t.Helper() - var before, after runtime.MemStats - runtime.GC() - runtime.GC() - runtime.ReadMemStats(&before) - - build() - - runtime.GC() - runtime.GC() - runtime.ReadMemStats(&after) - return int64(after.HeapAlloc) - int64(before.HeapAlloc) -} diff --git a/syft/internal/fileresolver/archive_index_glob.go b/syft/internal/fileresolver/archive_index_glob.go deleted file mode 100644 index 7fe1c60c0..000000000 --- a/syft/internal/fileresolver/archive_index_glob.go +++ /dev/null @@ -1,544 +0,0 @@ -package fileresolver - -import ( - "sort" - "strings" - - "github.com/bmatcuk/doublestar/v4" - - syftindex "github.com/anchore/syft/internal/index" - "github.com/anchore/syft/syft/file" -) - -// The search here is ported from the dirscan prototype the key-split index came from. The shape is -// the prototype's: walk the pattern segment by segment, and wherever a segment fixes enough text, -// answer it from an index instead of from the children of the directory the walk has reached. -// -// Three deliberate departures from the prototype: -// -// - doublestar decides every segment the indexes cannot answer, rather than the prototype's -// hand-built regex. syft's globs are doublestar's dialect, and a segment carrying `?` or a `[...]` -// class has to mean there what it means everywhere else in syft. -// - a pattern the prototype panics on is answered instead: an embedded `**` (`a**b`) is matched as -// an ordinary segment, and a trailing `**` collects everything below. -// - alternations are expanded over the whole pattern rather than one segment at a time, and on `,` -// rather than the prototype's `|`. syft's catalogers write `{status,status.d/**}`, where the -// alternatives carry separators, and `|` is not an alternation to doublestar at all. -const ( - anyDir = "**" - sep = "/" - wildcard = '*' -) - -// FilesByGlob returns the files matching any of the patterns. -// -// Patterns are archive-relative. A leading slash is the archive's own root rather than a filesystem -// root, so `/META-INF/*` and `META-INF/*` name the same entries. -func (r *ArchiveIndex) FilesByGlob(patterns ...string) ([]file.Location, error) { - found := map[*indexNode]struct{}{} - for _, pattern := range patterns { - // braces are expanded before splitting into segments, since an alternative may carry its own - // separators: `{status,status.d/**}` is one segment and two, and splitting first would cut the - // group in half. Past maxAlternatives the pattern is searched as written - see segmentProvider. - alternatives, ok := expandBraces(pattern) - if !ok { - alternatives = []string{pattern} - } - for _, alternative := range alternatives { - trimmed := strings.Trim(alternative, sep) - if trimmed == "" { - continue - } - r.search(found, r.root, strings.Split(trimmed, sep), nil, true) - } - } - - out := make([]file.Location, 0, len(found)) - for target, access := range collapseToOnePathPerFile(found) { - out = append(out, r.accessedLocation(target, access)) - } - // location order decides how a cataloger merges its findings, and the search accumulates into a map. - // This sort is only for a stable answer; which path names each file is settled by preferredAccess. - sort.Slice(out, func(i, j int) bool { return out[i].RealPath < out[j].RealPath }) - return out, nil -} - -// collapseToOnePathPerFile reduces the matched nodes to one path per file, keyed by the node holding -// that file's content. -// -// Several paths can name one file - a link and its target, or two links to it - and a resolver answers -// with one of them (see file.Resolver: "if multiple paths to the same file are found, the best single -// match should be returned"). A dangling link names no content, so it stands alone under its own path. -func collapseToOnePathPerFile(found map[*indexNode]struct{}) map[*indexNode]*indexNode { - best := make(map[*indexNode]*indexNode, len(found)) - for node := range found { - target := node.target - if target == nil { - target = node - } - current, seen := best[target] - if !seen || preferredAccess(node, current, target) == node { - best[target] = node - } - } - return best -} - -// preferredAccess picks which of two paths to one file the answer carries: the path that is the file -// itself, and otherwise the lower-sorting one. -// -// Both clauses read only the candidates, never their arrival order, so the winner is stable across -// runs regardless of map iteration order or how results are later sorted. -func preferredAccess(a, b, target *indexNode) *indexNode { - switch { - case a == target: - return a - case b == target: - return b - case a.path < b.path: - return a - } - return b -} - -// childProvider yields the nodes of one directory that a segment can reach. A provider may ignore the -// directory it is handed and answer from a global index instead, which is how `**` is skipped over. -type childProvider func(parent *indexNode) []*indexNode - -// search matches segments against the tree from parent, collecting every file the pattern reaches. -// -// visited is how `**` avoids reporting the same file twice when the directories it expands to overlap, -// and checkVisited is false on the recursions that are not a descent - an alternation being retried -// against the same directory, or `**` matching the directory it starts from. -func (r *ArchiveIndex) search(found map[*indexNode]struct{}, parent *indexNode, segments []string, visited []*indexNode, checkVisited bool) { - if len(segments) == 0 { - return - } - - if checkVisited { - for _, seen := range visited { - if seen == parent { - return - } - } - visited = append(visited, parent) - } - - segment := segments[0] - - if segment == anyDir { - r.searchAnyDir(found, parent, segments, visited) - return - } - - provider := r.segmentProvider(segment) - - if len(segments) == 1 { - for _, child := range provider(parent) { - keepFile(found, child) - } - return - } - - for _, child := range provider(parent) { - if !child.isDir { - continue - } - r.search(found, child, segments[1:], visited, true) - } -} - -// searchAnyDir handles a `**` segment. -// -// `**` matches the directory it sits at and every directory below it, so the general answer is to -// re-run what follows from each of those. The index is what makes that avoidable: when the pattern -// starts with `**`, the segment after it can often be looked up by name across the whole archive, -// which reaches the same directories - or, at the end of a pattern, the same files - without walking. -func (r *ArchiveIndex) searchAnyDir(found map[*indexNode]struct{}, parent *indexNode, segments []string, visited []*indexNode) { - rest := segments[1:] - - if len(rest) == 0 { - // a pattern ending in `**`: everything below here. The prototype rejects this shape; syft's - // catalogers do write it. - r.collect(found, parent) - return - } - - // The segment after a `**` is looked up across the whole archive, then confined to what sits below - // parent. The global lookup is what the name indexes key on; the filter is what lets a `**` appear - // anywhere in the pattern, not just as the leading segment. - // - // This matters for split patterns: `/{usr,opt}/**/*.jar` becomes `/usr/**/*.jar` and `/opt/**/*.jar`, - // each reaching its `**` with a directory already selected, where the prototype would walk every - // directory beneath it. - // - // Alternations are already expanded by FilesByGlob, so `{go,go.exe}` reaches here as two exact names. - if len(rest) == 1 { - // `**/`, the shape most cataloger globs take - if nodes, ok := r.byName(&r.fileNames, rest[0]); ok { - for _, node := range nodes { - if under(node, parent) { - keepFile(found, node) - } - } - return - } - } else if nodes, ok := r.byName(&r.dirNames, rest[0]); ok { - // `**//`: the directory segment is looked up across the archive, and the search - // continues from each match with both segments consumed - for _, node := range nodes { - if under(node, parent) { - r.search(found, node, rest[1:], visited, true) - } - } - return - } - - // `**` matches this directory and every directory beneath it - r.search(found, parent, rest, visited, false) - for _, dir := range r.dirsBeneath(parent) { - r.search(found, dir, rest, visited, true) - } -} - -func dedupeNodes(nodes []*indexNode) []*indexNode { - if len(nodes) < 2 { - return nodes - } - seen := make(map[*indexNode]struct{}, len(nodes)) - out := nodes[:0] - for _, node := range nodes { - if _, dup := seen[node]; dup { - continue - } - seen[node] = struct{}{} - out = append(out, node) - } - return out -} - -// byName answers one segment from a global name index, reporting false when the segment fixes too -// little for any lookup to reach every name it could match. -// -// Three shapes the index answers outright: an exact name, a single leading `*`, and a single trailing -// `*`. Everything else goes to byNameNarrowed, which either reduces the segment to a set of those or -// narrows by the literal text it fixes and lets doublestar decide. -func (r *ArchiveIndex) byName(idx *syftindex.PrefixSuffix[[]*indexNode], segment string) ([]*indexNode, bool) { - if strings.Contains(segment, anyDir) { - // `a**b` crosses directories, so no base-name lookup can answer it - return nil, false - } - if strings.ContainsAny(segment, nonWildcardMeta) { - // not a literal-with-one-`*`, so the plain lookups below would read the metacharacters as text - return r.byNameNarrowed(idx, segment) - } - - first := strings.IndexRune(segment, wildcard) - last := strings.LastIndexByte(segment, byte(wildcard)) - - switch { - case first < 0: - // an exact name, e.g. **/pom.properties - return idx.Get(segment), true - case first == 0 && last == 0: - // `*something`, e.g. **/*.jar - what the reverse index is keyed for. A lone `*` gives the empty - // suffix, which reaches every name, and that is the right answer for `**/*`. - return flattenNodes(idx.BySuffix(segment[1:])), true - case last == len(segment)-1 && first == last: - // `something*`, e.g. **/pom* - return flattenNodes(idx.ByPrefix(segment[:len(segment)-1])), true - } - - // anything else - a `?`, a class, a wildcard in the middle - is not a lookup on its own, but the - // literal text it fixes still narrows one - return r.byNameNarrowed(idx, segment) -} - -// byNameNarrowed answers a segment the three shapes above cannot, by narrowing the index to a -// candidate set and letting doublestar decide over it. It reports false when the segment fixes nothing -// to narrow by, which leaves the caller to walk. -// -// Two ways in. A segment whose only metacharacters are alternations and classes denotes a finite set -// of names, so it reduces to exact lookups and doublestar is not needed at all. Anything else is -// narrowed by the literal text it fixes at its head or tail - `libstd-????????????????.so` fixes -// "libstd-" and ".so" - and doublestar runs over what that leaves. -func (r *ArchiveIndex) byNameNarrowed(idx *syftindex.PrefixSuffix[[]*indexNode], segment string) ([]*indexNode, bool) { - if lookups := enumerateLookups(segment); len(lookups) > 0 { - // each lookup is exactly its alternative: `*` stands for any run of characters and a base name - // holds no separator for it to stop at - return byLookups(idx, lookups), true - } - - candidates, ok := r.narrowByBounds(idx, segment) - if !ok { - return nil, false - } - return matchingNodes(candidates, segment), true -} - -func byLookups(idx *syftindex.PrefixSuffix[[]*indexNode], lookups []nameLookup) []*indexNode { - var out []*indexNode - for _, lookup := range lookups { - switch lookup.kind { - case segExact: - out = append(out, idx.Get(lookup.literal)...) - case segPrefix: - out = append(out, flattenNodes(idx.ByPrefix(lookup.literal))...) - case segSuffix: - out = append(out, flattenNodes(idx.BySuffix(lookup.literal))...) - } - } - if len(lookups) < 2 { - return out - } - // alternatives can overlap, e.g. `{lib,l}*` looks up "lib" and "l" - return dedupeNodes(out) -} - -// boundProbe is how many names a bound may reach before it is treated as the broad side, capping what -// the probe below can waste: a side that gives up costs this much, not the size of its full result. -// -// Sized for what a cataloger's glob seeks, not what an archive holds. A bound reaching more names than -// this is not narrowing much. -const boundProbe = 64 - -// narrowByBounds returns the nodes whose name carries one of the literal bounds the segment fixes, or -// false when it fixes nothing to look up by. -// -// A segment fixing text at both ends has two possible lookups, and the text alone does not say which -// is selective: among one `libstd-.so` beside 1,500 `libstd-NNNN.a`, "libstd-" reaches all 1,501 -// and ".so" reaches one, so the longer literal is the worse lookup. -// -// The selective side is found by running one lookup bounded: one that completes is both the answer and -// proof it was cheap; one that gives up costs only boundProbe. So choosing costs the probe, not the -// broad side's full result. -func (r *ArchiveIndex) narrowByBounds(idx *syftindex.PrefixSuffix[[]*indexNode], segment string) ([]*indexNode, bool) { - prefixes, suffixes := segmentBounds(segment) - - switch { - case len(prefixes) == 0 && len(suffixes) == 0: - return nil, false - case len(suffixes) == 0: - return gather(prefixes, idx.ByPrefix), true - case len(prefixes) == 0: - return gather(suffixes, idx.BySuffix), true - } - - // the tail first: a name's extension is usually the more selective end of it - if nodes, ok := gatherUpTo(suffixes, idx.BySuffixUpTo); ok { - return nodes, true - } - if nodes, ok := gatherUpTo(prefixes, idx.ByPrefixUpTo); ok { - return nodes, true - } - - // neither end narrows to less than the probe, so nothing distinguishes them and doublestar has the - // same work either way - return gather(prefixes, idx.ByPrefix), true -} - -// gatherUpTo runs a set of bounds under one shared probe budget, reporting false as soon as they -// reach past it. What it returns on false is partial and is discarded. -func gatherUpTo(values []string, lookup func(string, int) ([][]*indexNode, bool)) ([]*indexNode, bool) { - var out []*indexNode - for _, value := range values { - groups, complete := lookup(value, boundProbe) - if !complete { - return nil, false - } - out = append(out, flattenNodes(groups)...) - if len(out) > boundProbe { - // one name per key, but a name repeats across directories - the budget is on what comes back - return nil, false - } - } - if len(values) < 2 { - return out, true - } - return dedupeNodes(out), true -} - -func gather(values []string, lookup func(string) [][]*indexNode) []*indexNode { - var out []*indexNode - for _, value := range values { - out = append(out, flattenNodes(lookup(value))...) - } - if len(values) < 2 { - return out - } - return dedupeNodes(out) -} - -// matchingNodes keeps the candidates whose name the segment actually matches: the bounds narrow, -// doublestar decides. -func matchingNodes(candidates []*indexNode, segment string) []*indexNode { - var out []*indexNode - for _, node := range candidates { - if ok, err := doublestar.Match(segment, node.name); err == nil && ok { - out = append(out, node) - } - } - return out -} - -// nonWildcardMeta are the glob metacharacters the index lookups cannot reason about. A lookup reads a -// segment as literal text around at most one `*`, so a segment carrying any of these goes to -// doublestar rather than being looked up as a literal name that matches nothing. -const nonWildcardMeta = `?[]{}\` - -// segmentProvider returns how to reach the children of a directory that one segment matches, choosing -// the narrowest answer the segment allows. -func (r *ArchiveIndex) segmentProvider(segment string) childProvider { - if strings.ContainsAny(segment, nonWildcardMeta) || strings.Contains(segment, anyDir) { - return narrowedChildren(segment) - } - - first := strings.IndexRune(segment, wildcard) - switch { - case first < 0: - // an exact name: one lookup in this directory's index - return func(parent *indexNode) []*indexNode { - if child := parent.children.Get(segment); child != nil { - return []*indexNode{child} - } - return nil - } - case segment == string(wildcard): - return func(parent *indexNode) []*indexNode { return parent.childList } - case first == len(segment)-1: - // `name*`, which this directory's index answers by prefix - prefix := segment[:len(segment)-1] - return func(parent *indexNode) []*indexNode { return parent.children.ByPrefix(prefix) } - } - - return narrowedChildren(segment) -} - -// narrowedChildren is the fallback, narrowing a directory's children as far as the segment allows -// before doublestar sees them. -// -// A directory's children are indexed forwards only, so unlike the global indexes only a leading bound -// is usable here - which is why a segment that reduces to a set of exact and prefix lookups is worth -// detecting separately, and why `*tail` falls back to comparing each child. -func narrowedChildren(segment string) childProvider { - if lookups := enumerateLookups(segment); len(lookups) > 0 { - return func(parent *indexNode) []*indexNode { - return childrenByLookups(parent, lookups) - } - } - - prefixes, _ := segmentBounds(segment) - if len(prefixes) == 0 { - return matchingChildren(segment, func(parent *indexNode) []*indexNode { return parent.childList }) - } - return matchingChildren(segment, func(parent *indexNode) []*indexNode { - var out []*indexNode - for _, prefix := range prefixes { - out = append(out, parent.children.ByPrefix(prefix)...) - } - if len(prefixes) < 2 { - return out - } - return dedupeNodes(out) - }) -} - -// childrenByLookups answers a segment that reduces to a finite set of lookups. Each lookup is exactly -// its alternative, so nothing here needs doublestar - see byLookups. -func childrenByLookups(parent *indexNode, lookups []nameLookup) []*indexNode { - var out []*indexNode - for _, lookup := range lookups { - switch lookup.kind { - case segExact: - if child := parent.children.Get(lookup.literal); child != nil { - out = append(out, child) - } - case segPrefix: - out = append(out, parent.children.ByPrefix(lookup.literal)...) - case segSuffix: - // children are indexed forwards only, so a fixed tail is compared per child - for _, child := range parent.childList { - if strings.HasSuffix(child.name, lookup.literal) { - out = append(out, child) - } - } - } - } - if len(lookups) < 2 { - return out - } - return dedupeNodes(out) -} - -// matchingChildren runs doublestar over whatever candidates narrows down to. -func matchingChildren(segment string, narrow childProvider) childProvider { - return func(parent *indexNode) []*indexNode { - var out []*indexNode - for _, child := range narrow(parent) { - if ok, err := doublestar.Match(segment, child.name); err == nil && ok { - out = append(out, child) - } - } - return out - } -} - -// under reports whether node sits strictly below dir, which is what confines a global name lookup to -// the subtree a `**` is standing in. -// -// Paths are archive-rooted and cleaned, so this is a prefix test with the separator checked: without -// it "/usrlocal/x" would read as below "/usr". The archive root is above everything. -func under(node, dir *indexNode) bool { - if dir.path == "/" { - return true - } - return len(node.path) > len(dir.path) && - node.path[len(dir.path)] == '/' && - strings.HasPrefix(node.path, dir.path) -} - -// keepFile records a node if it is a file this archive actually holds. Directories are never a glob's -// answer, and a synthesized directory has no entry to read. -func keepFile(found map[*indexNode]struct{}, node *indexNode) { - if node == nil || node.isDir || node.entry == nil { - return - } - found[node] = struct{}{} -} - -// dirsBeneath returns every directory below a node, which is what `**` has to consider when more -// pattern follows it. -func (r *ArchiveIndex) dirsBeneath(node *indexNode) []*indexNode { - var out []*indexNode - var walk func(*indexNode) - walk = func(n *indexNode) { - for _, child := range n.childList { - if !child.isDir { - continue - } - out = append(out, child) - walk(child) - } - } - walk(node) - return out -} - -// collect records every file at or below a node. -func (r *ArchiveIndex) collect(found map[*indexNode]struct{}, node *indexNode) { - for _, child := range node.childList { - if child.isDir { - r.collect(found, child) - continue - } - keepFile(found, child) - } -} - -func flattenNodes(groups [][]*indexNode) []*indexNode { - var out []*indexNode - for _, group := range groups { - out = append(out, group...) - } - return out -} diff --git a/syft/pkg/cataloger/java/archive_entries.go b/syft/pkg/cataloger/java/archive_entries.go index dccb801d7..a24b0a31b 100644 --- a/syft/pkg/cataloger/java/archive_entries.go +++ b/syft/pkg/cataloger/java/archive_entries.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" - "github.com/bmatcuk/doublestar/v4" "github.com/scylladb/go-set/strset" "github.com/anchore/syft/internal" @@ -14,30 +13,19 @@ import ( "github.com/anchore/syft/syft/file" ) -// archiveEntries is where an archive parser reads a java archive's entries from. -// -// A parser needs three things of an archive: which entries match a glob, the contents of some of -// them, and the digest of the archive as a whole. Everything else - which manifest wins, how -// pom.properties decides identity, license discovery - is logic over those answers, so naming the -// three lets the same logic run over an archive file and over one already extracted to a filesystem. -// -// Entry names are archive-relative with no leading slash, matching zip entry names, since they become -// the map keys and pom paths identity is derived from. An implementation over a filesystem must strip -// the root to match. +// archiveEntries is where the archive parser reads a java archive's entries from, so the same parser +// runs over an archive file and over an archive already extracted to a filesystem. Entry names are +// archive-relative with no leading slash, as zip entry names are. type archiveEntries interface { - glob(caseInsensitive bool, patterns ...string) []string + // glob returns the entries matching any pattern. Patterns are written against a leading slash + // ("/META-INF/*"); the names returned have none. + glob(patterns ...string) []string - // contents reads the named entries, returning a map keyed by the same names. + // contents reads the named entries, keyed by the same names. contents(ctx context.Context, names ...string) (map[string]string, error) - - // digests returns the digests of the archive as a whole, a property of the archive file rather - // than of anything inside it. - digests(ctx context.Context) ([]file.Digest, error) } -// zipEntries reads entries out of a java archive on disk, for scans where nothing has already -// extracted it. It copies the archive to a temp file, reads its central directory once, and reopens -// it for each set of entries it is asked for. +// zipEntries reads entries out of a java archive file on disk. type zipEntries struct { archivePath string fileManifest intFile.ZipFileManifest @@ -51,41 +39,20 @@ func newZipEntries(ctx context.Context, archivePath string) (*zipEntries, error) return &zipEntries{archivePath: archivePath, fileManifest: fileManifest}, nil } -func (z *zipEntries) glob(caseInsensitive bool, patterns ...string) []string { - return z.fileManifest.GlobMatch(caseInsensitive, patterns...) +func (z *zipEntries) glob(patterns ...string) []string { + return z.fileManifest.GlobMatch(patterns...) } func (z *zipEntries) contents(ctx context.Context, names ...string) (map[string]string, error) { return intFile.ContentsFromZip(ctx, z.archivePath, names...) } -func (z *zipEntries) digests(ctx context.Context) ([]file.Digest, error) { - return getDigestsFromArchive(ctx, z.archivePath) -} - -// resolverEntries reads entries out of an archive already extracted and indexed by the archive -// cataloger task. Nothing is copied or reopened. The archive's own digests come from whoever -// extracted it, since the archive file is not present in its own contents. +// resolverEntries reads entries out of an archive the archive cataloger task has extracted. type resolverEntries struct { - resolver file.Resolver - archiveDigests []file.Digest + resolver file.Resolver } -func newResolverEntries(resolver file.Resolver, archiveDigests []file.Digest) *resolverEntries { - return &resolverEntries{ - resolver: resolver, - archiveDigests: archiveDigests, - } -} - -func (r *resolverEntries) glob(caseInsensitive bool, patterns ...string) []string { - if caseInsensitive { - return r.globCaseInsensitive(patterns...) - } - - // deduplicated within this call only: the parser globs the same pattern more than once (identity - // resolution and aux package discovery both ask for the poms), so remembering across calls would - // return nothing the second time and lose every package derived from a bundled pom.properties +func (r *resolverEntries) glob(patterns ...string) []string { seen := strset.New() var names []string for _, pattern := range patterns { @@ -106,37 +73,6 @@ func (r *resolverEntries) glob(caseInsensitive bool, patterns ...string) []strin return names } -// globCaseInsensitive matches the way zipEntries does, by lowercasing both sides. The resolver's own -// glob is case sensitive, so this lists the archive's files once and matches them here. License -// discovery is the only caller. -func (r *resolverEntries) globCaseInsensitive(patterns ...string) []string { - all, err := r.resolver.FilesByGlob("**/*") - if err != nil { - log.WithFields("error", err).Trace("unable to list archive contents") - return nil - } - seen := strset.New() - var names []string - for _, pattern := range patterns { - lowered := strings.ToLower(pattern) - for _, location := range all { - // matched with a leading slash, since the patterns are written that way ("/META-INF/*") and - // zipEntries normalizes the same way; an extracted archive's resolver reports names without - // one, so matching raw would match nothing - name := strings.TrimPrefix(location.Path(), "/") - if ok, matchErr := doublestar.Match(lowered, "/"+strings.ToLower(name)); matchErr != nil || !ok { - continue - } - if seen.Has(name) { - continue - } - seen.Add(name) - names = append(names, name) - } - } - return names -} - func (r *resolverEntries) contents(_ context.Context, names ...string) (map[string]string, error) { out := make(map[string]string, len(names)) for _, name := range names { @@ -156,15 +92,7 @@ func (r *resolverEntries) contents(_ context.Context, names ...string) (map[stri return out, nil } -func (r *resolverEntries) digests(_ context.Context) ([]file.Digest, error) { - return r.archiveDigests, nil -} - -// read reads one entry's content to the end and closes the stream, returning a string because every -// consumer parses text. -// -// The copy is bounded the way zipEntries bounds its own: an entry of an extracted archive is still -// attacker-supplied, and reading it into a string is the allocation a decompression bomb targets. +// read returns one entry's content, bounded against decompression bombs as zipEntries bounds its own. func (r *resolverEntries) read(location file.Location) (string, error) { reader, err := r.resolver.FileContentsByLocation(location) if err != nil { diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 5b469826a..4232d07a1 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -17,8 +17,10 @@ import ( "golang.org/x/exp/maps" "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/archive" intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/tmpdir" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -56,27 +58,25 @@ var javaArchiveHashes = []crypto.Hash{ crypto.SHA1, } -// archiveParser reads one java archive, whichever source it came from: everything it does is logic -// over an archiveSource's entries and identity, so it does not know whether this cataloger opened an -// archive file or the archive cataloger task already extracted one. +// archiveParser reads one java archive, whether this cataloger opened the archive file or the archive +// cataloger task already extracted it. type archiveParser struct { - archiveSource + entries archiveEntries + // location is the archive's location in its parent filesystem; archive.VirtualPath(location) is its + // identity with the chain of containing archives prepended: "app.war:WEB-INF/lib/dep.jar" + location file.Location + fileInfo archiveFilename + // archiveDigests are of the archive file as a whole. When the archive cataloger task extracted the + // archive they come from that extraction, since the archive is not inside itself. + archiveDigests []file.Digest + // contentPath is the directory nested archives are unzipped into. Set only when this cataloger + // opened the archive itself, since only then does it recurse. + contentPath string detectNested bool cfg ArchiveCatalogerConfig maven *maven.Resolver } -// newArchiveParser builds the parser for one archive. detectNested is whether this parser recurses -// into the java archives inside this one, which only a file-backed source can do. -func newArchiveParser(src archiveSource, detectNested bool, cfg ArchiveCatalogerConfig) *archiveParser { - return &archiveParser{ - archiveSource: src, - detectNested: detectNested, - cfg: cfg, - maven: maven.NewResolver(nil, cfg.mavenConfig()), - } -} - type genericArchiveParserAdapter struct { cfg ArchiveCatalogerConfig } @@ -92,9 +92,7 @@ func (gap genericArchiveParserAdapter) parseJavaArchive(ctx context.Context, _ f // processJavaArchive processes an archive for java contents, returning all Java libraries and nested archives func (gap genericArchiveParserAdapter) processJavaArchive(ctx context.Context, reader file.LocationReadCloser, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) { - // when the archive cataloger task owns recursion, unarchiving here would catalog nested archives - // twice - parser, cleanupFn, err := newJavaArchiveParser(ctx, reader, !gap.cfg.NestedArchivesHandledExternally, gap.cfg) + parser, cleanupFn, err := newJavaArchiveParser(ctx, reader, !archive.NestedCatalogingEnabled(ctx), gap.cfg) // note: even on error, we should always run cleanup functions defer cleanupFn() if err != nil { @@ -111,14 +109,65 @@ func uniquePkgKey(groupID string, p *pkg.Package) string { return fmt.Sprintf("%s|%s|%s", groupID, p.Name, p.Version) } -// newJavaArchiveParser returns a new java archive parser for an archive file this cataloger opens -// itself. Can be configured to discover and parse nested archives or ignore them. +// newJavaArchiveParser returns a parser for an archive file this cataloger opens itself, copied to a +// temp dir. Can be configured to discover and parse nested archives or ignore them. The returned +// cleanup must run even when an error is returned. func newJavaArchiveParser(ctx context.Context, reader file.LocationReadCloser, detectNested bool, cfg ArchiveCatalogerConfig) (*archiveParser, func(), error) { - src, cleanupFn, err := newFileArchiveSource(ctx, reader) + // fetch the last element of the virtual path + virtualElements := strings.Split(archive.VirtualPath(reader.Location), ":") + currentFilepath := virtualElements[len(virtualElements)-1] + + td := tmpdir.FromContext(ctx) + if td == nil { + return nil, func() {}, fmt.Errorf("no temp dir factory in context") + } + contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(td, currentFilepath, reader) + if err != nil { + return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err) + } + + entries, err := newZipEntries(ctx, archivePath) if err != nil { return nil, cleanupFn, err } - return newArchiveParser(src, detectNested, cfg), cleanupFn, nil + + digests, err := getDigestsFromArchive(ctx, archivePath) + if err != nil { + return nil, cleanupFn, err + } + + return &archiveParser{ + entries: entries, + location: reader.Location, + fileInfo: newJavaArchiveFilename(currentFilepath), + archiveDigests: digests, + contentPath: contentPath, + detectNested: detectNested, + cfg: cfg, + maven: maven.NewResolver(nil, cfg.mavenConfig()), + }, cleanupFn, nil +} + +// newExtractedArchiveParser returns a parser for the archive whose extracted contents the resolver +// holds. Its identity comes from the traversal, since none of it is inside the archive, and it never +// recurses: archives nested inside it are the archive cataloger task's to find. +func newExtractedArchiveParser(trav *archive.Traversal, resolver file.Resolver, cfg ArchiveCatalogerConfig) (*archiveParser, error) { + if resolver == nil { + return nil, fmt.Errorf("no resolver for the contents of %q", archive.VirtualPath(trav.Location)) + } + return &archiveParser{ + entries: &resolverEntries{resolver: resolver}, + location: trav.Location, + fileInfo: newJavaArchiveFilename(path.Base(trav.Location.Path())), + archiveDigests: trav.Digests, + cfg: cfg, + maven: maven.NewResolver(nil, cfg.mavenConfig()), + }, nil +} + +// virtualPath is the archive's path with the chain of containing archives prepended. +func (j *archiveParser) virtualPath() string { + return archive.VirtualPath(j.location) } // parse the loaded archive and return all packages found. @@ -177,11 +226,10 @@ func (j *archiveParser) parse(ctx context.Context, parentPkg *pkg.Package) ([]pk } pkgs = append(pkgs, nestedPkgs...) relationships = append(relationships, nestedRelationships...) - } else if !j.cfg.NestedArchivesHandledExternally { - // nothing will catalog these: this parser is not recursing and the archive cataloger task is - // disabled, so record them as unknowns + } else if !archive.NestedCatalogingEnabled(ctx) { + // nothing will catalog these, so record them as unknowns // .jar and .war files are present in archives, are others? or generally just consider them top-level? - nestedArchives := j.entries.glob(true, "**/*.jar", "**/*.war") + nestedArchives := j.entries.glob("**/*.jar", "**/*.war") if len(nestedArchives) > 0 { slices.Sort(nestedArchives) errs = unknown.Appendf(errs, j.location, "nested archives not cataloged: %v", strings.Join(nestedArchives, ", ")) @@ -214,7 +262,7 @@ func finalizePackage(p *pkg.Package) { // discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages. func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package, error) { // search and parse java manifest files - manifestMatches := j.entries.glob(false, manifestGlob) + manifestMatches := j.entries.glob(manifestGlob) if len(manifestMatches) > 1 { return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches) } else if len(manifestMatches) == 0 { @@ -230,7 +278,7 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package, // parse the manifest file into a rich object manifestContents := contents[manifestMatches[0]] - manifest, err := parseJavaManifest(j.virtualPath, strings.NewReader(manifestContents)) + manifest, err := parseJavaManifest(j.virtualPath(), strings.NewReader(manifestContents)) if err != nil { log.Debugf("failed to parse java manifest (%s): %+v", j.location, err) return nil, nil @@ -243,12 +291,6 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package, return nil, nil } - // grab and assign digest for the entire archive - digests, err := j.entries.digests(ctx) - if err != nil { - return nil, err - } - name, version, lics, parsedPom := j.discoverNameVersionLicense(ctx, manifest) var pkgPomProject *pkg.JavaPomProject if parsedPom != nil { @@ -266,10 +308,10 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package, ), Type: j.fileInfo.pkgType(), Metadata: pkg.JavaArchive{ - VirtualPath: j.virtualPath, + VirtualPath: j.virtualPath(), Manifest: manifest, PomProject: pkgPomProject, - ArchiveDigests: digests, + ArchiveDigests: j.archiveDigests, }, }, nil } @@ -382,8 +424,8 @@ type parsedPomProject struct { // discoverMainPackageFromPomInfo attempts to resolve maven groupId, artifactId, version and other info from found pom information func (j *archiveParser) discoverMainPackageFromPomInfo(ctx context.Context) (group, name, version string, parsedPom *parsedPomProject) { // Find the pom.properties/pom.xml if the names seem like a plausible match - properties, _ := pomPropertiesByParentPath(ctx, j.entries, j.location, j.entries.glob(false, pomPropertiesGlob)) - projects, _ := pomProjectByParentPath(ctx, j.entries, j.location, j.entries.glob(false, pomXMLGlob)) + properties, _ := pomPropertiesByParentPath(ctx, j.entries, j.location, j.entries.glob(pomPropertiesGlob)) + projects, _ := pomProjectByParentPath(ctx, j.entries, j.location, j.entries.glob(pomXMLGlob)) artifactsMap := j.buildArtifactsMap(properties) pomProperties, parsedPom := j.findBestPomMatch(properties, projects, artifactsMap) @@ -514,13 +556,13 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, paren var pkgs []pkg.Package // pom.properties - properties, err := pomPropertiesByParentPath(ctx, j.entries, j.location, j.entries.glob(false, pomPropertiesGlob)) + properties, err := pomPropertiesByParentPath(ctx, j.entries, j.location, j.entries.glob(pomPropertiesGlob)) if err != nil { return nil, err } // pom.xml - projects, err := pomProjectByParentPath(ctx, j.entries, j.location, j.entries.glob(false, pomXMLGlob)) + projects, err := pomProjectByParentPath(ctx, j.entries, j.location, j.entries.glob(pomXMLGlob)) if err != nil { return nil, err } @@ -531,7 +573,7 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, paren parsedPom = proj } - pkgFromPom := newPackageFromMavenData(ctx, j.maven, propertiesObj, parsedPom, parentPkg, j.location, j.virtualPath) + pkgFromPom := newPackageFromMavenData(ctx, j.maven, propertiesObj, parsedPom, parentPkg, j.location) if pkgFromPom != nil { pkgs = append(pkgs, *pkgFromPom) } @@ -563,7 +605,7 @@ func (j *archiveParser) getLicenseFromFileInArchive(ctx context.Context) []pkg.L for _, glob := range []string{"/META-INF/*", "/*"} { var licenseMatches []string - for _, f := range j.entries.glob(true, glob) { + for _, f := range j.entries.glob(glob) { if licenses.IsLicenseFile(path.Base(f)) { licenseMatches = append(licenseMatches, f) } @@ -614,7 +656,7 @@ func (j *archiveParser) versionFromPropertiesFile(ctx context.Context, manifest return "" } - matches := j.entries.glob(true, "/version.properties") + matches := j.entries.glob("/version.properties") if len(matches) != 1 { return "" } @@ -641,9 +683,9 @@ func (j *archiveParser) versionFromPropertiesFile(ctx context.Context, manifest return "" } -// discoverPkgsFromNestedArchives recurses into the java archives inside this one. Only zipEntries -// applies, since recursion needs an archive file to unzip; with resolverEntries the archive cataloger -// task has already extracted everything and meets those nested archives itself. +// discoverPkgsFromNestedArchives recurses into the java archives inside this one, which needs an +// archive file to unzip; an already-extracted archive's nested archives are the archive cataloger +// task's to find. func (j *archiveParser) discoverPkgsFromNestedArchives(ctx context.Context, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) { zipped, ok := j.entries.(*zipEntries) if !ok { @@ -657,7 +699,7 @@ func (j *archiveParser) discoverPkgsFromNestedArchives(ctx context.Context, pare // associating each discovered package to the given parent package. func discoverPkgsFromZip(ctx context.Context, location file.Location, archivePath, contentPath string, fileManifest intFile.ZipFileManifest, parentPkg *pkg.Package, cfg ArchiveCatalogerConfig) ([]pkg.Package, []artifact.Relationship, error) { // search and parse pom.properties files & fetch the contents - openers, err := intFile.ExtractFromZipToUniqueTempFile(ctx, archivePath, contentPath, fileManifest.GlobMatch(false, archiveFormatGlobs...)...) + openers, err := intFile.ExtractFromZipToUniqueTempFile(ctx, archivePath, contentPath, fileManifest.GlobMatch(archiveFormatGlobs...)...) if err != nil { return nil, nil, fmt.Errorf("unable to extract files from zip: %w", err) } @@ -778,7 +820,7 @@ func pomProjectByParentPath(ctx context.Context, entries archiveEntries, locatio // newPackageFromMavenData processes a single Maven POM properties for a given parent package, returning all listed Java packages found and // associating each discovered package to the given parent package. Note the pom.xml is optional, the pom.properties is not. -func newPackageFromMavenData(ctx context.Context, r *maven.Resolver, pomProperties pkg.JavaPomProperties, parsedPom *parsedPomProject, parentPkg *pkg.Package, location file.Location, virtualPath string) *pkg.Package { +func newPackageFromMavenData(ctx context.Context, r *maven.Resolver, pomProperties pkg.JavaPomProperties, parsedPom *parsedPomProject, parentPkg *pkg.Package, location file.Location) *pkg.Package { // keep the artifact name within the virtual path if this package does not match the parent package vPathSuffix := "" groupID := "" @@ -798,7 +840,7 @@ func newPackageFromMavenData(ctx context.Context, r *maven.Resolver, pomProperti // https://github.com/anchore/syft/issues/1944 vPathSuffix += ":" + pomProperties.GroupID + ":" + pomProperties.ArtifactID } - virtualPath += vPathSuffix + virtualPath := archive.VirtualPath(location) + vPathSuffix var pkgPomProject *pkg.JavaPomProject diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index 454f868fe..84167c5ef 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -1155,7 +1155,7 @@ func Test_newPackageFromMavenData(t *testing.T) { test.expectedParent.Locations = locations r := maven.NewResolver(nil, maven.DefaultConfig()) - actualPackage := newPackageFromMavenData(pkgtest.Context(t), r, test.props, test.project, test.parent, file.NewLocation(virtualPath), virtualPath) + actualPackage := newPackageFromMavenData(pkgtest.Context(t), r, test.props, test.project, test.parent, file.NewLocation(virtualPath)) if test.expectedPackage == nil { require.Nil(t, actualPackage) } else { diff --git a/syft/pkg/cataloger/java/archive_parser_traversal_test.go b/syft/pkg/cataloger/java/archive_parser_traversal_test.go index ac8c6a8d6..8b3695198 100644 --- a/syft/pkg/cataloger/java/archive_parser_traversal_test.go +++ b/syft/pkg/cataloger/java/archive_parser_traversal_test.go @@ -3,6 +3,7 @@ package java import ( "archive/zip" "bytes" + "context" "fmt" "io" "os" @@ -20,68 +21,12 @@ import ( "github.com/anchore/syft/syft/source/directorysource" ) -func Test_virtualPathFromArchiveTraversal(t *testing.T) { - jarBytes := makeTestZip(t, map[string][]byte{ - "META-INF/MANIFEST.MF": testManifest("example-app", "1.0.0"), - }) - - parse := func(t *testing.T, traversal *archive.Traversal, readerPath string) pkg.Package { - ctx := pkgtest.Context(t) - if traversal != nil { - ctx = archive.WithTraversal(ctx, traversal) - } - gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{}) - pkgs, _, err := gap.processJavaArchive(ctx, file.LocationReadCloser{ - Location: file.NewLocation(readerPath), - ReadCloser: io.NopCloser(bytes.NewReader(jarBytes)), - }, nil) - require.NoError(t, err) - require.Len(t, pkgs, 1) - return pkgs[0] - } - - t.Run("no traversal keeps reader path as virtual path", func(t *testing.T) { - p := parse(t, nil, "some/dir/example-app-1.0.0.jar") - metadata, ok := p.Metadata.(pkg.JavaArchive) - require.True(t, ok) - assert.Equal(t, "some/dir/example-app-1.0.0.jar", metadata.VirtualPath) - assert.Equal(t, "example-app", p.Name) - assert.Equal(t, "1.0.0", p.Version) - }) - - t.Run("traversal prepends containing archive chain", func(t *testing.T) { - trav := &archive.Traversal{ - Location: file.NewLocation("/app.war"), - VirtualPath: "/app.war", - } - p := parse(t, trav, "/WEB-INF/lib/example-app-1.0.0.jar") - metadata, ok := p.Metadata.(pkg.JavaArchive) - require.True(t, ok) - assert.Equal(t, "/app.war:WEB-INF/lib/example-app-1.0.0.jar", metadata.VirtualPath) - // name/version inference from the last virtual path element is unaffected by the chain - assert.Equal(t, "example-app", p.Name) - assert.Equal(t, "1.0.0", p.Version) - }) - - t.Run("multi-level traversal chains all parents", func(t *testing.T) { - outer := &archive.Traversal{VirtualPath: "/dist.zip"} - inner := &archive.Traversal{ - VirtualPath: outer.VirtualPathOf("/app.war"), - } - p := parse(t, inner, "/WEB-INF/lib/example-app-1.0.0.jar") - metadata, ok := p.Metadata.(pkg.JavaArchive) - require.True(t, ok) - assert.Equal(t, "/dist.zip:app.war:WEB-INF/lib/example-app-1.0.0.jar", metadata.VirtualPath) - }) -} - // Test_virtualPathFromArchiveTraversal_colonInEntryPath is the java-cataloger-level counterpart of // internal/archive's escaping tests: a package reached through an entry whose path contains a colon -// must carry it escaped as %3A end to end in its virtual path -// (decisions.md#colon-in-an-entry-path-is-escaped), while the maven-derived ":groupID:artifactID" -// suffix that distinguishes a pom.properties package from its containing archive is still appended -// with real, unescaped delimiters. -func Test_virtualPathFromArchiveTraversal_colonInEntryPath(t *testing.T) { +// must carry it escaped as %3A end to end in its virtual path, while the maven-derived +// ":groupID:artifactID" suffix that distinguishes a pom.properties package from its containing archive +// is still appended with real, unescaped delimiters. +func Test_virtualPathFromArchiveLocation_colonInEntryPath(t *testing.T) { jarBytes := makeTestZip(t, map[string][]byte{ "META-INF/MANIFEST.MF": testManifest("outer-app", "1.0.0"), "META-INF/maven/com.example/different-lib/pom.properties": []byte( @@ -89,15 +34,13 @@ func Test_virtualPathFromArchiveTraversal_colonInEntryPath(t *testing.T) { ), }) - trav := &archive.Traversal{ - Location: file.NewLocation("/outer.zip"), - VirtualPath: "outer.zip", - } - ctx := archive.WithTraversal(pkgtest.Context(t), trav) + // as the archive cataloger task's resolver would present an entry of outer.zip + location := file.NewLocation("/lib/dep:special.jar") + location.ArchivePath = "outer.zip" gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{}) - pkgs, _, err := gap.processJavaArchive(ctx, file.LocationReadCloser{ - Location: file.NewLocation("/lib/dep:special.jar"), + pkgs, _, err := gap.processJavaArchive(pkgtest.Context(t), file.LocationReadCloser{ + Location: location, ReadCloser: io.NopCloser(bytes.NewReader(jarBytes)), }, nil) require.NoError(t, err) @@ -134,9 +77,9 @@ func Test_nestedArchiveOwnershipSwitch(t *testing.T) { "lib/nested-lib-2.0.jar": nestedJar, }) - parse := func(t *testing.T, cfg ArchiveCatalogerConfig) []pkg.Package { + parse := func(t *testing.T, ctx context.Context, cfg ArchiveCatalogerConfig) []pkg.Package { gap := newGenericArchiveParserAdapter(cfg) - pkgs, _, err := gap.processJavaArchive(pkgtest.Context(t), file.LocationReadCloser{ + pkgs, _, err := gap.processJavaArchive(ctx, file.LocationReadCloser{ Location: file.NewLocation("example-app-1.0.0.jar"), ReadCloser: io.NopCloser(bytes.NewReader(outerJar)), }, nil) @@ -145,7 +88,7 @@ func Test_nestedArchiveOwnershipSwitch(t *testing.T) { } t.Run("nested archive cataloging disabled: this cataloger unarchives nested jars itself", func(t *testing.T) { - pkgs := parse(t, DefaultArchiveCatalogerConfig()) // MaxDepth defaults to 0 (disabled) + pkgs := parse(t, pkgtest.Context(t), DefaultArchiveCatalogerConfig()) // MaxDepth defaults to 0 (disabled) require.Len(t, pkgs, 2) var names, virtualPaths []string @@ -163,12 +106,11 @@ func Test_nestedArchiveOwnershipSwitch(t *testing.T) { }) t.Run("nested archive cataloging enabled: this cataloger does not unarchive", func(t *testing.T) { - cfg := DefaultArchiveCatalogerConfig() - // only CreateSBOMConfig sets this in production; it is not reachable from user config - cfg.NestedArchivesHandledExternally = true + // only syft.CreateSBOM marks the context in production; it is not reachable from user config + ctx := archive.WithNestedCataloging(pkgtest.Context(t)) // no error expected: the nested jar is deliberately left to the archive cataloger task - pkgs := parse(t, cfg) + pkgs := parse(t, ctx, DefaultArchiveCatalogerConfig()) require.Len(t, pkgs, 1) assert.Equal(t, "example-app", pkgs[0].Name) }) @@ -179,9 +121,8 @@ func Test_nestedArchiveOwnershipSwitch(t *testing.T) { // wrong silently drops nested jars for every consumer that is not the CLI. cfg := DefaultArchiveCatalogerConfig() cfg.ArchiveSearchConfig = cfg.ArchiveSearchConfig.WithMaxDepth(2) - require.False(t, cfg.NestedArchivesHandledExternally) - pkgs := parse(t, cfg) + pkgs := parse(t, pkgtest.Context(t), cfg) require.Len(t, pkgs, 2, "this cataloger must still recurse itself") }) } @@ -195,7 +136,7 @@ func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "placeholder.txt"), []byte("x"), 0o600)) - queriedGlobs := func(t *testing.T, cfg ArchiveCatalogerConfig) *pkgtest.ObservingResolver { + queriedGlobs := func(t *testing.T, ctx context.Context, cfg ArchiveCatalogerConfig) *pkgtest.ObservingResolver { t.Helper() src, err := directorysource.NewFromPath(dir) require.NoError(t, err) @@ -203,7 +144,7 @@ func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) { require.NoError(t, err) observed := pkgtest.NewObservingResolver(resolver) - _, _, err = NewArchiveCataloger(cfg).Catalog(pkgtest.Context(t), observed) + _, _, err = NewArchiveCataloger(cfg).Catalog(ctx, observed) require.NoError(t, err) return observed } @@ -212,9 +153,8 @@ func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) { cfg := DefaultArchiveCatalogerConfig() cfg.IncludeIndexedArchives = true cfg.IncludeUnindexedArchives = true - cfg.NestedArchivesHandledExternally = true - observed := queriedGlobs(t, cfg) + observed := queriedGlobs(t, archive.WithNestedCataloging(pkgtest.Context(t)), cfg) // outside a traversal the cataloger opens no files at all: the task brings every archive back // as an extracted filesystem, and each is described there assert.False(t, observed.ObservedPathQuery("**/*.zip"), "no generic zip parser may be registered") @@ -227,7 +167,7 @@ func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) { cfg.IncludeIndexedArchives = true cfg.IncludeUnindexedArchives = true - observed := queriedGlobs(t, cfg) + observed := queriedGlobs(t, pkgtest.Context(t), cfg) assert.True(t, observed.ObservedPathQuery("**/*.zip"), "existing behavior must be unchanged") assert.True(t, observed.ObservedPathQuery("**/*.tar"), "existing behavior must be unchanged") }) @@ -237,7 +177,7 @@ func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) { cfg.IncludeIndexedArchives = false cfg.IncludeUnindexedArchives = false - observed := queriedGlobs(t, cfg) + observed := queriedGlobs(t, pkgtest.Context(t), cfg) assert.False(t, observed.ObservedPathQuery("**/*.zip")) assert.False(t, observed.ObservedPathQuery("**/*.tar")) }) @@ -246,8 +186,8 @@ func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) { func Test_mavenVirtualPathSuffixUnderTraversal(t *testing.T) { // a package derived from a pom.properties whose artifact does not match the containing archive gets // a trailing "::" appended to the archive's virtual path. That suffix form is in - // published SBOMs, and once the archive cataloger drives the parser the base comes from the traversal - // rather than the reader path - so the two must compose rather than one clobbering the other. + // published SBOMs, and once the archive cataloger drives the parser the base carries the location's + // archive path - so the two must compose rather than one clobbering the other. jarBytes := makeTestZip(t, map[string][]byte{ "META-INF/MANIFEST.MF": testManifest("example-app", "1.0.0"), "META-INF/maven/com.example.other/other-lib/pom.properties": []byte( @@ -255,15 +195,13 @@ func Test_mavenVirtualPathSuffixUnderTraversal(t *testing.T) { ), }) - parse := func(t *testing.T, traversal *archive.Traversal, readerPath string) []pkg.Package { + parse := func(t *testing.T, archivePath, readerPath string) []pkg.Package { t.Helper() - ctx := pkgtest.Context(t) - if traversal != nil { - ctx = archive.WithTraversal(ctx, traversal) - } + location := file.NewLocation(readerPath) + location.ArchivePath = archivePath gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{}) - pkgs, _, err := gap.processJavaArchive(ctx, file.LocationReadCloser{ - Location: file.NewLocation(readerPath), + pkgs, _, err := gap.processJavaArchive(pkgtest.Context(t), file.LocationReadCloser{ + Location: location, ReadCloser: io.NopCloser(bytes.NewReader(jarBytes)), }, nil) require.NoError(t, err) @@ -284,14 +222,14 @@ func Test_mavenVirtualPathSuffixUnderTraversal(t *testing.T) { return "" } - t.Run("no traversal: suffix hangs off the reader path", func(t *testing.T) { - pkgs := parse(t, nil, "/example-app-1.0.0.jar") + t.Run("outside an archive: suffix hangs off the reader path", func(t *testing.T) { + pkgs := parse(t, "", "/example-app-1.0.0.jar") assert.Equal(t, "/example-app-1.0.0.jar:com.example.other:other-lib", virtualPathOf(t, pkgs, "other-lib")) }) - t.Run("under traversal: suffix hangs off the archive chain", func(t *testing.T) { - pkgs := parse(t, &archive.Traversal{VirtualPath: "app.war"}, "/WEB-INF/lib/example-app-1.0.0.jar") + t.Run("inside an archive: suffix hangs off the archive chain", func(t *testing.T) { + pkgs := parse(t, "app.war", "/WEB-INF/lib/example-app-1.0.0.jar") assert.Equal(t, "app.war:WEB-INF/lib/example-app-1.0.0.jar:com.example.other:other-lib", virtualPathOf(t, pkgs, "other-lib")) }) diff --git a/syft/pkg/cataloger/java/archive_source.go b/syft/pkg/cataloger/java/archive_source.go deleted file mode 100644 index 29e0ee4c5..000000000 --- a/syft/pkg/cataloger/java/archive_source.go +++ /dev/null @@ -1,119 +0,0 @@ -package java - -import ( - "context" - "fmt" - "strings" - - "github.com/bmatcuk/doublestar/v4" - - "github.com/anchore/syft/internal/archive" - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/internal/tmpdir" - "github.com/anchore/syft/syft/file" -) - -// archiveSource is one java archive to describe: where its entries are read from, and the identity -// that does not live inside them - the archive's own location, the colon-delimited virtual path -// SBOMs carry, and the file name that name and version fall back to. -// -// Two of these exist because two things can own extraction. When this cataloger owns it, the archive -// is a file it copies to a temp dir and reads as a zip. When the archive cataloger task owns it, the -// archive has already been extracted and indexed, and arrives as a resolver plus the traversal -// describing the file it came from. Everything past this point - which manifest wins, how -// pom.properties decides identity, license discovery - is logic over a source, so it runs unchanged -// either way. -type archiveSource struct { - entries archiveEntries - location file.Location - virtualPath string - fileInfo archiveFilename - - // contentPath is the directory nested archives are extracted into. Set only when this cataloger - // opened the archive itself: recursing needs somewhere to unzip into, and an already-extracted - // archive has none. The archive file itself is held by zipEntries. - contentPath string -} - -// newFileArchiveSource copies the archive to a temp dir and reads its central directory, for a scan -// where nothing has already extracted it. The returned cleanup must run even when an error is -// returned. -func newFileArchiveSource(ctx context.Context, reader file.LocationReadCloser) (archiveSource, func(), error) { - // the full virtual path of this archive: when the archive cataloger task drove us here, the context - // traversal supplies the containing chain; otherwise the reader path already carries any - // colon-delimited nesting from this cataloger's own recursion - virtualPath := archive.TraversalFromContext(ctx).VirtualPathOf(reader.Path()) - - // fetch the last element of the virtual path - virtualElements := strings.Split(virtualPath, ":") - currentFilepath := virtualElements[len(virtualElements)-1] - - td := tmpdir.FromContext(ctx) - if td == nil { - return archiveSource{}, func() {}, fmt.Errorf("no temp dir factory in context") - } - contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(td, currentFilepath, reader) - if err != nil { - return archiveSource{}, cleanupFn, fmt.Errorf("unable to process java archive: %w", err) - } - - entries, err := newZipEntries(ctx, archivePath) - if err != nil { - return archiveSource{}, cleanupFn, err - } - - return archiveSource{ - entries: entries, - location: reader.Location, - virtualPath: virtualPath, - fileInfo: newJavaArchiveFilename(currentFilepath), - contentPath: contentPath, - }, cleanupFn, nil -} - -// newExtractedArchiveSource describes the archive whose extracted contents the given resolver holds. -// Nothing is copied or reopened, and identity comes from the traversal: the location, the chain the -// archive cataloger task composed, the file name, and the digests taken at extraction - none of which -// are inside the archive. -func newExtractedArchiveSource(trav *archive.Traversal, resolver file.Resolver) (archiveSource, error) { - if resolver == nil { - return archiveSource{}, fmt.Errorf("no resolver for the contents of %q", trav.VirtualPath) - } - return archiveSource{ - entries: newResolverEntries(resolver, trav.Digests), - location: trav.Location, - virtualPath: trav.VirtualPath, - fileInfo: newJavaArchiveFilename(archiveFileNameOf(trav)), - }, nil -} - -// archiveFileNameOf returns the archive's own file name, where a jar's name and version come from -// when its manifest does not say. It takes the last element of the virtual path rather than the real -// path, so a nested chain and a plain scan agree. -func archiveFileNameOf(trav *archive.Traversal) string { - name := trav.VirtualPath - if name == "" { - name = trav.Location.Path() - } - for i := len(name) - 1; i >= 0; i-- { - if name[i] == ':' || name[i] == '/' { - return name[i+1:] - } - } - return name -} - -// isJavaArchiveName reports whether the path is one of the java archive formats, by the same globs -// this cataloger registers against when it is the one opening files. -func isJavaArchiveName(path string) bool { - for _, glob := range archiveFormatGlobs { - if ok, err := doublestar.Match(glob, path); err == nil && ok { - return true - } - if ok, err := doublestar.Match(glob, "/"+path); err == nil && ok { - return true - } - } - log.WithFields("path", path).Trace("archive contents are not a java archive") - return false -} diff --git a/syft/pkg/cataloger/java/archive_source_test.go b/syft/pkg/cataloger/java/archive_source_test.go index e9ef020ad..fbfcb31cf 100644 --- a/syft/pkg/cataloger/java/archive_source_test.go +++ b/syft/pkg/cataloger/java/archive_source_test.go @@ -55,9 +55,8 @@ func Test_archiveContentsCataloger_agreesWithTheArchiveReadingCataloger(t *testi digests := digestsOfFile(t, jarPath) trav := &archive.Traversal{ - Location: file.NewLocation(archivePath), - VirtualPath: archivePath, - Digests: digests, + Location: file.NewLocation(archivePath), + Digests: digests, } ctx := archive.WithTraversal(context.Background(), trav) fromContents, _, err := NewArchiveCataloger(cfg).Catalog(ctx, contentsResolver) @@ -70,16 +69,14 @@ func Test_archiveContentsCataloger_reportsNothingOutsideAnArchive(t *testing.T) // with the archive cataloger task owning extraction, every archive comes back with a traversal // naming it. Outside one there is nothing to describe: opening archive files here as well would // catalog each of them a second time. - cfg := DefaultArchiveCatalogerConfig() - cfg.NestedArchivesHandledExternally = true - dir := t.TempDir() jarPath := writeJar(t, dir, "example-lib-1.2.3.jar", jarEntries()) require.FileExists(t, jarPath) resolver, err := fileresolver.NewFromDirectory(dir, "") require.NoError(t, err) - pkgs, _, err := NewArchiveCataloger(cfg).Catalog(context.Background(), resolver) + ctx := archive.WithNestedCataloging(context.Background()) + pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) require.NoError(t, err) assert.Empty(t, pkgs) } @@ -93,8 +90,7 @@ func Test_archiveContentsCataloger_reportsNothingForAnArchiveThatIsNotJava(t *te require.NoError(t, err) ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ - Location: file.NewLocation("/bundle.zip"), - VirtualPath: "/bundle.zip", + Location: file.NewLocation("/bundle.zip"), }) pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) require.NoError(t, err) @@ -117,8 +113,7 @@ func Test_archiveContentsCataloger_noManifestIsNoPackage(t *testing.T) { require.NoError(t, err) ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ - Location: file.NewLocation("no-manifest-1.0.0.jar"), - VirtualPath: "no-manifest-1.0.0.jar", + Location: file.NewLocation("no-manifest-1.0.0.jar"), }) pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) assert.Empty(t, pkgs) @@ -135,11 +130,6 @@ func Test_archiveContentsCataloger_readsWhatWasExtractedAndNoMore(t *testing.T) // extraction produced. An archive truncated by a limit yields a package built from the entries that // were written, so identity degrades rather than disappearing - the manifest is there, the maven // metadata is not. - // - // Observed on a real artifact: jenkins-core-2.578.jar holds more than the default 10,000-entry bound, - // so its META-INF/maven pom is never extracted and its group id falls back from org.jenkins-ci.main - // to the artifact id. The archive-reading cataloger never felt this, reading the archive's own - // central directory with no bound on it. entries := map[string]string{ "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n" + "Implementation-Title: example-lib\nImplementation-Version: 1.2.3\n", @@ -153,8 +143,7 @@ func Test_archiveContentsCataloger_readsWhatWasExtractedAndNoMore(t *testing.T) require.NoError(t, err) ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ - Location: file.NewLocation("example-lib-1.2.3.jar"), - VirtualPath: "example-lib-1.2.3.jar", + Location: file.NewLocation("example-lib-1.2.3.jar"), }) pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) require.NoError(t, err) @@ -249,9 +238,7 @@ func joinSorted(in []string) string { func digestsOfFile(t *testing.T, path string) []file.Digest { t.Helper() - entries, err := newZipEntries(context.Background(), path) - require.NoError(t, err) - digests, err := entries.digests(context.Background()) + digests, err := getDigestsFromArchive(context.Background(), path) require.NoError(t, err) require.NotEmpty(t, digests) return digests diff --git a/syft/pkg/cataloger/java/cataloger.go b/syft/pkg/cataloger/java/cataloger.go index 8663fd55a..fedd88298 100644 --- a/syft/pkg/cataloger/java/cataloger.go +++ b/syft/pkg/cataloger/java/cataloger.go @@ -6,6 +6,8 @@ package java import ( "context" + "github.com/bmatcuk/doublestar/v4" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" @@ -14,46 +16,39 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/generic" ) -const ArchiveCatalogerName = "java-archive-cataloger" - // NewArchiveCataloger returns a new Java archive cataloger object for detecting packages with archives (jar, war, ear, par, sar, jpi, hpi, and native-image formats) func NewArchiveCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger { gap := newGenericArchiveParserAdapter(cfg) - files := generic.NewCataloger(ArchiveCatalogerName). + c := generic.NewCataloger("java-archive-cataloger"). WithParserByGlobs(gap.parseJavaArchive, archiveFormatGlobs...) if cfg.IncludeIndexedArchives { // java archives wrapped within zip files gzp := newGenericZipWrappedJavaArchiveParser(cfg) - files.WithParserByGlobs(gzp.parseZipWrappedJavaArchive, genericZipGlobs...) + c.WithParserByGlobs(gzp.parseZipWrappedJavaArchive, genericZipGlobs...) } if cfg.IncludeUnindexedArchives { // java archives wrapped within tar files gtp := newGenericTarWrappedJavaArchiveParser(cfg) - files.WithParserByGlobs(gtp.parseTarWrappedJavaArchive, genericTarGlobs...) + c.WithParserByGlobs(gtp.parseTarWrappedJavaArchive, genericTarGlobs...) } - return &archiveCataloger{cfg: cfg, files: files} + return &archiveCataloger{cfg: cfg, extractingCataloger: c} } -// archiveCataloger describes java archives from whichever source this scan makes them available in. -// -// Which source that is turns on who owns extraction, and there are only two answers. Inside an -// archive the archive cataloger task extracted, the resolver is that archive's contents and the -// traversal names the file it came from, so the archive to describe is the one this resolver *is*. -// Everywhere else the archives are files in the resolver, and the cataloger opens each itself. -// -// One cataloger rather than one per source, so a package carries the same FoundBy and answers to the -// same --select-catalogers name however the scan was configured. +// archiveCataloger catalogs java archives two ways. Inside an archive the archive cataloger task has +// extracted, the resolver is that archive's contents and the traversal on the context names the +// archive, so the archive to describe is the one the resolver holds. Everywhere else, archives are +// files in the resolver that this cataloger opens itself. type archiveCataloger struct { - cfg ArchiveCatalogerConfig - files *generic.Cataloger + cfg ArchiveCatalogerConfig + extractingCataloger *generic.Cataloger } func (c *archiveCataloger) Name() string { - return ArchiveCatalogerName + return c.extractingCataloger.Name() } func (c *archiveCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { @@ -61,42 +56,46 @@ func (c *archiveCataloger) Catalog(ctx context.Context, resolver file.Resolver) return c.catalogExtracted(ctx, trav, resolver) } - if c.cfg.NestedArchivesHandledExternally { - // every archive in this scan comes back through the archive cataloger task with a traversal - // naming it, and is described above. Opening archive files here as well would catalog each of - // them a second time. + if archive.NestedCatalogingEnabled(ctx) { + // the archive cataloger task extracts every archive and runs this cataloger inside each; opening + // them here too would catalog each twice return nil, nil, nil } - return c.files.Catalog(ctx, resolver) + // process archives directly, using the original locally managed java archive extraction + return c.extractingCataloger.Catalog(ctx, resolver) } -// catalogExtracted reports the java package of the archive whose contents this resolver holds, if it -// is a java archive at all. -// -// It does not glob for the archives inside it: the task that extracted this one meets those on its -// own, at the next nesting level. See syft/java-archive-nesting#single-owner-of-archive-recursion. +// catalogExtracted reports the java package of the archive whose contents the resolver holds, if it +// is a java archive. Archives nested inside it are left to the archive cataloger task. func (c *archiveCataloger) catalogExtracted(ctx context.Context, trav *archive.Traversal, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { if !isJavaArchiveName(trav.Location.Path()) { - // the rule the globs apply when this cataloger opens files: a .zip is an archive but not a java - // archive return nil, nil, nil } - src, err := newExtractedArchiveSource(trav, resolver) + parser, err := newExtractedArchiveParser(trav, resolver, c.cfg) if err != nil { return nil, nil, err } - pkgs, relationships, err := newArchiveParser(src, false, c.cfg).parse(ctx, nil) + pkgs, relationships, err := parser.parse(ctx, nil) if err != nil { - // attributed to the archive rather than returned bare, as the generic cataloger attributes a - // parser's failure to the file it was parsing, so the scan carries on + // attributed to the archive, as the generic cataloger attributes a parser failure to its file return pkgs, relationships, unknown.New(trav.Location, err) } return pkgs, relationships, nil } +// isJavaArchiveName reports whether the path matches the globs this cataloger opens files by. +func isJavaArchiveName(path string) bool { + for _, glob := range archiveFormatGlobs { + if ok, _ := doublestar.Match(glob, path); ok { + return true + } + } + return false +} + // NewPomCataloger returns a cataloger capable of parsing dependencies from a pom.xml file. // Pom files list dependencies that maybe not be locally installed yet. func NewPomCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger { diff --git a/syft/pkg/cataloger/java/config.go b/syft/pkg/cataloger/java/config.go index 460a31b76..652ca827b 100644 --- a/syft/pkg/cataloger/java/config.go +++ b/syft/pkg/cataloger/java/config.go @@ -33,15 +33,6 @@ type ArchiveCatalogerConfig struct { // ResolveTransitiveDependencies enables resolving transitive dependencies for java packages found within archives. // app-config: java.resolve-transitive-dependencies ResolveTransitiveDependencies bool `yaml:"resolve-transitive-dependencies" json:"resolve-transitive-dependencies" mapstructure:"resolve-transitive-dependencies"` - - // NestedArchivesHandledExternally indicates that another mechanism owns recursion into nested - // archives, so this cataloger must not unarchive them itself. It has no yaml/json/mapstructure - // key, so no config file, flag or environment variable reaches it. - // - // Only syft.CreateSBOMConfig writes it, deriving it from the nested-archive cataloging depth so - // the two cannot disagree. Callers building this config should leave it alone; the zero value - // recurses, this cataloger's default. - NestedArchivesHandledExternally bool `yaml:"-" json:"-" mapstructure:"-"` } func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig { diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 4c56ac108..8b4ef6d56 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -3,7 +3,6 @@ package java import ( "context" "errors" - "path/filepath" "strings" "github.com/anchore/syft/internal" @@ -309,8 +308,7 @@ func pomParent(ctx context.Context, r *maven.Resolver, pom *maven.Project) *pkg. // The java-archive-cataloger identifies these archives from pom.properties, whose values are always // resolved, so skipping them here avoids phantom dependency packages. func isArchiveMetaPom(location file.Location) bool { - p := filepath.ToSlash(location.Path()) - return strings.Contains(p, "META-INF/maven/") + return strings.Contains(location.Path(), "META-INF/maven/") } func cleanDescription(original string) (cleaned string) { diff --git a/syft/pkg/cataloger/java/version_properties_test.go b/syft/pkg/cataloger/java/version_properties_test.go index 72e289c43..7f416878a 100644 --- a/syft/pkg/cataloger/java/version_properties_test.go +++ b/syft/pkg/cataloger/java/version_properties_test.go @@ -17,13 +17,13 @@ import ( func readManifest(t *testing.T, ctx context.Context, parser *archiveParser) *pkg.JavaManifest { t.Helper() - matches := parser.entries.glob(false, manifestGlob) + matches := parser.entries.glob(manifestGlob) require.Len(t, matches, 1) contents, err := parser.entries.contents(ctx, matches...) require.NoError(t, err) - manifest, err := parseJavaManifest(parser.virtualPath, strings.NewReader(contents[matches[0]])) + manifest, err := parseJavaManifest(parser.virtualPath(), strings.NewReader(contents[matches[0]])) require.NoError(t, err) return manifest diff --git a/syft/source/directorysource/directory_source.go b/syft/source/directorysource/directory_source.go index 7288215ea..0b270b466 100644 --- a/syft/source/directorysource/directory_source.go +++ b/syft/source/directorysource/directory_source.go @@ -118,9 +118,7 @@ func (s *directorySource) FileResolver(_ source.Scope) (file.Resolver, error) { return s.resolver, nil } -// ExcludedPaths returns a copy of the exclusion patterns this source was configured with, so a -// consumer indexing content taken from it can honor the same patterns. A copy, because -// GetDirectoryExclusionFunctions rewrites the patterns it is given. +// ExcludedPaths returns the exclusion patterns this source was configured with. func (s directorySource) ExcludedPaths() []string { return slices.Clone(s.config.Exclude.Paths) } @@ -138,13 +136,8 @@ func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileres return nil, nil } - // the indexer reports an absolute, symlink-resolved path to every visitor (the resolver normalizes - // its root with EvalSymlinks and the walk takes filepath.Abs of it), so patterns must anchor to a - // root derived the same way. Otherwise on macOS - where a directory under /var is really under - // /private/var - they exclude nothing. - // - // An unresolvable root is left alone rather than failing the scan: EvalSymlinks needs the path to - // exist, and the resolver reports a missing root better than this can. + // the indexer reports symlink-resolved absolute paths to every visitor, so the patterns must be + // anchored to a root resolved the same way, or a root under a symlink (macOS's /var) excludes nothing if resolved, err := filepath.EvalSymlinks(root); err == nil { root = resolved } @@ -161,9 +154,7 @@ func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileres root += "/" } - // a new slice, not a rewrite in place: these patterns belong to the source's config and are read - // elsewhere (see source.PathExcluder), so building the resolver must not replace what the user - // configured with scan-root-absolute patterns + // the given patterns are the source's configuration (see source.PathExcluder), so they are not rewritten in place rooted := make([]string, 0, len(exclusions)) var errors []string for _, exclusion := range exclusions { diff --git a/syft/source/exclude.go b/syft/source/exclude.go index 64ddd06bd..812cc1575 100644 --- a/syft/source/exclude.go +++ b/syft/source/exclude.go @@ -4,14 +4,8 @@ type ExcludeConfig struct { Paths []string } -// PathExcluder is optionally implemented by a source configured with path exclusion patterns. It -// publishes them so a consumer that indexes source content as its own filesystem can honor the scan's -// exclusions without a second setting. The nested archive cataloger is that consumer: it indexes every -// archive independently. All patterns are published; which are depth-independent enough to be -// meaningful is the consumer's business. -// -// Implementations must return a copy: sources that build directory resolvers pass their patterns to -// GetDirectoryExclusionFunctions, which rewrites them absolute against the scan root. +// PathExcluder is implemented by a source configured with path exclusion patterns, so the archive +// cataloger can apply the same exclusions inside the archives it extracts. type PathExcluder interface { ExcludedPaths() []string } diff --git a/syft/source/filesource/file_source.go b/syft/source/filesource/file_source.go index 488740fd9..63c34c1c8 100644 --- a/syft/source/filesource/file_source.go +++ b/syft/source/filesource/file_source.go @@ -176,9 +176,7 @@ func (s fileSource) FileResolver(_ source.Scope) (file.Resolver, error) { return s.resolver, nil } -// ExcludedPaths returns a copy of the exclusion patterns this source was configured with, so a -// consumer indexing content taken from it can honor the same patterns. A copy, because -// directorysource.GetDirectoryExclusionFunctions rewrites the patterns it is given. +// ExcludedPaths returns the exclusion patterns this source was configured with. func (s fileSource) ExcludedPaths() []string { return slices.Clone(s.config.Exclude.Paths) } diff --git a/syft/source/stereoscopesource/image_source.go b/syft/source/stereoscopesource/image_source.go index 8557e8c0a..da8c5e18b 100644 --- a/syft/source/stereoscopesource/image_source.go +++ b/syft/source/stereoscopesource/image_source.go @@ -126,12 +126,7 @@ func (s stereoscopeImageSource) FileResolver(scope source.Scope) (file.Resolver, return res, nil } -// ExcludedPaths returns a copy of the exclusion patterns this source was configured with, so a -// consumer indexing content taken from it can honor the same patterns. -// -// An image filters its own tree after indexing (NewExcludingDecorator above) rather than with an -// index visitor, since stereoscope builds that tree. Content taken out of the image is an ordinary -// index and is filtered like one, so the patterns travel unchanged. +// ExcludedPaths returns the exclusion patterns this source was configured with. func (s stereoscopeImageSource) ExcludedPaths() []string { return slices.Clone(s.config.Exclude.Paths) }