From b36234cfe775680afa8905e1c5bf42aaad7231bc Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 22 Sep 2026 11:32:30 -0400 Subject: [PATCH] feat: simplify+improve resolver implementation / glob search Signed-off-by: Keith Zantow --- internal/archive/README.md | 108 ++++-- internal/archive/content.go | 158 -------- internal/archive/content_test.go | 204 ---------- internal/archive/entry.go | 75 ---- internal/archive/entrystore.go | 169 --------- internal/archive/entrystore_test.go | 143 ------- internal/archive/extract.go | 128 +++---- internal/archive/extract_test.go | 299 +++++++++------ internal/archive/glob.go | 390 ++++++++++++++++++++ internal/archive/glob_test.go | 330 +++++++++++++++++ internal/archive/helpers_test.go | 324 ++++++++++------ internal/archive/index.go | 350 ------------------ internal/archive/index_bench_test.go | 63 ---- internal/archive/index_glob_test.go | 135 ------- internal/archive/index_links_test.go | 298 --------------- internal/archive/index_test.go | 271 -------------- internal/archive/limits.go | 67 ++-- internal/archive/limits_test.go | 189 +++++----- internal/archive/resolver.go | 482 ++++++++++++++++++++++++ internal/archive/resolver_test.go | 512 ++++++++++++++++++++++++++ internal/archive/storage.go | 193 ++++++++++ internal/archive/storage_test.go | 306 ++++++++++----- internal/archive/workdir.go | 71 ---- internal/archive/workdir_test.go | 67 ---- internal/task/archive_nesting_test.go | 40 +- internal/task/archive_tasks.go | 6 +- 26 files changed, 2780 insertions(+), 2598 deletions(-) delete mode 100644 internal/archive/content.go delete mode 100644 internal/archive/content_test.go delete mode 100644 internal/archive/entry.go delete mode 100644 internal/archive/entrystore.go delete mode 100644 internal/archive/entrystore_test.go create mode 100644 internal/archive/glob.go create mode 100644 internal/archive/glob_test.go delete mode 100644 internal/archive/index.go delete mode 100644 internal/archive/index_bench_test.go delete mode 100644 internal/archive/index_glob_test.go delete mode 100644 internal/archive/index_links_test.go delete mode 100644 internal/archive/index_test.go create mode 100644 internal/archive/resolver.go create mode 100644 internal/archive/resolver_test.go create mode 100644 internal/archive/storage.go delete mode 100644 internal/archive/workdir.go delete mode 100644 internal/archive/workdir_test.go diff --git a/internal/archive/README.md b/internal/archive/README.md index 2110b5df1..b4965f32c 100644 --- a/internal/archive/README.md +++ b/internal/archive/README.md @@ -7,56 +7,98 @@ does everything for a single archive. ## Flow ``` -Extract(ctx, reader, fileSystemID, archivePath, limiter, exclusions) - acquireContent read in place if the reader is already seekable, else hold in memory / write to disk - identifyFormat mholt/archives sniffing first, end-of-central-directory fallback for stub-prefixed zips - extractInto every entry -> EntryStore (excluded entries are never stored) - NewIndex file.Resolver over the store's entries +Extract(ctx, reader, fileSystemID, archivePath, limiter, exclusions) -> *Resolver + acquire read in place if the reader is already seekable, else hold in memory / spill to disk + identifyFormat internal/file.IdentifyArchive first, end-of-central-directory fallback for stub-prefixed zips + Resolver.extract every entry -> entryHeader -> Resolver.add (excluded entries are never stored) digestsOf SHA-1 of the archive itself ``` -`Extract` returns `nil` for content that is not an archive, `ErrDiskLimitReached` when the archive's -own bytes cannot be placed, and otherwise an `Extracted` whose `Cleanup` releases everything. +`Extract` returns `nil` for content that is not an archive (a bare compression format such as a +gzipped file that is not a tar counts as not an archive), `ErrDiskLimitReached` when the archive's +own bytes cannot be placed, and otherwise a `Resolver` whose `Cleanup` releases everything. + +## Resolver + +One type does the work: `Resolver` is fed entries by `Extract`, holds their content, indexes them, +and implements `file.Resolver` over the result. Each entry is a `node` carrying its header and a +`blob`; the archive's own bytes, when they have to be held at all, are one more blob. ## Where bytes live -| what | in memory while... | otherwise | -|---|---|---| -| the archive's own bytes | the reader is not seekable and memory admits them | `/archive` | -| its entries' content | memory admits them | `/entries`, one file for all entries, located by offset | +Blobs are held in memory while the memory limit admits them and in one spill file per archive +(`archive-spill-*`) once it does not; when a chunk is refused, everything held in memory moves to +the file first. The file is created on first use, so most archives never touch the filesystem, and +blobs are located in it by offset with no framing of their own. It goes under the scan's temp root +when the context carries one (`internal/tmpdir`), otherwise the system temp directory. -Anything already random-access (an `*os.File`, or a nested archive read out of its parent's store) -is used in place and costs nothing. The work directory is created only when something is written, -so most archives never touch the filesystem. Held archive bytes are released as soon as extraction -and digesting finish. +Anything already random-access (an `*os.File`, or a nested archive read out of its parent's +resolver) is used in place and costs nothing, so only a top-level stream is ever copied. Its memory +is refunded once it is extracted and digested; bytes it spilled wait for `Cleanup` with the entries. ## Limits `Limits` bound what one scan holds *at once*; each falls as archives are released. For both limits, positive is the bound, zero forbids the resource, negative is unbounded. -- `Limiter` is scan-wide; `Charge` is one archive's draw on it. Bytes are charged as they land, +- `Limiter` is scan-wide; a `charge` is one archive's draw on it. Bytes are charged as they land, never from declared sizes. - Memory refused → content moves to disk. Disk refused → the archive's own bytes cannot be placed - (`ErrDiskLimitReached`, archive skipped) or an entry cannot be stored (`Extracted.Truncated`, - the entries stored so far are still cataloged). -- Index bookkeeping is approximated as `approxIndexBytesPerEntry` (2 KiB) plus name length per - entry, charged to memory and falling back to disk so a zero memory limit still indexes. + (`ErrDiskLimitReached`, archive skipped) or an entry cannot be stored (`Resolver.Truncated`, the + entries stored so far are still cataloged). +- Index bookkeeping is approximated as `approxIndexBytesPerEntry` (2 KiB) per node, listed or + implied by an entry's path, plus name length. It lives in memory, so it is charged to the memory + limit, spilling held content to make room; only a zero memory limit lets it fall back to the disk + budget so that configuration still indexes. An entry whose index cost is refused is dropped like + one whose content is, and marks the resolver `Truncated`. +- `Limiter.Peak` reports the most held at once over the scan, for reporting. -## Index (the resolver) +## Hostile archives -Paths are archive-relative with no leading slash. Directories implied by entry paths exist for -`HasPath`. Links resolve inside the archive only; a glob matching a link and its target returns one -location. `FilesByGlob` narrows by the pattern's last segment through a base-name index -(`internal/index`) when it is a literal, `*suffix` or `prefix*`, and otherwise matches every file -with doublestar. +Nothing an archive declares is trusted. Content is charged chunk by chunk as it decompresses, so a +deflate or gzip bomb stops at the disk limit with the entry dropped whole. Every node is charged +against the memory limit, including directories implied by deep paths, so an archive of millions of +tiny entries or of a few entries with 4 KiB names stops at the memory limit. Names longer than +PATH_MAX are refused, link chains stop after `maxLinkHops`, and nesting stops at the task's +`MaxDepth`. See the bomb tests in `extract_test.go`. + +## Lookup + +Paths are archive-relative with no leading slash; a leading slash on a query is tolerated. Directories +implied by entry paths exist for `HasPath` but are never files, and every node is linked to its +parent and children. The first entry at a path wins over later duplicates. + +Links resolve inside the archive only, up to `maxLinkHops` hops, which also ends a cycle. A glob or +path lookup matching a link and its target returns one location: the file's own path when it +matched, otherwise the lowest-sorting link, with `AccessPath` naming the path matched and `RealPath` +the file read. A dangling link keeps its own path and reads as empty. `FilesByMIMEType` answers only +with files holding bytes, since a link has nothing to sniff; `AllLocations` enumerates every path, +links included. Answers are sorted by path. + +`FilesByGlob` (glob.go) parses a pattern once, cached (bounded by `maxCachedGlobs`), into segments. +A brace group inside a segment, `g{o,o.exe}`, makes that segment a set of alternatives; only a group +spanning `/` expands the whole pattern. The literal text at either end of every alternative is looked +up in a base-name index (`internal/index`): files for the last segment, directories for the rest, so +`**/one/two/**/[a-z]thing` is narrowed to what lies under directories named `two` or to files ending +in `thing`, whichever is smaller (a file-name lookup of `narrowEnough` or fewer is taken as is). Each +candidate is then verified segment by segment up its parent chain with `path.Match`. `**/`, +`**/*`, `**/*` and brace sets of those are answered from the index alone. Only a +pattern with no literal text anywhere, such as `**/[lm]*`, is matched against every file. ## Other pieces -- `Exclusions`: the scan's `**/` exclusion patterns, matched against each entry path and its parent - directories at extraction time. Sourced from `source.PathExcluder`. -- `Traversal`: put on the context by the task so a cataloger (java) knows which archive it is inside. - `VirtualPath(location)` builds the `outer.war:WEB-INF/lib/inner.jar` style chain from the - `ArchivePath` the index stamps on every location. -- `entryHeader`: cleans entry names to archive-relative paths, drops device nodes and fifos, and - refuses names longer than PATH_MAX. +- `Exclusions`: the scan's exclusion patterns, of which only the `**/` shape means the same thing + re-rooted at an archive. Matched against each entry path and its parent directories at extraction + time. `syft.CreateSBOM` passes the source's patterns to the task, which builds one `Exclusions` + for the scan. +- `Traversal`: put on the context by the task so a cataloger (java) knows which archive it is inside + and what its digests are. `VirtualPath(location)` builds the `outer.war:WEB-INF/lib/inner.jar` + style chain from the `ArchivePath` the resolver stamps on every location, escaping colons in entry + names. +- `WithNestedCataloging` / `NestedCatalogingEnabled`: `syft.CreateSBOM` marks the context when the + archive task is running, so the java cataloger leaves nested archives to the task instead of + opening them itself. +- `entryHeader`: keeps regular files, directories and links only (no devices or fifos), cleans entry names to + archive-relative paths, refuses names longer than PATH_MAX or containing NUL, and drops header + fields nothing reads (PAX records, user and group names, sub-second and access times) since the + header is held for the life of the archive. diff --git a/internal/archive/content.go b/internal/archive/content.go deleted file mode 100644 index b2c66a52a..000000000 --- a/internal/archive/content.go +++ /dev/null @@ -1,158 +0,0 @@ -package archive - -import ( - "bytes" - "errors" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/anchore/syft/internal/log" -) - -// copyChunkSize is how many bytes are read and charged at a time, bounding how far a charge runs -// ahead of what is actually held. -const copyChunkSize = 32 * 1024 - -// ReaderAtSeeker is the random access an archive format needs: a zip is read from its central -// directory at the end of the stream. -type ReaderAtSeeker interface { - io.Reader - io.ReaderAt - io.Seeker -} - -// Content is one archive's bytes, ready for random access. -type Content struct { - ReaderAtSeeker - - // release gives back what holding the bytes cost; nil when they are read in place - release func() -} - -// Release drops the held bytes and refunds their charge. Safe to call more than once. -func (c *Content) Release() { - if c.release != nil { - c.release() - c.release = nil - } -} - -// acquireContent makes an archive's bytes random-access. Bytes that already are, such as a file on -// disk or an entry of a parent archive, are read in place and cost nothing. Anything else is held -// in memory or written to disk within the limits. -func acquireContent(r io.Reader, name string, workDir *WorkDir, charge *Charge) (Content, error) { - if ras, ok := r.(ReaderAtSeeker); ok { - return Content{ReaderAtSeeker: ras}, nil - } - return holdContent(r, name, workDir, charge) -} - -// holdContent holds an archive's bytes in memory while the memory limit admits them and writes them -// to disk once it does not. Reaching the disk limit yields ErrDiskLimitReached. -func holdContent(r io.Reader, name string, workDir *WorkDir, charge *Charge) (Content, error) { - held, rest, err := readWhileMemoryAdmits(r, charge) - if err != nil { - return Content{}, fmt.Errorf("unable to read archive content: %w", err) - } - if rest == nil { - return Content{ - ReaderAtSeeker: bytes.NewReader(held), - release: func() { charge.RefundMemory(int64(len(held))) }, - }, nil - } - - log.WithFields("archive", name).Debug("archive content does not fit in memory; writing it to disk") - charge.RefundMemory(int64(len(held))) - return writeContentToDisk(io.MultiReader(bytes.NewReader(held), rest), workDir, charge) -} - -// readWhileMemoryAdmits reads r into memory, charging each chunk before holding it so a stream larger -// than the memory limit is never held in full. When a chunk is refused it returns what is held so -// far, still charged, and a reader over the refused chunk followed by the rest of r. rest is nil when -// all of r fit. -func readWhileMemoryAdmits(r io.Reader, charge *Charge) (held []byte, rest io.Reader, err error) { - chunk := make([]byte, copyChunkSize) - for { - n, readErr := r.Read(chunk) - if n > 0 { - if !charge.Memory(int64(n)) { - return held, io.MultiReader(bytes.NewReader(chunk[:n]), r), nil - } - held = append(held, chunk[:n]...) - } - if errors.Is(readErr, io.EOF) { - return held, nil, nil - } - if readErr != nil { - return nil, nil, readErr - } - } -} - -// contentFileName is the file an archive's own bytes are written to when they do not fit in memory. -const contentFileName = "archive" - -// writeContentToDisk writes r into the work directory, charging the disk limit as bytes land, and -// returns a reader over the file. Reaching the disk limit removes the file and yields -// ErrDiskLimitReached. -func writeContentToDisk(r io.Reader, workDir *WorkDir, charge *Charge) (Content, error) { - dir, err := workDir.Path() - if err != nil { - return Content{}, err - } - path := filepath.Join(dir, contentFileName) - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) - if err != nil { - return Content{}, fmt.Errorf("unable to create archive content file: %w", err) - } - - written, err := copyCharged(f, r, charge) - release := func() { - charge.RefundDisk(written) - if err := f.Close(); err != nil { - log.WithFields("path", path, "error", err).Trace("unable to close archive content file") - } - if err := os.Remove(path); err != nil { - log.WithFields("path", path, "error", err).Trace("unable to remove archive content file") - } - } - if err == nil { - _, err = f.Seek(0, io.SeekStart) - } - if err != nil { - release() - if errors.Is(err, ErrDiskLimitReached) { - return Content{}, err - } - return Content{}, fmt.Errorf("unable to write archive content file: %w", err) - } - return Content{ReaderAtSeeker: f, release: release}, nil -} - -// copyCharged copies src to dst, charging each chunk to the disk limit before writing it. It stops -// with ErrDiskLimitReached at the first refused chunk; bytes written before that stay charged. -func copyCharged(dst io.Writer, src io.Reader, charge *Charge) (written int64, err error) { - chunk := make([]byte, copyChunkSize) - for { - n, readErr := src.Read(chunk) - if n > 0 { - if !charge.Disk(int64(n)) { - return written, ErrDiskLimitReached - } - w, writeErr := dst.Write(chunk[:n]) - written += int64(w) - charge.RefundDisk(int64(n - w)) - if writeErr != nil { - return written, writeErr - } - } - if errors.Is(readErr, io.EOF) { - return written, nil - } - if readErr != nil { - return written, readErr - } - } -} diff --git a/internal/archive/content_test.go b/internal/archive/content_test.go deleted file mode 100644 index d6b8ab946..000000000 --- a/internal/archive/content_test.go +++ /dev/null @@ -1,204 +0,0 @@ -package archive - -import ( - "bytes" - "io" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestAcquireContent_randomAccessReadersAreUsedInPlace(t *testing.T) { - workDir := t.TempDir() - charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).Charge() - reader := bytes.NewReader([]byte("already random access")) - - content, err := acquireContent(reader, "app.zip", WorkDirAt(workDir), charge) - require.NoError(t, err) - - assert.Same(t, reader, content.ReaderAtSeeker) - assert.Empty(t, filesIn(t, workDir)) - mem, disk := charge.held() - assert.Zero(t, mem, "nothing is held, so nothing is charged") - assert.Zero(t, disk) - content.Release() -} - -func TestHoldContent_heldWhileTheLimitAdmitsIt(t *testing.T) { - workDir := t.TempDir() - body := []byte(strings.Repeat("a", 100)) - charge := NewLimiter(Limits{MaxMemoryBytes: 1000, MaxDiskBytes: 1000}).Charge() - - content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) - require.NoError(t, err) - - assert.Equal(t, body, readAll(t, content)) - assert.Empty(t, filesIn(t, workDir)) - mem, disk := charge.held() - assert.Equal(t, int64(100), mem) - assert.Zero(t, disk) - - content.Release() - mem, _ = charge.held() - assert.Zero(t, mem, "releasing held content refunds its memory") -} - -func TestHoldContent_writtenToDiskWhenTheLimitDoesNotAdmitIt(t *testing.T) { - workDir := t.TempDir() - body := []byte(strings.Repeat("b", 250)) - charge := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 10_000}).Charge() - - content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) - require.NoError(t, err) - - assert.Equal(t, []string{contentFileName}, filesIn(t, workDir)) - assert.Equal(t, body, readAll(t, content), "the bytes read before the limit refused must not be lost") - mem, disk := charge.held() - assert.Zero(t, mem, "nothing stays charged to a limit that refused") - assert.Equal(t, int64(250), disk) - - content.Release() - assert.Empty(t, filesIn(t, workDir), "releasing removes the file") - _, disk = charge.held() - assert.Zero(t, disk, "and refunds it") -} - -// peakTrackingReader records the highest memory charge observed while being read. -type peakTrackingReader struct { - r io.Reader - charge *Charge - peak int64 -} - -func (p *peakTrackingReader) Read(b []byte) (int, error) { - n, err := p.r.Read(b) - if mem, _ := p.charge.held(); mem > p.peak { - p.peak = mem - } - return n, err -} - -func TestHoldContent_largeArchiveNeverBuffersBeyondTheLimit(t *testing.T) { - workDir := t.TempDir() - const limit = int64(3 * copyChunkSize) - body := []byte(strings.Repeat("c", 10*copyChunkSize)) - charge := NewLimiter(Limits{MaxMemoryBytes: limit, MaxDiskBytes: int64(len(body)) + 1}).Charge() - tracked := &peakTrackingReader{r: bytes.NewReader(body), charge: charge} - - content, err := holdContent(tracked, "app.zip", WorkDirAt(workDir), charge) - require.NoError(t, err) - t.Cleanup(content.Release) - - assert.Equal(t, []string{contentFileName}, filesIn(t, workDir)) - assert.Equal(t, body, readAll(t, content)) - assert.LessOrEqual(t, tracked.peak, limit, "memory charged must never exceed the limit, even transiently") - mem, disk := charge.held() - assert.Zero(t, mem) - assert.Equal(t, int64(len(body)), disk) -} - -func TestHoldContent_diskLimitIsTerminal(t *testing.T) { - tests := []struct { - name string - limits Limits - }{ - {"a disk limit that is exceeded", Limits{MaxDiskBytes: 100}}, - {"a zero disk limit", Limits{MaxMemoryBytes: 10, MaxDiskBytes: 0}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - workDir := t.TempDir() - charge := NewLimiter(tt.limits).Charge() - - _, err := holdContent(bytes.NewReader([]byte(strings.Repeat("d", 500))), "app.zip", WorkDirAt(workDir), charge) - require.ErrorIs(t, err, ErrDiskLimitReached) - - assert.Empty(t, filesIn(t, workDir), "the partial file must not be left behind") - mem, disk := charge.held() - assert.Zero(t, mem) - assert.Zero(t, disk, "bytes no longer on disk must be refunded") - }) - } -} - -func TestHoldContent_threeStateLimits(t *testing.T) { - body := []byte(strings.Repeat("e", 500)) - - t.Run("a zero memory limit sends everything to disk", func(t *testing.T) { - workDir := t.TempDir() - charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10_000}).Charge() - - content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) - require.NoError(t, err) - t.Cleanup(content.Release) - - assert.Equal(t, []string{contentFileName}, filesIn(t, workDir)) - mem, disk := charge.held() - assert.Zero(t, mem) - assert.Equal(t, int64(500), disk) - }) - - t.Run("a negative memory limit holds content regardless of how much is already held", func(t *testing.T) { - workDir := t.TempDir() - charge := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: 10_000}).Charge() - require.True(t, charge.Memory(1_000_000)) - - content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) - require.NoError(t, err) - t.Cleanup(content.Release) - - assert.Empty(t, filesIn(t, workDir)) - mem, disk := charge.held() - assert.Equal(t, int64(1_000_000+len(body)), mem) - assert.Zero(t, disk) - }) - - t.Run("a zero disk limit still holds what memory admits", func(t *testing.T) { - workDir := t.TempDir() - charge := NewLimiter(Limits{MaxMemoryBytes: 10_000, MaxDiskBytes: 0}).Charge() - - content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) - require.NoError(t, err) - t.Cleanup(content.Release) - - assert.Empty(t, filesIn(t, workDir)) - mem, disk := charge.held() - assert.Equal(t, int64(len(body)), mem) - assert.Zero(t, disk) - }) - - t.Run("a negative disk limit writes with no ceiling", func(t *testing.T) { - workDir := t.TempDir() - charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}).Charge() - - content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), charge) - require.NoError(t, err) - t.Cleanup(content.Release) - - assert.Equal(t, []string{contentFileName}, filesIn(t, workDir)) - assert.Equal(t, body, readAll(t, content)) - }) -} - -func TestHoldContent_nilChargeHoldsEverythingInMemory(t *testing.T) { - workDir := t.TempDir() - body := []byte("small enough") - - content, err := holdContent(bytes.NewReader(body), "app.zip", WorkDirAt(workDir), nil) - require.NoError(t, err) - t.Cleanup(content.Release) - - assert.Equal(t, body, readAll(t, content)) - assert.Empty(t, filesIn(t, workDir)) -} - -func readAll(t *testing.T, c Content) []byte { - t.Helper() - _, err := c.Seek(0, io.SeekStart) - require.NoError(t, err) - body, err := io.ReadAll(c) - require.NoError(t, err) - return body -} diff --git a/internal/archive/entry.go b/internal/archive/entry.go deleted file mode 100644 index 9d629d68d..000000000 --- a/internal/archive/entry.go +++ /dev/null @@ -1,75 +0,0 @@ -package archive - -import ( - "archive/tar" - "io/fs" - "path" - "strings" - "time" - - "github.com/mholt/archives" - - "github.com/anchore/syft/internal/log" -) - -// Entry is one archive entry: its header and where its content is. Content starts in memory and -// moves to the store's overflow file when memory runs out; the move changes nothing an index over -// the entry holds. -type Entry struct { - Header tar.Header - - mem []byte // content while held in memory - offset int64 // where content starts in the overflow file once written there - size int64 -} - -// maxEntryNameBytes is the longest entry name stored: PATH_MAX. Every path component becomes an index -// node, so a longer name is a cheap way to inflate the index and no filesystem could hold it anyway. -const maxEntryNameBytes = 4096 - -// entryHeader builds the header stored for one archive entry, or reports false for an entry not worth -// carrying: device nodes, fifos and sockets, which no cataloger reads, and names that cannot be a path -// inside the archive. -// -// The name is cleaned to an archive-relative path with no leading slash, so "../x" names x at the -// archive root rather than anything outside it. Symlinks are stored as headers, never followed on the -// host; the index resolves their targets inside the archive. -func entryHeader(f archives.FileInfo) (tar.Header, bool) { - mode := f.Mode() - if !mode.IsRegular() && !mode.IsDir() && mode.Type()&fs.ModeSymlink == 0 { - log.WithFields("entry", f.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry") - return tar.Header{}, false - } - - name, ok := entryName(f.NameInArchive) - if !ok { - log.WithFields("entry-name-bytes", len(f.NameInArchive)).Trace("skipping archive entry with an unusable name") - return tar.Header{}, false - } - - hdr, err := tar.FileInfoHeader(f.FileInfo, f.LinkTarget) - if err != nil { - log.WithFields("entry", f.NameInArchive, "error", err).Debug("skipping archive entry with an unusable header") - return tar.Header{}, false - } - hdr.Name = name - hdr.Linkname = f.LinkTarget - - // nothing reads these, and the header is held for the life of the archive - hdr.PAXRecords = nil - hdr.Uname, hdr.Gname = "", "" - hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} - hdr.ModTime = hdr.ModTime.Truncate(time.Second) - - return *hdr, true -} - -// entryName cleans an archive entry's name to a path relative to the archive root, reporting false -// when it names nothing inside the archive. -func entryName(name string) (string, bool) { - if len(name) > maxEntryNameBytes || strings.ContainsRune(name, 0) { - return "", false - } - cleaned := strings.TrimPrefix(path.Clean("/"+name), "/") - return cleaned, cleaned != "" -} diff --git a/internal/archive/entrystore.go b/internal/archive/entrystore.go deleted file mode 100644 index b7d0c5cdc..000000000 --- a/internal/archive/entrystore.go +++ /dev/null @@ -1,169 +0,0 @@ -package archive - -import ( - "archive/tar" - "bytes" - "errors" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/anchore/syft/internal/log" -) - -// approxIndexBytesPerEntry estimates what holding one entry costs beyond its content: the entry and -// header themselves, and the index node, path map slot and name index slot the resolver builds over -// it. It is charged so an archive of many tiny entries is still bounded by the limits. -const approxIndexBytesPerEntry = 2 * 1024 - -func approxIndexBytes(hdr tar.Header) int64 { - return approxIndexBytesPerEntry + int64(len(hdr.Name)+len(hdr.Linkname)) -} - -// entriesFileName is the single file an archive's entries overflow into. Entries are located by -// offset; the file has no framing of its own. -const entriesFileName = "entries" - -// EntryStore holds one archive's entries: in memory while the memory limit admits them, and in one -// overflow file once it does not. Everything held is charged to one Charge. -type EntryStore struct { - name string - workDir *WorkDir - charge *Charge - - entries []*Entry - file *os.File - written int64 -} - -// NewEntryStore returns an empty store for the named archive that overflows into workDir. -func NewEntryStore(name string, workDir *WorkDir, charge *Charge) *EntryStore { - return &EntryStore{name: name, workDir: workDir, charge: charge} -} - -// Add stores one entry, reading its content in full now since archive readers are sequential. -// Reaching the disk limit stores nothing for this entry and returns ErrDiskLimitReached; entries -// stored before it remain usable. -func (s *EntryStore) Add(hdr tar.Header, content io.Reader) error { - if !s.charge.Index(approxIndexBytes(hdr)) { - return ErrDiskLimitReached - } - - entry := &Entry{Header: hdr} - if hdr.Typeflag == tar.TypeReg && content != nil { - held, rest, err := readWhileMemoryAdmits(content, s.charge) - if err != nil { - return fmt.Errorf("unable to read archive entry %q: %w", hdr.Name, err) - } - entry.mem, entry.size = held, int64(len(held)) - if rest != nil { - if err := s.overflow(entry, rest); err != nil { - return err - } - } - } - - s.entries = append(s.entries, entry) - return nil -} - -// overflow writes every entry held in memory into the overflow file, followed by the new entry's held -// bytes and the rest of its content, charging the disk limit as bytes land and refunding the memory -// they leave. -func (s *EntryStore) overflow(entry *Entry, rest io.Reader) error { - if err := s.openFile(); err != nil { - return err - } - - moved := 0 - for _, held := range s.entries { - if held.mem == nil { - continue - } - if err := s.moveToDisk(held); err != nil { - return err - } - moved++ - } - - if err := s.moveToDisk(entry); err != nil { - return err - } - written, err := copyCharged(io.NewOffsetWriter(s.file, s.written), rest, s.charge) - s.written += written - entry.size += written - - log.WithFields("archive", s.name, "entries", moved+1, "bytes", s.written). - Debug("archive entries do not fit in memory; written to disk") - - if err != nil && !errors.Is(err, ErrDiskLimitReached) { - return fmt.Errorf("unable to write archive entry %q to disk: %w", entry.Header.Name, err) - } - return err -} - -func (s *EntryStore) openFile() error { - if s.file != nil { - return nil - } - dir, err := s.workDir.Path() - if err != nil { - return err - } - f, err := os.OpenFile(filepath.Join(dir, entriesFileName), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) - if err != nil { - return fmt.Errorf("unable to create archive entries file: %w", err) - } - s.file = f - return nil -} - -// moveToDisk appends an entry's in-memory content to the overflow file and points the entry there. -func (s *EntryStore) moveToDisk(entry *Entry) error { - n := int64(len(entry.mem)) - if !s.charge.Disk(n) { - return ErrDiskLimitReached - } - if _, err := s.file.WriteAt(entry.mem, s.written); err != nil { - return fmt.Errorf("unable to write archive entry %q to disk: %w", entry.Header.Name, err) - } - entry.offset, entry.size = s.written, n - s.written += n - s.charge.RefundMemory(n) - entry.mem = nil - return nil -} - -// Entries returns the stored entries in the order they were added. -func (s *EntryStore) Entries() []*Entry { - return s.entries -} - -// Open returns a reader over an entry's content, wherever it is. The reader supports Read, Seek and -// ReadAt, so a nested archive can be read where it lies. -func (s *EntryStore) Open(entry *Entry) ReaderAtSeeker { - switch { - case entry.mem != nil: - return bytes.NewReader(entry.mem) - case entry.size == 0: - return bytes.NewReader(nil) - default: - return io.NewSectionReader(s.file, entry.offset, entry.size) - } -} - -// OnDisk reports how many bytes of entry content have been written to the overflow file. -func (s *EntryStore) OnDisk() int64 { - return s.written -} - -// Close releases the overflow file. Readers from Open must not be used after this. -func (s *EntryStore) Close() error { - if s.file == nil { - return nil - } - err := s.file.Close() - s.file = nil - return err -} diff --git a/internal/archive/entrystore_test.go b/internal/archive/entrystore_test.go deleted file mode 100644 index 1323daad6..000000000 --- a/internal/archive/entrystore_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package archive - -import ( - "archive/tar" - "bytes" - "io" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestEntryStore_holdsSmallContentInMemory(t *testing.T) { - dir := t.TempDir() - s := NewEntryStore("app.zip", WorkDirAt(dir), memCharge(1024)) - t.Cleanup(func() { require.NoError(t, s.Close()) }) - - require.NoError(t, s.Add(regularHeader("a.txt", 5), bytes.NewReader([]byte("hello")))) - - assert.Equal(t, int64(5), s.heldInMemory()) - assert.NoFileExists(t, filepath.Join(dir, entriesFileName)) - assert.Equal(t, "hello", readEntry(t, s.Open(s.Entries()[0]))) -} - -func TestEntryStore_overflowsToDiskAndKeepsTheSameEntries(t *testing.T) { - // an index over these entries is not rebuilt when content moves - dir := t.TempDir() - s := NewEntryStore("app.zip", WorkDirAt(dir), memCharge(8)) - t.Cleanup(func() { require.NoError(t, s.Close()) }) - - require.NoError(t, s.Add(regularHeader("a.txt", 4), bytes.NewReader([]byte("aaaa")))) - first := s.Entries()[0] - require.Equal(t, int64(4), s.heldInMemory()) - - require.NoError(t, s.Add(regularHeader("b.txt", 6), bytes.NewReader([]byte("bbbbbb")))) - - assert.Zero(t, s.heldInMemory(), "everything moves out, not just the entry that did not fit") - assert.FileExists(t, filepath.Join(dir, entriesFileName)) - assert.Equal(t, int64(10), s.OnDisk()) - - entries := s.Entries() - require.Len(t, entries, 2) - assert.Same(t, first, entries[0], "entry identity survives the move") - assert.Equal(t, "aaaa", readEntry(t, s.Open(entries[0]))) - assert.Equal(t, "bbbbbb", readEntry(t, s.Open(entries[1]))) -} - -func TestEntryStore_zeroMemoryHoldsNothing(t *testing.T) { - dir := t.TempDir() - s := NewEntryStore("app.zip", WorkDirAt(dir), memCharge(0)) - t.Cleanup(func() { require.NoError(t, s.Close()) }) - - require.NoError(t, s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")))) - - assert.Zero(t, s.heldInMemory()) - assert.FileExists(t, filepath.Join(dir, entriesFileName)) - assert.Equal(t, "abc", readEntry(t, s.Open(s.Entries()[0]))) -} - -func TestEntryStore_negativeMemoryNeverOverflows(t *testing.T) { - dir := t.TempDir() - s := NewEntryStore("app.zip", WorkDirAt(dir), memCharge(-1)) - t.Cleanup(func() { require.NoError(t, s.Close()) }) - - for _, body := range []string{"one", "two", "three"} { - require.NoError(t, s.Add(regularHeader(body+".txt", int64(len(body))), bytes.NewReader([]byte(body)))) - } - - assert.Equal(t, int64(11), s.heldInMemory()) - assert.NoFileExists(t, filepath.Join(dir, entriesFileName)) -} - -func TestEntryStore_refusedEntryIsNotStored(t *testing.T) { - // a half-stored pom a cataloger parses is worse than an absent one - s := storeFor(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).Charge()) - - err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc"))) - assert.ErrorIs(t, err, ErrDiskLimitReached) - assert.Empty(t, s.Entries()) -} - -func TestEntryStore_directoryEntriesHoldNoContent(t *testing.T) { - s := storeFor(t, memCharge(1024)) - - require.NoError(t, s.Add(tar.Header{Name: "lib", Typeflag: tar.TypeDir, Mode: 0o755}, nil)) - - assert.Zero(t, s.heldInMemory()) - assert.Empty(t, readEntry(t, s.Open(s.Entries()[0]))) -} - -func TestEntryStore_indexCostIsChargedEvenForEmptyContent(t *testing.T) { - // the index estimate is what bounds an archive of many empty entries - limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}) - s := storeFor(t, limiter.Charge()) - - hdr := regularHeader("a.txt", 0) - require.NoError(t, s.Add(hdr, bytes.NewReader(nil))) - - mem, _ := limiter.InUse() - assert.Equal(t, approxIndexBytes(hdr), mem) - assert.Zero(t, s.heldInMemory()) -} - -func TestEntryStore_indexCostFallsBackToDiskWhenMemoryIsZero(t *testing.T) { - limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) - s := storeFor(t, limiter.Charge()) - - hdr := regularHeader("a.txt", 0) - require.NoError(t, s.Add(hdr, bytes.NewReader(nil))) - - mem, disk := limiter.InUse() - assert.Zero(t, mem) - assert.Equal(t, approxIndexBytes(hdr), disk) -} - -func TestEntryStore_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) { - room := approxIndexBytes(regularHeader("x", 0)) * 3 - s := storeFor(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).Charge()) - - var err error - admitted := 0 - for i := 0; i < 100 && err == nil; i++ { - if err = s.Add(regularHeader("x", 0), bytes.NewReader(nil)); err == nil { - admitted++ - } - } - require.ErrorIs(t, err, ErrDiskLimitReached) - assert.Equal(t, 3, admitted) -} - -func TestEntryStore_readsAfterCloseFail(t *testing.T) { - s := storeFor(t, memCharge(0)) - require.NoError(t, s.Add(regularHeader("a.txt", 5), bytes.NewReader([]byte("hello")))) - entry := s.Entries()[0] - require.Equal(t, "hello", readEntry(t, s.Open(entry))) - - require.NoError(t, s.Close()) - - _, err := io.ReadAll(s.Open(entry)) - assert.Error(t, err) - assert.NoError(t, s.Close(), "Close is idempotent") -} diff --git a/internal/archive/extract.go b/internal/archive/extract.go index 01ae1ef4a..90031b0a9 100644 --- a/internal/archive/extract.go +++ b/internal/archive/extract.go @@ -1,7 +1,6 @@ package archive import ( - "archive/tar" "bytes" "context" "crypto" @@ -18,32 +17,6 @@ import ( "github.com/anchore/syft/syft/file" ) -// Extracted is one archive opened as its own filesystem. -type Extracted struct { - // Resolver reads the archive's entries. Its locations carry the archive's FileSystemID and - // ArchivePath. - Resolver file.Resolver - - // Digests are of the archive file itself. - Digests []file.Digest - - // Truncated reports that the disk limit stopped extraction early. Resolver covers the entries - // stored before that. - Truncated bool - - cleanup func() -} - -// Cleanup removes everything the archive holds on disk and releases its charge against the limiter. -// Safe to call more than once. -func (e *Extracted) Cleanup() { - if e == nil || e.cleanup == nil { - return - } - e.cleanup() - e.cleanup = nil -} - // Extract opens the archive read from r as its own filesystem, holding its bytes and entries within // the limiter's bounds. It returns nil when the content is not an archive it can read, and // ErrDiskLimitReached when the archive's own bytes cannot be placed within the disk limit. The caller @@ -51,46 +24,49 @@ func (e *Extracted) Cleanup() { // // fileSystemID and archivePath are stamped onto every location the resolver returns: the filesystem // the archive was found on, and the colon-delimited chain of archives from the scan root to this one. -func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string, limiter *Limiter, exclusions Exclusions) (*Extracted, error) { - workDir := NewWorkDir(ctx) - charge := limiter.Charge() - store := NewEntryStore(archivePath, workDir, charge) - var content Content - release := func() { - content.Release() - if err := store.Close(); err != nil { - log.WithFields("archive", archivePath, "error", err).Trace("unable to close archive entries file") - } - workDir.Remove() - charge.Release() - } +func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string, limiter *Limiter, exclusions Exclusions) (*Resolver, error) { + charge := limiter.charge() + resolver := newResolver(ctx, fileSystemID, archivePath, charge) - content, err := acquireContent(r, archivePath, workDir, charge) + content, release, err := resolver.acquire(r) if err != nil { - release() + resolver.Cleanup() return nil, err } // the archive's own bytes are needed only until they are extracted and digested - defer content.Release() + defer release() format := identifyFormat(ctx, archiveFileName(archivePath), content) if format == nil { - release() + resolver.Cleanup() return nil, nil } - truncated, err := extractInto(ctx, format, content, store, exclusions) - if err != nil { - release() + if err := resolver.extract(ctx, format, content, exclusions); err != nil { + resolver.Cleanup() return nil, fmt.Errorf("unable to extract archive %q: %w", archivePath, err) } + resolver.Digests = digestsOf(ctx, content, archivePath) + return resolver, nil +} - return &Extracted{ - Resolver: NewIndex(store, fileSystemID, archivePath), - Digests: digestsOf(ctx, content, archivePath), - Truncated: truncated, - cleanup: release, - }, nil +// acquire makes the archive's own bytes random-access, which identifying, extracting and digesting +// them needs. Bytes that already are, such as a file on disk or an entry of a parent archive, are +// read in place and cost nothing. Anything else is held within the limits like an entry, and the +// returned release gives back what memory it takes; bytes spilled to disk stay until Cleanup. +// Reaching the disk limit yields ErrDiskLimitReached. +func (r *Resolver) acquire(content io.Reader) (ReaderAtSeeker, func(), error) { + if ras, ok := content.(ReaderAtSeeker); ok { + return ras, func() {}, nil + } + var b blob + if err := r.put(&b, content); err != nil { + if errors.Is(err, ErrDiskLimitReached) { + return nil, nil, err + } + return nil, nil, fmt.Errorf("unable to read archive content: %w", err) + } + return r.open(&b), func() { r.discard(&b) }, nil } // archiveFileName returns the archive's own file name from the end of its archive path, which format @@ -121,40 +97,38 @@ func identifyFormat(ctx context.Context, name string, content ReaderAtSeeker) ar return nil } -// extractInto reads every entry of the archive into the store. Reaching the disk limit stops the -// walk and reports truncation rather than an error. -func extractInto(ctx context.Context, format archives.Extractor, content ReaderAtSeeker, store *EntryStore, exclusions Exclusions) (truncated bool, err error) { +// extract adds every entry of the archive. Reaching the disk limit stops the walk and marks the +// resolver truncated rather than failing. +func (r *Resolver) extract(ctx context.Context, format archives.Extractor, content ReaderAtSeeker, exclusions Exclusions) error { if _, err := content.Seek(0, io.SeekStart); err != nil { - return false, err + return err } - err = format.Extract(ctx, content, func(_ context.Context, f archives.FileInfo) error { + err := format.Extract(ctx, content, func(_ context.Context, f archives.FileInfo) error { hdr, ok := entryHeader(f) if !ok || exclusions.Excludes(hdr.Name) { return nil } - return storeEntry(store, hdr, f) + if !f.Mode().IsRegular() { + return r.add(hdr, nil) + } + entry, err := f.Open() + if err != nil { + log.WithFields("entry", hdr.Name, "error", err).Trace("unable to open archive entry, skipping it") + return nil + } + defer func() { + if err := entry.Close(); err != nil { + log.WithFields("entry", hdr.Name, "error", err).Trace("unable to close archive entry") + } + }() + return r.add(hdr, entry) }) + r.finish() if errors.Is(err, ErrDiskLimitReached) { - return true, nil - } - return false, err -} - -func storeEntry(store *EntryStore, hdr tar.Header, f archives.FileInfo) error { - if !f.Mode().IsRegular() { - return store.Add(hdr, nil) - } - content, err := f.Open() - if err != nil { - log.WithFields("entry", hdr.Name, "error", err).Trace("unable to open archive entry, skipping it") + r.Truncated = true return nil } - defer func() { - if err := content.Close(); err != nil { - log.WithFields("entry", hdr.Name, "error", err).Trace("unable to close archive entry") - } - }() - return store.Add(hdr, content) + return err } // archiveDigestHashes are the digests taken of an archive as a whole: SHA-1, matching what the java diff --git a/internal/archive/extract_test.go b/internal/archive/extract_test.go index 714f704db..75227bdd6 100644 --- a/internal/archive/extract_test.go +++ b/internal/archive/extract_test.go @@ -2,7 +2,9 @@ package archive import ( "archive/tar" + "archive/zip" "bytes" + "compress/gzip" "context" "fmt" "io" @@ -54,49 +56,39 @@ func Test_identifyFormat(t *testing.T) { } } -func Test_extractInto_storesEveryEntryInMemoryWhenUnbounded(t *testing.T) { +func TestResolver_extract_storesEveryEntryInMemoryWhenUnbounded(t *testing.T) { for name, data := range map[string][]byte{ "test.zip": zipBytes(t, sampleFiles), "test.tar.gz": tarGzBytes(t, sampleFiles), } { t.Run(name, func(t *testing.T) { - store, truncated := extractBytes(t, data, name, nil) - assert.False(t, truncated) - assert.Zero(t, store.OnDisk()) - assert.Empty(t, filesIn(t, storeDir(t, store))) + r, root := extractBytes(t, data, name, nil) + assert.False(t, r.Truncated) + assert.Zero(t, r.written) + assert.Empty(t, filesIn(t, root)) - entries := readStore(t, store) + entries := readStore(t, r) assert.Equal(t, "content1", entries["file1.txt"].body) assert.Equal(t, "content2", entries["dir/file2.txt"].body) }) } } -func Test_extractInto_diskLimitTruncatesAtTheEntryThatDoesNotFit(t *testing.T) { +func TestResolver_extract_diskLimitTruncatesAtTheEntryThatDoesNotFit(t *testing.T) { // each entry costs its index estimate plus its content on disk record := approxIndexBytes(tar.Header{Name: "file1.txt"}) data := zipBytes(t, map[string]string{"file1.txt": "a", "file2.txt": "b", "file3.txt": "c"}) - store, truncated := extractBytes(t, data, "test.zip", diskCharge(2*(record+1)+record)) + r, _ := extractBytes(t, data, "test.zip", diskCharge(2*(record+1)+record)) - assert.True(t, truncated) - assert.Equal(t, []string{"file1.txt", "file2.txt"}, storedNames(t, store), "the refused entry is not stored") - entries := readStore(t, store) + assert.True(t, r.Truncated) + assert.Equal(t, []string{"file1.txt", "file2.txt"}, storedNames(t, r), "the refused entry is not stored") + entries := readStore(t, r) assert.Equal(t, "a", entries["file1.txt"].body) assert.Equal(t, "b", entries["file2.txt"].body) } -func Test_extractInto_anEntryLargerThanTheBudgetIsDroppedWhole(t *testing.T) { - data := zipBytes(t, map[string]string{"file1.txt": strings.Repeat("x", 100)}) - - store, truncated := extractBytes(t, data, "test.zip", diskCharge(50)) - - assert.True(t, truncated) - assert.Empty(t, store.Entries()) - assert.Zero(t, store.OnDisk()) -} - -func Test_extractInto_diskLimitBitesMidEntry(t *testing.T) { +func TestResolver_extract_diskLimitBitesMidEntry(t *testing.T) { // a tar is walked entry by entry, so the limit can bite part way through an entry data := tarGzBytes(t, map[string]string{ "a/first.txt": strings.Repeat("a", 40*1024), @@ -105,45 +97,43 @@ func Test_extractInto_diskLimitBitesMidEntry(t *testing.T) { const budget = 80 * 1024 charge := diskCharge(budget) - store, truncated := extractBytes(t, data, "test.tar.gz", charge) + r, root := extractBytes(t, data, "test.tar.gz", charge) - assert.True(t, truncated) - assert.Equal(t, []string{"a/first.txt"}, storedNames(t, store)) - assert.Greater(t, store.OnDisk(), int64(40*1024), "the first entry landed whole and the second part way") + assert.True(t, r.Truncated) + assert.Equal(t, []string{"a/first.txt"}, storedNames(t, r)) + assert.Greater(t, r.written, int64(40*1024), "the first entry landed whole and the second part way") _, disk := charge.held() assert.LessOrEqual(t, disk, int64(budget)) - assert.Greater(t, disk, store.OnDisk(), "the index estimate is charged to disk too") + assert.Greater(t, disk, r.written, "the index estimate is charged to disk too") - info, err := os.Stat(filepath.Join(storeDir(t, store), entriesFileName)) + info, err := os.Stat(filepath.Join(root, spillFile(t, root))) require.NoError(t, err) - assert.Equal(t, store.OnDisk(), info.Size()) + assert.Equal(t, r.written, info.Size()) } -func Test_extractInto_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) { +func TestResolver_extract_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) { data := zipBytes(t, map[string]string{"a.txt": strings.Repeat("a", 200)}) oneArchive := approxIndexBytes(tar.Header{Name: "a.txt"}) + 200 limiter := NewLimiter(Limits{MaxDiskBytes: oneArchive * 3 / 2}) - first := limiter.Charge() - store, truncated := extractBytes(t, data, "test.zip", first) - require.False(t, truncated) - require.Equal(t, int64(200), store.OnDisk()) + first, _ := extractBytes(t, data, "test.zip", limiter.charge()) + require.False(t, first.Truncated) + require.Equal(t, int64(200), first.written) - second := limiter.Charge() - _, truncated = extractBytes(t, data, "test.zip", second) - assert.True(t, truncated, "a second archive is bounded while the first is still held") - second.Release() + second, _ := extractBytes(t, data, "test.zip", limiter.charge()) + assert.True(t, second.Truncated, "a second archive is bounded while the first is still held") + second.Cleanup() - first.Release() + first.Cleanup() _, disk := limiter.InUse() require.Zero(t, disk) - store, truncated = extractBytes(t, data, "test.zip", limiter.Charge()) - assert.False(t, truncated, "and extracts in full once the first is released") - assert.Equal(t, int64(200), store.OnDisk()) + third, _ := extractBytes(t, data, "test.zip", limiter.charge()) + assert.False(t, third.Truncated, "and extracts in full once the first is released") + assert.Equal(t, int64(200), third.written) } -func Test_extractInto_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) { +func TestResolver_extract_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) { var entries []testEntry for i := range 50 { entries = append(entries, testEntry{name: fmt.Sprintf("d%04d/", i), dir: true}) @@ -155,14 +145,14 @@ func Test_extractInto_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testi "dirs.tar.gz": tarGzFromEntries(t, entries), } { t.Run(name, func(t *testing.T) { - store, truncated := extractBytes(t, data, name, diskCharge(room)) - assert.True(t, truncated) - assert.Len(t, store.Entries(), 10) + r, _ := extractBytes(t, data, name, diskCharge(room)) + assert.True(t, r.Truncated) + assert.Len(t, storedNames(t, r), 10) }) } } -func Test_extractInto_entryNamesAreCleanedAndNothingIsWrittenUnderThem(t *testing.T) { +func TestResolver_extract_entryNamesAreCleanedAndNothingIsWrittenUnderThem(t *testing.T) { entries := []testEntry{ {name: "../../etc/passwd", body: "evil"}, {name: "/etc/hosts", body: "pwned"}, @@ -178,15 +168,15 @@ func Test_extractInto_entryNamesAreCleanedAndNothingIsWrittenUnderThem(t *testin "evil.tar.gz": tarGzFromEntries(t, entries), } { t.Run(name, func(t *testing.T) { - store, truncated := extractBytes(t, data, name, nil) - require.False(t, truncated) + r, root := extractBytes(t, data, name, nil) + require.False(t, r.Truncated) assert.Equal(t, []string{"META-INF", "META-INF/MANIFEST.MF", "bin/link.txt", "etc/hosts", "etc/passwd", "passwd", "shadow"}, - storedNames(t, store)) - assert.Empty(t, filesIn(t, storeDir(t, store)), "entries live in the store, never under their own names") + storedNames(t, r)) + assert.Empty(t, filesIn(t, root), "entries live in the store, never under their own names") assert.NoFileExists(t, filepath.Join(t.TempDir(), "..", "etc", "passwd")) - got := readStore(t, store) + got := readStore(t, r) assert.Equal(t, "evil", got["etc/passwd"].body) assert.Equal(t, "pwned", got["etc/hosts"].body) assert.Equal(t, byte(tar.TypeDir), got["META-INF"].typeflag) @@ -205,18 +195,18 @@ func Test_extractInto_entryNamesAreCleanedAndNothingIsWrittenUnderThem(t *testin } } -func Test_extractInto_dataEdgeCases(t *testing.T) { +func TestResolver_extract_dataEdgeCases(t *testing.T) { t.Run("an empty archive is not an error", func(t *testing.T) { - store, truncated := extractBytes(t, zipBytes(t, map[string]string{}), "test.zip", nil) - assert.Empty(t, store.Entries()) - assert.False(t, truncated) + r, _ := extractBytes(t, zipBytes(t, map[string]string{}), "test.zip", nil) + assert.Empty(t, r.files) + assert.False(t, r.Truncated) }) t.Run("a zero-byte entry costs no disk", func(t *testing.T) { - store, _ := extractBytes(t, zipBytes(t, map[string]string{"empty.txt": ""}), "test.zip", memCharge(0)) - assert.Equal(t, []string{"empty.txt"}, storedNames(t, store)) - assert.Zero(t, store.OnDisk()) - assert.Empty(t, readStore(t, store)["empty.txt"].body) + r, _ := extractBytes(t, zipBytes(t, map[string]string{"empty.txt": ""}), "test.zip", memCharge(0)) + assert.Equal(t, []string{"empty.txt"}, storedNames(t, r)) + assert.Zero(t, r.written) + assert.Empty(t, readStore(t, r)["empty.txt"].body) }) t.Run("unicode entry names survive", func(t *testing.T) { @@ -225,8 +215,8 @@ func Test_extractInto_dataEdgeCases(t *testing.T) { "日本語/ファイル.txt": "japanese", "emoji-🎉.txt": "emoji", } - store, _ := extractBytes(t, zipBytes(t, names), "test.zip", nil) - entries := readStore(t, store) + r, _ := extractBytes(t, zipBytes(t, names), "test.zip", nil) + entries := readStore(t, r) for name, want := range names { assert.Equal(t, want, entries[name].body, name) } @@ -237,27 +227,27 @@ func Test_extractInto_dataEdgeCases(t *testing.T) { strings.Repeat("a/", maxEntryNameBytes) + "f": "deep", "ok.txt": "fine", }) - store, _ := extractBytes(t, data, "test.zip", nil) - assert.Equal(t, []string{"ok.txt"}, storedNames(t, store)) + r, _ := extractBytes(t, data, "test.zip", nil) + assert.Equal(t, []string{"ok.txt"}, storedNames(t, r)) }) } -func Test_extractInto_excludedEntriesAreNeverStored(t *testing.T) { +func TestResolver_extract_excludedEntriesAreNeverStored(t *testing.T) { data := zipBytes(t, map[string]string{ "keep.txt": "kept", "vendor/lib.go": "excluded by directory", "deep/down/pkg.rpm": "excluded by name", }) limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}) - store := storeFor(t, limiter.Charge()) + r, _ := resolverIn(t, limiter.charge()) content := bytes.NewReader(data) - truncated, err := extractInto(context.Background(), identifyFormat(context.Background(), "test.zip", content), content, store, + err := r.extract(context.Background(), identifyFormat(context.Background(), "test.zip", content), content, NewExclusions([]string{"**/vendor", "**/*.rpm"})) require.NoError(t, err) - require.False(t, truncated) + require.False(t, r.Truncated) - assert.Equal(t, []string{"keep.txt"}, storedNames(t, store)) + assert.Equal(t, []string{"keep.txt"}, storedNames(t, r)) mem, _ := limiter.InUse() assert.Equal(t, approxIndexBytes(tar.Header{Name: "keep.txt"})+4, mem, "an excluded entry costs nothing") } @@ -274,13 +264,13 @@ func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) { require.Len(t, extracted.Digests, 1) assert.Equal(t, "sha1", extracted.Digests[0].Algorithm) - locations, err := extracted.Resolver.FilesByGlob("**/*.txt") + locations, err := extracted.FilesByGlob("**/*.txt") require.NoError(t, err) require.Len(t, locations, 1) assert.Equal(t, file.Coordinates{RealPath: "dir/hello.txt", FileSystemID: "parentFS", ArchivePath: "app.war:some/path/app.zip"}, locations[0].Coordinates) - reader, err := extracted.Resolver.FileContentsByLocation(locations[0]) + reader, err := extracted.FileContentsByLocation(locations[0]) require.NoError(t, err) body, err := io.ReadAll(reader) require.NoError(t, err) @@ -293,42 +283,26 @@ func TestExtract_notAnArchive(t *testing.T) { assert.Nil(t, extracted) } -func TestExtract_writesUnderTheScansTempRootAndCleansUp(t *testing.T) { +func TestExtract_spillsUnderTheScansTempRootAndCleansUp(t *testing.T) { ctx, root := scanContext(t) - limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) + limiter := NewLimiter(spillingLimits) extracted, err := Extract(ctx, bytes.NewReader(zipBytes(t, sampleFiles)), "", "app.zip", limiter, nil) require.NoError(t, err) require.NotNil(t, extracted) t.Cleanup(extracted.Cleanup) - workDirs := workDirsUnder(t, root) - require.Len(t, workDirs, 1) - assert.Equal(t, []string{entriesFileName}, filesIn(t, workDirs[0])) + require.Len(t, filesIn(t, root), 1) _, disk := limiter.InUse() assert.Positive(t, disk) extracted.Cleanup() - assert.NoDirExists(t, workDirs[0]) + assert.Empty(t, filesIn(t, root)) _, disk = limiter.InUse() assert.Zero(t, disk, "cleanup releases everything the archive held") extracted.Cleanup() } -func TestExtract_anArchiveThatNeverWritesCreatesNoWorkDir(t *testing.T) { - ctx, root := scanContext(t) - - extracted, err := Extract(ctx, bytes.NewReader(zipBytes(t, sampleFiles)), "", "app.zip", - NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), nil) - require.NoError(t, err) - require.NotNil(t, extracted) - t.Cleanup(extracted.Cleanup) - - assert.Empty(t, workDirsUnder(t, root)) - extracted.Cleanup() - assert.Empty(t, workDirsUnder(t, root)) -} - func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) { // the archive's own bytes are only needed to extract and digest it; afterwards the limiter holds // the entries and the index estimate alone @@ -344,7 +318,9 @@ func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) { assert.Zero(t, disk) } -func TestExtract_heldContentWrittenToDiskIsRemovedOnceExtracted(t *testing.T) { +func TestExtract_heldContentWrittenToDiskStaysUntilCleanup(t *testing.T) { + // only a top-level stream is ever held; a nested archive is read out of its parent, which is held + // for the whole descent anyway, so spilled bytes wait for Cleanup with the entries ctx, root := scanContext(t) data := zipBytes(t, map[string]string{"hello.txt": "hi"}) limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) @@ -353,11 +329,14 @@ func TestExtract_heldContentWrittenToDiskIsRemovedOnceExtracted(t *testing.T) { require.NoError(t, err) t.Cleanup(extracted.Cleanup) - workDirs := workDirsUnder(t, root) - require.Len(t, workDirs, 1) - assert.Equal(t, []string{entriesFileName}, filesIn(t, workDirs[0]), "the archive's own bytes are gone") + require.Len(t, filesIn(t, root), 1, "the archive's bytes and its entries share one file") _, disk := limiter.InUse() - assert.Equal(t, approxIndexBytes(tar.Header{Name: "hello.txt"})+2, disk) + assert.Equal(t, approxIndexBytes(tar.Header{Name: "hello.txt"})+2+int64(len(data)), disk) + + extracted.Cleanup() + assert.Empty(t, filesIn(t, root)) + _, disk = limiter.InUse() + assert.Zero(t, disk) } func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) { @@ -373,30 +352,11 @@ func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) { require.NotNil(t, extracted) t.Cleanup(extracted.Cleanup) assert.True(t, extracted.Truncated) - locations, err := extracted.Resolver.FilesByGlob("**/*") + locations, err := extracted.FilesByGlob("**/*") require.NoError(t, err) assert.Empty(t, locations) } -func TestExtract_truncationKeepsWhatWasStored(t *testing.T) { - data := zipBytes(t, map[string]string{ - "a/small.txt": strings.Repeat("s", 50), - "b/big.txt": strings.Repeat("b", 500), - }) - records := approxIndexBytes(tar.Header{Name: "a/small.txt"}) + approxIndexBytes(tar.Header{Name: "b/big.txt"}) - limiter := NewLimiter(Limits{MaxDiskBytes: records + 50 + 100}) - - extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "app.zip", limiter, nil) - require.NoError(t, err) - t.Cleanup(extracted.Cleanup) - - assert.True(t, extracted.Truncated) - locations, err := extracted.Resolver.FilesByGlob("**/*.txt") - require.NoError(t, err) - require.Len(t, locations, 1) - assert.Equal(t, "a/small.txt", locations[0].RealPath) -} - func TestExtract_aZipBehindALauncherScript(t *testing.T) { // a Spring Boot executable jar sniffs as a shell script; archive/zip still finds the central directory data := append([]byte("#!/bin/bash\necho launcher\nexit 0\n"), zipBytes(t, map[string]string{ @@ -410,7 +370,7 @@ func TestExtract_aZipBehindALauncherScript(t *testing.T) { t.Cleanup(extracted.Cleanup) assert.False(t, extracted.Truncated) - locations, err := extracted.Resolver.FilesByGlob("**/*") + locations, err := extracted.FilesByGlob("**/*") require.NoError(t, err) assert.Len(t, locations, 2) } @@ -465,3 +425,108 @@ func Test_MayHideAnAppendedArchive(t *testing.T) { assert.Equal(t, prose, rest) }) } + +// zeros is a reader of n zero bytes: the most compressible content there is. +func zeros(n int64) io.Reader { + return io.LimitReader(zeroSource{}, n) +} + +type zeroSource struct{} + +func (zeroSource) Read(p []byte) (int, error) { + clear(p) + return len(p), nil +} + +func TestExtract_decompressionBombIsBoundedByTheLimits(t *testing.T) { + // 64 MiB of zeros compresses to a few hundred KiB. The declared size is never trusted: bytes are + // charged as they land, so the limits hold whatever the archive claims + const inflated = 64 << 20 + limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: 4 << 20} + + var zipBomb bytes.Buffer + zw := zip.NewWriter(&zipBomb) + w, err := zw.Create("bomb.bin") + require.NoError(t, err) + _, err = io.Copy(w, zeros(inflated)) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + var tarBomb bytes.Buffer + gw := gzip.NewWriter(&tarBomb) + tw := tar.NewWriter(gw) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "bomb.bin", Mode: 0o644, Size: inflated, Typeflag: tar.TypeReg})) + _, err = io.Copy(tw, zeros(inflated)) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + + for name, data := range map[string][]byte{"bomb.zip": zipBomb.Bytes(), "bomb.tar.gz": tarBomb.Bytes()} { + t.Run(name, func(t *testing.T) { + require.Less(t, len(data), 1<<20, "the fixture must be a bomb: small on disk, huge inflated") + ctx, root := scanContext(t) + limiter := NewLimiter(limits) + + r, err := Extract(ctx, bytes.NewReader(data), "", name, limiter, nil) + require.NoError(t, err) + require.NotNil(t, r) + t.Cleanup(r.Cleanup) + + assert.True(t, r.Truncated) + locations, err := r.FilesByGlob("**/*") + require.NoError(t, err) + assert.Empty(t, locations, "the entry that did not fit is not cataloged in part") + + peakMemory, peakDisk := limiter.Peak() + assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes) + assert.LessOrEqual(t, peakDisk, limits.MaxDiskBytes) + + r.Cleanup() + assert.Empty(t, filesIn(t, root)) + }) + } +} + +func TestExtract_entryCountBombIsBoundedByTheMemoryLimit(t *testing.T) { + // every entry is a node in memory whatever the disk limit says, so a tiny archive of many entries + // is bounded by the memory limit alone + limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1} + limiter := NewLimiter(limits) + + r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), "", "many.zip", limiter, nil) + require.NoError(t, err) + t.Cleanup(r.Cleanup) + + assert.True(t, r.Truncated) + assert.Less(t, len(r.files), 20_000) + assert.LessOrEqual(t, int64(len(r.byPath))*approxIndexBytesPerEntry, limits.MaxMemoryBytes+approxIndexBytesPerEntry, + "nodes held must not exceed what the memory limit paid for") + peakMemory, _ := limiter.Peak() + assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes) +} + +func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) { + // 100 entries whose 4 KiB names each imply 500 distinct directories would be 50,000 nodes; the + // implied directories are charged as nodes, so the memory limit bounds them too + files := map[string]string{} + for i := range 100 { + var b strings.Builder + for j := range 500 { + fmt.Fprintf(&b, "%03d-%03d/", i, j) + } + b.WriteString("f") + require.LessOrEqual(t, b.Len(), maxEntryNameBytes) + files[b.String()] = "x" + } + limits := Limits{MaxMemoryBytes: 8 << 20, MaxDiskBytes: -1} + limiter := NewLimiter(limits) + + r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), "", "deep.zip", limiter, nil) + require.NoError(t, err) + t.Cleanup(r.Cleanup) + + assert.True(t, r.Truncated) + assert.LessOrEqual(t, int64(len(r.byPath))*approxIndexBytesPerEntry, limits.MaxMemoryBytes+approxIndexBytesPerEntry) + peakMemory, _ := limiter.Peak() + assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes) +} diff --git a/internal/archive/glob.go b/internal/archive/glob.go new file mode 100644 index 000000000..ea9d7e410 --- /dev/null +++ b/internal/archive/glob.go @@ -0,0 +1,390 @@ +package archive + +import ( + "path" + "strings" + "sync" + "sync/atomic" + + syftindex "github.com/anchore/syft/internal/index" +) + +// A glob is parsed once into segments, cached by pattern since catalogers ask the same few hundred +// globs of every archive. A brace group inside a segment, `g{o,o.exe}`, makes that segment a set of +// alternatives, each looked up on its own; only a group spanning `/` expands the whole pattern. +// The literal text at either end of every alternative is looked up in the base-name indexes: files +// for the last segment, directories for the rest. The smallest set found is the candidate set, and +// each candidate is verified segment by segment up its parent chain. Only a pattern with no literal +// text anywhere falls back to every file. + +// globSegment is one `/`-delimited piece of a pattern. +type globSegment struct { + alts []globAlt // the texts a name may match; one unless the segment has a brace group + any bool // `**`: zero or more directories + + // indexed reports every alternative has literal text to look up; exact that every lookup is the + // whole answer for its alternative: a plain name, `prefix*` or `*suffix` + indexed, exact bool +} + +// globAlt is one alternative text of a segment. +type globAlt struct { + text string // ready for path.Match + + // prefix and suffix are the literal text before the first and after the last metacharacter, the + // necessary conditions a name index can check. For a plain text prefix is the whole of it. + prefix, suffix string + plain bool +} + +// maxCachedGlobs bounds the parse cache, which is sized for the catalogers' own patterns. +const maxCachedGlobs = 4096 + +var ( + parsedGlobs sync.Map // pattern -> [][]globSegment + parsedGlobCount atomic.Int64 +) + +// parseGlob returns the segment lists a pattern spells out: one, unless a brace group spans `/`. +// Leading slashes are dropped since paths are archive-relative; an empty pattern has none. +func parseGlob(pattern string) [][]globSegment { + if cached, ok := parsedGlobs.Load(pattern); ok { + return cached.([][]globSegment) + } + var out [][]globSegment + if braceSpansSeparator(pattern) { + for _, alternative := range expandBraces(pattern) { + out = append(out, parseSegments(alternative)) + } + } else if segments := parseSegments(pattern); segments != nil { + out = [][]globSegment{segments} + } + if parsedGlobCount.Add(1) <= maxCachedGlobs { + parsedGlobs.Store(pattern, out) + } + return out +} + +func parseSegments(pattern string) []globSegment { + pattern = strings.TrimLeft(pattern, "/") + if pattern == "" { + return nil + } + var segments []globSegment + for _, text := range strings.Split(pattern, "/") { + s := globSegment{any: text == "**", indexed: true, exact: true} + if !s.any { + texts := []string{text} + if strings.IndexByte(text, '{') >= 0 { + texts = expandBraces(text) + } + for _, t := range texts { + a := globAlt{text: strings.ReplaceAll(t, "[!", "[^")} // doublestar negation, for path.Match + a.prefix, a.suffix, a.plain = literalEnds(a.text) + s.indexed = s.indexed && (a.prefix != "" || a.suffix != "") + s.exact = s.exact && (a.plain || a.text == a.prefix+"*" || a.text == "*"+a.suffix) + s.alts = append(s.alts, a) + } + } + segments = append(segments, s) + } + return segments +} + +// braceSpansSeparator reports whether a `/` sits inside a brace group, so the group's alternatives +// have different segment counts. +func braceSpansSeparator(pattern string) bool { + depth := 0 + for i := 0; i < len(pattern); i++ { + switch pattern[i] { + case '\\': + i++ + case '{': + depth++ + case '}': + depth = max(depth-1, 0) + case '/': + if depth > 0 { + return true + } + } + } + return false +} + +// literalEnds returns the literal text before the first metacharacter and after the last one, and +// whether there were none. A character class is one metacharacter however long, as is an escape pair. +func literalEnds(seg string) (prefix, suffix string, plain bool) { + first, lastEnd := -1, -1 + mark := func(start, end int) { + if first < 0 { + first = start + } + lastEnd = end + } + for i := 0; i < len(seg); { + switch seg[i] { + case '*', '?': + mark(i, i+1) + i++ + case '\\': + mark(i, min(i+2, len(seg))) + i += 2 + case '[': + // a `]` right after `[` or `[^` is a member of the class, not its end + j := i + 1 + if j < len(seg) && seg[j] == '^' { + j++ + } + if j < len(seg) && seg[j] == ']' { + j++ + } + end := strings.IndexByte(seg[j:], ']') + if end < 0 { + mark(i, len(seg)) + i = len(seg) + } else { + mark(i, j+end+1) + i = j + end + 1 + } + default: + i++ + } + } + if first < 0 { + return seg, "", true + } + return seg[:first], seg[lastEnd:], false +} + +// expandBraces returns every pattern a brace group spells out, nested groups included. A pattern +// without a complete group is returned as is. +func expandBraces(pattern string) []string { + start, depth := -1, 0 + for i := 0; i < len(pattern); i++ { + switch pattern[i] { + case '\\': + i++ + case '{': + if depth == 0 { + start = i + } + depth++ + case '}': + if depth == 0 { + continue + } + depth-- + if depth == 0 { + var out []string + for _, alt := range splitAlternatives(pattern[start+1 : i]) { + out = append(out, expandBraces(pattern[:start]+alt+pattern[i+1:])...) + } + return out + } + } + } + return []string{pattern} +} + +// splitAlternatives splits a brace group's body on the commas at its top level. +func splitAlternatives(body string) []string { + var out []string + depth, last := 0, 0 + for i := 0; i < len(body); i++ { + switch body[i] { + case '\\': + i++ + case '{': + depth++ + case '}': + depth-- + case ',': + if depth == 0 { + out = append(out, body[last:i]) + last = i + 1 + } + } + } + return append(out, body[last:]) +} + +// lookup finds the nodes a segment admits in a base-name index: for each alternative the smaller of +// what its prefix and suffix find, or the name itself when it is plain. It reports false for a +// segment with an alternative that has no literal text, which admits every name. +func lookup(idx *syftindex.PrefixSuffix[[]*node], s globSegment) ([]*node, bool) { + if !s.indexed { + return nil, false + } + var out []*node + for _, a := range s.alts { + switch { + case a.plain: + out = append(out, idx.Get(a.text)...) + case a.suffix == "": + out = append(out, flatten(idx.ByPrefix(a.prefix))...) + case a.prefix == "": + out = append(out, flatten(idx.BySuffix(a.suffix))...) + default: + byPrefix, bySuffix := flatten(idx.ByPrefix(a.prefix)), flatten(idx.BySuffix(a.suffix)) + if len(bySuffix) < len(byPrefix) { + byPrefix = bySuffix + } + out = append(out, byPrefix...) + } + } + return out, true +} + +func flatten(groups [][]*node) []*node { + var out []*node + for _, group := range groups { + out = append(out, group...) + } + return out +} + +// exactByName reports whether the pattern is `**/` with a segment the file-name index answers +// completely, so its lookup needs no verification. +func exactByName(segments []globSegment) bool { + return len(segments) == 2 && segments[0].any && segments[1].exact +} + +// narrowEnough is a candidate count small enough that looking for a smaller set is not worth it. +const narrowEnough = 64 + +// globCandidates returns the files a pattern can match, and whether they are exactly the answer. +func (r *Resolver) globCandidates(segments []globSegment) (candidates []*node, exact bool) { + last := segments[len(segments)-1] + best := r.files + if !last.any { + if files, ok := lookup(&r.names, last); ok { + if exactByName(segments) || len(files) <= narrowEnough { + return files, exactByName(segments) + } + best = files + } + } + + // a directory segment narrows to what lies under the directories it names; take it only when + // that is smaller than what the file name gave, stopping the walk as soon as it is not + for _, s := range segments[:len(segments)-1] { + if s.any { + continue + } + dirs, ok := lookup(&r.dirs, s) + if !ok || len(dirs) >= len(best) { + continue + } + if under := filesUnder(dirs, len(best)-1); under != nil { + best = under + } + } + return best, false +} + +// filesUnder collects the files anywhere beneath the given directories, or nil once more than limit +// are found. +func filesUnder(dirs []*node, limit int) []*node { + var out []*node + stack := append([]*node(nil), dirs...) + for len(stack) > 0 { + n := stack[len(stack)-1] + stack = stack[:len(stack)-1] + for _, child := range n.children { + if child.isDir { + stack = append(stack, child) + continue + } + if child.header != nil { + if len(out) >= limit { + return nil + } + out = append(out, child) + } + } + } + return out +} + +// matchNode reports whether the segments match the node's path, read up its parent chain. +func matchNode(segments []globSegment, n *node) (bool, error) { + var buf [16]string + names := buf[:0] + for m := n; m.parent != nil; m = m.parent { + names = append(names, m.name) + } + for i, j := 0, len(names)-1; i < j; i, j = i+1, j-1 { + names[i], names[j] = names[j], names[i] + } + return matchSegments(segments, names) +} + +// matchSegments matches segments against path components. The segments after the last `**` sit at +// fixed positions from the end, so they are matched against the tail first, which settles the common +// `**/x` in one comparison; what remains ends in `**` and is matched greedily, backing up to the most +// recent `**` on a mismatch. +func matchSegments(segments []globSegment, names []string) (bool, error) { + lastAny := -1 + for i, s := range segments { + if s.any { + lastAny = i + } + } + tail := segments[lastAny+1:] + if len(names) < len(tail) { + return false, nil + } + names, tailNames := names[:len(names)-len(tail)], names[len(names)-len(tail):] + for i, s := range tail { + if ok, err := s.match(tailNames[i]); !ok || err != nil { + return false, err + } + } + if lastAny < 0 { + return len(names) == 0, nil + } + + segments = segments[:lastAny+1] + si, ni := 0, 0 + starSeg, starName := -1, 0 + for ni < len(names) { + if si < len(segments) { + if segments[si].any { + starSeg, starName = si, ni + si++ + continue + } + ok, err := segments[si].match(names[ni]) + if err != nil { + return false, err + } + if ok { + si++ + ni++ + continue + } + } + if starSeg < 0 { + return false, nil + } + starName++ + si, ni = starSeg+1, starName + } + for ; si < len(segments); si++ { + if !segments[si].any { + return false, nil + } + } + return true, nil +} + +// match reports whether any of the segment's alternatives matches a name. +func (s globSegment) match(name string) (bool, error) { + for _, a := range s.alts { + if ok, err := path.Match(a.text, name); ok || err != nil { + return ok, err + } + } + return false, nil +} diff --git a/internal/archive/glob_test.go b/internal/archive/glob_test.go new file mode 100644 index 000000000..e72e5479a --- /dev/null +++ b/internal/archive/glob_test.go @@ -0,0 +1,330 @@ +package archive + +import ( + "fmt" + "sort" + "strings" + "testing" + + "github.com/bmatcuk/doublestar/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var globFixture = map[string]string{ + "one/two/athing": "a", + "one/two/deep/er/zthing": "z", + "one/two/Athing": "A", + "two/athing": "not under one", + "x/one/two/bthing": "b", + "onefoo/bartwo/deep/something": "s", + "onefoo/bartwo/nothing": "n", + "one/bartwo/something": "no one* dir", + "usr/local/go/bin/go": "bin", + "usr/local/go/bin/go.exe": "bin", + "usr/lib/rustlib/libstd-0123.so": "lib", + "usr/lib/libstd-0123.a": "lib", + "var/lib/dpkg/status": "dpkg", + "var/lib/dpkg/status.d/base": "dpkg", + "var/lib/dpkg/status.d/deep/thing": "dpkg", + "opt/a.jar": "PK", + "opt/b.war": "PK", + "opt/c.ear": "PK", + "opt/notes.txt": "prose", + "LICENSE": "root file", + ".hidden/secret.jar": "dotdir", + "META-INF/maven/g/a/pom.xml": "pom", + "META-INF/maven/g/a/pom.properties": "pom", + "weird/a]b/x": "class edge", + "weird/a-b/x": "class edge", + "java/openjdk-21/release": "jvm", + "jvm/zulu-17/release": "jvm", + "go": "root binary", + "gopher.txt": "prose", + "opt/c.rar": "not a jar", + "opt/app/lib/inner.jar": "PK", + "usr/lib/libstd-short.so": "lib", + "usr/lib/libz.so": "lib", + "usr/bin/mariadb": "bin", + "usr/bin/mysql": "bin", + "usr/bin/postgres": "bin", + "var/lib/rpm/Packages": "rpm", + "usr/lib/sysimage/rpm/rpmdb.sqlite": "rpm", + "usr/lib/firefox/firefox": "bin", + "META-INF/MANIFEST.MF": "manifest", + "WEB-INF/lib/dep.jar": "PK", + ".hidden.jar": "dotfile", + "xyz": "class edge", + "xaz": "class edge", + "lib.so": "root lib", + "log.txt": "prose", + "mn.txt": "prose", +} + +// every shape the catalogers use, plus the ones that exercise narrowing and its edges +var globPatterns = []string{ + "**/one/two/**/[a-z]thing", + "**/one*/*two/**/[s]omething", + "**/one/two/*", + "**/one/two/**", + "**/two/**", + "/one/**", + "one/two/athing", + "/*", + "*", + "**/*", + "**", + "**/{go,go.exe}", + "**/*.{jar,war}", + "**/{opt,usr}/**/*.{jar,so}", + "**/var/lib/dpkg/{status,status.d/**}", + "**/{a,{b,c}}.{jar,war,ear}", + "**/g{o,o.exe}", + "**/{[a]b,cd}", + "**/*{,.exe}", + "**/{one,x/one}/two/*thing", + "**/*.[jw]ar", + "**/[!a]thing", + "**/[^a]thing", + "**/libstd-????.so", + "**/libstd-*", + "**/*thing", + "**/pom.*", + "**/maven/*/*/pom.xml", + "**/{java,jvm}/*/release", + "**/*-17/release", + "**/a]b/x", + "**/.hidden/*.jar", + "**/*.jar", + "**/status", + "**/status.d/**", + "**/deep/**/*thing", + "**/nothing/**", + "nope/**", + "", + "/", + "**/one//two/athing", + "**/a\\]b/x", + "**/pom*", + "/opt/*", + "/META-INF/*", + "**/*.?ar", + "**/*o*", + "**/?o", + "**/{mariadb,mysql}", + "**/[lm]*", + "**/pom.{xml,properties}", + "**/{lib,libz}.so", + "**/[a-c]*", + "**/libstd-[s0]*.so", + "**/x[!y]z", + "**/{go,*}", + "**/{lib,l}*", + "**/{lo,m}*", + "**/{*.jar,pom.xml}", + "/{META-INF,WEB-INF}/**/*.xml", + "/usr/{bin,lib}/*", + "/opt/*/lib/*.jar", + "/us?/lib/libstd-*.so", + "**/lib/**/*.jar", + // the name looked up also exists outside the selected directory + "/opt/**/*.jar", + "/opt/**/lib/*.jar", + "/usr/**/go", + "/{opt,WEB-INF}/**/*.jar", + "/META-INF/**/pom.*", + "/usr/**/lib*.so", + // alternations whose branches carry separators, as syft's catalogers write them + "**/var/lib/dpkg/{status,status.d/*}", + "**/{var/lib,usr/share,usr/lib/sysimage}/rpm/{Packages,Packages.db,rpmdb.sqlite}", + "**/{firefox,firefox.exe}", + "**/**", + "/**/*.so", +} + +// every pattern shape is checked against matching every path the slow way, since a name-index +// shortcut that is too eager fails silently by returning fewer files +func TestGlob_agreesWithDoublestar(t *testing.T) { + r := resolverOver(t, 1<<20, globFixture) + var all []string + for _, n := range r.files { + all = append(all, reportedPath(n.path)) + } + + for _, pattern := range globPatterns { + t.Run(pattern, func(t *testing.T) { + // paths are archive-relative, so a leading slash anchors at the archive root + var want []string + for _, p := range all { + ok, err := doublestar.Match(strings.TrimLeft(pattern, "/"), p) + require.NoError(t, err) + if ok { + want = append(want, p) + } + } + sort.Strings(want) + + locations, err := r.FilesByGlob(pattern) + require.NoError(t, err) + assert.Equal(t, want, realPaths(locations)) + }) + } +} + +func TestGlob_narrowsByDirectory(t *testing.T) { + // enough files share the last segment's suffix that the directories in the pattern narrow better + entries := map[string]string{} + for k, v := range globFixture { + entries[k] = v + } + for i := range 100 { + entries[fmt.Sprintf("noise/f%03dthing", i)] = "x" + entries[fmt.Sprintf("noise/d%03d/something", i)] = "x" + } + r := resolverOver(t, 1<<20, entries) + + tests := []struct { + pattern string + want int // candidates considered + }{ + {"**/one/two/**/[a-z]thing", 5}, // everything under a directory named `one` + {"**/one*/*two/**/[s]omething", 7}, + {"**/dpkg/**", 3}, + {"**/[a-z]thing", 209}, // the file-name suffix alone + {"**/[lm]*", len(r.files)}, + {"**/*", len(r.files)}, + {"**/g{o,o.exe}", 3}, + {"**/{d00*,f00*}/**", 10}, // ten directories d000-d009 narrow; f00* names no directory + {"**/{*,cd}", len(r.files)}, + } + for _, tt := range tests { + t.Run(tt.pattern, func(t *testing.T) { + candidates, _ := r.globCandidates(parseGlob(tt.pattern)[0]) + assert.Len(t, candidates, tt.want) + }) + } +} + +func TestGlob_exactLookupsSkipVerification(t *testing.T) { + r := resolverOver(t, 1<<20, globFixture) + for pattern, exact := range map[string]bool{ + "**/pom.xml": true, + "**/*.jar": true, + "**/libstd-*": true, + "**/pom.*": true, + "**/p*m.xml": false, // narrowed by an end, but the middle still has to be checked + "**/g{o,o.exe}": true, + "**/*.{jar,war}": true, + "**/{[a]b,cd}": false, + "**/{*,cd}": false, + "**/*thing": true, + "**/[a-z]thing": false, + "/opt/*.jar": false, + "**/maven/pom.xml": false, + } { + _, got := r.globCandidates(parseGlob(pattern)[0]) + assert.Equal(t, exact, got, pattern) + } +} + +func Test_expandBraces(t *testing.T) { + tests := []struct { + pattern string + want []string + }{ + {"**/*.jar", []string{"**/*.jar"}}, + {"**/{go,go.exe}", []string{"**/go", "**/go.exe"}}, + {"**/{a,{b,c}}.{jar,war}", []string{"**/a.jar", "**/a.war", "**/b.jar", "**/b.war", "**/c.jar", "**/c.war"}}, + {"**/dpkg/{status,status.d/**}", []string{"**/dpkg/status", "**/dpkg/status.d/**"}}, + {"**/{lone", []string{"**/{lone"}}, + {"**/lone}", []string{"**/lone}"}}, + {"**/\\{not,a,group\\}", []string{"**/\\{not,a,group\\}"}}, + {"{,a}", []string{"", "a"}}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, expandBraces(tt.pattern), tt.pattern) + } +} + +func Test_literalEnds(t *testing.T) { + tests := []struct { + seg string + prefix, suffix string + plain bool + }{ + {"pom.xml", "pom.xml", "", true}, + {"*.jar", "", ".jar", false}, + {"libstd-*", "libstd-", "", false}, + {"[a-z]thing", "", "thing", false}, + {"libstd-????.so", "libstd-", ".so", false}, + {"a*b*c", "a", "c", false}, + {"*", "", "", false}, + {"a]b", "a]b", "", true}, + {"a[]-]b", "a", "b", false}, + {"a[^]]b", "a", "b", false}, + {"a\\*b", "a", "b", false}, + {"a\\", "a", "", false}, + {"a[b", "a", "", false}, + } + for _, tt := range tests { + prefix, suffix, plain := literalEnds(tt.seg) + assert.Equal(t, []any{tt.prefix, tt.suffix, tt.plain}, []any{prefix, suffix, plain}, tt.seg) + } +} + +// every pattern but the last two is one syft's own catalogers register +func BenchmarkResolver_FilesByGlob(b *testing.B) { + r := benchArchive(b) + + patterns := map[string]string{ + "exact-name": "**/go", + "extension": "**/*.jar", + "alternation": "**/{go,go.exe}", + "question-marks": "**/libstd-????????????????.so", + "alternation-suffix": "**/*.{jar,war}", + "alternation-with-sep": "**/var/lib/dpkg/{status,status.d/**}", + "alternation-both-ends": "**/{var/lib,usr/share,usr/lib/sysimage}/rpm/" + + "{Packages,Packages.db,rpmdb.sqlite}", + "alternation-mid-segment": "**/{java,jvm}/*/release", + "alternation-mid-path": "/{usr,opt}/**/*.jar", + "class-in-extension": "**/*.[jw]ar", + "many-matches": "**/[lm]*", + } + + for name, pattern := range patterns { + b.Run(name, func(b *testing.B) { + for b.Loop() { + if _, err := r.FilesByGlob(pattern); err != nil { + b.Fatal(err) + } + } + }) + } +} + +// benchArchive is shaped like a fat application image layer: thousands of entries with distinct base +// names, plus decoys sharing the prefixes and suffixes the patterns fix, so no lookup looks free. +func benchArchive(tb testing.TB) *Resolver { + tb.Helper() + entries := map[string]string{ + "usr/local/go/bin/go": "binary", + "usr/lib/rustlib/libstd-0123456789abcdef.so": "binary", + "usr/bin/mariadb": "binary", + "opt/app/lib/app.jar": "PK\x03\x04", + "opt/app/lib/dep.war": "PK\x03\x04", + "var/lib/dpkg/status": "dpkg", + "var/lib/dpkg/status.d/base": "dpkg", + "var/lib/rpm/Packages": "rpm", + "usr/lib/jvm/java-21/release": "jvm", + } + for i := range 1500 { + entries[fmt.Sprintf("var/cache/archive%04d.tar", i)] = "data" + entries[fmt.Sprintf("var/cache/bundle%04d.rar", i)] = "data" + entries[fmt.Sprintf("usr/share/cal%04dendar", i)] = "data" + entries[fmt.Sprintf("usr/share/gopher%04d.md", i)] = "docs" + entries[fmt.Sprintf("usr/lib/libstd-%04d.a", i)] = "archive" + entries[fmt.Sprintf("usr/lib/mod%04d.py", i)] = "code" + entries[fmt.Sprintf("usr/share/doc/pkg%04d/NOTES%04d.md", i, i)] = "docs" + } + return resolverOver(tb, 1<<26, entries) +} diff --git a/internal/archive/helpers_test.go b/internal/archive/helpers_test.go index c64f00210..9b66e9be1 100644 --- a/internal/archive/helpers_test.go +++ b/internal/archive/helpers_test.go @@ -6,61 +6,66 @@ import ( "bytes" "compress/gzip" "context" + "fmt" "io" "io/fs" "os" - "path/filepath" + "path" "sort" - "strings" "testing" "github.com/stretchr/testify/require" "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/syft/file" ) -// WorkDirAt returns a WorkDir over a directory that already exists, so a test can inspect it -// afterwards; Remove leaves it alone. -func WorkDirAt(dir string) *WorkDir { - return &WorkDir{created: true, path: dir} -} - -func (w *WorkDir) wasCreated() bool { - return w != nil && w.remove != nil -} - -// heldInMemory reports the entry content the store is still holding in memory. -func (s *EntryStore) heldInMemory() int64 { - var total int64 - for _, entry := range s.entries { - total += int64(len(entry.mem)) - } - return total -} - // held reports what this charge is currently holding, in memory and on disk. -func (c *Charge) held() (memory, disk int64) { +func (c *charge) held() (memory, disk int64) { if c == nil { return 0, 0 } c.limiter.mu.Lock() defer c.limiter.mu.Unlock() - return c.memory, c.disk + return c.inMemory, c.onDisk +} + +// heldInMemory reports the content the resolver is still holding in memory. +func (r *Resolver) heldInMemory() int64 { + var total int64 + for _, b := range r.held { + total += int64(len(b.mem)) + } + return total } // unseekable hides a reader's Seek and ReadAt, forcing content through the held path. type unseekable struct{ io.Reader } -// testEntry is one entry of a fixture archive: a directory when dir is set, a symlink when link is -// set, otherwise a regular file. +// testEntry is one entry of a fixture archive: a directory when dir is set, a link when link is set +// (hard when hard is set, otherwise symbolic), otherwise a regular file. type testEntry struct { name string body string link string + hard bool dir bool mode fs.FileMode } +// header is the tar header this entry would carry. +func (e testEntry) header() tar.Header { + switch { + case e.dir: + return tar.Header{Name: e.name, Mode: int64(e.modeOr(0o755)), Typeflag: tar.TypeDir} + case e.link != "" && e.hard: + return tar.Header{Name: e.name, Mode: 0o644, Typeflag: tar.TypeLink, Linkname: e.link} + case e.link != "": + return tar.Header{Name: e.name, Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: e.link} + } + return tar.Header{Name: e.name, Mode: int64(e.modeOr(0o644)), Size: int64(len(e.body)), Typeflag: tar.TypeReg} +} + func fileEntries(files map[string]string) []testEntry { names := make([]string, 0, len(files)) for name := range files { @@ -114,14 +119,8 @@ func tarGzFromEntries(t testing.TB, entries []testEntry) []byte { gw := gzip.NewWriter(&buf) tw := tar.NewWriter(gw) for _, e := range entries { - hdr := &tar.Header{Name: e.name, Mode: int64(e.modeOr(0o644)), Size: int64(len(e.body))} - switch { - case e.dir: - hdr.Typeflag, hdr.Mode, hdr.Size = tar.TypeDir, int64(e.modeOr(0o755)), 0 - case e.link != "": - hdr.Typeflag, hdr.Linkname, hdr.Mode, hdr.Size = tar.TypeSymlink, e.link, 0o777, 0 - } - require.NoError(t, tw.WriteHeader(hdr)) + hdr := e.header() + require.NoError(t, tw.WriteHeader(&hdr)) _, err := tw.Write([]byte(e.body)) require.NoError(t, err) } @@ -141,71 +140,11 @@ func regularHeader(name string, size int64) tar.Header { return tar.Header{Name: name, Size: size, Mode: 0o600, Typeflag: tar.TypeReg} } -// storeFor makes a store in the test's own temp space with the given charge. -func storeFor(t testing.TB, charge *Charge) *EntryStore { +// scanContext returns a context whose temp root is a fresh directory, and that directory. +func scanContext(t testing.TB) (context.Context, string) { t.Helper() - s := NewEntryStore("test-archive", WorkDirAt(t.TempDir()), charge) - t.Cleanup(func() { _ = s.Close() }) - return s -} - -// storeDir is the directory a store spills into. -func storeDir(t testing.TB, s *EntryStore) string { - t.Helper() - dir, err := s.workDir.Path() - require.NoError(t, err) - return dir -} - -func readEntry(t testing.TB, r ReaderAtSeeker) string { - t.Helper() - _, err := r.Seek(0, io.SeekStart) - require.NoError(t, err) - b, err := io.ReadAll(r) - require.NoError(t, err) - return string(b) -} - -// storedEntry is one entry read back out of a store. -type storedEntry struct { - body string - linkTarget string - mode fs.FileMode - typeflag byte -} - -func readStore(t testing.TB, s *EntryStore) map[string]storedEntry { - t.Helper() - out := map[string]storedEntry{} - for _, entry := range s.Entries() { - out[entry.Header.Name] = storedEntry{ - body: readEntry(t, s.Open(entry)), - linkTarget: entry.Header.Linkname, - mode: entry.Header.FileInfo().Mode(), - typeflag: entry.Header.Typeflag, - } - } - return out -} - -func storedNames(t testing.TB, s *EntryStore) []string { - t.Helper() - var names []string - for _, entry := range s.Entries() { - names = append(names, entry.Header.Name) - } - sort.Strings(names) - return names -} - -// memCharge is a charge against a memory bound with disk unbounded. -func memCharge(maxMemory int64) *Charge { - return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).Charge() -} - -// diskCharge is a charge against a disk bound with memory refusing everything. -func diskCharge(maxDisk int64) *Charge { - return NewLimiter(Limits{MaxDiskBytes: maxDisk}).Charge() + root := t.TempDir() + return tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)), root } // filesIn lists the regular files directly in dir. @@ -222,35 +161,186 @@ func filesIn(t testing.TB, dir string) []string { return names } -// scanContext returns a context whose temp root is a fresh directory, and that directory. -func scanContext(t testing.TB) (context.Context, string) { +// spillFile is the one file a resolver has written, or "" when it has not. +func spillFile(t testing.TB, dir string) string { t.Helper() - root := t.TempDir() - return tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)), root + files := filesIn(t, dir) + require.LessOrEqual(t, len(files), 1) + if len(files) == 0 { + return "" + } + return files[0] } -// workDirsUnder returns the archive work directories directly under a scan temp root. -func workDirsUnder(t testing.TB, root string) []string { +// resolverIn makes an empty resolver over the given charge that spills into dir, along with that +// directory. Its entries are added by the test. +func resolverIn(t testing.TB, charge *charge) (*Resolver, string) { t.Helper() - children, err := os.ReadDir(root) + root := t.TempDir() + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)) + r := newResolver(ctx, "", "test-archive", charge) + t.Cleanup(r.Cleanup) + return r, root +} + +// addAll adds entries in the order given and finishes the resolver. +func addAll(t testing.TB, r *Resolver, entries []testEntry) { + t.Helper() + for _, e := range entries { + require.NoError(t, r.add(e.header(), bytes.NewReader([]byte(e.body)))) + } + r.finish() +} + +// resolverOver builds a resolver holding the given files within a memory bound. +func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Resolver { + t.Helper() + r, _ := resolverIn(t, memCharge(maxInMemory)) + r.archivePath = "outer.jar" + addAll(t, r, fileEntries(files)) + return r +} + +// resolverFrom builds a resolver holding the given entries, links and directories included, with +// ample memory. +func resolverFrom(t testing.TB, entries ...testEntry) *Resolver { + t.Helper() + r, _ := resolverIn(t, memCharge(1<<20)) + r.archivePath = "outer.jar" + addAll(t, r, entries) + return r +} + +// indexCostOf is what add charges for the given entries in name order: every listed node and every +// directory their paths imply. +func indexCostOf(entries map[string]string) int64 { + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + seen := map[string]bool{} + var total int64 + for _, name := range names { + p := path.Clean("/" + name) + for i := 1; i < len(p); i++ { + if p[i] == '/' && !seen[p[:i]] { + seen[p[:i]] = true + total += approxIndexBytesPerEntry + } + } + seen[p] = true + total += approxIndexBytes(tar.Header{Name: name}) + } + return total +} + +// extractBytes extracts an archive into a resolver built over the given charge. +func extractBytes(t testing.TB, data []byte, name string, charge *charge) (*Resolver, string) { + t.Helper() + r, root := resolverIn(t, charge) + content := bytes.NewReader(data) + format := identifyFormat(context.Background(), name, content) + require.NotNil(t, format, "fixture must identify as an archive") + require.NoError(t, r.extract(context.Background(), format, content, nil)) + return r, root +} + +func readEntry(t testing.TB, r ReaderAtSeeker) string { + t.Helper() + _, err := r.Seek(0, io.SeekStart) require.NoError(t, err) - var out []string - for _, child := range children { - if child.IsDir() && strings.HasPrefix(child.Name(), workDirName) { - out = append(out, filepath.Join(root, child.Name())) + b, err := io.ReadAll(r) + require.NoError(t, err) + return string(b) +} + +// storedEntry is one entry read back out of a resolver. +type storedEntry struct { + body string + linkTarget string + mode fs.FileMode + typeflag byte +} + +// readStore reads every listed entry, files and directories alike, by name. +func readStore(t testing.TB, r *Resolver) map[string]storedEntry { + t.Helper() + out := map[string]storedEntry{} + for _, n := range r.byPath { + if n.header == nil { + continue + } + out[n.header.Name] = storedEntry{ + body: readEntry(t, r.open(&n.content)), + linkTarget: n.header.Linkname, + mode: n.header.FileInfo().Mode(), + typeflag: n.header.Typeflag, } } return out } -// extractBytes extracts an archive into a store built over the given charge. -func extractBytes(t testing.TB, data []byte, name string, charge *Charge) (*EntryStore, bool) { +// storedNames lists every listed entry's name, files and directories alike. +func storedNames(t testing.TB, r *Resolver) []string { t.Helper() - store := storeFor(t, charge) - content := bytes.NewReader(data) - format := identifyFormat(context.Background(), name, content) - require.NotNil(t, format, "fixture must identify as an archive") - truncated, err := extractInto(context.Background(), format, content, store, nil) - require.NoError(t, err) - return store, truncated + var names []string + for _, n := range r.byPath { + if n.header != nil { + names = append(names, n.header.Name) + } + } + sort.Strings(names) + return names } + +// memCharge is a charge against a memory bound with disk unbounded. +func memCharge(maxMemory int64) *charge { + return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).charge() +} + +// diskCharge is a charge against a disk bound with memory refusing everything. +func diskCharge(maxDisk int64) *charge { + return NewLimiter(Limits{MaxDiskBytes: maxDisk}).charge() +} + +// realPaths lists the real path of every location, sorted. +func realPaths(locations []file.Location) []string { + var out []string + for _, loc := range locations { + out = append(out, loc.RealPath) + } + sort.Strings(out) + return out +} + +// manyEntryZip is a zip of count one-byte files. +func manyEntryZip(t testing.TB, count int) []byte { + t.Helper() + files := make(map[string]string, count) + for i := range count { + files[fmt.Sprintf("e%06d.txt", i)] = "x" + } + return zipBytes(t, files) +} + +// extractedResolver extracts one archive through Extract and returns the resolver, along with the scan +// temp root its spill file would appear under. +func extractedResolver(t testing.TB, data []byte, limits Limits) (*Resolver, string) { + t.Helper() + ctx, root := scanContext(t) + + r, err := Extract(ctx, bytes.NewReader(data), "", "app.zip", NewLimiter(limits), nil) + require.NoError(t, err) + require.NotNil(t, r) + t.Cleanup(r.Cleanup) + + return r, root +} + +var ( + unboundedLimits = Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1} + + // spillingLimits hold nothing in memory, so every entry lands in the spill file + spillingLimits = Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1} +) diff --git a/internal/archive/index.go b/internal/archive/index.go deleted file mode 100644 index 8567d8585..000000000 --- a/internal/archive/index.go +++ /dev/null @@ -1,350 +0,0 @@ -package archive - -import ( - "archive/tar" - "context" - "fmt" - "io" - "path" - "sort" - "strings" - - "github.com/bmatcuk/doublestar/v4" - - stereoscopeFile "github.com/anchore/stereoscope/pkg/file" - syftindex "github.com/anchore/syft/internal/index" - "github.com/anchore/syft/syft/file" -) - -// Index is a file.Resolver over one extracted archive's entries. -// -// Paths are archive-relative. Nearly every cataloger glob is `**/` or `**/*`, so files -// are indexed by base name and a pattern whose last segment fixes a name, prefix or suffix is -// narrowed to those files before doublestar decides; any other pattern is matched against every file. -type Index struct { - fileSystemID string - archivePath string - store *EntryStore - - // byPath holds every file and directory by rooted path, including directories the archive never - // listed but that entry paths imply - byPath map[string]*node - - // files holds every file the archive listed, in path order - files []*node - - // names indexes files by base name - names syftindex.PrefixSuffix[[]*node] -} - -type node struct { - path string // rooted, e.g. "/META-INF/MANIFEST.MF" - isDir bool - entry *Entry // nil for a directory the archive never listed - metadata file.Metadata - - // target is the node whose entry holds this node's content: itself for a regular file, the - // resolved node for a link, and nil for a link that leads nowhere inside this archive - target *node -} - -var _ file.Resolver = (*Index)(nil) - -// NewIndex builds a resolver over the store's entries whose locations carry the given fileSystemID -// and archivePath. -func NewIndex(store *EntryStore, fileSystemID, archivePath string) *Index { - r := &Index{ - fileSystemID: fileSystemID, - archivePath: archivePath, - store: store, - byPath: map[string]*node{"/": {path: "/", isDir: true}}, - } - for _, entry := range store.Entries() { - r.add(entry) - } - sort.Slice(r.files, func(i, j int) bool { return r.files[i].path < r.files[j].path }) - for _, n := range r.byPath { - n.target = r.linkTarget(n) - } - return r -} - -// add places an entry at its path, creating the directories the path implies. The first entry at a -// path wins. -func (r *Index) add(entry *Entry) { - entryPath := path.Clean("/" + entry.Header.Name) - if entryPath == "/" { - return - } - for i := 1; i < len(entryPath); i++ { - if entryPath[i] == '/' { - r.nodeAt(entryPath[:i], true) - } - } - - n := r.nodeAt(entryPath, entry.Header.FileInfo().IsDir()) - if n.entry != nil { - return - } - n.entry = entry - n.metadata = r.metadataOf(entry, entryPath) - if n.isDir { - return - } - r.files = append(r.files, n) - r.names.Update(path.Base(entryPath), func(current *syftindex.Node[[]*node]) []*node { - return append(current.Value(), n) - }) -} - -func (r *Index) nodeAt(p string, isDir bool) *node { - if n, ok := r.byPath[p]; ok { - return n - } - n := &node{path: p, isDir: isDir} - r.byPath[p] = n - return n -} - -// maxLinkHops bounds a chain of links pointing at links, which is also what ends a cycle. -const maxLinkHops = 8 - -func (r *Index) linkTarget(n *node) *node { - for hops := 0; hops <= maxLinkHops; hops++ { - if n.entry == nil || n.isDir { - return nil - } - if n.entry.Header.Linkname == "" { - return n - } - next, ok := r.byPath[linkDestination(n.path, n.entry.Header)] - if !ok { - return nil - } - n = next - } - return nil -} - -// linkDestination is the rooted path a link points at inside the archive. A hard link names its -// target from the archive root, as does an absolute symlink, so "/etc/passwd" is an entry of this -// archive and never the host's. A relative symlink is joined onto the link's own directory, and -// cleaning against the root keeps "../../x" inside the archive. -func linkDestination(linkPath string, hdr tar.Header) string { - if hdr.Typeflag == tar.TypeLink || path.IsAbs(hdr.Linkname) { - return path.Join("/", hdr.Linkname) - } - return path.Join(path.Dir(linkPath), hdr.Linkname) -} - -func (r *Index) metadataOf(entry *Entry, entryPath string) file.Metadata { - metadata := stereoscopeFile.NewMetadata(entry.Header, r.store.Open(entry)) - // NewMetadata copies the header and points FileInfo into the copy; share the one the store holds - metadata.FileInfo = entry.Header.FileInfo() - metadata.Path = reportedPath(entryPath) - if entry.Header.Linkname != "" { - metadata.LinkDestination = reportedPath(linkDestination(entryPath, entry.Header)) - } - return metadata -} - -// reportedPath is how paths leave the resolver: archive-relative with no leading slash. -func reportedPath(rooted string) string { - return strings.TrimPrefix(rooted, "/") -} - -func rootedPath(userPath string) string { - return path.Clean("/" + userPath) -} - -func (r *Index) coordinates(n *node) file.Coordinates { - return file.Coordinates{ - RealPath: reportedPath(n.path), - FileSystemID: r.fileSystemID, - ArchivePath: r.archivePath, - } -} - -// locationOf names a node by its own path. -func (r *Index) locationOf(n *node) file.Location { - return file.NewLocationFromCoordinates(r.coordinates(n)) -} - -// resolvedLocation names the file behind a node: a link is reported at its target's path with the -// link's own path as the access path, and a link leading nowhere is reported as itself. -func (r *Index) resolvedLocation(n *node) file.Location { - if n.target == nil || n.target == n { - return r.locationOf(n) - } - return file.NewVirtualLocationFromCoordinates(r.coordinates(n.target), reportedPath(n.path)) -} - -func (r *Index) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { - n, ok := r.byPath[rootedPath(location.RealPath)] - if !ok || n.entry == nil { - return nil, fmt.Errorf("no archive entry for path=%q", location.RealPath) - } - return entryReader{r.store.Open(n.entry)}, nil -} - -func (r *Index) FileMetadataByLocation(location file.Location) (file.Metadata, error) { - n, ok := r.byPath[rootedPath(location.RealPath)] - if !ok { - return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.RealPath) - } - return n.metadata, nil -} - -func (r *Index) HasPath(userPath string) bool { - _, ok := r.byPath[rootedPath(userPath)] - return ok -} - -func (r *Index) FilesByPath(paths ...string) ([]file.Location, error) { - var out []file.Location - seen := map[*node]struct{}{} - for _, userPath := range paths { - n, ok := r.byPath[rootedPath(userPath)] - if !ok || n.isDir || n.entry == nil { - continue - } - if _, dup := seen[n]; dup { - continue - } - seen[n] = struct{}{} - out = append(out, r.resolvedLocation(n)) - } - return out, nil -} - -func (r *Index) FilesByGlob(patterns ...string) ([]file.Location, error) { - matched := map[*node]struct{}{} - for _, pattern := range patterns { - pattern = strings.TrimLeft(pattern, "/") - if pattern == "" { - continue - } - for _, n := range r.globCandidates(pattern) { - ok, err := doublestar.Match(pattern, reportedPath(n.path)) - if err != nil { - return nil, fmt.Errorf("invalid glob %q: %w", pattern, err) - } - if ok { - matched[n] = struct{}{} - } - } - } - - var out []file.Location - for _, n := range onePathPerFile(matched) { - out = append(out, r.resolvedLocation(n)) - } - sort.Slice(out, func(i, j int) bool { - if out[i].RealPath != out[j].RealPath { - return out[i].RealPath < out[j].RealPath - } - return out[i].AccessPath < out[j].AccessPath - }) - return out, nil -} - -// globMetacharacters are the glob metacharacters other than `*` itself. -const globMetacharacters = `?[]{}\` - -// globCandidates narrows the files a pattern can match by its last segment: an exact name, `*suffix` -// or `prefix*` is looked up in the name index. Any other shape has every file as a candidate. -func (r *Index) globCandidates(pattern string) []*node { - last := pattern[strings.LastIndexByte(pattern, '/')+1:] - if strings.ContainsAny(last, globMetacharacters) || strings.Count(last, "*") > 1 { - return r.files - } - switch star := strings.IndexByte(last, '*'); { - case star < 0: - return r.names.Get(last) - case star == 0: - return flatten(r.names.BySuffix(last[1:])) - case star == len(last)-1: - return flatten(r.names.ByPrefix(last[:star])) - } - return r.files -} - -func flatten(groups [][]*node) []*node { - var out []*node - for _, group := range groups { - out = append(out, group...) - } - return out -} - -// onePathPerFile keeps one matched path per file. Several paths can name the same content, such as -// a link and its target, and a resolver answers with one: the file's own path when it matched, -// otherwise the lowest-sorting link. A link leading nowhere stands as its own file. -func onePathPerFile(matched map[*node]struct{}) []*node { - best := map[*node]*node{} - for n := range matched { - key := n.target - if key == nil { - key = n - } - current, seen := best[key] - if !seen || n == key || (current != key && n.path < current.path) { - best[key] = n - } - } - out := make([]*node, 0, len(best)) - for _, n := range best { - out = append(out, n) - } - return out -} - -func (r *Index) FilesByMIMEType(types ...string) ([]file.Location, error) { - wanted := map[string]struct{}{} - for _, t := range types { - wanted[t] = struct{}{} - } - var out []file.Location - for _, n := range r.files { - // only a file holding bytes has a type of its own; a link's target answers for itself - if n.target != n { - continue - } - if _, ok := wanted[n.metadata.MIMEType]; ok { - out = append(out, r.locationOf(n)) - } - } - return out, nil -} - -// RelativeFileByPath resolves against the archive root, the only frame an archive has. -func (r *Index) RelativeFileByPath(_ file.Location, userPath string) *file.Location { - locations, _ := r.FilesByPath(userPath) - if len(locations) == 0 { - return nil - } - return &locations[0] -} - -func (r *Index) AllLocations(ctx context.Context) <-chan file.Location { - out := make(chan file.Location) - go func() { - defer close(out) - for _, n := range r.files { - select { - case out <- r.locationOf(n): - case <-ctx.Done(): - return - } - } - }() - return out -} - -// entryReader gives an entry's reader the Close the resolver interface requires. The store owns the -// bytes, so there is nothing to close. -type entryReader struct { - ReaderAtSeeker -} - -func (entryReader) Close() error { return nil } diff --git a/internal/archive/index_bench_test.go b/internal/archive/index_bench_test.go deleted file mode 100644 index f4a4c6eb5..000000000 --- a/internal/archive/index_bench_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package archive - -import ( - "fmt" - "testing" -) - -// every pattern but the last two is one syft's own catalogers register -func BenchmarkIndex_FilesByGlob(b *testing.B) { - r := benchArchive(b) - - patterns := map[string]string{ - "exact-name": "**/go", - "extension": "**/*.jar", - "alternation": "**/{go,go.exe}", - "question-marks": "**/libstd-????????????????.so", - "alternation-suffix": "**/*.{jar,war}", - "alternation-with-sep": "**/var/lib/dpkg/{status,status.d/**}", - "alternation-both-ends": "**/{var/lib,usr/share,usr/lib/sysimage}/rpm/" + - "{Packages,Packages.db,rpmdb.sqlite}", - "alternation-mid-segment": "**/{java,jvm}/*/release", - "alternation-mid-path": "/{usr,opt}/**/*.jar", - "class-in-extension": "**/*.[jw]ar", - "many-matches": "**/[lm]*", - } - - for name, pattern := range patterns { - b.Run(name, func(b *testing.B) { - for b.Loop() { - if _, err := r.FilesByGlob(pattern); err != nil { - b.Fatal(err) - } - } - }) - } -} - -// benchArchive is shaped like a fat application image layer: thousands of entries with distinct base -// names, plus decoys sharing the prefixes and suffixes the patterns fix, so no lookup looks free. -func benchArchive(tb testing.TB) *Index { - tb.Helper() - entries := map[string]string{ - "usr/local/go/bin/go": "binary", - "usr/lib/rustlib/libstd-0123456789abcdef.so": "binary", - "usr/bin/mariadb": "binary", - "opt/app/lib/app.jar": "PK\x03\x04", - "opt/app/lib/dep.war": "PK\x03\x04", - "var/lib/dpkg/status": "dpkg", - "var/lib/dpkg/status.d/base": "dpkg", - "var/lib/rpm/Packages": "rpm", - "usr/lib/jvm/java-21/release": "jvm", - } - for i := range 1500 { - entries[fmt.Sprintf("var/cache/archive%04d.tar", i)] = "data" - entries[fmt.Sprintf("var/cache/bundle%04d.rar", i)] = "data" - entries[fmt.Sprintf("usr/share/cal%04dendar", i)] = "data" - entries[fmt.Sprintf("usr/share/gopher%04d.md", i)] = "docs" - entries[fmt.Sprintf("usr/lib/libstd-%04d.a", i)] = "archive" - entries[fmt.Sprintf("usr/lib/mod%04d.py", i)] = "code" - entries[fmt.Sprintf("usr/share/doc/pkg%04d/NOTES%04d.md", i, i)] = "docs" - } - return indexOver(tb, 1<<26, entries) -} diff --git a/internal/archive/index_glob_test.go b/internal/archive/index_glob_test.go deleted file mode 100644 index 1cb978a19..000000000 --- a/internal/archive/index_glob_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package archive - -import ( - "sort" - "testing" - - "github.com/bmatcuk/doublestar/v4" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// every pattern shape is checked against matching every path the slow way, since a name-index -// shortcut that is too eager fails silently by returning fewer files -func Test_FilesByGlob_agreesWithBruteForce(t *testing.T) { - paths := []string{ - "LICENSE", - "go", - "go.exe", - "gopher.txt", - "opt/a.jar", - "opt/b.war", - "opt/c.rar", - "opt/notes.txt", - "opt/app/lib/inner.jar", - "usr/local/go/bin/go", - "usr/lib/libstd-0123456789abcdef.so", - "usr/lib/libstd-short.so", - "usr/lib/libz.so", - "usr/bin/mariadb", - "usr/bin/mysql", - "usr/bin/postgres", - "var/lib/dpkg/status", - "var/lib/dpkg/status.d/base", - "var/lib/dpkg/status.d/nested/deep", - "var/lib/rpm/Packages", - "usr/lib/sysimage/rpm/rpmdb.sqlite", - "usr/lib/firefox/firefox", - "META-INF/MANIFEST.MF", - "META-INF/maven/com.example/lib/pom.xml", - "META-INF/maven/com.example/lib/pom.properties", - "WEB-INF/lib/dep.jar", - ".hidden.jar", - "xyz", - "xaz", - "lib.so", - "log.txt", - "mn.txt", - } - - entries := make(map[string]string, len(paths)) - for _, p := range paths { - entries[p] = "body of " + p - } - r := indexOver(t, 1<<20, entries) - - patterns := []string{ - "**/pom.xml", "**/*.jar", "**/pom*", "**/*", "/*", "/opt/*", - "**/{go,go.exe}", - "**/libstd-????????????????.so", - "**/*.[jw]ar", - "**/libstd-*.so", - "**/*.?ar", - "**/*o*", - "**/?o", - "**/{a,b}.{jar,war}", - "**/{mariadb,mysql}", - "**/[lm]*", - "**/*.{jar,war}", - "**/pom.{xml,properties}", - "**/{lib,libz}.so", - "**/[a-c]*", - "**/libstd-[s0]*.so", - "**/x[!y]z", - "**/{go,*}", - "**/{lib,l}*", - "**/{lo,m}*", - "**/{*.jar,pom.xml}", - "/{META-INF,WEB-INF}/**/*.xml", - "/usr/{bin,lib}/*", - "/opt/*/lib/*.jar", - "/us?/lib/libstd-*.so", - "**/lib/**/*.jar", - // the name looked up also exists outside the selected directory - "/opt/**/*.jar", - "/opt/**/lib/*.jar", - "/usr/**/go", - "/{opt,WEB-INF}/**/*.jar", - "/META-INF/**/pom.*", - "/usr/**/lib*.so", - // alternations whose branches carry separators, as syft's catalogers write them - "**/var/lib/dpkg/{status,status.d/**}", - "**/var/lib/dpkg/{status,status.d/*}", - "**/{var/lib,usr/share,usr/lib/sysimage}/rpm/{Packages,Packages.db,rpmdb.sqlite}", - "**/{firefox,firefox.exe}", - "**/{java,jvm}/*/release", - "**", "**/**", "/**/*.so", - } - - for _, pattern := range patterns { - t.Run(pattern, func(t *testing.T) { - locations, err := r.FilesByGlob(pattern) - require.NoError(t, err) - - var got []string - for _, loc := range locations { - got = append(got, loc.RealPath) - } - sort.Strings(got) - - assert.Equal(t, bruteForceMatches(t, pattern, paths), got) - }) - } -} - -func bruteForceMatches(t *testing.T, pattern string, paths []string) []string { - t.Helper() - trimmed := trimArchiveRoot(pattern) - var out []string - for _, p := range paths { - matched, err := doublestar.Match(trimmed, p) - require.NoError(t, err) - if matched { - out = append(out, p) - } - } - sort.Strings(out) - return out -} - -func trimArchiveRoot(pattern string) string { - for len(pattern) > 0 && pattern[0] == '/' { - pattern = pattern[1:] - } - return pattern -} diff --git a/internal/archive/index_links_test.go b/internal/archive/index_links_test.go deleted file mode 100644 index c7af26e84..000000000 --- a/internal/archive/index_links_test.go +++ /dev/null @@ -1,298 +0,0 @@ -package archive - -import ( - "archive/tar" - "bytes" - "context" - "io" - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// linkEntry is a regular file when linkname is empty, otherwise a symlink or, with hard set, a hard link. -type linkEntry struct { - name string - body string - linkname string - hard bool -} - -func indexWithLinks(t testing.TB, entries ...linkEntry) *Index { - t.Helper() - store := storeFor(t, memCharge(1<<20)) - for _, e := range entries { - hdr := regularHeader(e.name, int64(len(e.body))) - switch { - case e.linkname != "" && e.hard: - hdr = tar.Header{Name: e.name, Mode: 0o644, Typeflag: tar.TypeLink, Linkname: e.linkname} - case e.linkname != "": - hdr = tar.Header{Name: e.name, Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: e.linkname} - } - require.NoError(t, store.Add(hdr, bytes.NewReader([]byte(e.body)))) - } - return NewIndex(store, "", "outer.tar") -} - -func globPaths(t *testing.T, r *Index, pattern string) []string { - t.Helper() - locations, err := r.FilesByGlob(pattern) - require.NoError(t, err) - var out []string - for _, l := range locations { - out = append(out, l.RealPath) - } - return out -} - -// several paths to one file answer as one location: the file's own path when it matched, otherwise -// the lowest-sorting link -func TestIndex_linksCollapseToOnePath(t *testing.T) { - tests := []struct { - name string - entries []linkEntry - pattern string - want []string - reason string - }{ - { - name: "the file itself beats every link to it", - entries: []linkEntry{ - {name: "opt/real.jar", body: "PK\x03\x04"}, - {name: "opt/a-sym.jar", linkname: "real.jar"}, - {name: "opt/z-sym.jar", linkname: "real.jar"}, - {name: "opt/hard.jar", linkname: "opt/real.jar", hard: true}, - }, - pattern: "**/*.jar", - want: []string{"opt/real.jar"}, - reason: "one file, so one location - and it is named by the path that holds it", - }, - { - name: "with no candidate naming the file, the lowest-sorting link wins", - entries: []linkEntry{ - {name: "opt/real.bin", body: "PK\x03\x04"}, - {name: "opt/z-sym.jar", linkname: "real.bin"}, - {name: "opt/a-sym.jar", linkname: "real.bin"}, - {name: "opt/m-sym.jar", linkname: "real.bin"}, - }, - pattern: "**/*.jar", - want: []string{"opt/real.bin"}, - reason: "the content is the target's, so that is the real path the answer carries", - }, - { - name: "distinct files stay distinct", - entries: []linkEntry{ - {name: "opt/one.jar", body: "one"}, - {name: "opt/two.jar", body: "two"}, - {name: "opt/one-sym.jar", linkname: "one.jar"}, - }, - pattern: "**/*.jar", - want: []string{"opt/one.jar", "opt/two.jar"}, - reason: "collapsing is per file, not per pattern", - }, - { - name: "a dangling link keeps its own path", - entries: []linkEntry{ - {name: "opt/real.jar", body: "PK\x03\x04"}, - {name: "opt/gone.jar", linkname: "missing.jar"}, - }, - pattern: "**/*.jar", - want: []string{"opt/gone.jar", "opt/real.jar"}, - reason: "it names no content, so it belongs to no other file's group", - }, - { - name: "two dangling links are two paths, not one", - entries: []linkEntry{ - {name: "opt/a.jar", linkname: "missing.jar"}, - {name: "opt/b.jar", linkname: "missing.jar"}, - }, - pattern: "**/*.jar", - want: []string{"opt/a.jar", "opt/b.jar"}, - reason: "nothing resolves them to a common file, so neither may absorb the other", - }, - { - name: "a chain of links collapses onto the file at its end", - entries: []linkEntry{ - {name: "opt/real.jar", body: "PK\x03\x04"}, - {name: "opt/mid.jar", linkname: "real.jar"}, - {name: "opt/outer.jar", linkname: "mid.jar"}, - }, - pattern: "**/*.jar", - want: []string{"opt/real.jar"}, - reason: "following one hop at a time reaches the same file from every path", - }, - { - name: "a cycle resolves to nothing rather than looping", - entries: []linkEntry{ - {name: "opt/a.jar", linkname: "b.jar"}, - {name: "opt/b.jar", linkname: "a.jar"}, - }, - pattern: "**/*.jar", - want: []string{"opt/a.jar", "opt/b.jar"}, - reason: "neither reaches content, so each stands under its own path", - }, - { - name: "a hard link resolves from the archive root, not from its own directory", - entries: []linkEntry{ - {name: "opt/real.jar", body: "PK\x03\x04"}, - {name: "opt/nested/hard.jar", linkname: "opt/real.jar", hard: true}, - }, - pattern: "**/*.jar", - want: []string{"opt/real.jar"}, - reason: "read as relative it would resolve to opt/nested/opt/real.jar and dangle", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := indexWithLinks(t, tt.entries...) - assert.Equal(t, tt.want, globPaths(t, r, tt.pattern), tt.reason) - }) - } -} - -// the choice must not depend on the order the search found the candidates in -func TestIndex_collapseIgnoresTheOrderItFinds(t *testing.T) { - r := indexWithLinks(t, - linkEntry{name: "opt/real.jar", body: "PK\x03\x04"}, - linkEntry{name: "opt/a-sym.jar", linkname: "real.jar"}, - linkEntry{name: "opt/m-sym.jar", linkname: "real.jar"}, - linkEntry{name: "opt/z-sym.jar", linkname: "real.jar"}, - ) - - candidates := []*node{ - r.byPath["/opt/real.jar"], r.byPath["/opt/a-sym.jar"], - r.byPath["/opt/m-sym.jar"], r.byPath["/opt/z-sym.jar"], - } - for _, n := range candidates { - require.NotNil(t, n) - } - - rng := rand.New(rand.NewSource(1)) - for range 64 { - rng.Shuffle(len(candidates), func(i, j int) { - candidates[i], candidates[j] = candidates[j], candidates[i] - }) - - found := map[*node]struct{}{} - for _, n := range candidates { - found[n] = struct{}{} - } - - best := onePathPerFile(found) - require.Len(t, best, 1, "four paths, one file") - assert.Equal(t, "/opt/real.jar", best[0].path, "the file itself must win from any order") - } - - // and with the file itself not among the candidates, the lowest-sorting link wins from any order - links := candidates[:0:0] - for _, n := range candidates { - if n.path != "/opt/real.jar" { - links = append(links, n) - } - } - for range 64 { - rng.Shuffle(len(links), func(i, j int) { links[i], links[j] = links[j], links[i] }) - - found := map[*node]struct{}{} - for _, n := range links { - found[n] = struct{}{} - } - - best := onePathPerFile(found) - require.Len(t, best, 1) - assert.Equal(t, "/opt/a-sym.jar", best[0].path, "the lowest-sorting link must win from any order") - } -} - -func TestIndex_collapsedLocationReadsTheTarget(t *testing.T) { - r := indexWithLinks(t, - linkEntry{name: "opt/real.bin", body: "PK\x03\x04 real contents"}, - linkEntry{name: "opt/a-sym.jar", linkname: "real.bin"}, - linkEntry{name: "opt/z-sym.jar", linkname: "real.bin"}, - ) - - locations, err := r.FilesByGlob("**/*.jar") - require.NoError(t, err) - require.Len(t, locations, 1) - - assert.Equal(t, "opt/real.bin", locations[0].RealPath, "content comes from the file") - assert.Equal(t, "opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched") - - reader, err := r.FileContentsByLocation(locations[0]) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, reader.Close()) }) - - body, err := io.ReadAll(reader) - require.NoError(t, err) - assert.Equal(t, "PK\x03\x04 real contents", string(body), - "a location standing for a link must still read the file's bytes, not the link's none") -} - -func TestIndex_danglingLinkHasNoContent(t *testing.T) { - r := indexWithLinks(t, linkEntry{name: "opt/gone.jar", linkname: "missing.jar"}) - - locations, err := r.FilesByGlob("**/*.jar") - require.NoError(t, err) - require.Len(t, locations, 1) - assert.Equal(t, "opt/gone.jar", locations[0].RealPath) - - reader, err := r.FileContentsByLocation(locations[0]) - require.NoError(t, err) - body, err := io.ReadAll(reader) - require.NoError(t, err) - assert.Empty(t, string(body), "a link to nothing holds nothing") - - metadata, err := r.FileMetadataByLocation(locations[0]) - require.NoError(t, err) - assert.Equal(t, "opt/missing.jar", metadata.LinkDestination, - "where it pointed is still reported, resolved against the link's own directory") -} - -// a link has no content to sniff a type from; the file it points at answers under its own path -func TestIndex_mimeTypeAnswersOnlyWithFiles(t *testing.T) { - r := indexWithLinks(t, - linkEntry{name: "opt/real.jar", body: "PK\x03\x04 zip contents here"}, - linkEntry{name: "opt/sym.jar", linkname: "real.jar"}, - linkEntry{name: "opt/hard.jar", linkname: "opt/real.jar", hard: true}, - linkEntry{name: "opt/gone.jar", linkname: "missing.jar"}, - linkEntry{name: "opt/notes.txt", body: "hello world"}, - ) - - byMIME := func(types ...string) []string { - t.Helper() - locations, err := r.FilesByMIMEType(types...) - require.NoError(t, err) - var out []string - for _, l := range locations { - out = append(out, l.RealPath) - } - return out - } - - assert.Equal(t, []string{"opt/real.jar"}, byMIME("application/zip"), - "the file answers, not the two links to it") - assert.Equal(t, []string{"opt/notes.txt"}, byMIME("text/plain")) - assert.Empty(t, byMIME(""), - "a link sniffs to no type, and must not be reachable by asking for that") - assert.Equal(t, []string{"opt/notes.txt", "opt/real.jar"}, byMIME("application/zip", "text/plain")) -} - -// AllLocations enumerates what the archive holds rather than searching for files: the file metadata -// cataloger records a row per path, links included -func TestIndex_allLocationsKeepsEveryPath(t *testing.T) { - r := indexWithLinks(t, - linkEntry{name: "opt/real.jar", body: "PK\x03\x04"}, - linkEntry{name: "opt/sym.jar", linkname: "real.jar"}, - linkEntry{name: "opt/gone.jar", linkname: "missing.jar"}, - ) - - var got []string - for l := range r.AllLocations(context.Background()) { - got = append(got, l.RealPath) - } - assert.Equal(t, []string{"opt/gone.jar", "opt/real.jar", "opt/sym.jar"}, got, - "every path the archive names, links included") -} diff --git a/internal/archive/index_test.go b/internal/archive/index_test.go deleted file mode 100644 index e02f13e0f..000000000 --- a/internal/archive/index_test.go +++ /dev/null @@ -1,271 +0,0 @@ -package archive - -import ( - "archive/tar" - "bytes" - "context" - "io" - "sort" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/anchore/syft/syft/file" -) - -func TestIndex_globByExtensionUsesTheNameIndex(t *testing.T) { - r := indexOver(t, 1<<20, jarEntriesFixture) - - locations, err := r.FilesByGlob("**/*.jar") - require.NoError(t, err) - assert.Equal(t, []string{"/BOOT-INF/lib/inner.jar"}, relPaths(t, r, locations)) -} - -func TestIndex_globByExactNameAtAnyDepth(t *testing.T) { - r := indexOver(t, 1<<20, jarEntriesFixture) - - locations, err := r.FilesByGlob("**/pom.properties") - require.NoError(t, err) - assert.Equal(t, []string{"/META-INF/maven/com.example/lib/pom.properties"}, relPaths(t, r, locations)) -} - -func TestIndex_globFindsRootLevelFiles(t *testing.T) { - r := indexOver(t, 1<<20, jarEntriesFixture) - - locations, err := r.FilesByGlob("/*") - require.NoError(t, err) - assert.Equal(t, []string{"/LICENSE"}, relPaths(t, r, locations)) -} - -func TestIndex_globOneDirectoryDeep(t *testing.T) { - r := indexOver(t, 1<<20, jarEntriesFixture) - - locations, err := r.FilesByGlob("/META-INF/*") - require.NoError(t, err) - assert.Equal(t, []string{"/META-INF/MANIFEST.MF"}, relPaths(t, r, locations), - "one level only: the maven files are deeper") -} - -func TestIndex_globAcrossSyntheticDirectories(t *testing.T) { - // the archive never lists META-INF/ or META-INF/maven/ - r := indexOver(t, 1<<20, jarEntriesFixture) - - locations, err := r.FilesByGlob("**/maven/*/*/pom.xml") - require.NoError(t, err) - assert.Equal(t, []string{"/META-INF/maven/com.example/lib/pom.xml"}, relPaths(t, r, locations)) -} - -func TestIndex_globHonorsNonWildcardMetacharacters(t *testing.T) { - // a segment with any metacharacter other than one `*` must reach doublestar rather than be looked - // up as a literal name - r := indexOver(t, 1<<20, map[string]string{ - "usr/local/go/bin/go": "binary", - "usr/lib/libstd-0123456789abcdef.so": "binary", - "opt/a.jar": "PK\x03\x04", - "opt/b.war": "PK\x03\x04", - "opt/notes.txt": "prose", - }) - - tests := []struct { - pattern string - want []string - }{ - {"**/{go,go.exe}", []string{"/usr/local/go/bin/go"}}, - {"**/libstd-????????????????.so", []string{"/usr/lib/libstd-0123456789abcdef.so"}}, - {"**/*.[jw]ar", []string{"/opt/a.jar", "/opt/b.war"}}, - {"**/{a,b}.{jar,war}", []string{"/opt/a.jar", "/opt/b.war"}}, - {"**/*.jar", []string{"/opt/a.jar"}}, - {"**/go", []string{"/usr/local/go/bin/go"}}, - {"**/notes*", []string{"/opt/notes.txt"}}, - } - for _, tt := range tests { - t.Run(tt.pattern, func(t *testing.T) { - locations, err := r.FilesByGlob(tt.pattern) - require.NoError(t, err) - assert.Equal(t, tt.want, relPaths(t, r, locations)) - }) - } -} - -func TestIndex_globMetacharactersInAMiddleSegment(t *testing.T) { - r := indexOver(t, 1<<20, map[string]string{ - "META-INF/maven/com.example/lib/pom.xml": "", - "WEB-INF/lib/dep.jar": "PK\x03\x04", - }) - - locations, err := r.FilesByGlob("/{META-INF,WEB-INF}/**/*.xml") - require.NoError(t, err) - assert.Equal(t, []string{"/META-INF/maven/com.example/lib/pom.xml"}, relPaths(t, r, locations)) -} - -func TestIndex_pathsAndContents(t *testing.T) { - r := indexOver(t, 1<<20, jarEntriesFixture) - - locations, err := r.FilesByPath("/META-INF/MANIFEST.MF") - require.NoError(t, err) - require.Len(t, locations, 1) - - assert.True(t, r.HasPath("/META-INF/MANIFEST.MF")) - assert.True(t, r.HasPath("META-INF/MANIFEST.MF"), "a path may arrive with or without the leading slash") - assert.False(t, r.HasPath("/nope")) - - reader, err := r.FileContentsByLocation(locations[0]) - require.NoError(t, err) - body, err := io.ReadAll(reader) - require.NoError(t, err) - require.NoError(t, reader.Close()) - assert.Equal(t, "Manifest-Version: 1.0\n", string(body)) -} - -func TestIndex_contentsSurviveTheMoveToDisk(t *testing.T) { - r := indexOver(t, 8, jarEntriesFixture) - - locations, err := r.FilesByGlob("**/*.jar") - require.NoError(t, err) - require.Len(t, locations, 1) - - reader, err := r.FileContentsByLocation(locations[0]) - require.NoError(t, err) - body, err := io.ReadAll(reader) - require.NoError(t, err) - require.NoError(t, reader.Close()) - assert.Equal(t, "PK\x03\x04 not really", string(body)) -} - -func TestIndex_mimeTypesAreSniffedFromContent(t *testing.T) { - r := indexOver(t, 1<<20, map[string]string{ - "notes.txt": "just some text", - "run.sh": "#!/bin/bash\necho hello\n", - }) - - locations, err := r.FilesByMIMEType("text/x-shellscript") - require.NoError(t, err) - assert.Equal(t, []string{"/run.sh"}, relPaths(t, r, locations)) -} - -func TestIndex_allLocationsCoversEveryFile(t *testing.T) { - r := indexOver(t, 1<<20, jarEntriesFixture) - - var got []string - for loc := range r.AllLocations(context.Background()) { - got = append(got, "/"+loc.RealPath) - } - sort.Strings(got) - - assert.Equal(t, []string{ - "/BOOT-INF/lib/inner.jar", - "/LICENSE", - "/META-INF/MANIFEST.MF", - "/META-INF/maven/com.example/lib/pom.properties", - "/META-INF/maven/com.example/lib/pom.xml", - "/com/example/Thing.class", - }, got) -} - -func TestIndex_answersAreOrdered(t *testing.T) { - // order decides which locations group into which package when results merge into the SBOM - entries := map[string]string{} - for _, name := range []string{ - "b/z.jar", "a/y.jar", "c/x.jar", "a/b/c/w.jar", "z.jar", "a.jar", - } { - entries[name] = "PK\x03\x04 not really" - } - r := indexOver(t, 1<<20, entries) - - first, err := r.FilesByGlob("**/*.jar") - require.NoError(t, err) - require.Len(t, first, len(entries)) - - byMIME, err := r.FilesByMIMEType("application/octet-stream") - require.NoError(t, err) - - for range 8 { - again, err := r.FilesByGlob("**/*.jar") - require.NoError(t, err) - assert.Equal(t, first, again) - - mimeAgain, err := r.FilesByMIMEType("application/octet-stream") - require.NoError(t, err) - assert.Equal(t, byMIME, mimeAgain) - - var all []string - for loc := range r.AllLocations(context.Background()) { - all = append(all, loc.RealPath) - } - assert.True(t, sort.StringsAreSorted(all)) - } - - var paths []string - for _, loc := range first { - paths = append(paths, loc.RealPath) - } - assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable") -} - -// storeWith builds an entry store holding the given entries in name order. A name ending in "/" with -// an empty body is a directory. -func storeWith(t testing.TB, maxInMemory int64, entries map[string]string) *EntryStore { - t.Helper() - store := storeFor(t, memCharge(maxInMemory)) - - names := make([]string, 0, len(entries)) - for name := range entries { - names = append(names, name) - } - sort.Strings(names) - - for _, name := range names { - body := entries[name] - if body == "" && name[len(name)-1] == '/' { - require.NoError(t, store.Add(tar.Header{Name: name, Mode: 0o755, Typeflag: tar.TypeDir}, nil)) - continue - } - require.NoError(t, store.Add(regularHeader(name, int64(len(body))), bytes.NewReader([]byte(body)))) - } - return store -} - -func indexOver(t testing.TB, maxInMemory int64, entries map[string]string) *Index { - t.Helper() - return NewIndex(storeWith(t, maxInMemory, entries), "", "outer.jar") -} - -func relPaths(t *testing.T, r *Index, locations []file.Location) []string { - t.Helper() - var out []string - for _, loc := range locations { - out = append(out, "/"+loc.RealPath) - } - sort.Strings(out) - return out -} - -var jarEntriesFixture = map[string]string{ - "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", - "META-INF/maven/com.example/lib/pom.properties": "groupId=com.example\n", - "META-INF/maven/com.example/lib/pom.xml": "", - "LICENSE": "Apache License 2.0", - "BOOT-INF/lib/inner.jar": "PK\x03\x04 not really", - "com/example/Thing.class": "bytecode", -} - -func TestIndex_impliedDirectoriesArePathsButNotFiles(t *testing.T) { - r := indexOver(t, 1<<20, jarEntriesFixture) - - assert.True(t, r.HasPath("META-INF")) - assert.True(t, r.HasPath("/META-INF/maven")) - - locations, err := r.FilesByPath("META-INF") - require.NoError(t, err) - assert.Empty(t, locations) - locations, err = r.FilesByGlob("**/maven") - require.NoError(t, err) - assert.Empty(t, locations) -} - -func TestIndex_invalidGlobIsAnError(t *testing.T) { - r := indexOver(t, 1<<20, jarEntriesFixture) - _, err := r.FilesByGlob("**/[") - assert.Error(t, err) -} diff --git a/internal/archive/limits.go b/internal/archive/limits.go index 11223953f..87ab321fe 100644 --- a/internal/archive/limits.go +++ b/internal/archive/limits.go @@ -59,21 +59,21 @@ func (l *Limiter) InUse() (memory, disk int64) { return l.memory, l.disk } -// Charge starts accounting for one archive. Releasing the charge gives back everything it took. -func (l *Limiter) Charge() *Charge { +// charge starts accounting for one archive. Releasing the charge gives back everything it took. +func (l *Limiter) charge() *charge { if l == nil { return nil } - return &Charge{limiter: l} + return &charge{limiter: l} } -// Charge is one archive's draw on a Limiter. A nil *Charge admits everything. +// charge is one archive's draw on a Limiter. A nil *charge admits everything. // // Bytes are charged as they land, never from the sizes an archive declares, since those can lie. -type Charge struct { - limiter *Limiter - memory int64 - disk int64 +type charge struct { + limiter *Limiter + inMemory int64 + onDisk int64 } func admits(limit, held, n int64) bool { @@ -87,9 +87,9 @@ func admits(limit, held, n int64) bool { } } -// Memory charges n bytes held in memory, or reports false (charging nothing) when the memory limit +// memory charges n bytes held in memory, or reports false (charging nothing) when the memory limit // does not admit them. -func (c *Charge) Memory(n int64) bool { +func (c *charge) memory(n int64) bool { if c == nil || n <= 0 { return true } @@ -100,14 +100,14 @@ func (c *Charge) Memory(n int64) bool { return false } l.memory += n - c.memory += n + c.inMemory += n l.peakMemory = max(l.peakMemory, l.memory) return true } -// Disk charges n bytes written to disk, or reports false (charging nothing) when the disk limit does +// disk charges n bytes written to disk, or reports false (charging nothing) when the disk limit does // not admit them. -func (c *Charge) Disk(n int64) bool { +func (c *charge) disk(n int64) bool { if c == nil || n <= 0 { return true } @@ -118,53 +118,56 @@ func (c *Charge) Disk(n int64) bool { return false } l.disk += n - c.disk += n + c.onDisk += n l.peakDisk = max(l.peakDisk, l.disk) return true } -// Index charges n bytes of index bookkeeping. The index always lives in memory, but a zero memory -// limit is a valid configuration that must still index archives, so it falls back to the disk -// budget when memory does not admit it. Reports false only when neither does. -func (c *Charge) Index(n int64) bool { - return c.Memory(n) || c.Disk(n) +// index charges n bytes of index bookkeeping, which lives in memory whatever the limits say. A zero +// memory limit is a valid configuration that must still index archives, so only then does the cost +// fall back to the disk budget; a bounded memory limit bounds the index too. +func (c *charge) index(n int64) bool { + if c.memory(n) { + return true + } + return c != nil && c.limiter.limits.MaxMemoryBytes == 0 && c.disk(n) } -// RefundMemory gives back n bytes no longer held in memory. -func (c *Charge) RefundMemory(n int64) { +// refundMemory gives back n bytes no longer held in memory. +func (c *charge) refundMemory(n int64) { if c == nil || n <= 0 { return } l := c.limiter l.mu.Lock() defer l.mu.Unlock() - n = min(n, c.memory) + n = min(n, c.inMemory) l.memory -= n - c.memory -= n + c.inMemory -= n } -// RefundDisk gives back n bytes no longer on disk. -func (c *Charge) RefundDisk(n int64) { +// refundDisk gives back n bytes no longer on disk. +func (c *charge) refundDisk(n int64) { if c == nil || n <= 0 { return } l := c.limiter l.mu.Lock() defer l.mu.Unlock() - n = min(n, c.disk) + n = min(n, c.onDisk) l.disk -= n - c.disk -= n + c.onDisk -= n } -// Release gives back everything this charge took. Safe to call more than once. -func (c *Charge) Release() { +// release gives back everything this charge took. Safe to call more than once. +func (c *charge) release() { if c == nil { return } l := c.limiter l.mu.Lock() defer l.mu.Unlock() - l.memory -= c.memory - l.disk -= c.disk - c.memory, c.disk = 0, 0 + l.memory -= c.inMemory + l.disk -= c.onDisk + c.inMemory, c.onDisk = 0, 0 } diff --git a/internal/archive/limits_test.go b/internal/archive/limits_test.go index d26fa2f51..7cdc85c86 100644 --- a/internal/archive/limits_test.go +++ b/internal/archive/limits_test.go @@ -11,89 +11,79 @@ import ( func TestLimiter_chargeAndRelease(t *testing.T) { limiter := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 200}) - first := limiter.Charge() - require.True(t, first.Memory(60)) - require.True(t, first.Disk(150)) + first := limiter.charge() + require.True(t, first.memory(60)) + require.True(t, first.disk(150)) mem, disk := limiter.InUse() assert.Equal(t, int64(60), mem) assert.Equal(t, int64(150), disk) - second := limiter.Charge() - assert.False(t, second.Memory(50), "60 + 50 is over the memory limit") - assert.False(t, second.Disk(60), "150 + 60 is over the disk limit") - assert.True(t, second.Memory(40), "and exactly at the limit is admitted") + second := limiter.charge() + assert.False(t, second.memory(50), "60 + 50 is over the memory limit") + assert.False(t, second.disk(60), "150 + 60 is over the disk limit") + assert.True(t, second.memory(40), "and exactly at the limit is admitted") mem, _ = limiter.InUse() assert.Equal(t, int64(100), mem, "a refused charge takes nothing") - first.Release() + first.release() mem, disk = limiter.InUse() assert.Equal(t, int64(40), mem, "release gives back exactly what that charge took") assert.Zero(t, disk) - first.Release() + first.release() mem, _ = limiter.InUse() assert.Equal(t, int64(40), mem, "releasing twice must not give back what was never taken") } func TestLimiter_refund(t *testing.T) { limiter := NewLimiter(Limits{MaxDiskBytes: 100}) - charge := limiter.Charge() + charge := limiter.charge() - require.True(t, charge.Disk(80)) - charge.RefundDisk(80) + require.True(t, charge.disk(80)) + charge.refundDisk(80) _, disk := limiter.InUse() assert.Zero(t, disk) - assert.True(t, charge.Disk(100), "the refunded room is available again") + assert.True(t, charge.disk(100), "the refunded room is available again") - charge.RefundDisk(500) + charge.refundDisk(500) _, disk = limiter.InUse() assert.Zero(t, disk, "a refund larger than what was charged cannot drive the limiter negative") } -func TestLimiter_boundsAreIndependent(t *testing.T) { - memoryBounded := NewLimiter(Limits{MaxMemoryBytes: 10, MaxDiskBytes: -1}).Charge() - assert.False(t, memoryBounded.Memory(50)) - assert.True(t, memoryBounded.Disk(1_000_000), "a negative disk limit admits anything") - - diskBounded := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10}).Charge() - assert.False(t, diskBounded.Memory(1_000_000), "a zero memory limit refuses everything rather than admitting anything") - assert.False(t, diskBounded.Disk(50)) -} - func TestLimiter_threeStateReadingIsUniform(t *testing.T) { t.Run("positive is the limit", func(t *testing.T) { - c := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 100}).Charge() - assert.True(t, c.Memory(100)) - assert.False(t, c.Memory(1)) - assert.True(t, c.Disk(100)) - assert.False(t, c.Disk(1)) + c := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 100}).charge() + assert.True(t, c.memory(100)) + assert.False(t, c.memory(1)) + assert.True(t, c.disk(100)) + assert.False(t, c.disk(1)) }) t.Run("zero refuses unconditionally on both", func(t *testing.T) { - c := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).Charge() - assert.False(t, c.Memory(1)) - assert.False(t, c.Disk(1)) + c := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).charge() + assert.False(t, c.memory(1)) + assert.False(t, c.disk(1)) }) t.Run("negative admits unconditionally on both", func(t *testing.T) { - c := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}).Charge() - assert.True(t, c.Memory(1_000_000_000)) - assert.True(t, c.Disk(1_000_000_000)) + c := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}).charge() + assert.True(t, c.memory(1_000_000_000)) + assert.True(t, c.disk(1_000_000_000)) }) } func TestLimiter_nilIsUnbounded(t *testing.T) { var limiter *Limiter - charge := limiter.Charge() + charge := limiter.charge() assert.Nil(t, charge) - assert.True(t, charge.Memory(1_000_000)) - assert.True(t, charge.Disk(1_000_000)) - charge.RefundDisk(10) - charge.Release() + assert.True(t, charge.memory(1_000_000)) + assert.True(t, charge.disk(1_000_000)) + charge.refundDisk(10) + charge.release() mem, disk := charge.held() assert.Zero(t, mem) @@ -102,21 +92,25 @@ func TestLimiter_nilIsUnbounded(t *testing.T) { mem, disk = limiter.InUse() assert.Zero(t, mem) assert.Zero(t, disk) + + mem, disk = limiter.Peak() + assert.Zero(t, mem) + assert.Zero(t, disk) } func TestLimiter_concurrentChargesAgree(t *testing.T) { limiter := NewLimiter(Limits{MaxMemoryBytes: 1_000_000, MaxDiskBytes: -1}) var wg sync.WaitGroup - charges := make([]*Charge, 50) + charges := make([]*charge, 50) for i := range charges { - charges[i] = limiter.Charge() + charges[i] = limiter.charge() wg.Add(1) - go func(c *Charge) { + go func(c *charge) { defer wg.Done() for range 100 { - c.Memory(1) - c.Disk(2) + c.memory(1) + c.disk(2) } }(charges[i]) } @@ -127,75 +121,58 @@ func TestLimiter_concurrentChargesAgree(t *testing.T) { assert.Equal(t, int64(50*100*2), disk) for _, c := range charges { - c.Release() + c.release() } mem, disk = limiter.InUse() assert.Zero(t, mem) assert.Zero(t, disk) } -func Test_Limiter_peakIsAHighWaterMark(t *testing.T) { - l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) +func TestLimiter_peak(t *testing.T) { + t.Run("is a high-water mark", func(t *testing.T) { + l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) - first := l.Charge() - require.True(t, first.Memory(100)) - require.True(t, first.Disk(300)) - first.Release() + first := l.charge() + require.True(t, first.memory(100)) + require.True(t, first.disk(300)) + first.release() - second := l.Charge() - require.True(t, second.Memory(40)) - require.True(t, second.Disk(50)) + second := l.charge() + require.True(t, second.memory(40)) + require.True(t, second.disk(50)) - memory, disk := l.InUse() - assert.Equal(t, int64(40), memory) - assert.Equal(t, int64(50), disk) + memory, disk := l.InUse() + assert.Equal(t, int64(40), memory) + assert.Equal(t, int64(50), disk) - peakMemory, peakDisk := l.Peak() - assert.Equal(t, int64(100), peakMemory, "the peak is the most held at once, not the most recently held") - assert.Equal(t, int64(300), peakDisk) -} - -func Test_Limiter_peakCountsConcurrentHoldersTogether(t *testing.T) { - l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) - - outer := l.Charge() - require.True(t, outer.Memory(100)) - inner := l.Charge() - require.True(t, inner.Memory(60)) - inner.Release() - outer.Release() - - peakMemory, _ := l.Peak() - assert.Equal(t, int64(160), peakMemory) -} - -func Test_Limiter_peakOfANilLimiterIsZero(t *testing.T) { - var l *Limiter - memory, disk := l.Peak() - assert.Zero(t, memory) - assert.Zero(t, disk) -} - -func Test_Limiter_refusedChargeDoesNotMoveThePeak(t *testing.T) { - l := NewLimiter(Limits{MaxMemoryBytes: 50, MaxDiskBytes: 50}) - - c := l.Charge() - assert.False(t, c.Memory(500)) - assert.False(t, c.Disk(500)) - - peakMemory, peakDisk := l.Peak() - assert.Zero(t, peakMemory) - assert.Zero(t, peakDisk) -} - -func TestCharge_indexFallsBackToDisk(t *testing.T) { - memoryOnly := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 0}).Charge() - assert.True(t, memoryOnly.Index(100)) - assert.False(t, memoryOnly.Index(1), "neither budget admits it") - - diskOnly := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 100}).Charge() - assert.True(t, diskOnly.Index(100)) - mem, disk := diskOnly.held() - assert.Zero(t, mem) - assert.Equal(t, int64(100), disk) + peakMemory, peakDisk := l.Peak() + assert.Equal(t, int64(100), peakMemory, "the peak is the most held at once, not the most recently held") + assert.Equal(t, int64(300), peakDisk) + }) + + t.Run("counts concurrent holders together", func(t *testing.T) { + l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) + + outer := l.charge() + require.True(t, outer.memory(100)) + inner := l.charge() + require.True(t, inner.memory(60)) + inner.release() + outer.release() + + peakMemory, _ := l.Peak() + assert.Equal(t, int64(160), peakMemory) + }) + + t.Run("a refused charge does not move it", func(t *testing.T) { + l := NewLimiter(Limits{MaxMemoryBytes: 50, MaxDiskBytes: 50}) + + c := l.charge() + assert.False(t, c.memory(500)) + assert.False(t, c.disk(500)) + + peakMemory, peakDisk := l.Peak() + assert.Zero(t, peakMemory) + assert.Zero(t, peakDisk) + }) } diff --git a/internal/archive/resolver.go b/internal/archive/resolver.go new file mode 100644 index 000000000..04ae820cd --- /dev/null +++ b/internal/archive/resolver.go @@ -0,0 +1,482 @@ +package archive + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "sort" + "strings" + "time" + + "github.com/mholt/archives" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" + syftindex "github.com/anchore/syft/internal/index" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/syft/file" +) + +// Resolver is one archive opened as its own filesystem: a file.Resolver over the entries Extract fed +// into it, holding their content within the scan's limits. +// +// Paths are archive-relative with no leading slash. Files and directories are indexed by base name and +// linked into a tree, so a glob is narrowed to the names its segments fix and verified up the parent +// chain; see glob.go. +type Resolver struct { + // Digests are of the archive file itself. + Digests []file.Digest + + // Truncated reports that the disk limit stopped extraction early; the resolver covers the entries + // stored before that. + Truncated bool + + fileSystemID string + archivePath string + charge *charge + + // content is held in memory while the memory limit admits it and in one spill file once it does + // not. The file is created on first use, so an archive that stays in memory never touches the + // filesystem, and blobs are located in it by offset with no framing of their own. + tempDir *tmpdir.TempDir + held []*blob // blobs in memory, in the order they were added + file *os.File + remove func() + written int64 + chunk []byte + + // byPath holds every file and directory by rooted path, including directories the archive never + // listed but that entry paths imply + byPath map[string]*node + + // files holds every file the archive listed, in path order once finished + files []*node + + // names and dirs index files and directories by base name, for globs + names syftindex.PrefixSuffix[[]*node] + dirs syftindex.PrefixSuffix[[]*node] +} + +type node struct { + path string // rooted, e.g. "/META-INF/MANIFEST.MF" + name string // the last component of path + isDir bool + header *tar.Header // nil for a directory the archive never listed + content blob + metadata file.Metadata + + // the tree the paths spell out, so a glob is matched up the chain and a directory's files are + // found beneath it + parent *node + children []*node + + // target is the node whose content this node reads: itself for a regular file, the resolved node + // for a link, and nil for a link that leads nowhere inside this archive + target *node +} + +var _ file.Resolver = (*Resolver)(nil) + +// newResolver returns an empty resolver whose locations carry the given fileSystemID and archivePath, +// spilling into the scan's temp root when ctx carries one (see internal/tmpdir). +func newResolver(ctx context.Context, fileSystemID, archivePath string, charge *charge) *Resolver { + return &Resolver{ + fileSystemID: fileSystemID, + archivePath: archivePath, + charge: charge, + tempDir: tmpdir.FromContext(ctx), + byPath: map[string]*node{"/": {path: "/", isDir: true}}, + } +} + +// Cleanup removes everything the archive holds on disk and releases its charge against the limiter. +// Readers handed out before are no longer valid. Safe to call more than once. +func (r *Resolver) Cleanup() { + if r == nil { + return + } + r.releaseStorage() + r.charge.release() +} + +// approxIndexBytesPerEntry estimates what holding one node costs beyond its content: the node, its +// header, and its path map and name index slots. It is charged for every node an entry creates, +// including the directories its path implies, so an archive of many tiny entries or of a few entries +// with very deep paths is still bounded by the limits. +const approxIndexBytesPerEntry = 2 * 1024 + +func approxIndexBytes(hdr tar.Header) int64 { + return approxIndexBytesPerEntry + int64(len(hdr.Name)+len(hdr.Linkname)) +} + +// add stores and indexes one entry, reading its content in full now since archive readers are +// sequential. Reaching a limit stores nothing for this entry and returns ErrDiskLimitReached; entries +// added before it remain usable. The first entry at a path wins. +func (r *Resolver) add(hdr tar.Header, content io.Reader) error { + entryPath := path.Clean("/" + hdr.Name) + if entryPath == "/" { + return nil + } + + // directories the path implies that do not exist yet are nodes this entry pays for + var newDirs int64 + for i := 1; i < len(entryPath); i++ { + if entryPath[i] == '/' { + if _, ok := r.byPath[entryPath[:i]]; !ok { + newDirs++ + } + } + } + if !r.chargeIndex(approxIndexBytes(hdr) + newDirs*approxIndexBytesPerEntry) { + return ErrDiskLimitReached + } + + for i := 1; i < len(entryPath); i++ { + if entryPath[i] == '/' { + r.nodeAt(entryPath[:i], true) + } + } + _, existed := r.byPath[entryPath] + n := r.nodeAt(entryPath, hdr.FileInfo().IsDir()) + if n.header != nil { + return nil + } + + if hdr.Typeflag == tar.TypeReg && content != nil { + if err := r.put(&n.content, content); err != nil { + r.discard(&n.content) + if !existed { + delete(r.byPath, entryPath) + } + if !errors.Is(err, ErrDiskLimitReached) { + return fmt.Errorf("unable to read archive entry %q: %w", hdr.Name, err) + } + return err + } + } + + n.header = &hdr + n.metadata = r.metadataOf(n) + if n.isDir { + return nil + } + r.files = append(r.files, n) + r.names.Update(path.Base(entryPath), func(current *syftindex.Node[[]*node]) []*node { + return append(current.Value(), n) + }) + return nil +} + +// chargeIndex charges index bookkeeping, which lives in memory. Content held in memory can move to +// disk to make room for it; the index cannot, so when it is refused the entries are. +func (r *Resolver) chargeIndex(n int64) bool { + if r.charge.index(n) { + return true + } + if len(r.held) == 0 || r.spill() != nil { + return false + } + return r.charge.index(n) +} + +// nodeAt returns the node at a rooted path, creating it under its parent, which must exist already. +func (r *Resolver) nodeAt(p string, isDir bool) *node { + if n, ok := r.byPath[p]; ok { + return n + } + slash := strings.LastIndexByte(p, '/') + parentPath := p[:slash] + if parentPath == "" { + parentPath = "/" + } + parent := r.byPath[parentPath] + n := &node{path: p, name: p[slash+1:], isDir: isDir, parent: parent} + parent.children = append(parent.children, n) + r.byPath[p] = n + if isDir { + r.dirs.Update(n.name, func(current *syftindex.Node[[]*node]) []*node { + return append(current.Value(), n) + }) + } + return n +} + +// finish orders the files and resolves links once every entry is added. +func (r *Resolver) finish() { + sort.Slice(r.files, func(i, j int) bool { return r.files[i].path < r.files[j].path }) + for _, n := range r.byPath { + n.target = r.linkTarget(n) + } +} + +// maxLinkHops bounds a chain of links pointing at links, which is also what ends a cycle. +const maxLinkHops = 8 + +func (r *Resolver) linkTarget(n *node) *node { + for hops := 0; hops <= maxLinkHops; hops++ { + if n.header == nil || n.isDir { + return nil + } + if n.header.Linkname == "" { + return n + } + next, ok := r.byPath[linkDestination(n.path, n.header.Linkname, n.header.Typeflag == tar.TypeLink)] + if !ok { + return nil + } + n = next + } + return nil +} + +// linkDestination is the rooted path a link points at inside the archive. A hard link names its +// target from the archive root, as does an absolute symlink, so "/etc/passwd" is an entry of this +// archive and never the host's. A relative symlink is joined onto the link's own directory, and +// cleaning against the root keeps "../../x" inside the archive. +func linkDestination(linkPath, linkname string, hard bool) string { + if hard || path.IsAbs(linkname) { + return path.Join("/", linkname) + } + return path.Join(path.Dir(linkPath), linkname) +} + +func (r *Resolver) metadataOf(n *node) file.Metadata { + metadata := stereoscopeFile.NewMetadata(*n.header, r.open(&n.content)) + // NewMetadata copies the header and points FileInfo into the copy; share the one the node holds + metadata.FileInfo = n.header.FileInfo() + metadata.Path = reportedPath(n.path) + if n.header.Linkname != "" { + metadata.LinkDestination = reportedPath(linkDestination(n.path, n.header.Linkname, n.header.Typeflag == tar.TypeLink)) + } + return metadata +} + +// reportedPath is how paths leave the resolver: archive-relative with no leading slash. +func reportedPath(rooted string) string { + return strings.TrimPrefix(rooted, "/") +} + +func rootedPath(userPath string) string { + return path.Clean("/" + userPath) +} + +func (r *Resolver) coordinates(n *node) file.Coordinates { + return file.Coordinates{ + RealPath: reportedPath(n.path), + FileSystemID: r.fileSystemID, + ArchivePath: r.archivePath, + } +} + +// locationOf names a node by its own path. +func (r *Resolver) locationOf(n *node) file.Location { + return file.NewLocationFromCoordinates(r.coordinates(n)) +} + +// resolvedLocation names the file behind a node: a link is reported at its target's path with the +// link's own path as the access path, and a link leading nowhere is reported as itself. +func (r *Resolver) resolvedLocation(n *node) file.Location { + if n.target == nil || n.target == n { + return r.locationOf(n) + } + return file.NewVirtualLocationFromCoordinates(r.coordinates(n.target), reportedPath(n.path)) +} + +func (r *Resolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { + n, ok := r.byPath[rootedPath(location.RealPath)] + if !ok || n.header == nil { + return nil, fmt.Errorf("no archive entry for path=%q", location.RealPath) + } + return entryReader{r.open(&n.content)}, nil +} + +func (r *Resolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) { + n, ok := r.byPath[rootedPath(location.RealPath)] + if !ok { + return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.RealPath) + } + return n.metadata, nil +} + +func (r *Resolver) HasPath(userPath string) bool { + _, ok := r.byPath[rootedPath(userPath)] + return ok +} + +func (r *Resolver) FilesByPath(paths ...string) ([]file.Location, error) { + var out []file.Location + seen := map[*node]struct{}{} + for _, userPath := range paths { + n, ok := r.byPath[rootedPath(userPath)] + if !ok || n.isDir || n.header == nil { + continue + } + if _, dup := seen[n]; dup { + continue + } + seen[n] = struct{}{} + out = append(out, r.resolvedLocation(n)) + } + return out, nil +} + +func (r *Resolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + matched := map[*node]struct{}{} + for _, pattern := range patterns { + for _, segments := range parseGlob(pattern) { + candidates, exact := r.globCandidates(segments) + for _, n := range candidates { + if !exact { + ok, err := matchNode(segments, n) + if err != nil { + return nil, fmt.Errorf("invalid glob %q: %w", pattern, err) + } + if !ok { + continue + } + } + matched[n] = struct{}{} + } + } + } + + var out []file.Location + for _, n := range onePathPerFile(matched) { + out = append(out, r.resolvedLocation(n)) + } + sort.Slice(out, func(i, j int) bool { + if out[i].RealPath != out[j].RealPath { + return out[i].RealPath < out[j].RealPath + } + return out[i].AccessPath < out[j].AccessPath + }) + return out, nil +} + +// onePathPerFile keeps one matched path per file. Several paths can name the same content, such as +// a link and its target, and a resolver answers with one: the file's own path when it matched, +// otherwise the lowest-sorting link. A link leading nowhere stands as its own file. +func onePathPerFile(matched map[*node]struct{}) []*node { + best := map[*node]*node{} + for n := range matched { + key := n.target + if key == nil { + key = n + } + current, seen := best[key] + if !seen || n == key || (current != key && n.path < current.path) { + best[key] = n + } + } + out := make([]*node, 0, len(best)) + for _, n := range best { + out = append(out, n) + } + return out +} + +func (r *Resolver) FilesByMIMEType(types ...string) ([]file.Location, error) { + wanted := map[string]struct{}{} + for _, t := range types { + wanted[t] = struct{}{} + } + var out []file.Location + for _, n := range r.files { + // only a file holding bytes has a type of its own; a link's target answers for itself + if n.target != n { + continue + } + if _, ok := wanted[n.metadata.MIMEType]; ok { + out = append(out, r.locationOf(n)) + } + } + return out, nil +} + +// RelativeFileByPath resolves against the archive root, the only frame an archive has. +func (r *Resolver) RelativeFileByPath(_ file.Location, userPath string) *file.Location { + locations, _ := r.FilesByPath(userPath) + if len(locations) == 0 { + return nil + } + return &locations[0] +} + +func (r *Resolver) AllLocations(ctx context.Context) <-chan file.Location { + out := make(chan file.Location) + go func() { + defer close(out) + for _, n := range r.files { + select { + case out <- r.locationOf(n): + case <-ctx.Done(): + return + } + } + }() + return out +} + +// entryReader gives an entry's reader the Close the resolver interface requires. The resolver owns +// the bytes, so there is nothing to close. +type entryReader struct { + ReaderAtSeeker +} + +func (entryReader) Close() error { return nil } + +// maxEntryNameBytes is the longest entry name stored: PATH_MAX. Every path component becomes a node, +// so a longer name is a cheap way to inflate the index and no filesystem could hold it anyway. +const maxEntryNameBytes = 4096 + +// entryHeader builds the header stored for one archive entry, or reports false for an entry not worth +// carrying: device nodes, fifos and sockets, which no cataloger reads, and names that cannot be a path +// inside the archive. +// +// The name is cleaned to an archive-relative path with no leading slash, so "../x" names x at the +// archive root rather than anything outside it. Symlinks are stored as headers, never followed on the +// host; the resolver resolves their targets inside the archive. +func entryHeader(f archives.FileInfo) (tar.Header, bool) { + mode := f.Mode() + if !mode.IsRegular() && !mode.IsDir() && mode.Type()&fs.ModeSymlink == 0 { + log.WithFields("entry", f.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry") + return tar.Header{}, false + } + + name, ok := entryName(f.NameInArchive) + if !ok { + log.WithFields("entry-name-bytes", len(f.NameInArchive)).Trace("skipping archive entry with an unusable name") + return tar.Header{}, false + } + + hdr, err := tar.FileInfoHeader(f.FileInfo, f.LinkTarget) + if err != nil { + log.WithFields("entry", f.NameInArchive, "error", err).Debug("skipping archive entry with an unusable header") + return tar.Header{}, false + } + hdr.Name = name + hdr.Linkname = f.LinkTarget + + // nothing reads these, and the header is held for the life of the archive + hdr.PAXRecords = nil + hdr.Uname, hdr.Gname = "", "" + hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} + hdr.ModTime = hdr.ModTime.Truncate(time.Second) + + return *hdr, true +} + +// entryName cleans an archive entry's name to a path relative to the archive root, reporting false +// when it names nothing inside the archive. +func entryName(name string) (string, bool) { + if len(name) > maxEntryNameBytes || strings.ContainsRune(name, 0) { + return "", false + } + cleaned := strings.TrimPrefix(path.Clean("/"+name), "/") + return cleaned, cleaned != "" +} diff --git a/internal/archive/resolver_test.go b/internal/archive/resolver_test.go new file mode 100644 index 000000000..b3fda91ba --- /dev/null +++ b/internal/archive/resolver_test.go @@ -0,0 +1,512 @@ +package archive + +import ( + "archive/tar" + "bytes" + "context" + "fmt" + "io" + "math/rand" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var jarEntriesFixture = map[string]string{ + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", + "META-INF/maven/com.example/lib/pom.properties": "groupId=com.example\n", + "META-INF/maven/com.example/lib/pom.xml": "", + "LICENSE": "Apache License 2.0", + "BOOT-INF/lib/inner.jar": "PK\x03\x04 not really", + "com/example/Thing.class": "bytecode", +} + +func TestResolver_add_refusedEntryIsNotStored(t *testing.T) { + // a half-stored pom a cataloger parses is worse than an absent one + r, _ := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).charge()) + + err := r.add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc"))) + assert.ErrorIs(t, err, ErrDiskLimitReached) + assert.Empty(t, storedNames(t, r)) + assert.False(t, r.HasPath("a.txt")) +} + +func TestResolver_add_refusedContentLeavesNoNodeBehind(t *testing.T) { + // the index estimate is admitted, then the content is refused + record := approxIndexBytes(regularHeader("dir/a.txt", 0)) + approxIndexBytesPerEntry // the entry and the directory it implies + r, _ := resolverIn(t, diskCharge(record+2)) + + err := r.add(regularHeader("dir/a.txt", 3), bytes.NewReader([]byte("abc"))) + assert.ErrorIs(t, err, ErrDiskLimitReached) + assert.False(t, r.HasPath("dir/a.txt")) + assert.True(t, r.HasPath("dir"), "the directories it implied stay") +} + +func TestResolver_add_directoryEntriesHoldNoContent(t *testing.T) { + r, _ := resolverIn(t, memCharge(1<<20)) + + require.NoError(t, r.add(tar.Header{Name: "lib", Typeflag: tar.TypeDir, Mode: 0o755}, nil)) + + assert.Zero(t, r.heldInMemory()) + assert.Empty(t, readStore(t, r)["lib"].body) +} + +func TestResolver_add_indexCostIsChargedEvenForEmptyContent(t *testing.T) { + // the index estimate is what bounds an archive of many empty entries + limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}) + r, _ := resolverIn(t, limiter.charge()) + + hdr := regularHeader("a.txt", 0) + require.NoError(t, r.add(hdr, bytes.NewReader(nil))) + + mem, _ := limiter.InUse() + assert.Equal(t, approxIndexBytes(hdr), mem) + assert.Zero(t, r.heldInMemory()) +} + +func TestResolver_add_indexCostFallsBackToDiskWhenMemoryIsZero(t *testing.T) { + limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) + r, _ := resolverIn(t, limiter.charge()) + + hdr := regularHeader("a.txt", 0) + require.NoError(t, r.add(hdr, bytes.NewReader(nil))) + + mem, disk := limiter.InUse() + assert.Zero(t, mem) + assert.Equal(t, approxIndexBytes(hdr), disk) +} + +func TestResolver_add_indexDoesNotFallBackToDiskWhenMemoryIsBounded(t *testing.T) { + // the index lives in memory, so a bounded memory limit bounds it; only a zero limit defers to disk + limiter := NewLimiter(Limits{MaxMemoryBytes: 1024, MaxDiskBytes: -1}) + r, _ := resolverIn(t, limiter.charge()) + + err := r.add(regularHeader("a.txt", 0), bytes.NewReader(nil)) + assert.ErrorIs(t, err, ErrDiskLimitReached) + _, disk := limiter.InUse() + assert.Zero(t, disk) +} + +func TestResolver_add_indexMakesRoomBySpillingHeldContent(t *testing.T) { + record := approxIndexBytes(regularHeader("a.txt", 0)) + r, dir := resolverIn(t, memCharge(2*record+50)) + + require.NoError(t, r.add(regularHeader("a.txt", 100), bytes.NewReader(bytes.Repeat([]byte("a"), 100)))) + require.Equal(t, int64(100), r.heldInMemory()) + + // b's index does not fit beside a's content, so a's content moves to disk and b is indexed + require.NoError(t, r.add(regularHeader("b.txt", 3), bytes.NewReader([]byte("bbb")))) + r.finish() + + assert.Equal(t, int64(3), r.heldInMemory()) + assert.NotEmpty(t, spillFile(t, dir)) + assert.Equal(t, []string{"a.txt", "b.txt"}, storedNames(t, r)) + assert.Equal(t, strings.Repeat("a", 100), readStore(t, r)["a.txt"].body) +} + +func TestResolver_add_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) { + room := approxIndexBytes(regularHeader("x0", 0)) * 3 + r, _ := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).charge()) + + var err error + admitted := 0 + for i := 0; i < 100 && err == nil; i++ { + if err = r.add(regularHeader(fmt.Sprintf("x%d", i), 0), bytes.NewReader(nil)); err == nil { + admitted++ + } + } + require.ErrorIs(t, err, ErrDiskLimitReached) + assert.Equal(t, 3, admitted) +} + +func TestResolver_add_firstEntryAtAPathWins(t *testing.T) { + r, _ := resolverIn(t, memCharge(1<<20)) + + require.NoError(t, r.add(regularHeader("a.txt", 5), bytes.NewReader([]byte("first")))) + require.NoError(t, r.add(regularHeader("a.txt", 6), bytes.NewReader([]byte("second")))) + r.finish() + + assert.Equal(t, "first", readStore(t, r)["a.txt"].body) + assert.Len(t, r.files, 1) +} + +func TestResolver_pathsAndContents(t *testing.T) { + r := resolverOver(t, 1<<20, jarEntriesFixture) + + locations, err := r.FilesByPath("/META-INF/MANIFEST.MF") + require.NoError(t, err) + require.Len(t, locations, 1) + + assert.True(t, r.HasPath("/META-INF/MANIFEST.MF")) + assert.True(t, r.HasPath("META-INF/MANIFEST.MF"), "a path may arrive with or without the leading slash") + assert.False(t, r.HasPath("/nope")) + + reader, err := r.FileContentsByLocation(locations[0]) + require.NoError(t, err) + body, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Equal(t, "Manifest-Version: 1.0\n", string(body)) +} + +func TestResolver_contentsSurviveTheMoveToDisk(t *testing.T) { + // room for the index but only a few bytes of content, so the rest moves to disk + r := resolverOver(t, indexCostOf(jarEntriesFixture)+8, jarEntriesFixture) + + locations, err := r.FilesByGlob("**/*.jar") + require.NoError(t, err) + require.Len(t, locations, 1) + + reader, err := r.FileContentsByLocation(locations[0]) + require.NoError(t, err) + body, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Equal(t, "PK\x03\x04 not really", string(body)) +} + +func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) { + r := resolverOver(t, 1<<20, map[string]string{ + "notes.txt": "just some text", + "run.sh": "#!/bin/bash\necho hello\n", + }) + + locations, err := r.FilesByMIMEType("text/x-shellscript") + require.NoError(t, err) + assert.Equal(t, []string{"run.sh"}, realPaths(locations)) +} + +func TestResolver_allLocationsCoversEveryFile(t *testing.T) { + r := resolverOver(t, 1<<20, jarEntriesFixture) + + var got []string + for loc := range r.AllLocations(context.Background()) { + got = append(got, loc.RealPath) + } + sort.Strings(got) + + assert.Equal(t, []string{ + "BOOT-INF/lib/inner.jar", + "LICENSE", + "META-INF/MANIFEST.MF", + "META-INF/maven/com.example/lib/pom.properties", + "META-INF/maven/com.example/lib/pom.xml", + "com/example/Thing.class", + }, got) +} + +func TestResolver_answersAreOrdered(t *testing.T) { + // order decides which locations group into which package when results merge into the SBOM + entries := map[string]string{} + for _, name := range []string{ + "b/z.jar", "a/y.jar", "c/x.jar", "a/b/c/w.jar", "z.jar", "a.jar", + } { + entries[name] = "PK\x03\x04 not really" + } + r := resolverOver(t, 1<<20, entries) + + first, err := r.FilesByGlob("**/*.jar") + require.NoError(t, err) + require.Len(t, first, len(entries)) + + byMIME, err := r.FilesByMIMEType("application/octet-stream") + require.NoError(t, err) + + for range 8 { + again, err := r.FilesByGlob("**/*.jar") + require.NoError(t, err) + assert.Equal(t, first, again) + + mimeAgain, err := r.FilesByMIMEType("application/octet-stream") + require.NoError(t, err) + assert.Equal(t, byMIME, mimeAgain) + + var all []string + for loc := range r.AllLocations(context.Background()) { + all = append(all, loc.RealPath) + } + assert.True(t, sort.StringsAreSorted(all)) + } + + var paths []string + for _, loc := range first { + paths = append(paths, loc.RealPath) + } + assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable") +} + +func TestResolver_impliedDirectoriesArePathsButNotFiles(t *testing.T) { + r := resolverOver(t, 1<<20, jarEntriesFixture) + + assert.True(t, r.HasPath("META-INF")) + assert.True(t, r.HasPath("/META-INF/maven")) + + locations, err := r.FilesByPath("META-INF") + require.NoError(t, err) + assert.Empty(t, locations) + locations, err = r.FilesByGlob("**/maven") + require.NoError(t, err) + assert.Empty(t, locations) +} + +func TestResolver_invalidGlobIsAnError(t *testing.T) { + r := resolverOver(t, 1<<20, jarEntriesFixture) + _, err := r.FilesByGlob("**/[") + assert.Error(t, err) +} + +func globPaths(t *testing.T, r *Resolver, pattern string) []string { + t.Helper() + locations, err := r.FilesByGlob(pattern) + require.NoError(t, err) + return realPaths(locations) +} + +// several paths to one file answer as one location: the file's own path when it matched, otherwise +// the lowest-sorting link +func TestResolver_linksCollapseToOnePath(t *testing.T) { + tests := []struct { + name string + entries []testEntry + pattern string + want []string + reason string + }{ + { + name: "the file itself beats every link to it", + entries: []testEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/a-sym.jar", link: "real.jar"}, + {name: "opt/z-sym.jar", link: "real.jar"}, + {name: "opt/hard.jar", link: "opt/real.jar", hard: true}, + }, + pattern: "**/*.jar", + want: []string{"opt/real.jar"}, + reason: "one file, so one location - and it is named by the path that holds it", + }, + { + name: "with no candidate naming the file, the lowest-sorting link wins", + entries: []testEntry{ + {name: "opt/real.bin", body: "PK\x03\x04"}, + {name: "opt/z-sym.jar", link: "real.bin"}, + {name: "opt/a-sym.jar", link: "real.bin"}, + {name: "opt/m-sym.jar", link: "real.bin"}, + }, + pattern: "**/*.jar", + want: []string{"opt/real.bin"}, + reason: "the content is the target's, so that is the real path the answer carries", + }, + { + name: "distinct files stay distinct", + entries: []testEntry{ + {name: "opt/one.jar", body: "one"}, + {name: "opt/two.jar", body: "two"}, + {name: "opt/one-sym.jar", link: "one.jar"}, + }, + pattern: "**/*.jar", + want: []string{"opt/one.jar", "opt/two.jar"}, + reason: "collapsing is per file, not per pattern", + }, + { + name: "a dangling link keeps its own path", + entries: []testEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/gone.jar", link: "missing.jar"}, + }, + pattern: "**/*.jar", + want: []string{"opt/gone.jar", "opt/real.jar"}, + reason: "it names no content, so it belongs to no other file's group", + }, + { + name: "two dangling links are two paths, not one", + entries: []testEntry{ + {name: "opt/a.jar", link: "missing.jar"}, + {name: "opt/b.jar", link: "missing.jar"}, + }, + pattern: "**/*.jar", + want: []string{"opt/a.jar", "opt/b.jar"}, + reason: "nothing resolves them to a common file, so neither may absorb the other", + }, + { + name: "a chain of links collapses onto the file at its end", + entries: []testEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/mid.jar", link: "real.jar"}, + {name: "opt/outer.jar", link: "mid.jar"}, + }, + pattern: "**/*.jar", + want: []string{"opt/real.jar"}, + reason: "following one hop at a time reaches the same file from every path", + }, + { + name: "a cycle resolves to nothing rather than looping", + entries: []testEntry{ + {name: "opt/a.jar", link: "b.jar"}, + {name: "opt/b.jar", link: "a.jar"}, + }, + pattern: "**/*.jar", + want: []string{"opt/a.jar", "opt/b.jar"}, + reason: "neither reaches content, so each stands under its own path", + }, + { + name: "a hard link resolves from the archive root, not from its own directory", + entries: []testEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/nested/hard.jar", link: "opt/real.jar", hard: true}, + }, + pattern: "**/*.jar", + want: []string{"opt/real.jar"}, + reason: "read as relative it would resolve to opt/nested/opt/real.jar and dangle", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := resolverFrom(t, tt.entries...) + assert.Equal(t, tt.want, globPaths(t, r, tt.pattern), tt.reason) + }) + } +} + +// the choice must not depend on the order the search found the candidates in +func TestResolver_collapseIgnoresTheOrderItFinds(t *testing.T) { + r := resolverFrom(t, + testEntry{name: "opt/real.jar", body: "PK\x03\x04"}, + testEntry{name: "opt/a-sym.jar", link: "real.jar"}, + testEntry{name: "opt/m-sym.jar", link: "real.jar"}, + testEntry{name: "opt/z-sym.jar", link: "real.jar"}, + ) + + candidates := []*node{ + r.byPath["/opt/real.jar"], r.byPath["/opt/a-sym.jar"], + r.byPath["/opt/m-sym.jar"], r.byPath["/opt/z-sym.jar"], + } + for _, n := range candidates { + require.NotNil(t, n) + } + + rng := rand.New(rand.NewSource(1)) + for range 64 { + rng.Shuffle(len(candidates), func(i, j int) { + candidates[i], candidates[j] = candidates[j], candidates[i] + }) + + found := map[*node]struct{}{} + for _, n := range candidates { + found[n] = struct{}{} + } + + best := onePathPerFile(found) + require.Len(t, best, 1, "four paths, one file") + assert.Equal(t, "/opt/real.jar", best[0].path, "the file itself must win from any order") + } + + // and with the file itself not among the candidates, the lowest-sorting link wins from any order + links := candidates[:0:0] + for _, n := range candidates { + if n.path != "/opt/real.jar" { + links = append(links, n) + } + } + for range 64 { + rng.Shuffle(len(links), func(i, j int) { links[i], links[j] = links[j], links[i] }) + + found := map[*node]struct{}{} + for _, n := range links { + found[n] = struct{}{} + } + + best := onePathPerFile(found) + require.Len(t, best, 1) + assert.Equal(t, "/opt/a-sym.jar", best[0].path, "the lowest-sorting link must win from any order") + } +} + +func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) { + r := resolverFrom(t, + testEntry{name: "opt/real.bin", body: "PK\x03\x04 real contents"}, + testEntry{name: "opt/a-sym.jar", link: "real.bin"}, + testEntry{name: "opt/z-sym.jar", link: "real.bin"}, + ) + + locations, err := r.FilesByGlob("**/*.jar") + require.NoError(t, err) + require.Len(t, locations, 1) + + assert.Equal(t, "opt/real.bin", locations[0].RealPath, "content comes from the file") + assert.Equal(t, "opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched") + + reader, err := r.FileContentsByLocation(locations[0]) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reader.Close()) }) + + body, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "PK\x03\x04 real contents", string(body), + "a location standing for a link must still read the file's bytes, not the link's none") +} + +func TestResolver_danglingLinkHasNoContent(t *testing.T) { + r := resolverFrom(t, testEntry{name: "opt/gone.jar", link: "missing.jar"}) + + locations, err := r.FilesByGlob("**/*.jar") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, "opt/gone.jar", locations[0].RealPath) + + reader, err := r.FileContentsByLocation(locations[0]) + require.NoError(t, err) + body, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Empty(t, string(body), "a link to nothing holds nothing") + + metadata, err := r.FileMetadataByLocation(locations[0]) + require.NoError(t, err) + assert.Equal(t, "opt/missing.jar", metadata.LinkDestination, + "where it pointed is still reported, resolved against the link's own directory") +} + +// a link has no content to sniff a type from; the file it points at answers under its own path +func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) { + r := resolverFrom(t, + testEntry{name: "opt/real.jar", body: "PK\x03\x04 zip contents here"}, + testEntry{name: "opt/sym.jar", link: "real.jar"}, + testEntry{name: "opt/hard.jar", link: "opt/real.jar", hard: true}, + testEntry{name: "opt/gone.jar", link: "missing.jar"}, + testEntry{name: "opt/notes.txt", body: "hello world"}, + ) + + byMIME := func(types ...string) []string { + t.Helper() + locations, err := r.FilesByMIMEType(types...) + require.NoError(t, err) + return realPaths(locations) + } + + assert.Equal(t, []string{"opt/real.jar"}, byMIME("application/zip"), + "the file answers, not the two links to it") + assert.Equal(t, []string{"opt/notes.txt"}, byMIME("text/plain")) + assert.Empty(t, byMIME(""), + "a link sniffs to no type, and must not be reachable by asking for that") + assert.Equal(t, []string{"opt/notes.txt", "opt/real.jar"}, byMIME("application/zip", "text/plain")) +} + +// AllLocations enumerates what the archive holds rather than searching for files: the file metadata +// cataloger records a row per path, links included +func TestResolver_allLocationsKeepsEveryPath(t *testing.T) { + r := resolverFrom(t, + testEntry{name: "opt/real.jar", body: "PK\x03\x04"}, + testEntry{name: "opt/sym.jar", link: "real.jar"}, + testEntry{name: "opt/gone.jar", link: "missing.jar"}, + ) + + var got []string + for l := range r.AllLocations(context.Background()) { + got = append(got, l.RealPath) + } + assert.Equal(t, []string{"opt/gone.jar", "opt/real.jar", "opt/sym.jar"}, got, + "every path the archive names, links included") +} diff --git a/internal/archive/storage.go b/internal/archive/storage.go new file mode 100644 index 000000000..9a3c4ccae --- /dev/null +++ b/internal/archive/storage.go @@ -0,0 +1,193 @@ +package archive + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + + "github.com/anchore/syft/internal/log" +) + +// copyChunkSize is how many bytes are read and charged at a time, bounding how far a charge runs +// ahead of what is actually held. +const copyChunkSize = 32 * 1024 + +// spillFilePattern names the one file an archive's content spills into, as in os.CreateTemp. +const spillFilePattern = "archive-spill-*" + +// ReaderAtSeeker is the random access an archive format needs: a zip is read from its central +// directory at the end of the stream. +type ReaderAtSeeker interface { + io.Reader + io.ReaderAt + io.Seeker +} + +// blob is one piece of content the resolver holds: in memory, or at an offset in its spill file once +// memory refused it. Moving changes nothing that points at the blob. +type blob struct { + mem []byte // content while held in memory + off int64 // where content starts in the spill file once written there + size int64 +} + +// put reads content in full into b, since archive readers are sequential. Each chunk is charged before it +// is held, so a stream larger than the memory limit is never held in full. Reaching the disk limit +// returns ErrDiskLimitReached; what landed in the file before that stays there and stays charged. +func (r *Resolver) put(b *blob, content io.Reader) error { + if r.chunk == nil { + r.chunk = make([]byte, copyChunkSize) + } + for { + n, readErr := content.Read(r.chunk) + if n > 0 { + if err := r.write(b, r.chunk[:n]); err != nil { + return err + } + } + if readErr == nil { + continue + } + if errors.Is(readErr, io.EOF) { + break + } + return readErr + } + return nil +} + +// write appends p to b: into memory while the memory limit admits it, otherwise into the spill file +// after everything held in memory is moved there first. +func (r *Resolver) write(b *blob, p []byte) error { + n := int64(len(p)) + onDisk := b.mem == nil && b.size > 0 + if !onDisk { + if r.charge.memory(n) { + if b.mem == nil { + r.held = append(r.held, b) + } + b.mem = append(b.mem, p...) + b.size += n + return nil + } + if err := r.spill(); err != nil { + return err + } + } + + if err := r.openFile(); err != nil { + return err + } + if !r.charge.disk(n) { + return ErrDiskLimitReached + } + if b.size == 0 { + b.off = r.written + } + // b is always the blob most recently written, so it ends where the file does + w, err := r.file.WriteAt(p, r.written) + r.written += int64(w) + b.size += int64(w) + r.charge.refundDisk(n - int64(w)) + if err != nil { + return fmt.Errorf("unable to write archive content to disk: %w", err) + } + return nil +} + +// spill moves every blob held in memory into the spill file, charging the disk limit as bytes land +// and refunding the memory they leave. +func (r *Resolver) spill() error { + if len(r.held) == 0 { + return nil + } + if err := r.openFile(); err != nil { + return err + } + log.WithFields("archive", r.archivePath, "held", len(r.held), "bytes", r.written). + Debug("archive content does not fit in memory; writing it to disk") + + for len(r.held) > 0 { + b := r.held[0] + n := int64(len(b.mem)) + if !r.charge.disk(n) { + return ErrDiskLimitReached + } + if _, err := r.file.WriteAt(b.mem, r.written); err != nil { + return fmt.Errorf("unable to write archive content to disk: %w", err) + } + b.off, b.mem = r.written, nil + r.written += n + r.charge.refundMemory(n) + r.held = r.held[1:] + } + return nil +} + +func (r *Resolver) openFile() error { + if r.file != nil { + return nil + } + var err error + if r.tempDir != nil { + r.file, r.remove, err = r.tempDir.NewFile(spillFilePattern) + } else { + r.file, err = os.CreateTemp("", spillFilePattern) + r.remove = func() { + if err := os.Remove(r.file.Name()); err != nil { + log.WithFields("path", r.file.Name(), "error", err).Trace("unable to remove archive temp file") + } + } + } + if err != nil { + return fmt.Errorf("unable to create archive temp file: %w", err) + } + return nil +} + +// discard drops a blob's bytes held in memory and refunds them. Bytes it holds on disk stay until +// Cleanup. +func (r *Resolver) discard(b *blob) { + for i, held := range r.held { + if held == b { + r.held = append(r.held[:i], r.held[i+1:]...) + break + } + } + r.charge.refundMemory(int64(len(b.mem))) + b.mem, b.size = nil, 0 +} + +// open returns a reader over a blob's content, wherever it is. The reader supports Read, Seek and +// ReadAt, so a nested archive can be read where it lies. +func (r *Resolver) open(b *blob) ReaderAtSeeker { + switch { + case b.mem != nil: + return bytes.NewReader(b.mem) + case b.size == 0: + return bytes.NewReader(nil) + default: + return io.NewSectionReader(r.file, b.off, b.size) + } +} + +// releaseStorage refunds every blob held in memory and closes, removes and refunds the spill file. +// Readers from open must not be used after this. Safe to call more than once. +func (r *Resolver) releaseStorage() { + for _, b := range r.held { + r.charge.refundMemory(int64(len(b.mem))) + b.mem = nil + } + r.held = nil + if r.file == nil { + return + } + if err := r.file.Close(); err != nil { + log.WithFields("archive", r.archivePath, "error", err).Trace("unable to close archive temp file") + } + r.remove() + r.charge.refundDisk(r.written) + r.file, r.remove, r.written = nil, nil, 0 +} diff --git a/internal/archive/storage_test.go b/internal/archive/storage_test.go index c1a85f9ea..65ac029f7 100644 --- a/internal/archive/storage_test.go +++ b/internal/archive/storage_test.go @@ -5,7 +5,6 @@ import ( "bytes" "fmt" "io" - "io/fs" "os" "path/filepath" "strings" @@ -15,46 +14,228 @@ import ( "github.com/stretchr/testify/require" ) -var ( - unboundedLimits = Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1} +func TestResolver_put_holdsSmallContentInMemory(t *testing.T) { + s, dir := resolverIn(t, memCharge(1024)) - // spillingLimits hold nothing in memory, so every entry lands in the overflow file - spillingLimits = Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1} -) + var b blob + require.NoError(t, s.put(&b, bytes.NewReader([]byte("hello")))) + + assert.Equal(t, int64(5), s.heldInMemory()) + assert.Empty(t, filesIn(t, dir)) + assert.Equal(t, "hello", readEntry(t, s.open(&b))) +} + +func TestResolver_put_spillsToDiskAndKeepsTheSameBlobs(t *testing.T) { + // nodes pointing at a blob are not rebuilt when its content moves + s, dir := resolverIn(t, memCharge(8)) + + var first, second blob + require.NoError(t, s.put(&first, bytes.NewReader([]byte("aaaa")))) + require.Equal(t, int64(4), s.heldInMemory()) + + require.NoError(t, s.put(&second, bytes.NewReader([]byte("bbbbbb")))) + + assert.Zero(t, s.heldInMemory(), "everything moves out, not just the blob that did not fit") + assert.NotEmpty(t, spillFile(t, dir)) + assert.Equal(t, int64(10), s.written) + assert.Equal(t, "aaaa", readEntry(t, s.open(&first))) + assert.Equal(t, "bbbbbb", readEntry(t, s.open(&second))) +} + +func TestResolver_put_threeStateLimits(t *testing.T) { + body := []byte(strings.Repeat("e", 500)) + + t.Run("a zero memory limit sends everything to disk", func(t *testing.T) { + charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10_000}).charge() + s, dir := resolverIn(t, charge) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.Zero(t, s.heldInMemory()) + assert.NotEmpty(t, spillFile(t, dir)) + assert.Equal(t, body, []byte(readEntry(t, s.open(&b)))) + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Equal(t, int64(500), disk) + }) + + t.Run("a negative memory limit holds content regardless of how much is already held", func(t *testing.T) { + charge := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: 10_000}).charge() + require.True(t, charge.memory(1_000_000)) + s, dir := resolverIn(t, charge) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.Empty(t, filesIn(t, dir)) + mem, disk := charge.held() + assert.Equal(t, int64(1_000_000+len(body)), mem) + assert.Zero(t, disk) + }) + + t.Run("a zero disk limit still holds what memory admits", func(t *testing.T) { + charge := NewLimiter(Limits{MaxMemoryBytes: 10_000, MaxDiskBytes: 0}).charge() + s, dir := resolverIn(t, charge) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.Empty(t, filesIn(t, dir)) + mem, disk := charge.held() + assert.Equal(t, int64(len(body)), mem) + assert.Zero(t, disk) + }) + + t.Run("a negative disk limit writes with no ceiling", func(t *testing.T) { + s, dir := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}).charge()) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.NotEmpty(t, spillFile(t, dir)) + assert.Equal(t, body, []byte(readEntry(t, s.open(&b)))) + }) + + t.Run("a nil charge holds everything in memory", func(t *testing.T) { + s, dir := resolverIn(t, nil) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.Empty(t, filesIn(t, dir)) + assert.Equal(t, body, []byte(readEntry(t, s.open(&b)))) + }) +} + +func TestResolver_put_bytesReadBeforeTheLimitRefusedAreNotLost(t *testing.T) { + body := []byte(strings.Repeat("b", 250)) + charge := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 10_000}).charge() + s, _ := resolverIn(t, charge) + + var b blob + require.NoError(t, s.put(&b, bytes.NewReader(body))) + + assert.Equal(t, body, []byte(readEntry(t, s.open(&b)))) + mem, disk := charge.held() + assert.Zero(t, mem, "nothing stays charged to a limit that refused") + assert.Equal(t, int64(250), disk) +} + +// peakTrackingReader records the highest memory charge observed while being read. +type peakTrackingReader struct { + r io.Reader + charge *charge + peak int64 +} + +func (p *peakTrackingReader) Read(b []byte) (int, error) { + n, err := p.r.Read(b) + if mem, _ := p.charge.held(); mem > p.peak { + p.peak = mem + } + return n, err +} + +func TestResolver_put_neverBuffersBeyondTheMemoryLimit(t *testing.T) { + const limit = int64(3 * copyChunkSize) + body := []byte(strings.Repeat("c", 10*copyChunkSize)) + charge := NewLimiter(Limits{MaxMemoryBytes: limit, MaxDiskBytes: int64(len(body)) + 1}).charge() + tracked := &peakTrackingReader{r: bytes.NewReader(body), charge: charge} + s, _ := resolverIn(t, charge) + + var b blob + require.NoError(t, s.put(&b, tracked)) + + assert.Equal(t, body, []byte(readEntry(t, s.open(&b)))) + assert.LessOrEqual(t, tracked.peak, limit, "memory charged must never exceed the limit, even transiently") + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Equal(t, int64(len(body)), disk) +} + +func TestResolver_put_diskLimitIsTerminal(t *testing.T) { + tests := []struct { + name string + limits Limits + }{ + {"a disk limit that is exceeded", Limits{MaxDiskBytes: 100}}, + {"a zero disk limit", Limits{MaxMemoryBytes: 10, MaxDiskBytes: 0}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + charge := NewLimiter(tt.limits).charge() + s, dir := resolverIn(t, charge) + + var b blob + err := s.put(&b, bytes.NewReader([]byte(strings.Repeat("d", 500)))) + require.ErrorIs(t, err, ErrDiskLimitReached) + + s.Cleanup() + assert.Empty(t, filesIn(t, dir), "cleanup removes the partial file") + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Zero(t, disk, "and refunds it") + }) + } +} + +func TestResolver_cleanupRefundsEverythingAndIsIdempotent(t *testing.T) { + charge := NewLimiter(Limits{MaxMemoryBytes: 8, MaxDiskBytes: -1}).charge() + s, dir := resolverIn(t, charge) + + var spilled, held blob + require.NoError(t, s.put(&spilled, bytes.NewReader([]byte("spilled to disk")))) + require.NoError(t, s.put(&held, bytes.NewReader([]byte("held")))) + mem, disk := charge.held() + require.Equal(t, int64(4), mem) + require.Equal(t, int64(15), disk) + require.Equal(t, "held", readEntry(t, s.open(&held))) + + s.Cleanup() + + assert.Empty(t, filesIn(t, dir)) + mem, disk = charge.held() + assert.Zero(t, mem) + assert.Zero(t, disk) + _, err := io.ReadAll(s.open(&spilled)) + assert.Error(t, err, "readers over the spill file are no longer valid") + s.Cleanup() + mem, disk = charge.held() + assert.Zero(t, mem, "a second cleanup refunds nothing twice") + assert.Zero(t, disk) +} + +func TestResolver_put_discardRefundsWhatIsHeldInMemory(t *testing.T) { + charge := memCharge(1024) + s, _ := resolverIn(t, charge) + + var keep, drop blob + require.NoError(t, s.put(&keep, bytes.NewReader([]byte("keep")))) + require.NoError(t, s.put(&drop, bytes.NewReader([]byte("dropped")))) + + s.discard(&drop) + + mem, _ := charge.held() + assert.Equal(t, int64(4), mem) + assert.Equal(t, int64(4), s.heldInMemory()) + assert.Empty(t, readEntry(t, s.open(&drop))) + assert.Equal(t, "keep", readEntry(t, s.open(&keep))) +} func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) { - _, smallStore, smallRoot := extractedStore(t, manyEntryZip(t, 2), spillingLimits) - _, largeStore, largeRoot := extractedStore(t, manyEntryZip(t, 5000), spillingLimits) + small, smallRoot := extractedResolver(t, manyEntryZip(t, 2), spillingLimits) + large, largeRoot := extractedResolver(t, manyEntryZip(t, 5000), spillingLimits) - want := []string{entriesFileName} - assert.Equal(t, want, filesystemEntries(t, workDirIn(t, smallRoot))) - assert.Equal(t, want, filesystemEntries(t, workDirIn(t, largeRoot)), "5000 entries cost the same filesystem entries as 2") + assert.Len(t, filesIn(t, smallRoot), 1) + assert.Len(t, filesIn(t, largeRoot), 1, "5000 entries cost the same filesystem entries as 2") - assert.Len(t, smallStore.Entries(), 2) - assert.Len(t, largeStore.Entries(), 5000) + assert.Len(t, small.files, 2) + assert.Len(t, large.files, 5000) // entries cost their content and nothing else on disk: no headers, no padding - assert.Equal(t, int64(2), smallStore.OnDisk()) - assert.Equal(t, int64(5000), largeStore.OnDisk()) -} - -func TestStorage_anArchiveWithinTheMemoryBoundTouchesNoFilesystemAtAll(t *testing.T) { - _, store, root := extractedStore(t, manyEntryZip(t, 5000), unboundedLimits) - - assert.Len(t, store.Entries(), 5000) - assert.Zero(t, store.OnDisk()) - assert.Empty(t, filesystemEntries(t, root)) -} - -func TestStorage_cleanupRemovesTheWorkDirectory(t *testing.T) { - extracted, _, root := extractedStore(t, manyEntryZip(t, 5000), spillingLimits) - workDir := workDirIn(t, root) - require.Len(t, filesystemEntries(t, workDir), 1) - - extracted.Cleanup() - - assert.NoDirExists(t, workDir) - extracted.Cleanup() + assert.Equal(t, int64(2), small.written) + assert.Equal(t, int64(5000), large.written) } func TestStorage_anEntryAtTheEndIsReadBySeeking(t *testing.T) { @@ -76,18 +257,17 @@ func TestStorage_anEntryAtTheEndIsReadBySeeking(t *testing.T) { require.NoError(t, err) require.NoError(t, zw.Close()) - _, store, root := extractedStore(t, buf.Bytes(), spillingLimits) + r, root := extractedResolver(t, buf.Bytes(), spillingLimits) - body, err := os.ReadFile(filepath.Join(workDirIn(t, root), entriesFileName)) + body, err := os.ReadFile(filepath.Join(root, spillFile(t, root))) require.NoError(t, err) offset := int64(bytes.Index(body, []byte(wanted))) require.Greater(t, offset, int64(len(body))*9/10, "the last entry must really be at the end of the file") - entries := store.Entries() - entry := entries[len(entries)-1] - require.Equal(t, "zz-last.txt", entry.Header.Name) + n := r.byPath["/zz-last.txt"] + require.NotNil(t, n) - reader := store.Open(entry) + reader := r.open(&n.content) end, err := reader.Seek(0, io.SeekEnd) require.NoError(t, err) assert.Equal(t, int64(len(wanted)), end) @@ -97,49 +277,3 @@ func TestStorage_anEntryAtTheEndIsReadBySeeking(t *testing.T) { require.NoError(t, err) assert.Equal(t, wanted[:8], string(head)) } - -func manyEntryZip(t *testing.T, count int) []byte { - t.Helper() - files := make(map[string]string, count) - for i := range count { - files[fmt.Sprintf("e%06d.txt", i)] = "x" - } - return zipBytes(t, files) -} - -// extractedStore extracts one archive and returns the store its entries went into, along with the -// scan temp root its work directory would appear under. -func extractedStore(t *testing.T, data []byte, limits Limits) (*Extracted, *EntryStore, string) { - t.Helper() - ctx, root := scanContext(t) - - extracted, err := Extract(ctx, bytes.NewReader(data), "", "app.zip", NewLimiter(limits), nil) - require.NoError(t, err) - require.NotNil(t, extracted) - t.Cleanup(extracted.Cleanup) - - return extracted, extracted.Resolver.(*Index).store, root -} - -func workDirIn(t *testing.T, root string) string { - t.Helper() - dirs := workDirsUnder(t, root) - require.Len(t, dirs, 1) - return dirs[0] -} - -// filesystemEntries is every directory entry under dir, recursively. -func filesystemEntries(t *testing.T, dir string) []string { - t.Helper() - var out []string - require.NoError(t, filepath.WalkDir(dir, func(p string, _ fs.DirEntry, err error) error { - if err != nil { - return err - } - if rel, _ := filepath.Rel(dir, p); rel != "." { - out = append(out, filepath.ToSlash(rel)) - } - return nil - })) - return out -} diff --git a/internal/archive/workdir.go b/internal/archive/workdir.go deleted file mode 100644 index 8a7ffd6cf..000000000 --- a/internal/archive/workdir.go +++ /dev/null @@ -1,71 +0,0 @@ -package archive - -import ( - "context" - "fmt" - "os" - - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/internal/tmpdir" -) - -// workDirName prefixes the directory holding one archive's overflow files. -const workDirName = "syft-archive" - -// WorkDir is the scratch directory one archive writes overflow into. It is created on first use, so -// an archive that stays in memory never touches the filesystem. -type WorkDir struct { - tempDir *tmpdir.TempDir - - created bool - path string - remove func() - err error -} - -// NewWorkDir returns a work directory under the scan's temp root when ctx carries one (see -// internal/tmpdir), otherwise under the system temp directory. -func NewWorkDir(ctx context.Context) *WorkDir { - return &WorkDir{tempDir: tmpdir.FromContext(ctx)} -} - -// Path returns the directory, creating it on the first call. -func (w *WorkDir) Path() (string, error) { - if w == nil { - return "", fmt.Errorf("no work directory for archive content") - } - if !w.created { - w.created = true - w.path, w.remove, w.err = w.create() - } - return w.path, w.err -} - -func (w *WorkDir) create() (string, func(), error) { - if w.tempDir != nil { - dir, remove, err := w.tempDir.NewChild(workDirName) //nolint:gocritic // the cleanup is returned, not deferred - if err != nil { - return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err) - } - return dir, remove, nil - } - - dir, err := os.MkdirTemp("", workDirName+"-") - if err != nil { - return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err) - } - return dir, func() { - if err := os.RemoveAll(dir); err != nil { - log.WithFields("dir", dir, "error", err).Trace("unable to remove archive temp dir") - } - }, nil -} - -// Remove deletes the directory if it was created. Safe to call more than once. -func (w *WorkDir) Remove() { - if w == nil || w.remove == nil { - return - } - w.remove() - w.remove = nil -} diff --git a/internal/archive/workdir_test.go b/internal/archive/workdir_test.go deleted file mode 100644 index 10453d154..000000000 --- a/internal/archive/workdir_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package archive - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWorkDir_isNotCreatedUntilAskedFor(t *testing.T) { - ctx, root := scanContext(t) - w := NewWorkDir(ctx) - - assert.False(t, w.wasCreated()) - assert.Empty(t, workDirsUnder(t, root)) - - dir, err := w.Path() - require.NoError(t, err) - assert.DirExists(t, dir) - assert.True(t, w.wasCreated()) - assert.Equal(t, []string{dir}, workDirsUnder(t, root)) -} - -func TestWorkDir_pathIsStableAcrossCalls(t *testing.T) { - ctx, _ := scanContext(t) - w := NewWorkDir(ctx) - - first, err := w.Path() - require.NoError(t, err) - second, err := w.Path() - require.NoError(t, err) - assert.Equal(t, first, second) -} - -func TestWorkDir_removeIsANoOpWhenNothingWasCreated(t *testing.T) { - ctx, root := scanContext(t) - w := NewWorkDir(ctx) - - w.Remove() - w.Remove() - - assert.Empty(t, workDirsUnder(t, root)) -} - -func TestWorkDir_removeDeletesWhatWasCreated(t *testing.T) { - ctx, _ := scanContext(t) - w := NewWorkDir(ctx) - - dir, err := w.Path() - require.NoError(t, err) - require.NoError(t, os.WriteFile(filepath.Join(dir, "blob"), []byte("x"), 0o600)) - - w.Remove() - assert.NoDirExists(t, dir) - w.Remove() -} - -func TestWorkDir_nilHasNowhereToWrite(t *testing.T) { - var w *WorkDir - - _, err := w.Path() - assert.Error(t, err) - assert.False(t, w.wasCreated()) - w.Remove() -} diff --git a/internal/task/archive_nesting_test.go b/internal/task/archive_nesting_test.go index a9afab352..31a3aba17 100644 --- a/internal/task/archive_nesting_test.go +++ b/internal/task/archive_nesting_test.go @@ -88,9 +88,9 @@ type nestingRow struct { type nestingVisit struct { virtualPath string depth int - // heldFiles are the basenames of the files in the live extraction work directories at this moment, - // sampled only when the probe was given a temp dir to look in. An archive's own bytes are read where - // they lie, so only "entries" files appear, one per archive whose entries did not fit in memory. + // heldFiles are the spill files live at this moment, sampled only when the probe was given a temp + // dir to look in. A nested archive's own bytes are read where they lie, so one appears per archive + // whose entries did not fit in memory. heldFiles []string } @@ -316,15 +316,16 @@ func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing }) t.Run("with a memory limit the outermost archive's entries do not fit in", func(t *testing.T) { + // room for every level's index estimate, which must stay in memory, but not the outer entries probe := &nestingProbe{tempDir: tempDir} bounds := cataloging.DefaultArchiveSearchConfig(). - WithMaxMemoryBytes(int64(nested.sizes[2])). + WithMaxMemoryBytes(int64(nested.sizes[2] + 32*1024)). WithMaxDiskBytes(-1) s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) deepest := probe.deepestVisit() require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held") - assert.Contains(t, deepest.heldFiles, "entries", "entries that do not fit in memory are written to disk") + assert.Contains(t, deepest.heldFiles, "archive-spill", "entries that do not fit in memory are written to disk") assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(), "and every level is still cataloged") locs := leafLocations(s) @@ -335,8 +336,8 @@ func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) { // an archive's own bytes are read where they lie: a scanned file from disk, a nested archive from - // its parent's entries. Nothing copies them, so no "archive" file ever appears in a work directory - // whatever the memory limit; only entries are held, and only they can be written out. + // its parent's entries. Nothing copies them, so at most one spill file per level appears whatever + // the memory limit; only entries are held, and only they can be written out. scanDir := t.TempDir() tempDir := isolatedTempDir(t) nested := writeNestedArchive(t, scanDir, gradedChain()) @@ -353,7 +354,7 @@ func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) { { name: "with a memory limit of zero, so every level's entries are written out", memoryBytes: 0, - wantHeldFiles: []string{"entries", "entries", "entries"}, + wantHeldFiles: []string{"archive-spill", "archive-spill", "archive-spill"}, }, } { t.Run(tc.name, func(t *testing.T) { @@ -725,24 +726,17 @@ func (p *nestingProbe) deepestVisit() nestingVisit { return best } -// heldFiles names the files in the live extraction work directories under tempDir. Meaningful only -// while the archives are still held, so it is sampled from inside the sub-pipeline rather than after -// the scan. +// heldFiles names the spill files live under tempDir, one "archive-spill" per archive holding content +// that did not fit in memory. Meaningful only while the archives are still held, so it is sampled from +// inside the sub-pipeline rather than after the scan. func heldFiles(tempDir string) (names []string) { - _ = filepath.WalkDir(tempDir, func(path string, d fs.DirEntry, err error) error { - if err != nil || !d.IsDir() || !strings.HasPrefix(d.Name(), "syft-archive-") { + _ = filepath.WalkDir(tempDir, func(_ string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasPrefix(d.Name(), "archive-") { return nil //nolint:nilerr // an unreadable directory holds nothing this probe can report } - held, err := os.ReadDir(path) - if err != nil { - return filepath.SkipDir - } - for _, entry := range held { - if !entry.IsDir() { - names = append(names, entry.Name()) - } - } - return filepath.SkipDir + // the name is the pattern with os.CreateTemp's random suffix in place of the "*" + names = append(names, d.Name()[:strings.LastIndexByte(d.Name(), '-')]) + return nil }) sort.Strings(names) return names diff --git a/internal/task/archive_tasks.go b/internal/task/archive_tasks.go index f7d59196e..009ef6061 100644 --- a/internal/task/archive_tasks.go +++ b/internal/task/archive_tasks.go @@ -175,7 +175,7 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi if extracted.Truncated { appendUnknowns(builder, ArchiveCatalogerTaskName, []unknown.CoordinateError{{ Coordinates: location.Coordinates, - Reason: fmt.Errorf("archive cataloged from part of its contents: extraction stopped at the disk limit"), + Reason: fmt.Errorf("archive cataloged from part of its contents: extraction stopped at the configured memory or disk limit"), }}) } @@ -183,7 +183,7 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi ctx = archive.WithTraversal(ctx, traversal) c.progress.Increment() - scratch := c.runSubPipeline(ctx, extracted.Resolver, location.Coordinates) + scratch := c.runSubPipeline(ctx, extracted, location.Coordinates) mergeArchiveResults(location.Coordinates, scratch, builder) c.progress.AtomicStage.Set(fmt.Sprintf("%s archives (%s)", humanize.Comma(c.progress.Current()), archivePath)) @@ -192,7 +192,7 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi c.slowest, c.slowestPath = took, archivePath } - return c.catalog(ctx, extracted.Resolver, depth+1, builder) + return c.catalog(ctx, extracted, depth+1, builder) } // runSubPipeline runs every task against the resolver into a throwaway SBOM, so results can be