diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index 9a7072a7d..1701be24a 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -120,9 +120,8 @@ func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig { } func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig { - // the exclusion patterns come from the same --exclude flag that reaches the source, narrowed to - // the ones whose shape reaches inside an archive. CreateSBOM fills the field from the source only - // when it is empty, so this value wins. + // exclusion patterns come from --exclude, narrowed to those that reach inside an archive. CreateSBOM + // fills this field from the source only when empty, so this value wins. return cataloging.DefaultArchiveSearchConfig(). WithIncludeIndexedArchives(cfg.Package.SearchIndexedArchives). WithIncludeUnindexedArchives(cfg.Package.SearchUnindexedArchives). diff --git a/cmd/syft/internal/options/pkg.go b/cmd/syft/internal/options/pkg.go index 24f5f5a50..1cc238a01 100644 --- a/cmd/syft/internal/options/pkg.go +++ b/cmd/syft/internal/options/pkg.go @@ -26,18 +26,9 @@ note: enabling this may result in a performance impact since all discovered comp note: for now this only applies to the java package cataloger`) descriptions.Add(&o.ExcludeBinaryOverlapByOwnership, `allows users to exclude synthetic binary packages from the sbom these packages are removed if an overlap with a non-synthetic package is found`) - descriptions.Add(&o.NestedArchiveMaxDepth, `maximum depth to recursively catalog nested archives, treating each archive as its own filesystem (0 = disabled, -1 = unlimited) -when enabled, all archives (including java archives) are recursively cataloged by the archive cataloger -and the java cataloger does not unarchive nested archives itself`) - descriptions.Add(&o.NestedArchiveMaxMemoryBytes, `maximum bytes of nested archive content held in memory at once (0 = none, always overflow to disk; negative = unbounded) -this is an in-use limit: it falls as each archive is released, so it bounds a scan's peak rather than its total -content is held in memory while this limit admits it and overflowed to disk instead of being refused when it does not -there is no separate setting for the size at which content overflows; this limit alone decides it -only applies when nested-archive-max-depth is enabled`) - descriptions.Add(&o.NestedArchiveMaxDiskBytes, `maximum bytes of nested archive content on disk at once, overflow content plus extracted entries (0 = none, nothing overflows; negative = unbounded) -this is an in-use limit: it falls as each archive is released, so it bounds a scan's peak rather than its total -an archive that would exceed it is skipped, since there is nowhere further to overflow -only applies when nested-archive-max-depth is enabled`) + descriptions.Add(&o.NestedArchiveMaxDepth, `maximum depth to recursively catalog nested archives (0 = disabled, -1 = unlimited)`) + descriptions.Add(&o.NestedArchiveMaxMemoryBytes, `maximum bytes of nested archive content held in memory at once (0 = none, always overflow to disk; -1 = unbounded)`) + descriptions.Add(&o.NestedArchiveMaxDiskBytes, `maximum bytes of nested archive content on disk at once (0 = none, nothing overflows; -1 = unbounded)`) } func defaultPackageConfig() packageConfig { diff --git a/cmd/syft/internal/options/pkg_test.go b/cmd/syft/internal/options/pkg_test.go index 579254ec5..eb4a3f354 100644 --- a/cmd/syft/internal/options/pkg_test.go +++ b/cmd/syft/internal/options/pkg_test.go @@ -11,8 +11,6 @@ import ( ) func Test_packageConfigDefaultsMatchArchiveSearchConfig(t *testing.T) { - // the nested-archive settings must default to the values in syft/cataloging rather than - // restating them, so a config round trip with nothing set is a no-op want := cataloging.DefaultArchiveSearchConfig() got := defaultPackageConfig() @@ -24,9 +22,9 @@ func Test_packageConfigDefaultsMatchArchiveSearchConfig(t *testing.T) { } func Test_packageConfigNamesNoArchiveExclusionSetting(t *testing.T) { - // nested archive cataloging honors the scan's own --exclude patterns, so the application config - // must name no archive-specific exclusion key. Asserted over the yaml tags rather than over the - // Go field names, because the key is what a user writes and what a config dump shows. + // nested archive cataloging honors the scan's own --exclude patterns, so the application config must + // name no archive-specific exclusion key. Asserted over the yaml tags rather than the Go field names: + // the key is what a user writes and what a config dump shows. var keys []string typ := reflect.TypeOf(packageConfig{}) for i := 0; i < typ.NumField(); i++ { @@ -56,8 +54,8 @@ func Test_ToArchiveConfig(t *testing.T) { }) t.Run("every setting is threaded through", func(t *testing.T) { - // a key that parses but is not carried into the archive config is invisible in a - // config dump, so assert each one arrives + // a key that parses but is not carried into the archive config is invisible in a config + // dump, so assert each one arrives cfg := Catalog{Package: packageConfig{ NestedArchiveMaxDepth: 3, NestedArchiveMaxMemoryBytes: 111, @@ -75,11 +73,10 @@ func Test_ToArchiveConfig(t *testing.T) { }) t.Run("--exclude patterns arrive already filtered to the in-scope subset, with no source involved", func(t *testing.T) { - // every-boundary-populates-the-patterns: the CLI is one of the two boundaries that fills - // ExclusionPatterns, from Catalog.Exclusions, applying the same shape rule the library - // entry point applies to what the source publishes - only a pattern that reaches inside an - // archive (any-depth, "**/") is in scope. A root-anchored or one-level pattern is not, and - // no source is consulted to produce this: Catalog carries no source at all. + // every-boundary-populates-the-patterns: the CLI is one of the two boundaries filling + // ExclusionPatterns, from Catalog.Exclusions, applying the same shape rule the library entry + // point applies to what the source publishes - only an any-depth ("**/") pattern is in scope. + // A root-anchored or one-level pattern is not, and no source is consulted: Catalog carries none. cfg := Catalog{Exclusions: []string{"./root-anchored", "**/*.rpm", "*/one-level", "**/vendor"}} assert.Equal(t, []string{"**/*.rpm", "**/vendor"}, cfg.ToArchiveConfig().ExclusionPatterns) }) diff --git a/cmd/syft/internal/test/integration/package_catalogers_represented_test.go b/cmd/syft/internal/test/integration/package_catalogers_represented_test.go index 92d04c08d..ec7452eb8 100644 --- a/cmd/syft/internal/test/integration/package_catalogers_represented_test.go +++ b/cmd/syft/internal/test/integration/package_catalogers_represented_test.go @@ -50,13 +50,7 @@ func TestAllPackageCatalogersReachableInTasks(t *testing.T) { constructorCount += constructors.Size() } - // java has two archive catalogers - one that reads an archive file and one that reads an archive - // already extracted to a filesystem - and the java archive slot in the task list yields whichever - // the config selects, never both. So exactly one constructor has no task under the default config. - const configSelectedAlternates = 1 - - assert.Equal(t, len(taskTagsByName)+configSelectedAlternates, constructorCount, - "mismatch in number of cataloger constructors and task names") + assert.Equal(t, len(taskTagsByName), constructorCount, "mismatch in number of cataloger constructors and task names") exceptions := strset.New( // not reachable since they are deprecated diff --git a/cmd/syft/internal/ui/ui.go b/cmd/syft/internal/ui/ui.go index 0db0d32ca..9cb80fa21 100644 --- a/cmd/syft/internal/ui/ui.go +++ b/cmd/syft/internal/ui/ui.go @@ -186,10 +186,8 @@ func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) { func (m UI) View() string { v := m.frame.View() - // always end on an empty line: bubbletea erases the line the cursor rests on when the program stops, so - // a frame whose last line carries content loses it as the run finishes. Until now a log line in - // the footer happened to supply that newline, which is why this only showed on sources that log - // nothing at all. + // bubbletea erases the line the cursor rests on at exit, so end on a newline or a frame's last line + // of content is lost if !strings.HasSuffix(v, "\n") { return v + "\n" } diff --git a/cmd/syft/internal/ui/ui_test.go b/cmd/syft/internal/ui/ui_test.go index ec8b3ecd7..23e344814 100644 --- a/cmd/syft/internal/ui/ui_test.go +++ b/cmd/syft/internal/ui/ui_test.go @@ -14,18 +14,18 @@ var _ tea.Model = (*staticModel)(nil) type staticModel string -func (m staticModel) Init() tea.Cmd { return nil } - -func (m staticModel) Update(tea.Msg) (tea.Model, tea.Cmd) { return m, nil } - -func (m staticModel) View() string { return string(m) } - func Test_UI_viewEndsOnAnEmptyLine(t *testing.T) { // bubbletea erases the line the cursor rests on when the program stops, so a view whose last line - // carries content loses that line as the run finishes. Until a log line happened to be in the - // footer, that is what dropped the last row of the cataloging tree on any source that logs nothing. + // carries content loses it. Unless a log line happened to be in the footer, that dropped the last row + // of the cataloging tree on any source that logs nothing. subject := New(io.Discard, false) subject.frame.(*frame.Frame).AppendModel(staticModel("first row\nlast row")) require.Equal(t, "first row\nlast row\n", subject.View()) } + +func (m staticModel) Init() tea.Cmd { return nil } + +func (m staticModel) Update(tea.Msg) (tea.Model, tea.Cmd) { return m, nil } + +func (m staticModel) View() string { return string(m) } diff --git a/internal/archive/content.go b/internal/archive/content.go index ccf13ef70..c73a57f9a 100644 --- a/internal/archive/content.go +++ b/internal/archive/content.go @@ -6,19 +6,17 @@ import ( "fmt" "io" "os" - "path/filepath" intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" ) -// copyChunkSize is how much is read, charged and written at a time. It bounds how far a charge can -// be ahead of what has actually been written, and matches io.Copy's own buffer size. +// copyChunkSize is the read/charge/write chunk size, bounding how far a charge runs ahead of what is +// written. Matches io.Copy's buffer size. const copyChunkSize = 32 * 1024 -// ReaderAtSeeker is the random access an archive format needs. A zip reads its central directory -// from the end of the stream, so a plain io.Reader will not do; both a *bytes.Reader over content -// held in memory and an *os.File over content overflowed to disk satisfy it. +// ReaderAtSeeker is the random access an archive format needs: a zip reads its central directory from +// the end of the stream, so a plain io.Reader will not do. type ReaderAtSeeker interface { io.Reader io.ReaderAt @@ -26,9 +24,6 @@ type ReaderAtSeeker interface { } // Content is one archive's bytes, wherever the limiter routed them. -// -// Name is the archive's file name and is used for format identification only - never to open -// anything - so content held in memory needs no file on disk at all. type Content struct { Name string Reader ReaderAtSeeker @@ -45,43 +40,22 @@ func (c Content) Close() error { return c.closer.Close() } -// OpenFileContent opens an archive already on disk as Content. The caller closes it. -func OpenFileContent(path string) (Content, error) { - f, err := os.Open(path) - if err != nil { - return Content{}, err - } - return Content{Name: filepath.Base(path), Reader: f, closer: f}, nil -} - -// holdContent routes one archive's bytes: held in memory and charged to the memory limit while that -// limit admits them, overflowed into workDir and charged to the disk limit once it does not. Where the -// boundary falls is decided entirely by the memory limit - there is no separate configured size at -// which overflowing begins. +// holdContent holds one archive's bytes in memory while the memory limit admits them, and spills into +// workDir against the disk limit once it does not. Reads and charges copyChunkSize at a time so an +// oversized archive is never held past the limit. A zero memory limit spills in full; a negative one +// never refuses. // -// Content is read and charged incrementally, copyChunkSize at a time, rather than read speculatively -// into a buffer ahead of any charge: nothing is ever held beyond what the memory limit would admit at -// that moment, so a single archive far larger than the limit is never buffered past it. A zero -// memory limit refuses every chunk, so its content overflows to disk in full; a negative memory limit -// never refuses, so nothing overflows for want of memory. -// -// When the limit refuses a chunk part way through, what has already been read leads the overflow rather -// than being re-read from the start or lost, and the memory it was charged against is refunded since -// it is about to be charged to disk instead. -// -// Memory pressure degrades to disk - content that would take memory in use past the memory limit -// overflows instead of being refused, because there is somewhere for it to go. The disk limit is -// terminal: there is nowhere further, so content that would take it past the disk limit yields -// ErrDiskLimitReached and the archive is skipped. Nothing waits for capacity; see ExtractToResolver. -func holdContent(r io.Reader, workDir, name string, charge *Charge, notify Notify) (Content, error) { +// On a refused chunk, held bytes and the rest of the stream spill without re-reading, and the memory +// charge is refunded as it is charged to disk. The disk limit is terminal: exceeding it yields +// ErrDiskLimitReached and the archive is skipped (see ExtractToResolver). +func holdContent(r io.Reader, workDir *WorkDir, name string, charge *Charge, notify Notify) (Content, error) { var held bytes.Buffer chunk := make([]byte, copyChunkSize) for { n, readErr := r.Read(chunk) if n > 0 { if !charge.Memory(int64(n)) { - // the memory limit refused this chunk: overflow what is already held plus this chunk - // plus whatever remains, without re-reading from the start + // spill held bytes, this chunk, and the rest without re-reading if notify != nil { notify(ContentOverflowed{Archive: name, Bytes: int64(held.Len() + n), Reason: overflowReason(charge)}) } @@ -99,20 +73,24 @@ func holdContent(r io.Reader, workDir, name string, charge *Charge, notify Notif } } -// overflowContent writes an archive's bytes into workDir, charging the disk limit as they land, and -// returns a reader over the file positioned at its start. +// overflowContent writes an archive's bytes into its work directory, charging the disk limit as they +// land, and returns a reader over the file positioned at its start. The work directory is created +// here on first write, so an in-memory archive never creates one. // -// This is the only filesystem write left in the extraction path - an archive's entries are headers in -// one tar and traverse no path - and the name it writes under is derived from where the archive was -// found, which inside another archive is text the archive supplied. So the destination is checked the -// same two ways an entry's path used to be: lexically, and then against where it really lands on the -// filesystem. -func overflowContent(r io.Reader, workDir, name string, charge *Charge) (Content, error) { - dest, err := intFile.SafeJoin(workDir, name) +// This is the only filesystem write in the extraction path, and the name derives from where the +// archive was found - attacker-controlled inside another archive - so the destination is checked +// both lexically and against where it really lands. +func overflowContent(r io.Reader, workDir *WorkDir, name string, charge *Charge) (Content, error) { + dir, err := workDir.Path() + if err != nil { + return Content{}, err + } + + dest, err := intFile.SafeJoin(dir, name) if err != nil { return Content{}, fmt.Errorf("refusing to overflow archive content to %q: %w", name, err) } - if inside, err := resolvesInsideRoot(workDir, dest); err != nil || !inside { + if inside, err := resolvesInsideRoot(dir, dest); err != nil || !inside { return Content{}, fmt.Errorf("refusing to overflow archive content outside its work directory: %q", name) } f, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) @@ -144,10 +122,9 @@ func overflowContent(r io.Reader, workDir, name string, charge *Charge) (Content return Content{Name: name, Reader: f, closer: f}, nil } -// copyCharged copies src into dst, charging every chunk to the disk limit before it is written, so -// what bounds the copy is what actually lands rather than any size the archive declared. It stops at -// the first refused charge and says so, leaving the caller to decide whether that truncates one -// entry or skips a whole archive. +// copyCharged copies src into dst, charging each chunk to the disk limit before writing, so the copy +// is bounded by what lands rather than any declared size. It stops at the first refused charge and +// reports it; the caller decides whether that truncates one entry or skips the archive. func copyCharged(dst io.Writer, src io.Reader, charge *Charge) (written int64, limitReached bool, err error) { buf := make([]byte, copyChunkSize) for { diff --git a/internal/archive/content_test.go b/internal/archive/content_test.go index fe64fe405..ed701beb5 100644 --- a/internal/archive/content_test.go +++ b/internal/archive/content_test.go @@ -12,8 +12,212 @@ import ( "github.com/stretchr/testify/require" ) +func TestHoldContent_heldWhileTheLimitAdmitsIt(t *testing.T) { + // an archive small enough to hold needs no file on disk at all + workDir := t.TempDir() + body := []byte(strings.Repeat("a", 100)) + limiter := NewLimiter(Limits{MaxMemoryBytes: 1000, MaxDiskBytes: 1000}) + charge := limiter.Charge() + + held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, held.Close()) }) + + assert.IsType(t, &bytes.Reader{}, held.Reader) + assert.Equal(t, body, readAll(t, held)) + assert.Empty(t, filesIn(t, workDir), "content held in memory must write nothing to disk") + + mem, disk := charge.held() + assert.Equal(t, int64(100), mem) + assert.Zero(t, disk) +} + +func TestHoldContent_overflowsWhenTheLimitDoesNotAdmitIt(t *testing.T) { + // the memory limit alone decides where the boundary falls: an archive that does not fit spills in + // full, and the bytes already read to discover that lead the spill rather than being lost + workDir := t.TempDir() + body := []byte(strings.Repeat("b", 250)) + limiter := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 10_000}) + charge := limiter.Charge() + + held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, held.Close()) }) + + assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir)) + assert.Equal(t, body, readAll(t, held), + "the bytes read to decide the routing must lead the overflow rather than being lost") + + mem, disk := charge.held() + assert.Zero(t, mem, "nothing is charged to a limit that refused") + assert.Equal(t, int64(250), disk) +} + +// peakTrackingReader records the highest memory charge observed while reading, which is how a test +// sees that a large archive was never buffered beyond what the limit admits; the routing decision is +// not otherwise observable from outside holdContent. +type peakTrackingReader struct { + r io.Reader + charge *Charge + peak int64 +} + +func TestHoldContent_largeArchiveNeverBuffersBeyondTheLimit(t *testing.T) { + // an archive many times the memory limit must spill, and at no point may more than the limit admits + // be held while the routing is decided: it is never read speculatively + workDir := t.TempDir() + const limit = int64(3 * copyChunkSize) + body := []byte(strings.Repeat("c", 10*copyChunkSize)) + limiter := NewLimiter(Limits{MaxMemoryBytes: limit, MaxDiskBytes: int64(len(body)) + 1}) + charge := limiter.Charge() + tracked := &peakTrackingReader{r: bytes.NewReader(body), charge: charge} + + held, err := holdContent(tracked, WorkDirAt(workDir), "app.zip", charge, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, held.Close()) }) + + assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir), "an archive this much larger than the limit must overflow") + assert.Equal(t, body, readAll(t, held)) + assert.LessOrEqual(t, tracked.peak, limit, + "memory charged must never exceed the limit, even transiently, while routing is decided") + + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Equal(t, int64(len(body)), disk) +} + +func TestHoldContent_diskLimitIsTerminal(t *testing.T) { + // there is nowhere further to spill, so content that will not fit is refused and the caller skips the + // archive rather than waiting for capacity nothing will release + workDir := t.TempDir() + body := []byte(strings.Repeat("d", 500)) + limiter := NewLimiter(Limits{MaxDiskBytes: 100}) + charge := limiter.Charge() + + _, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) + require.ErrorIs(t, err, ErrDiskLimitReached) + + assert.Empty(t, filesIn(t, workDir), "the partial overflow must not be left behind") + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Zero(t, disk, "bytes no longer on disk must be refunded, or the limit only ever rises") +} + +func TestHoldContent_zeroDiskLimitSkipsRatherThanFails(t *testing.T) { + // a zero disk limit behaves like a positive one exceeded: ErrDiskLimitReached, so the caller skips + // the archive rather than the scan failing + workDir := t.TempDir() + body := []byte(strings.Repeat("f", 500)) + limiter := NewLimiter(Limits{MaxMemoryBytes: 10, MaxDiskBytes: 0}) + charge := limiter.Charge() + + _, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) + require.ErrorIs(t, err, ErrDiskLimitReached) + + assert.Empty(t, filesIn(t, workDir), "no partial overflow is left behind") + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Zero(t, disk) +} + +func TestHoldContent_threeStateLimits(t *testing.T) { + // both limits read positive/zero/negative the same way - the limit, none of that resource, unbounded + // - and a caller can bound one without the other + body := []byte(strings.Repeat("e", 500)) + + t.Run("a zero memory limit sends everything to disk", func(t *testing.T) { + // zero does not mean unlimited memory: the limit decides a routing question, so zero means the + // memory side of that routing is never taken + workDir := t.TempDir() + limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10_000}) + charge := limiter.Charge() + + held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, held.Close()) }) + + assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir), "a zero memory limit holds nothing in memory") + mem, disk := charge.held() + assert.Zero(t, mem) + assert.Equal(t, int64(500), disk, "and the disk limit still applies to what it overflowed") + }) + + t.Run("a negative memory limit holds content regardless of how much is already held", func(t *testing.T) { + workDir := t.TempDir() + limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: 10_000}) + charge := limiter.Charge() + require.True(t, charge.Memory(1_000_000), "a negative memory limit admits any amount already") + + held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, held.Close()) }) + + assert.Empty(t, filesIn(t, workDir), "nothing overflows for want of memory when the limit is negative") + mem, disk := charge.held() + assert.Equal(t, int64(1_000_000+len(body)), mem) + assert.Zero(t, disk) + }) + + t.Run("a zero disk limit writes nothing to disk while content the memory limit admits is still cataloged", func(t *testing.T) { + workDir := t.TempDir() + limiter := NewLimiter(Limits{MaxMemoryBytes: 10_000, MaxDiskBytes: 0}) + charge := limiter.Charge() + + held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, held.Close()) }) + + assert.Empty(t, filesIn(t, workDir), "content the memory limit admits never touches disk") + mem, disk := charge.held() + assert.Equal(t, int64(len(body)), mem) + assert.Zero(t, disk) + }) + + t.Run("a negative disk limit overflows with no ceiling", func(t *testing.T) { + workDir := t.TempDir() + limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) + charge := limiter.Charge() + + held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", charge, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, held.Close()) }) + + assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir)) + assert.Equal(t, body, readAll(t, held)) + }) +} + +func TestHoldContent_noLimitsAtAll(t *testing.T) { + // a nil charge is what an extraction with no configured bounds gets: it must route content rather + // than panic, holding everything in memory since nothing refuses it + workDir := t.TempDir() + body := []byte("small enough") + + held, err := holdContent(bytes.NewReader(body), WorkDirAt(workDir), "app.zip", nil, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, held.Close()) }) + + assert.Equal(t, body, readAll(t, held)) + assert.Empty(t, filesIn(t, workDir)) +} + +func TestOpenFileContent(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "app.zip") + require.NoError(t, os.WriteFile(path, []byte("body"), 0o600)) + + content, err := openFileContent(path) + require.NoError(t, err) + assert.Equal(t, "app.zip", content.Name) + assert.Equal(t, []byte("body"), readAll(t, content)) + require.NoError(t, content.Close()) + + _, err = openFileContent(filepath.Join(dir, "missing.zip")) + assert.Error(t, err) +} + // filesIn lists the regular files directly in dir, which is how a test tells content held in memory -// from content overflowed to disk. +// from content spilled to disk. func filesIn(t *testing.T, dir string) []string { t.Helper() entries, err := os.ReadDir(dir) @@ -36,220 +240,10 @@ func readAll(t *testing.T, c Content) []byte { return body } -func TestHoldContent_heldWhileTheLimitAdmitsIt(t *testing.T) { - // the referent for the memory limit: an archive small enough to hold needs no file on disk at - // all, where every archive used to be copied to a temp file purely so an extractor could seek it - workDir := t.TempDir() - body := []byte(strings.Repeat("a", 100)) - limiter := NewLimiter(Limits{MaxMemoryBytes: 1000, MaxDiskBytes: 1000}) - charge := limiter.Charge() - - held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.IsType(t, &bytes.Reader{}, held.Reader) - assert.Equal(t, body, readAll(t, held)) - assert.Empty(t, filesIn(t, workDir), "content held in memory must write nothing to disk") - - mem, disk := charge.Held() - assert.Equal(t, int64(100), mem) - assert.Zero(t, disk) -} - -func TestHoldContent_overflowsWhenTheLimitDoesNotAdmitIt(t *testing.T) { - // where the boundary falls is decided by the memory limit and nothing else: an archive that does - // not fit overflows in full, and the bytes already read to discover that lead the overflow rather than - // being lost - workDir := t.TempDir() - body := []byte(strings.Repeat("b", 250)) - limiter := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 10_000}) - charge := limiter.Charge() - - held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir)) - assert.Equal(t, body, readAll(t, held), - "the bytes read to decide the routing must lead the overflow rather than being lost") - - mem, disk := charge.Held() - assert.Zero(t, mem, "nothing is charged to a limit that refused") - assert.Equal(t, int64(250), disk) -} - -// peakTrackingReader records the highest memory charge observed at any point while reading, which is -// how a test tells that a large archive was never buffered beyond what the limit would admit - -// nothing about the routing decision is otherwise observable from outside holdContent. -type peakTrackingReader struct { - r io.Reader - charge *Charge - peak int64 -} - func (p *peakTrackingReader) Read(b []byte) (int, error) { n, err := p.r.Read(b) - if mem, _ := p.charge.Held(); mem > p.peak { + if mem, _ := p.charge.held(); mem > p.peak { p.peak = mem } return n, err } - -func TestHoldContent_largeArchiveNeverBuffersBeyondTheLimit(t *testing.T) { - // an archive many times the memory limit must overflow, and at no point along the way may more than - // the limit would admit be held while the routing is being decided - it is never read - // speculatively into memory that has no room for it - workDir := t.TempDir() - const limit = int64(3 * copyChunkSize) - body := []byte(strings.Repeat("c", 10*copyChunkSize)) - limiter := NewLimiter(Limits{MaxMemoryBytes: limit, MaxDiskBytes: int64(len(body)) + 1}) - charge := limiter.Charge() - tracked := &peakTrackingReader{r: bytes.NewReader(body), charge: charge} - - held, err := holdContent(tracked, workDir, "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir), "an archive this much larger than the limit must overflow") - assert.Equal(t, body, readAll(t, held)) - assert.LessOrEqual(t, tracked.peak, limit, - "memory charged must never exceed the limit, even transiently, while routing is decided") - - mem, disk := charge.Held() - assert.Zero(t, mem) - assert.Equal(t, int64(len(body)), disk) -} - -func TestHoldContent_diskLimitIsTerminal(t *testing.T) { - // there is nowhere further to overflow, so content that will not fit is refused and the caller skips - // the archive rather than waiting for capacity that nothing will release - workDir := t.TempDir() - body := []byte(strings.Repeat("d", 500)) - limiter := NewLimiter(Limits{MaxDiskBytes: 100}) - charge := limiter.Charge() - - _, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil) - require.ErrorIs(t, err, ErrDiskLimitReached) - - assert.Empty(t, filesIn(t, workDir), "the partial overflow must not be left behind") - mem, disk := charge.Held() - assert.Zero(t, mem) - assert.Zero(t, disk, "bytes no longer on disk must be refunded, or the limit only ever rises") -} - -func TestHoldContent_zeroDiskLimitSkipsRatherThanFails(t *testing.T) { - // a zero disk limit has the same outcome as a positive disk limit exceeded: content that will - // not fit anywhere yields ErrDiskLimitReached so the caller skips the archive, rather than the - // scan failing - workDir := t.TempDir() - body := []byte(strings.Repeat("f", 500)) - limiter := NewLimiter(Limits{MaxMemoryBytes: 10, MaxDiskBytes: 0}) - charge := limiter.Charge() - - _, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil) - require.ErrorIs(t, err, ErrDiskLimitReached) - - assert.Empty(t, filesIn(t, workDir), "no partial overflow is left behind") - mem, disk := charge.Held() - assert.Zero(t, mem) - assert.Zero(t, disk) -} - -func TestHoldContent_threeStateLimits(t *testing.T) { - // both limits read positive/zero/negative the same way: positive is the limit, zero means none - // of that resource at all, negative means unbounded. A caller can bound one without bounding the - // other. - body := []byte(strings.Repeat("e", 500)) - - t.Run("a zero memory limit sends everything to disk", func(t *testing.T) { - // zero does not mean unlimited memory: the memory limit decides a routing question rather - // than only capping an accumulator, so zero means the memory side of that routing is never - // taken - workDir := t.TempDir() - limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10_000}) - charge := limiter.Charge() - - held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir), "a zero memory limit holds nothing in memory") - mem, disk := charge.Held() - assert.Zero(t, mem) - assert.Equal(t, int64(500), disk, "and the disk limit still applies to what it overflowed") - }) - - t.Run("a negative memory limit holds content regardless of how much is already held", func(t *testing.T) { - workDir := t.TempDir() - limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: 10_000}) - charge := limiter.Charge() - require.True(t, charge.Memory(1_000_000), "a negative memory limit admits any amount already") - - held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Empty(t, filesIn(t, workDir), "nothing overflows for want of memory when the limit is negative") - mem, disk := charge.Held() - assert.Equal(t, int64(1_000_000+len(body)), mem) - assert.Zero(t, disk) - }) - - t.Run("a zero disk limit writes nothing to disk while content the memory limit admits is still cataloged", func(t *testing.T) { - workDir := t.TempDir() - limiter := NewLimiter(Limits{MaxMemoryBytes: 10_000, MaxDiskBytes: 0}) - charge := limiter.Charge() - - held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Empty(t, filesIn(t, workDir), "content the memory limit admits never touches disk") - mem, disk := charge.Held() - assert.Equal(t, int64(len(body)), mem) - assert.Zero(t, disk) - }) - - t.Run("a negative disk limit overflows with no ceiling", func(t *testing.T) { - workDir := t.TempDir() - limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) - charge := limiter.Charge() - - held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir)) - assert.Equal(t, body, readAll(t, held)) - }) -} - -func TestHoldContent_noLimitsAtAll(t *testing.T) { - // a nil charge is what an extraction with no configured bounds gets, and it must route content - // rather than panic - holding everything in memory, since nothing is there to refuse it - workDir := t.TempDir() - body := []byte("small enough") - - held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", nil, nil) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, held.Close()) }) - - assert.Equal(t, body, readAll(t, held)) - assert.Empty(t, filesIn(t, workDir)) -} - -func TestOpenFileContent(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "app.zip") - require.NoError(t, os.WriteFile(path, []byte("body"), 0o600)) - - content, err := OpenFileContent(path) - require.NoError(t, err) - assert.Equal(t, "app.zip", content.Name) - assert.Equal(t, []byte("body"), readAll(t, content)) - require.NoError(t, content.Close()) - - _, err = OpenFileContent(filepath.Join(dir, "missing.zip")) - assert.Error(t, err) -} diff --git a/internal/archive/entry_header.go b/internal/archive/entry_header.go new file mode 100644 index 000000000..febe562ef --- /dev/null +++ b/internal/archive/entry_header.go @@ -0,0 +1,47 @@ +package archive + +import ( + "archive/tar" + "io/fs" + "strings" + "time" + + "github.com/mholt/archives" + + "github.com/anchore/syft/internal/log" +) + +// entryHeader builds the tar header for one archive entry's metadata, or reports false when the entry +// is not worth carrying. The entry's bytes are held separately. +// +// Symlinks are recorded as headers, not real links; the target is resolved later inside the archive's +// own filetree, so one naming the host filesystem resolves to nothing. Device nodes, fifos and +// sockets are skipped - no cataloger reads them. +func entryHeader(f archives.FileInfo) (*tar.Header, bool) { + mode := f.Mode() + if !mode.IsRegular() && !mode.IsDir() && mode.Type()&fs.ModeSymlink == 0 { + log.WithFields("entry", f.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry") + return nil, false + } + + hdr, err := tar.FileInfoHeader(f.FileInfo, f.LinkTarget) + if err != nil { + log.WithFields("entry", f.NameInArchive, "error", err).Debug("skipping archive entry with an unusable header") + return nil, false + } + + // carried through as-is: nothing is written through this name, and the filetree sanitizes it + hdr.Name = f.NameInArchive + if f.IsDir() && !strings.HasSuffix(hdr.Name, "/") { + hdr.Name += "/" + } + hdr.Linkname = f.LinkTarget + + // dropped: no cataloger reads them, and the header is charged against the memory budget + hdr.PAXRecords = nil + hdr.Uname, hdr.Gname = "", "" + hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} + hdr.ModTime = hdr.ModTime.Truncate(time.Second) + + return hdr, true +} diff --git a/internal/archive/entrystore.go b/internal/archive/entrystore.go index 1225a3f99..d50f2327b 100644 --- a/internal/archive/entrystore.go +++ b/internal/archive/entrystore.go @@ -16,107 +16,80 @@ import ( "github.com/anchore/syft/internal/log" ) -// Entry is one archive entry held by an EntryStore: its header, and a reference to its bytes wherever -// they currently live. -// -// The bytes move; the entry does not. That is the point of this type - an index built over these -// entries stays valid when content is pushed out to disk, because what changes is the entry's -// reference and not its identity or its place in any index. +// Entry is one archive entry: its header and a reference to its bytes. The bytes move, the entry does +// not, so an index built over these entries stays valid when content spills to disk. type Entry struct { Header tar.Header - // ref names where this entry's bytes are right now. Guarded by the store's mutex, because a swap - // to disk rewrites it while readers may be asking for it. + // ref locates this entry's bytes. Guarded by the store's mutex: a spill rewrites it while readers + // may be asking for it. ref blobRef } -// blobRef is either bytes held in memory or a region of the store's overflow file. Exactly one of the -// two is live: mem is nil once the entry has been written out. +// blobRef is either bytes held in memory or a region of the store's overflow file; mem is nil once +// the entry has been written out. type blobRef struct { mem []byte offset int64 length int64 } -// EntryStore holds the entries of one extracted archive, in memory while the memory limit admits them -// and in a single overflow file once it does not. -// -// This replaces writing every archive's entries into a tar and indexing that tar by seek offset. A -// tar had to be on disk before there was anything to index, so a four-kilobyte jar paid for a file -// write and a tar walk before a cataloger could read a byte of it. Here the index is over the entries -// themselves, so a small archive never touches the disk at all, and a large one moves its bytes -// without the index noticing. -// -// The store owns the overflow file and closes it on Close. Readers handed out by Open borrow it, so -// they are only valid until then - which matches the lifetime the archive cataloger already gives an -// extracted archive. +// EntryStore holds one extracted archive's entries in memory while the memory limit admits them, and +// in a single overflow blob once it does not; an archive that fits in memory never touches disk. The +// store owns the overflow file and closes it on Close; readers from Open are valid only until then. type EntryStore struct { mu sync.RWMutex entries []*Entry - // heldInMemory is the bytes currently held by entries whose ref is memory-backed. It falls to zero - // when the store spills. - heldInMemory int64 - - workDir string + workDir *WorkDir file *os.File written int64 - // archive names this store's archive in the events it sends, so a spill can be attributed to the - // file that caused it rather than reported as an anonymous byte count. + // archive names this store's archive in the events it sends, attributing a spill to its cause. archive string notify Notify } -// overflowBlobName is the file an archive's entries are written into once they no longer fit in -// memory. Unlike the tar it replaces it has no framing: the entries' offsets live in the store, so -// nothing has to be parsed to find them again. +// overflowBlobName is the file entries spill into. It has no framing: entry offsets live in the store. const overflowBlobName = "contents.blob" -// indexRecordOverhead approximates the fixed memory kept per entry no matter its content: the *Entry, -// its tar.Header, and the resolver node built over it. It is an over-estimate of the fixed part, with -// the variable part - the entry's name and link target - added on top, so the bound errs towards -// charging more rather than letting the index outrun what was accounted for. See Charge.IndexRecord. -var indexRecordOverhead = int64(reflect.TypeOf((*tar.Header)(nil)).Elem().Size()) +// indexRecordOverhead is the fixed memory kept per entry regardless of content: the *Entry (tar.Header +// plus blobRef), its slot in the entries slice, and append's headroom. +// +// It covers only what the store keeps. The resolver built over these entries keeps several times as +// much in nodes the store cannot count, and charges for that itself; see Charge.IndexRecord. +var indexRecordOverhead = int64(reflect.TypeFor[Entry]().Size()) + 24 -// indexRecordCost is what one entry's index record charges: the fixed overhead plus the bytes of the -// two strings the header carries by value. +// indexRecordCost is what one entry's record charges: the fixed overhead plus the two header strings. func indexRecordCost(hdr tar.Header) int64 { return indexRecordOverhead + int64(len(hdr.Name)) + int64(len(hdr.Linkname)) } var _ EntrySink = (*EntryStore)(nil) -// NewEntryStore returns a store that writes its overflow blob into workDir, naming archive in the -// events it sends. +// NewEntryStore returns a store that spills into workDir, naming archive in its events. The directory +// is created only on spill, so an in-memory store creates nothing. // -// How much it holds in memory is not a property of the store: it asks the scan's limiter, entry by -// entry, and moves what it is holding out to disk the first time the limiter says no. That keeps one -// answer to "how much memory is this scan using" rather than a per-archive budget that no one set. -func NewEntryStore(workDir, archive string, notify Notify) *EntryStore { +// The store has no memory budget of its own: it asks the limiter entry by entry and spills everything +// it holds on the first refusal. +func NewEntryStore(workDir *WorkDir, archive string, notify Notify) *EntryStore { return &EntryStore{workDir: workDir, archive: archive, notify: notify} } -// Add stores one entry's header and content, and reports the entry. +// Add stores one entry's header and content. Content is read now, not referenced: the archive's +// reader is sequential. // -// Content is read here rather than referenced: an archive's reader is sequential, so an entry that is -// not read now cannot be read later. Where it lands is decided as it arrives, which is what keeps the -// decision out of every read path downstream. -// -// It is read and charged copyChunkSize at a time rather than into a buffer sized by anything the -// entry declared, for the reason holdContent reads an archive's own bytes that way: a header's size -// is the archive's claim about itself, so an entry far larger than the memory limit would be held -// past it in full before the limiter was ever asked. Here the limiter is asked per chunk, and the -// first refusal moves this entry and everything the store is holding out to the overflow blob. +// It reads and charges copyChunkSize at a time rather than into a header-sized buffer (whose size is +// the archive's own claim), so an entry larger than the memory limit is not held in full before the +// limiter is asked. The first refusal spills this entry and everything else the store holds. func (s *EntryStore) Add(hdr tar.Header, contents io.Reader, charge *Charge) (*Entry, error) { s.mu.Lock() defer s.mu.Unlock() - // charge this entry's index record before it is held, so an archive of millions of near-empty - // entries is bounded by the same budgets its content is rather than growing the index without - // limit. The record overflows to disk when memory is full and is only refused when neither budget - // admits it; a refusal leaves the archive truncated at the entries already stored. + // charge the record before holding the entry, so an archive of many near-empty entries is bounded + // by the same budgets its content is. Refused only when neither budget admits it, truncating the + // archive at the entries already stored. if !charge.IndexRecord(indexRecordCost(hdr)) { return nil, ErrDiskLimitReached } @@ -134,10 +107,8 @@ func (s *EntryStore) Add(hdr tar.Header, contents io.Reader, charge *Charge) (*E n, readErr := contents.Read(chunk) if n > 0 { if !charge.Memory(int64(n)) { - // memory is full: what is held of this entry joins the spill, and the rest of it - // streams straight into the blob behind it rather than being read into memory first + // memory full: held bytes join the spill, the rest streams straight into the blob entry.ref = memRef(&held) - s.heldInMemory += entry.ref.length rest := io.MultiReader(bytes.NewReader(chunk[:n]), contents) return entry, s.spillThroughLocked(entry, rest, charge) } @@ -152,12 +123,11 @@ func (s *EntryStore) Add(hdr tar.Header, contents io.Reader, charge *Charge) (*E } entry.ref = memRef(&held) - s.heldInMemory += entry.ref.length return entry, nil } -// memRef is a reference to bytes held in memory. The slice is taken non-nil even when it is empty, -// because nil is how blobRef says "these bytes are on disk" and a zero-length entry is not. +// memRef references bytes held in memory. The slice stays non-nil even when empty, since a nil +// blobRef.mem means the bytes are on disk - which a zero-length entry's are not. func memRef(held *bytes.Buffer) blobRef { body := held.Bytes() if body == nil { @@ -166,12 +136,11 @@ func memRef(held *bytes.Buffer) blobRef { return blobRef{mem: body, length: int64(len(body))} } -// spillThroughLocked moves everything the store is holding out to the overflow blob and writes the -// rest of entry's content in behind it, charging the disk limit as it lands. +// spillThroughLocked spills everything the store holds to the overflow blob and writes the rest of +// entry's content in behind it, charging the disk limit as it lands. // -// entry is the newest, so spillLocked writes it last and its region ends where the blob does - which -// is exactly where the rest of its content is about to go, so the two are one contiguous region and -// the entry keeps a single reference to it. +// entry is the newest, so spillLocked writes it last, ending its region right where the rest of its +// content goes. The two are contiguous, so the entry keeps one reference. func (s *EntryStore) spillThroughLocked(entry *Entry, rest io.Reader, charge *Charge) error { spilled := entry.ref.length moved, movedBytes, err := s.spillLocked(charge) @@ -183,8 +152,7 @@ func (s *EntryStore) spillThroughLocked(entry *Entry, rest io.Reader, charge *Ch s.written += written entry.ref = blobRef{offset: s.written - written - spilled, length: spilled + written} - // reported once the whole move is done, so the event counts the bytes that streamed through as - // well as the ones that were being held: they all went to disk for the same reason. + // reported after the move so the event counts streamed bytes as well as held ones if s.notify != nil && moved > 0 { s.notify(EntriesOverflowed{ Archive: s.archive, @@ -204,15 +172,16 @@ func (s *EntryStore) spillThroughLocked(entry *Entry, rest io.Reader, charge *Ch } // spillLocked writes every memory-backed entry into the overflow blob and repoints it there, -// reporting what it moved. The index over these entries is untouched: an entry's identity is the -// pointer, and only its reference to its bytes changes. -// -// It does not send the event for the move: the entry that triggered the spill is still streaming in -// behind it, and an event sent from here would report a byte count short by whatever that entry has -// left. spillThroughLocked sends it once both halves have landed. +// reporting what it moved. Any index over these entries is untouched: only blobRef changes. It sends +// no event; spillThroughLocked sends one once the triggering entry has also landed. func (s *EntryStore) spillLocked(charge *Charge) (moved int, movedBytes int64, err error) { if s.file == nil { - f, err := os.OpenFile(filepath.Join(s.workDir, overflowBlobName), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) + // first spill creates the work directory + dir, err := s.workDir.Path() + if err != nil { + return 0, 0, err + } + f, err := os.OpenFile(filepath.Join(dir, overflowBlobName), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { return 0, 0, fmt.Errorf("unable to create overflow blob: %w", err) } @@ -227,7 +196,6 @@ func (s *EntryStore) spillLocked(charge *Charge) (moved int, movedBytes int64, e if !charge.Disk(int64(len(body))) { return moved, movedBytes, ErrDiskLimitReached } - charged := charge != nil if _, err := s.file.WriteAt(body, s.written); err != nil { return moved, movedBytes, fmt.Errorf("unable to write archive entry %q to the overflow blob: %w", entry.Header.Name, err) } @@ -235,18 +203,14 @@ func (s *EntryStore) spillLocked(charge *Charge) (moved int, movedBytes int64, e s.written += int64(len(body)) moved++ movedBytes += int64(len(body)) - if charged { - // the bytes are on disk now, so the memory they were charged for is no longer held - charge.RefundMemory(int64(len(body))) - } + // on disk now, so refund the memory it was charged + charge.RefundMemory(int64(len(body))) } - s.heldInMemory = 0 return moved, movedBytes, nil } -// overflowReason distinguishes an archive pushed to disk by pressure from one that was never going to -// be held at all, which are different findings: the first says the scan is at its memory bound, the -// second says the bound is zero. +// overflowReason distinguishes an archive spilled under memory pressure from one never held because +// the memory limit is zero. func overflowReason(charge *Charge) OverflowReason { if charge != nil && charge.limiter != nil && charge.limiter.limits.MaxMemoryBytes == 0 { return MemoryHoldsNothing @@ -263,9 +227,8 @@ func (s *EntryStore) Entries() []*Entry { return out } -// Open returns a reader over one entry's content, from memory or from the overflow blob as the entry -// currently stands. The reader is Read, Seek and ReadAt either way, so a nested archive is handed to -// an archive format where it lies rather than being copied out first. +// Open returns a reader over one entry's content, from memory or the overflow blob. It supports Read, +// Seek and ReadAt either way, so a nested archive is extracted in place rather than copied out. func (s *EntryStore) Open(entry *Entry) (ReaderAtSeeker, error) { s.mu.RLock() defer s.mu.RUnlock() @@ -282,14 +245,6 @@ func (s *EntryStore) Open(entry *Entry) (ReaderAtSeeker, error) { return io.NewSectionReader(s.file, entry.ref.offset, entry.ref.length), nil } -// HeldInMemory reports the entry content currently held in memory, for tests and for the scan's own -// statistics. -func (s *EntryStore) HeldInMemory() int64 { - s.mu.RLock() - defer s.mu.RUnlock() - return s.heldInMemory -} - // Close releases the overflow blob. Readers handed out by Open do not outlive it. func (s *EntryStore) Close() error { s.mu.Lock() @@ -302,13 +257,9 @@ func (s *EntryStore) Close() error { return err } -// AddEntry stores one archive entry, so an EntryStore can stand where an OverflowTar stands. -// -// Note what is NOT here: no header encoding, no block padding, and no end-of-archive marker, because -// there is no framing to write. The disk limit is charged only when content actually reaches the disk, -// which for a small archive is never - so an archive that fits in memory costs the limiter nothing on -// disk and costs the machine no write at all. -func (s *EntryStore) AddEntry(f archives.FileInfo, result *ExtractionResult, limits ExtractionLimits) error { +// AddEntry stores one archive entry. There is no framing, so the disk limit is charged only when +// content reaches disk - never, for an archive that fits in memory. +func (s *EntryStore) AddEntry(f archives.FileInfo, result *ExtractionResult, charge *Charge) error { hdr, ok := entryHeader(f) if !ok { return nil @@ -325,33 +276,29 @@ func (s *EntryStore) AddEntry(f archives.FileInfo, result *ExtractionResult, lim contents = opened } - entry, err := s.Add(*hdr, contents, limits.Charge) + _, err := s.Add(*hdr, contents, charge) if contents != nil { if closeErr := contents.Close(); closeErr != nil { log.WithFields("entry", f.NameInArchive, "error", closeErr).Trace("unable to close archive entry") } } + switch { case errors.Is(err, ErrDiskLimitReached): - // nowhere left to put this archive's content: the entries stored so far stay, and the archive - // is reported as truncated rather than discarded + // keep the entries stored so far rather than discard the archive result.Truncation = TruncatedByDiskLimit return errTruncated case err != nil: return err } - result.FilesExtracted++ - result.BytesWritten = s.OnDisk() - _ = entry return nil } -// Finish has nothing to write: a store's entries are complete as they are added, with no framing to -// close off. -func (s *EntryStore) Finish(*ExtractionResult, ExtractionLimits) {} +// Finish is a no-op: entries are complete as added, with no framing to close off. +func (s *EntryStore) Finish(*ExtractionResult) {} -// OnDisk reports the entry content written out so far, which is what the disk limit was charged for. +// OnDisk reports the entry content written out so far, which is what the disk limit was charged. func (s *EntryStore) OnDisk() int64 { s.mu.RLock() defer s.mu.RUnlock() diff --git a/internal/archive/entrystore_test.go b/internal/archive/entrystore_test.go index f23813a96..6b5c56004 100644 --- a/internal/archive/entrystore_test.go +++ b/internal/archive/entrystore_test.go @@ -11,29 +11,16 @@ import ( "github.com/stretchr/testify/require" ) -func regularHeader(name string, size int64) tar.Header { - return tar.Header{Name: name, Size: size, Mode: 0o600, Typeflag: tar.TypeReg} -} - -func readEntry(t *testing.T, r ReaderAtSeeker) string { - t.Helper() - _, err := r.Seek(0, io.SeekStart) - require.NoError(t, err) - b, err := io.ReadAll(r) - require.NoError(t, err) - return string(b) -} - func TestEntryStore_holdsSmallContentInMemory(t *testing.T) { dir := t.TempDir() - s := NewEntryStore(dir, "app.zip", nil) + s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) charge := memCharge(1024) t.Cleanup(func() { require.NoError(t, s.Close()) }) entry, err := s.Add(regularHeader("a.txt", 5), bytes.NewReader([]byte("hello")), charge) require.NoError(t, err) - assert.Equal(t, int64(5), s.HeldInMemory()) + assert.Equal(t, int64(5), s.heldInMemory()) assert.NoFileExists(t, filepath.Join(dir, overflowBlobName), "a small archive must not touch the disk at all - that is the whole point of the store") @@ -44,21 +31,21 @@ func TestEntryStore_holdsSmallContentInMemory(t *testing.T) { func TestEntryStore_spillsToDiskAndKeepsTheSameEntries(t *testing.T) { // the index over these entries is not rebuilt when content moves, so the entry pointers and their - // contents must both survive the move + // contents must survive it dir := t.TempDir() - s := NewEntryStore(dir, "app.zip", nil) + s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) charge := memCharge(8) t.Cleanup(func() { require.NoError(t, s.Close()) }) first, err := s.Add(regularHeader("a.txt", 4), bytes.NewReader([]byte("aaaa")), charge) require.NoError(t, err) - require.Equal(t, int64(4), s.HeldInMemory()) + require.Equal(t, int64(4), s.heldInMemory()) // the second entry takes the store past its memory bound, so everything held moves out second, err := s.Add(regularHeader("b.txt", 6), bytes.NewReader([]byte("bbbbbb")), charge) require.NoError(t, err) - assert.Zero(t, s.HeldInMemory(), "everything must have moved out, not just the entry that did not fit") + assert.Zero(t, s.heldInMemory(), "everything must have moved out, not just the entry that did not fit") assert.FileExists(t, filepath.Join(dir, overflowBlobName)) // the same entry pointers still read their own content, from their new home @@ -76,14 +63,14 @@ func TestEntryStore_spillsToDiskAndKeepsTheSameEntries(t *testing.T) { func TestEntryStore_zeroMemoryHoldsNothing(t *testing.T) { // the reading a zero memory limit has everywhere else: none of that resource may be used dir := t.TempDir() - s := NewEntryStore(dir, "app.zip", nil) + s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) charge := memCharge(0) t.Cleanup(func() { require.NoError(t, s.Close()) }) entry, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), charge) require.NoError(t, err) - assert.Zero(t, s.HeldInMemory()) + assert.Zero(t, s.heldInMemory()) assert.FileExists(t, filepath.Join(dir, overflowBlobName)) r, err := s.Open(entry) @@ -93,7 +80,7 @@ func TestEntryStore_zeroMemoryHoldsNothing(t *testing.T) { func TestEntryStore_negativeMemoryNeverSpills(t *testing.T) { dir := t.TempDir() - s := NewEntryStore(dir, "app.zip", nil) + s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) charge := memCharge(-1) t.Cleanup(func() { require.NoError(t, s.Close()) }) @@ -102,15 +89,15 @@ func TestEntryStore_negativeMemoryNeverSpills(t *testing.T) { require.NoError(t, err) } - assert.Equal(t, int64(11), s.HeldInMemory()) + assert.Equal(t, int64(11), s.heldInMemory()) assert.NoFileExists(t, filepath.Join(dir, overflowBlobName)) } func TestEntryStore_spillIsRefusedWhenDiskIsNotAvailable(t *testing.T) { - // content that does not fit in memory and cannot be written has nowhere to go, and the archive is + // content that does not fit in memory and cannot be written has nowhere to go, so the archive is // skipped rather than cataloged as if it were empty dir := t.TempDir() - s := NewEntryStore(dir, "app.zip", nil) + s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) t.Cleanup(func() { require.NoError(t, s.Close()) }) nowhereToPutIt := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).Charge() @@ -120,30 +107,24 @@ func TestEntryStore_spillIsRefusedWhenDiskIsNotAvailable(t *testing.T) { func TestEntryStore_directoryEntriesHoldNoContent(t *testing.T) { dir := t.TempDir() - s := NewEntryStore(dir, "app.zip", nil) + s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) charge := memCharge(1024) t.Cleanup(func() { require.NoError(t, s.Close()) }) entry, err := s.Add(tar.Header{Name: "lib/", Typeflag: tar.TypeDir, Mode: 0o755}, nil, charge) require.NoError(t, err) - assert.Zero(t, s.HeldInMemory()) + assert.Zero(t, s.heldInMemory()) r, err := s.Open(entry) require.NoError(t, err) assert.Empty(t, readEntry(t, r)) } -// memCharge is a charge against a memory bound with disk left unbounded, which is the shape the store -// reads: hold while memory admits, move out when it does not. -func memCharge(maxMemory int64) *Charge { - return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).Charge() -} - func TestEntryStore_chargesIndexRecordEvenForEmptyContent(t *testing.T) { - // an empty entry still charges its index record - the header and node held to reach it - which is - // what bounds entry count, the dimension the content-byte budgets never bounded. + // an empty entry still charges its index record - the header and node held to reach it - which bounds + // entry count, the dimension the content-byte budgets do not cover dir := t.TempDir() - s := NewEntryStore(dir, "app.zip", nil) + s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}) charge := limiter.Charge() t.Cleanup(func() { require.NoError(t, s.Close()) }) @@ -154,15 +135,15 @@ func TestEntryStore_chargesIndexRecordEvenForEmptyContent(t *testing.T) { mem, _ := limiter.InUse() assert.Equal(t, indexRecordCost(hdr), mem, "the index record is charged to memory") - assert.Equal(t, int64(0), s.HeldInMemory(), "no content was held") + assert.Equal(t, int64(0), s.heldInMemory(), "no content was held") } func TestEntryStore_indexOverflowsToDiskWhenMemoryIsZero(t *testing.T) { - // the index cannot be refused just because memory is zero: a resolver still needs the records to - // reach content that lives entirely on disk. With disk unbounded, every entry is admitted and its - // record charged to disk instead. + // the index cannot be refused just because memory is zero: a resolver needs the records to reach + // content living entirely on disk. With disk unbounded, every entry is admitted and its record + // charged to disk instead. dir := t.TempDir() - s := NewEntryStore(dir, "app.zip", nil) + s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) charge := limiter.Charge() t.Cleanup(func() { require.NoError(t, s.Close()) }) @@ -177,11 +158,11 @@ func TestEntryStore_indexOverflowsToDiskWhenMemoryIsZero(t *testing.T) { } func TestEntryStore_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) { - // with memory zero and only a few index records' worth of disk, the store admits a bounded number - // of entries and then refuses - an archive of millions of empty entries can no longer grow the - // index without limit. + // with memory zero and only a few index records' worth of disk, the store admits a bounded number of + // entries and then refuses, so an archive of millions of empty entries cannot grow the index without + // limit dir := t.TempDir() - s := NewEntryStore(dir, "app.zip", nil) + s := NewEntryStore(WorkDirAt(dir), "app.zip", nil) room := indexRecordCost(regularHeader("x", 0)) * 3 charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).Charge() t.Cleanup(func() { require.NoError(t, s.Close()) }) @@ -197,3 +178,22 @@ func TestEntryStore_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) { require.ErrorIs(t, err, ErrDiskLimitReached) assert.Equal(t, 3, admitted, "exactly the index records that fit the disk budget were admitted") } + +func regularHeader(name string, size int64) tar.Header { + return tar.Header{Name: name, Size: size, Mode: 0o600, Typeflag: tar.TypeReg} +} + +func readEntry(t *testing.T, r ReaderAtSeeker) string { + t.Helper() + _, err := r.Seek(0, io.SeekStart) + require.NoError(t, err) + b, err := io.ReadAll(r) + require.NoError(t, err) + return string(b) +} + +// memCharge is a charge against a memory bound with disk unbounded, the shape the store reads: hold +// while memory admits, move out when it does not. +func memCharge(maxMemory int64) *Charge { + return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).Charge() +} diff --git a/internal/archive/extractor.go b/internal/archive/extractor.go index a259b46f6..488910c61 100644 --- a/internal/archive/extractor.go +++ b/internal/archive/extractor.go @@ -18,111 +18,56 @@ import ( "github.com/anchore/syft/syft/cataloging" ) -// Extractor extracts archive contents to a destination directory. +// Extractor reads the entries of one archive into an EntrySink. type Extractor interface { - // CanExtract returns true if this extractor can handle the given archive content. CanExtract(ctx context.Context, content Content) bool - // Extract writes the archive's entries into sink - one store per archive, not one file per entry - - // respecting the given limits. Returns how many entries were written and how many bytes that took. - Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error) + Extract(ctx context.Context, content Content, sink EntrySink, charge *Charge) (ExtractionResult, error) } // EntrySink is where an extractor puts the entries it reads out of an archive. -// -// Two implementations exist because they store an archive differently and the difference is -// measurable: OverflowTar writes every entry into one tar on disk and is indexed by seek offset, -// while EntryStore holds entries in memory until a bound says otherwise and is indexed by the entries -// themselves. The extractors do not care which they were given, with one exception - see -// VerbatimSink. type EntrySink interface { - // AddEntry stores one entry of the archive. - AddEntry(f archives.FileInfo, result *ExtractionResult, limits ExtractionLimits) error + AddEntry(f archives.FileInfo, result *ExtractionResult, charge *Charge) error - // Finish is called once the last entry has been offered, for whatever a sink has to write after - // its content. - Finish(result *ExtractionResult, limits ExtractionLimits) + // Finish is called once the last entry has been offered. + Finish(result *ExtractionResult) } -// VerbatimSink is an EntrySink that stores archives in tar form, so a tar-family archive can be -// copied into it as it stands rather than decoded and re-encoded entry by entry. -// -// This is the one place the storage format shows through the extractor interface, and it is worth the -// leak: a tar.gz copied verbatim is one streamed decompression, where walking it costs a read per -// entry. A sink that does not store tars does not implement this and gets the entry-by-entry path. -type VerbatimSink interface { - EntrySink - CopyVerbatim(r io.Reader, result *ExtractionResult, limits ExtractionLimits) error -} - -// ExtractionLimits defines safety limits for extracting one archive. -type ExtractionLimits struct { - - // Charge is this archive's draw on the scan's resource limiter. Entry bytes are charged to it as - // they land, never from the size the entry declared, and a refused charge truncates the - // extraction. A nil Charge enforces no byte bound. - // - // It is a handle rather than a remaining-bytes value because what it charges has to be given back - // when the archive is released, and a value copied into a call cannot be. - Charge *Charge -} - -// TruncationReason names the configured bound that stopped an extraction early. The empty value -// means the archive was extracted in full. +// TruncationReason names the bound that stopped an extraction early; empty means extracted in full. type TruncationReason string const ( - // TruncatedByDiskLimit means placing more of this archive's content would have taken the scan's - // disk usage past the disk limit. Named rather than reported as a bare truncation because a breach - // has to be attributed to the archive that triggered it, and saying which bound was reached is - // half of that attribution. + // TruncatedByDiskLimit: storing more content would exceed the scan's disk limit. TruncatedByDiskLimit TruncationReason = "disk limit" - // TruncatedByLimit means a bound stopped the extraction but the walker replaced the error that - // said which one. Defensive: both bounds record their own reason on the result before stopping. + // TruncatedByIndexLimit: indexing more entries would exceed the record budget. Distinct from the + // disk limit because the index is never written to disk, though the disk budget backstops it - see + // Charge.IndexRecord. + TruncatedByIndexLimit TruncationReason = "index limit" + + // TruncatedByLimit: a bound stopped extraction but the walker replaced the error naming which one. TruncatedByLimit TruncationReason = "configured limit" ) -// errTruncated stops an in-progress archive walk when a configured limit is reached. It never -// escapes this package: the extractors convert it into a successful, truncated ExtractionResult. +// errTruncated stops an in-progress walk at a configured limit. It never escapes this package: +// extractors convert it into a successful, truncated ExtractionResult. var errTruncated = errors.New("archive extraction stopped at a configured limit") -// ExtractionResult holds the result of an extraction operation. -// -// Reaching a configured limit is NOT an error: it is a truncation. Whatever was written before the -// limit is left on disk and is usable, so callers can still catalog what was extracted rather than -// discarding the whole archive. Check Truncated to tell a complete extraction from a partial one; -// a non-nil error from Extract means the extraction failed and nothing usable was produced. +// ExtractionResult holds the result of an extraction. Reaching a limit is a truncation, not an error: +// content stored before the limit stays usable. A non-nil error from Extract means nothing usable was +// produced. type ExtractionResult struct { - // FilesExtracted counts the entries this archive's filesystem holds - the records its overflow tar - // was indexed into: regular files, symlinks, and directory entries. It is the count after indexing - // rather than the count written, so an entry dropped by an exclusion pattern is not counted as - // present. Reported, not bounded. - FilesExtracted int - - // BytesWritten is what this archive put on disk: every byte of the overflow tar, headers and block - // padding included, because that is what the disk limit was charged for. - BytesWritten int64 - - // Truncation names the limit that stopped extraction early, or "" when the archive was - // extracted in full. + // Truncation names the limit that stopped extraction early, or "" if none did. Truncation TruncationReason } -// Truncated reports whether a configured limit stopped this extraction before the end of the -// archive. func (r ExtractionResult) Truncated() bool { return r.Truncation != "" } -// finishExtraction turns a walk that stopped at a configured limit into a successful, truncated -// result. -// -// The limit signal is carried on the result rather than only in the returned error because the -// underlying archive walkers are free to wrap or replace a handler's error, in which case -// errors.Is(err, errTruncated) would not match and a truncation would be reported as a failure - -// discarding an archive syft could have cataloged. The error is still checked, so a walker that -// passes it through cleanly is handled too. +// finishExtraction turns a walk stopped at a configured limit into a successful, truncated result. +// Both result and error are checked: walkers may wrap or replace the handler's error, so +// errors.Is(err, errTruncated) alone would report some truncations as failures. func finishExtraction(result ExtractionResult, err error) (ExtractionResult, error) { if result.Truncated() || errors.Is(err, errTruncated) { if !result.Truncated() { @@ -134,16 +79,6 @@ func finishExtraction(result ExtractionResult, err error) (ExtractionResult, err return result, err } -// DefaultExtractionLimits returns the per-archive limits from the given config. -// -// Nothing is read from the config: every bound this scan enforces is an in-use limit held by the -// scan's limiter, and the only per-archive field left is the Charge the caller draws against it. -// The parameter stays so the two Default* constructors are called the same way, and so a per-archive -// bound that is genuinely per-archive has somewhere to land. -func DefaultExtractionLimits(_ cataloging.ArchiveSearchConfig) ExtractionLimits { - return ExtractionLimits{} -} - // DefaultLimits returns the scan-wide in-use limits from the given config. func DefaultLimits(cfg cataloging.ArchiveSearchConfig) Limits { return Limits{ @@ -178,27 +113,16 @@ func FindExtractor(ctx context.Context, extractors []Extractor, content Content) // ZipExtractor extracts zip-based archives (zip, jar, war, ear, etc.). type ZipExtractor struct{} -// CanExtract reports whether this content is a zip, asking two questions in an order that matters. +// CanExtract reports whether this content is a zip. Test order matters: head identification wins, +// because a tar holding one jar ends in that jar's EOCD record, so reading it as a zip would lose the +// rest of the tar. The EOCD record is consulted only when nothing identifies from the head, where it +// is the only test that sees a zip concatenated onto a launcher stub (e.g. a Spring Boot executable +// jar). // -// First, what does the content say it is, reading from the start? A format that identifies itself at -// offset zero is what this content is, and if that format is not zip then it is not a zip - a tar -// holding one jar ends in that jar's end-of-central-directory record, and reading it as a zip -// instead of as a tar would lose everything else the tar holds. -// -// Only when nothing identifies from the head is the end-of-central-directory record consulted. That -// record is what defines a zip - a reader locates the central directory through it - and it is the -// only thing that can see an archive whose head is something else entirely. A self-extracting -// archive is a launcher stub with a zip concatenated onto it, so no magic at offset zero can find -// it; the canonical case is a Spring Boot executable jar, a shell script followed by a whole zip. -// -// Identification is intFile.IdentifyArchive, which matches on the filename as well as the content, -// and it is left that way. Dropping the name makes the answer content's alone, which reads better - -// but archives.Identify with no path has to try every format's content matcher against every -// candidate, and measured on a stock ruby image that took the scan from 17 seconds to 45. The name -// cannot admit a non-archive in practice: a file only reaches an extractor after its sniffed content -// type put it in the candidate set, so the prose named notes.zip that this would wrongly identify is -// never offered here at all. `#content-based-archive-detection` is upheld by the candidate set, and -// Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError is where that is asserted. +// IdentifyArchive matches on filename as well as content; dropping the name would run every format's +// content matcher against every candidate, far slower. The name cannot admit a non-archive, since +// files reach an extractor only after content sniffing (see +// Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError). func (z *ZipExtractor) CanExtract(ctx context.Context, content Content) bool { format, _, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader) if err == nil { @@ -212,21 +136,18 @@ func (z *ZipExtractor) CanExtract(ctx context.Context, content Content) bool { return HasZipEndOfCentralDirectory(content.Reader) } -func (z *ZipExtractor) Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error) { +func (z *ZipExtractor) Extract(ctx context.Context, content Content, sink EntrySink, charge *Charge) (ExtractionResult, error) { var result ExtractionResult if _, err := content.Reader.Seek(0, io.SeekStart); err != nil { return result, fmt.Errorf("unable to seek zip archive %q: %w", content.Name, err) } - // mholt/archives imposes no extraction target: the FileHandler is the caller's, so an entry can - // be written into a tar header rather than onto the filesystem. Nothing about the library had to - // be customized - the old code wrote files because the handler wrote files. err := archives.Zip{}.Extract(ctx, content.Reader, func(_ context.Context, file archives.FileInfo) error { - return sink.AddEntry(file, &result, limits) + return sink.AddEntry(file, &result, charge) }) - sink.Finish(&result, limits) + sink.Finish(&result) return finishExtraction(result, err) } @@ -234,20 +155,21 @@ func (z *ZipExtractor) Extract(ctx context.Context, content Content, sink EntryS // TarExtractor extracts tar-based archives (tar, tar.gz, tar.bz2, tar.xz, tar.zst). type TarExtractor struct{} +// CanExtract reports whether the content identifies as an extractable non-zip format. func (t *TarExtractor) CanExtract(ctx context.Context, content Content) bool { format, _, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader) if err != nil { return false } - // mholt/archives returns a compound type for tar+compression, check if it's an extractor but not a zip if _, isZip := format.(archives.Zip); isZip { return false } + // tar+compression comes back as a compound type, so test for the Extractor interface _, ok := format.(archives.Extractor) return ok } -func (t *TarExtractor) Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error) { +func (t *TarExtractor) Extract(ctx context.Context, content Content, sink EntrySink, charge *Charge) (ExtractionResult, error) { var result ExtractionResult if _, err := content.Reader.Seek(0, io.SeekStart); err != nil { @@ -270,47 +192,24 @@ func (t *TarExtractor) Extract(ctx context.Context, content Content, sink EntryS extractReader = content.Reader } - // a tar handed to a sink that stores tars is not walked at all: a compressed tar is decompressed - // once into the sink's file and a plain one is copied, and the entries are first looked at when - // the file is indexed. Where the sink stores entries rather than a tar there is nothing to copy - // into, so the tar is walked like any other archive. - if verbatim, isVerbatim := sink.(VerbatimSink); isVerbatim { - if plain, ok := tarStream(format, extractReader); ok { - err = verbatim.CopyVerbatim(plain, &result, limits) - if closeErr := plain.Close(); closeErr != nil { - log.WithFields("archive", content.Name, "error", closeErr).Trace("unable to close decompressed tar stream") - } - return finishExtraction(result, err) - } - } - - // anything else - 7z, rar, or a tar going into a sink that does not store tars - is walked entry - // by entry err = extractor.Extract(ctx, extractReader, func(_ context.Context, file archives.FileInfo) error { - return sink.AddEntry(file, &result, limits) + return sink.AddEntry(file, &result, charge) }) - sink.Finish(&result, limits) + sink.Finish(&result) return finishExtraction(result, err) } -// resolvesInsideRoot reports whether writing to path really lands inside root, following any symlink -// that already exists along the way the same way the filesystem will. +// resolvesInsideRoot reports whether writing to path lands inside root, following existing symlinks +// as the filesystem would. It guards the one filesystem write left in this package: an archive +// spilling into its work directory under an attacker-controlled name. // -// The one filesystem write left in this package is the archive's own bytes overflowing into its work -// directory, under a name derived from where the archive was found - which inside another archive is -// attacker-controlled text. That is what this now guards; entries are tar headers and traverse no -// path at all. See overflowContent. -// -// It walks path one component at a time rather than calling filepath.EvalSymlinks on the whole -// thing, because the leaf of an entry being extracted does not exist yet, which makes EvalSymlinks -// fail rather than answer. The first component that does not exist ends the walk: nothing below it -// exists either, so whatever is created there is created fresh under a path already known to be -// inside the root. +// The walk is component by component rather than one EvalSymlinks call because the leaf does not yet +// exist, which makes EvalSymlinks fail rather than answer. The first missing component ends the walk: +// anything created below it lands under an already-checked path. func resolvesInsideRoot(root, path string) (bool, error) { - // the root itself may sit under a symlinked directory (a temp dir on macOS, for one), so - // containment is judged against its real location + // the root itself may sit under a symlinked directory (e.g. a macOS temp dir) realRoot, err := filepath.EvalSymlinks(root) if err != nil { return false, fmt.Errorf("unable to resolve extraction root: %w", err) @@ -355,8 +254,8 @@ func resolvesInsideRoot(root, path string) (bool, error) { return isWithin(current, realRoot), nil } -// resolveLink returns where a symlink really points, falling back to the lexically cleaned target -// when the target does not exist yet - a dangling link still redirects a write. +// resolveLink returns where a symlink points, falling back to the lexically cleaned target when it +// does not exist yet - a dangling link still redirects a write. func resolveLink(path string) (string, error) { target, err := os.Readlink(path) if err != nil { @@ -375,8 +274,8 @@ func resolveLink(path string) (string, error) { return resolved, nil } -// isWithin reports whether path is root or sits beneath it, compared on a path boundary rather than -// as a string prefix so a sibling sharing the prefix does not pass. +// isWithin reports whether path is root or beneath it, compared on path boundaries so a sibling +// sharing the prefix does not pass. func isWithin(path, root string) bool { rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path)) if err != nil { @@ -385,93 +284,30 @@ func isWithin(path, root string) bool { return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))) } -// writeSafeSymlink creates a symlink at destPath only if target is relative and really resolves, -// relative to destPath's directory, to a location inside destDir. -// -// Absolute targets are rejected outright: os.Symlink writes the literal target -// string, so an absolute target like "/etc/passwd" would resolve on the host -// filesystem when read, regardless of any safety check we performed against -// destDir at extraction time. -// -// A relative target is resolved against the REAL location of destPath's directory, not against its -// lexical one, because two cooperating entries can otherwise each pass on their own and together -// point outside: a link to "." followed by a link to ".." inside it resolves lexically to the root -// while really landing above it, and repeating the pair climbs a level each time. The parent of -// every write is checked the same way before the write happens, so a link that slipped through here -// still could not be written through - this is the earlier of two refusals, not the only one. -// -// It has no caller since an archive's entries became headers in one tar rather than files on disk: a -// symlink entry is now a header write, which creates nothing and traverses nothing, and where the -// link points is decided inside the archive's own filetree. It is kept rather than retired because -// that is a judgement about every path that could write a link, not about this one call site, and -// because the reasoning above is the expensive part to reconstruct. -// -//nolint:unused // retained deliberately; see the note above -func writeSafeSymlink(target, destPath, destDir string) error { - if target == "" { - return fmt.Errorf("empty link target") - } - if filepath.IsAbs(target) { - return fmt.Errorf("absolute symlink target not allowed") - } - - realRoot, err := filepath.EvalSymlinks(destDir) - if err != nil { - return fmt.Errorf("unable to resolve extraction root: %w", err) - } - realParent, err := filepath.EvalSymlinks(filepath.Dir(destPath)) - if err != nil { - return fmt.Errorf("unable to resolve link parent: %w", err) - } - if !isWithin(realParent, realRoot) { - return fmt.Errorf("symlink parent escapes extraction root") - } - - resolved := filepath.Join(realParent, target) - if !isWithin(resolved, realRoot) { - return fmt.Errorf("symlink target escapes extraction root") - } - // the target itself may run through links planted by earlier entries - if existing, err := filepath.EvalSymlinks(resolved); err == nil && !isWithin(existing, realRoot) { - return fmt.Errorf("symlink target resolves outside extraction root") - } - - return os.Symlink(target, destPath) -} - -// zipEOCDSignature is the end-of-central-directory record's signature. A zip is defined by this -// record: it is what a reader seeks to in order to find the central directory, which is why a zip -// with something concatenated in front of it still opens. +// zipEOCDSignature is the end-of-central-directory record's signature. Readers seek to this record +// to find the central directory, which is why a zip with something prepended still opens. var zipEOCDSignature = []byte{'P', 'K', 0x05, 0x06} -// maxZipEOCDSearch is how far back the record can be. The EOCD is 22 bytes and ends with a comment -// of up to 65535 bytes, so it starts no earlier than this many bytes from the end. +// maxZipEOCDSearch is how far back from the end the EOCD record can start: a 22 byte record plus a +// comment of up to 65535 bytes. const maxZipEOCDSearch = 22 + 65535 // zipLocalFileHeaderSignature starts every entry in a zip. In a self-extracting archive the first -// one follows the stub, so it is what a bounded read of the head can find. +// one follows the stub, within reach of a bounded read of the head. var zipLocalFileHeaderSignature = []byte{'P', 'K', 0x03, 0x04} -// MaxAppendedArchiveStubBytes is how far into a file a hidden archive is looked for. A Spring Boot -// launcher stub is about 9 KB, so this is generous by a wide margin; a stub longer than this hides -// its archive from the walk, which is the cost of not reading every candidate to the end. -const MaxAppendedArchiveStubBytes = 64 * 1024 +// maxAppendedArchiveStubBytes is how far into a file an appended archive is looked for. A Spring +// Boot launcher stub is about 9 KB; a longer stub hides its archive from the walk. +const maxAppendedArchiveStubBytes = 64 * 1024 // MayHideAnAppendedArchive reports whether the head of this stream contains a zip local file header, // and returns a reader that replays what it consumed. // -// It exists to make the candidate set affordable. A file whose head sniffs as a script, an -// executable, or as nothing recognizable may still be an archive wearing a stub, but almost none of -// them are - and the authoritative test, the end-of-central-directory record, is at the far end of -// the file, so applying it to every such candidate means reading every executable in a scan to its -// end. Measured on a stock ruby image that doubled the scan. -// -// So the head is checked first, cheaply and with a bound, and only a file that shows an entry -// signature there is read in full and put to the real test. This narrows what is examined; it never -// admits anything, because a file that passes here is still refused by the extractor unless it ends -// in a valid central directory. +// A cheap, bounded pre-filter for HasZipEndOfCentralDirectory: reading every script, executable and +// unrecognized file to its end roughly doubles a scan, so only files showing an entry signature in +// their head are read in full. It narrows what is examined; it admits nothing on its own. func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) { - head := make([]byte, MaxAppendedArchiveStubBytes) + head := make([]byte, maxAppendedArchiveStubBytes) n, err := io.ReadFull(r, head) if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { return io.MultiReader(bytes.NewReader(head[:n]), r), false @@ -481,17 +317,9 @@ func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) { } // HasZipEndOfCentralDirectory reports whether the content ends in a zip end-of-central-directory -// record, which is the definition of a zip and the only part of one whose position is knowable -// without parsing. -// -// This is what identifies an archive that content sniffing types by its head: a self-extracting -// archive is a launcher stub with a zip concatenated onto it, so its first bytes are the stub's and -// no magic-at-offset-zero test can see the archive. The record at the end is unambiguous, and -// mholt/archives extracts such a file correctly once it is asked to - it reads through -// archive/zip, which locates the central directory from the end. -// -// It is a content test, so it cannot make a non-archive extractable: a file named `notes.zip` that -// is prose has no such record and is refused here exactly as it is refused by format identification. +// record - the only part of a zip whose position is knowable without parsing. It identifies archives +// that head-based sniffing types as something else, such as a launcher stub with a zip appended, +// which mholt/archives extracts correctly via archive/zip. func HasZipEndOfCentralDirectory(r ReaderAtSeeker) bool { size, err := r.Seek(0, io.SeekEnd) if err != nil || size < int64(len(zipEOCDSignature)) { diff --git a/internal/archive/extractor_test.go b/internal/archive/extractor_test.go index 0aa7b4739..97393a29f 100644 --- a/internal/archive/extractor_test.go +++ b/internal/archive/extractor_test.go @@ -6,7 +6,6 @@ import ( "bytes" "compress/gzip" "context" - "errors" "fmt" "io" "io/fs" @@ -22,25 +21,957 @@ import ( "github.com/anchore/syft/syft/cataloging" ) -// fileContent opens an archive already on disk as Content, which is what the extractors take now -// that an archive's bytes may be held in memory instead. Registered for close so a test that opens -// several does not leak handles. +// storedEntry is one entry read back out of an EntryStore. Assertions run against these rather than +// files in a directory, since an archive's entries are headers and bytes in a store. +type storedEntry struct { + name string + body string + linkTarget string + mode fs.FileMode + typeflag byte +} + +func TestZipExtractor_CanExtract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) + + ext := &ZipExtractor{} + + assert.True(t, ext.CanExtract(ctx, fileContent(t, zipPath))) + + nonZipPath := filepath.Join(dir, "notazip.txt") + require.NoError(t, os.WriteFile(nonZipPath, []byte("just text"), 0o644)) + + assert.False(t, ext.CanExtract(ctx, fileContent(t, nonZipPath))) +} + +func TestZipExtractor_Extract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": "content1", + "dir/file2.txt": "content2", + } + zipPath := createTestZip(t, dir, files) + + ext := &ZipExtractor{} + overflow := storeFor(t) + + _, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, nil) + require.NoError(t, err) + assert.Len(t, overflow.Entries(), 2) + + // both entries are in the one store, under their own names and with their own content + entries := readStore(t, overflow) + assert.Equal(t, "content1", entries["file1.txt"].body) + assert.Equal(t, "content2", entries["dir/file2.txt"].body) + + // and with no bound to push them out, nothing was written to disk at all + assert.Zero(t, overflow.OnDisk()) + assert.NoFileExists(t, filepath.Join(storeDir(t, overflow), overflowBlobName)) +} + +func TestZipExtractor_Extract_FileLimitReached(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": "a", + "file2.txt": "b", + "file3.txt": "c", + } + zipPath := createTestZip(t, dir, files) + + ext := &ZipExtractor{} + overflow := storeFor(t) + + // reaching a limit truncates rather than fails: what was stored stays catalogable. Each entry costs + // its index record plus its one byte of content, so two entries' worth of room refuses the third. + record := indexRecordCost(tar.Header{Name: "file1.txt"}) + result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, diskCharge(2*(record+1)+record)) + require.NoError(t, err) + assert.True(t, result.Truncated()) + assert.Equal(t, TruncatedByDiskLimit, result.Truncation) + + entries := readStore(t, overflow) + assert.Equal(t, "a", entries["file1.txt"].body) + assert.Equal(t, "b", entries["file2.txt"].body, + "the two entries stored before the limit must still be readable") + assert.Empty(t, entries["file3.txt"].body, "the entry the limit refused holds no content") +} + +func TestZipExtractor_Extract_SizeLimitReached(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": strings.Repeat("x", 100), + } + zipPath := createTestZip(t, dir, files) + + ext := &ZipExtractor{} + overflow := storeFor(t) + + result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, diskCharge(50)) + require.NoError(t, err) + assert.True(t, result.Truncated()) + assert.Equal(t, TruncatedByDiskLimit, result.Truncation) + + // the single entry blew the budget on its own, so it is dropped rather than left half-written: a + // partial jar or pom a cataloger parses anyway is worse than an absent one. The budget goes on the + // entry's header before any content, a header being bytes on disk too. + assert.Empty(t, overflow.Entries()) + assert.Zero(t, overflow.OnDisk()) + assert.Empty(t, storedNames(t, overflow), "the over-limit entry must not be left behind") +} + +func TestTarExtractor_CanExtract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) + + ext := &TarExtractor{} + assert.True(t, ext.CanExtract(ctx, fileContent(t, tarPath))) +} + +func TestTarExtractor_Extract(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{ + "file1.txt": "content1", + "dir/file2.txt": "content2", + } + tarPath := createTestTarGz(t, dir, files) + + ext := &TarExtractor{} + overflow := storeFor(t) + + _, err := ext.Extract(ctx, fileContent(t, tarPath), overflow, nil) + require.NoError(t, err) + + // a tar-family archive is walked entry by entry like any other, and with no bound pushing them out + // its entries stay in memory + assert.Len(t, overflow.Entries(), 2) + assert.Zero(t, overflow.OnDisk()) + + entries := readStore(t, overflow) + assert.Equal(t, "content1", entries["file1.txt"].body) + assert.Equal(t, "content2", entries["dir/file2.txt"].body) +} + +func TestFindExtractor(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + extractors := DefaultExtractors() + + zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) + ext := FindExtractor(ctx, extractors, fileContent(t, zipPath)) + require.NotNil(t, ext) + assert.IsType(t, &ZipExtractor{}, ext) + + tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) + ext = FindExtractor(ctx, extractors, fileContent(t, tarPath)) + require.NotNil(t, ext) + assert.IsType(t, &TarExtractor{}, ext) + + textPath := filepath.Join(dir, "plain.txt") + require.NoError(t, os.WriteFile(textPath, []byte("just text"), 0o644)) + ext = FindExtractor(ctx, extractors, fileContent(t, textPath)) + assert.Nil(t, ext) +} + +func TestFindExtractor_NoReaders(t *testing.T) { + // content no extractor claims yields nil rather than an error, so a file matching a broad mime + // filter but not an archive is skipped + ctx := context.Background() + dir := t.TempDir() + path := filepath.Join(dir, "plain.txt") + require.NoError(t, os.WriteFile(path, []byte("just text"), 0o644)) + + ext := FindExtractor(ctx, DefaultExtractors(), fileContent(t, path)) + assert.Nil(t, ext) +} + +func TestTarExtractor_CanExtract_NonTarFile(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + path := filepath.Join(dir, "plain.txt") + require.NoError(t, os.WriteFile(path, []byte("not a tar"), 0o644)) + + ext := &TarExtractor{} + assert.False(t, ext.CanExtract(ctx, fileContent(t, path))) +} + +func TestTarExtractor_CanExtract_ZipFileReturnsFalse(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) + + ext := &TarExtractor{} + + assert.False(t, ext.CanExtract(ctx, fileContent(t, zipPath))) +} + +func TestDefaultLimits(t *testing.T) { + cfg := cataloging.ArchiveSearchConfig{ + MaxMemoryBytes: 111, + MaxDiskBytes: 222, + } + limits := DefaultLimits(cfg) + assert.Equal(t, int64(111), limits.MaxMemoryBytes) + assert.Equal(t, int64(222), limits.MaxDiskBytes) +} + +func TestZipExtractor_Extract_ZipSlipPrevented(t *testing.T) { + // an entry named "../../etc/passwd" becomes a field in a tar header, and writing a header traverses + // no path, so extraction succeeds and nothing is created anywhere. The name still matters to the + // path the SBOM reports, which is clamped at index time - see the archive resolver's own tests. + dir := t.TempDir() + ctx := context.Background() + + zipPath := filepath.Join(dir, "evil.zip") + f, err := os.Create(zipPath) + require.NoError(t, err) + + w := zip.NewWriter(f) + fw, err := w.Create("../../etc/passwd") + require.NoError(t, err) + _, err = fw.Write([]byte("evil")) + require.NoError(t, err) + require.NoError(t, w.Close()) + require.NoError(t, f.Close()) + + ext := &ZipExtractor{} + overflow := storeFor(t) + + _, err = ext.Extract(ctx, fileContent(t, zipPath), overflow, nil) + require.NoError(t, err) + assert.Len(t, overflow.Entries(), 1) + + // nothing is created outside the tar, at the traversed path or anywhere else + _, statErr := os.Stat(filepath.Join(dir, "etc", "passwd")) + assert.True(t, os.IsNotExist(statErr)) + _, statErr = os.Stat(filepath.Join(storeDir(t, overflow), "etc")) + assert.True(t, os.IsNotExist(statErr)) + + // the name is carried through unsanitized, this being the archive's own record of what it held; + // sanitizing happens where the logical filetree is built + assert.Equal(t, []string{"../../etc/passwd"}, storedNames(t, overflow)) +} + +func TestTarExtractor_Extract_SymlinkIsRecordedAsALink(t *testing.T) { + // a symlink entry is recorded as a symlink header carrying its target - not a regular file holding + // the target as bytes, and not a link on the real filesystem. Nothing is created, so nothing can be + // redirected; where the link points is decided in the archive's own filetree. + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "with-link.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + target := []byte("real content") + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "real.txt", + Mode: 0o644, + Size: int64(len(target)), + })) + _, err = tw.Write(target) + require.NoError(t, err) + + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "link.txt", + Linkname: "real.txt", + Typeflag: tar.TypeSymlink, + Mode: 0o777, + })) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + overflow := storeFor(t) + + _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, nil) + require.NoError(t, err) + + entries := readStore(t, overflow) + link := entries["link.txt"] + assert.Equal(t, byte(tar.TypeSymlink), link.typeflag, "link.txt must stay a link, not become a file") + assert.Equal(t, "real.txt", link.linkTarget) + assert.Empty(t, link.body, "a link carries a target, not content") + assert.Equal(t, "real content", entries["real.txt"].body) + + // and no link exists on the real filesystem beside the tar + _, statErr := os.Lstat(filepath.Join(storeDir(t, overflow), "link.txt")) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestTarExtractor_Extract_SymlinkEscapingRootCreatesNothing(t *testing.T) { + // a link target climbing out of the archive is recorded as the archive gave it, the entry being a + // header with nothing created for a write to be redirected through. The target is clamped to the + // archive's own root when the filetree is built - see the archive resolver's own tests. + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "evil-link.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "passwd", + Linkname: "../../../../etc/passwd", + Typeflag: tar.TypeSymlink, + Mode: 0o777, + })) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + overflow := storeFor(t) + + _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, nil) + require.NoError(t, err) + + // nothing on the real filesystem beside the tar + _, statErr := os.Lstat(filepath.Join(storeDir(t, overflow), "passwd")) + assert.True(t, os.IsNotExist(statErr), "no link may be created for an escaping symlink") + + entries := readStore(t, overflow) + assert.Equal(t, byte(tar.TypeSymlink), entries["passwd"].typeflag) + assert.Equal(t, "../../../../etc/passwd", entries["passwd"].linkTarget) +} + +func TestTarExtractor_Extract_AbsoluteSymlinkTargetCreatesNothing(t *testing.T) { + // an absolute target creates no link either: a header write resolves nowhere, and the target is + // re-rooted inside the archive when the filetree is built + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "absolute-link.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "shadow", + Linkname: "/etc/shadow", + Typeflag: tar.TypeSymlink, + Mode: 0o777, + })) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + overflow := storeFor(t) + + _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, nil) + require.NoError(t, err) + + _, statErr := os.Lstat(filepath.Join(storeDir(t, overflow), "shadow")) + assert.True(t, os.IsNotExist(statErr), "no link may be created for an absolute target") + + entries := readStore(t, overflow) + assert.Equal(t, "/etc/shadow", entries["shadow"].linkTarget) +} + +func TestZipExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { + // jars may hold directory entries with read-only permissions (e.g. META-INF/ at mode 0o555); the + // extractor must still write files into them + dir := t.TempDir() + ctx := context.Background() + + zipPath := filepath.Join(dir, "readonly-dirs.zip") + f, err := os.Create(zipPath) + require.NoError(t, err) + + w := zip.NewWriter(f) + + dirHeader := &zip.FileHeader{ + Name: "META-INF/", + } + dirHeader.SetMode(0o555) + _, err = w.CreateHeader(dirHeader) + require.NoError(t, err) + + fileHeader := &zip.FileHeader{ + Name: "META-INF/MANIFEST.MF", + } + fileHeader.SetMode(0o644) + fw, err := w.CreateHeader(fileHeader) + require.NoError(t, err) + _, err = fw.Write([]byte("Manifest-Version: 1.0")) + require.NoError(t, err) + + require.NoError(t, w.Close()) + require.NoError(t, f.Close()) + + ext := &ZipExtractor{} + overflow := storeFor(t) + + _, err = ext.Extract(ctx, fileContent(t, zipPath), overflow, nil) + require.NoError(t, err) + // two entries: the directory and the file. A directory carries no bytes of its own, but its index + // record is charged, which is what bounds an archive of nothing but directories. + assert.Len(t, overflow.Entries(), 2) + + // nothing is written into a directory, so its mode decides nothing, but both entries must be + // recorded, modes and all + entries := readStore(t, overflow) + assert.Equal(t, "Manifest-Version: 1.0", entries["META-INF/MANIFEST.MF"].body) + assert.Equal(t, fs.FileMode(0o555), entries["META-INF/"].mode.Perm()) + assert.Equal(t, byte(tar.TypeDir), entries["META-INF/"].typeflag) +} + +func TestTarExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { + // tars may hold directory entries with read-only permissions; the extractor must still write files + // into them + dir := t.TempDir() + ctx := context.Background() + + tarPath := filepath.Join(dir, "readonly-dirs.tar.gz") + f, err := os.Create(tarPath) + require.NoError(t, err) + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "META-INF/", + Typeflag: tar.TypeDir, + Mode: 0o555, + })) + + content := []byte("Manifest-Version: 1.0") + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "META-INF/MANIFEST.MF", + Mode: 0o644, + Size: int64(len(content)), + })) + _, err = tw.Write(content) + require.NoError(t, err) + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + require.NoError(t, f.Close()) + + ext := &TarExtractor{} + overflow := storeFor(t) + + _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, nil) + require.NoError(t, err) + + entries := readStore(t, overflow) + assert.Equal(t, "Manifest-Version: 1.0", entries["META-INF/MANIFEST.MF"].body) + assert.Equal(t, fs.FileMode(0o555), entries["META-INF/"].mode.Perm()) +} + +func TestZipExtractor_Extract_nonPositiveLimitDisablesOnlyThatLimit(t *testing.T) { + // <= 0 means "no limit", per limit, so a caller can opt out of one without the rest; substituting a + // default would re-impose a limit the caller turned off + dir := t.TempDir() + ctx := context.Background() + + files := map[string]string{} + for i := range 20 { + files[fmt.Sprintf("file%02d.txt", i)] = "x" + } + files["big.txt"] = strings.Repeat("y", 500) + zipPath := createTestZip(t, dir, files) + + t.Run("the disk limit bites", func(t *testing.T) { + result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), storeFor(t), diskCharge(100)) + require.NoError(t, err) + assert.Equal(t, TruncatedByDiskLimit, result.Truncation, "the disk limit must still bite") + }) + + t.Run("unbounded extracts everything", func(t *testing.T) { + overflow := storeFor(t) + + result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, nil) + require.NoError(t, err) + assert.False(t, result.Truncated()) + assert.Len(t, overflow.Entries(), len(files)) + }) +} + +func TestZipExtractor_Extract_dataEdgeCases(t *testing.T) { + ctx := context.Background() + + t.Run("empty archive is a non-event", func(t *testing.T) { + dir := t.TempDir() + zipPath := createTestZip(t, dir, map[string]string{}) + + overflow := storeFor(t) + + result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, nil) + require.NoError(t, err, "a valid archive containing nothing is not a failure") + assert.Empty(t, overflow.Entries()) + assert.False(t, result.Truncated()) + }) + + t.Run("zero-byte entry is recorded", func(t *testing.T) { + dir := t.TempDir() + zipPath := createTestZip(t, dir, map[string]string{"empty.txt": ""}) + overflow := storeFor(t) + + _, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, nil) + require.NoError(t, err) + assert.Len(t, overflow.Entries(), 1) + + // a store has no framing to write, so an entry carrying no content costs no disk at all + assert.Zero(t, overflow.OnDisk()) + + entries := readStore(t, overflow) + require.Contains(t, entries, "empty.txt") + assert.Empty(t, entries["empty.txt"].body) + }) + + t.Run("absolute entry path is contained", func(t *testing.T) { + // distinct from ../ traversal: an entry literally named /etc/hosts must land inside the + // extraction dir, never at the host path + dir := t.TempDir() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("/etc/hosts") + require.NoError(t, err) + _, err = w.Write([]byte("pwned")) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + zipPath := filepath.Join(dir, "abs.zip") + require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0o600)) + + overflow := storeFor(t) + + _, err = (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, nil) + require.NoError(t, err) + + // one entry in the tar and nothing at the host path; the leading slash is dropped when the + // filetree is built, so it reads as a path inside this archive + assert.Equal(t, "pwned", readStore(t, overflow)["/etc/hosts"].body) + _, statErr := os.Stat(filepath.Join(dir, "etc", "hosts")) + assert.True(t, os.IsNotExist(statErr)) + }) + + t.Run("unicode and non-ascii entry names survive", func(t *testing.T) { + dir := t.TempDir() + names := map[string]string{ + "café/naïve.txt": "accents", + "日本語/ファイル.txt": "japanese", + "emoji-🎉.txt": "emoji", + } + zipPath := createTestZip(t, dir, names) + overflow := storeFor(t) + + _, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, nil) + require.NoError(t, err) + assert.Len(t, overflow.Entries(), len(names)) + + entries := readStore(t, overflow) + for name, want := range names { + require.Contains(t, entries, name, "entry %q must be recorded under its own name", name) + assert.Equal(t, want, entries[name].body) + } + }) +} + +func TestExtract_diskLimitTruncatesOneArchive(t *testing.T) { + // the disk limit is enforced as entries land, not between archives: a bound checked only between + // archives lets one archive overshoot by everything it writes. + // + // The budget is built from what a store charges, there being no framing: each entry's index record + // plus the content bytes that reach the blob. It admits both records and the small entry whole, and + // refuses the big one. + ctx := context.Background() + files := map[string]string{ + "a/small.txt": strings.Repeat("s", 50), + "b/big.txt": strings.Repeat("b", 500), + } + + dir := t.TempDir() + archivePath := createTestZip(t, dir, files) + overflow := storeFor(t) + + records := indexRecordCost(tar.Header{Name: "a/small.txt"}) + indexRecordCost(tar.Header{Name: "b/big.txt"}) + charge := diskCharge(records + 50 + 100) + result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, archivePath), overflow, charge) + require.NoError(t, err, "reaching a limit is a truncation, not a failure") + + assert.Equal(t, TruncatedByDiskLimit, result.Truncation, + "a limit breach reports its own reason, so the log can name which bound was reached") + assert.Equal(t, int64(50), overflow.OnDisk(), + "only the small entry was stored whole, and a store writes content and no framing") + + _, disk := charge.held() + assert.Equal(t, records+50, disk, "the limiter holds the index records plus what content landed") + + // what was stored before the limit filled stays readable + entries := readStore(t, overflow) + assert.Len(t, entries["a/small.txt"].body, 50) + + // KNOWN GAP: the entry the limit refused stays in the store, its header claiming the size the + // archive declared with none of the content behind it. A half-written pom a cataloger parses and + // believes is worse than an absent one. Asserted as it stands, so a fix changes a test. + require.Contains(t, entries, "b/big.txt") + assert.Empty(t, entries["b/big.txt"].body, "none of the refused entry's content landed") +} + +func TestTarExtractor_Extract_diskLimitStopsMidEntry(t *testing.T) { + // a tar-family archive is walked entry by entry, so the disk limit can bite mid-entry rather than on + // a boundary: the walk stops, says why, and the limiter holds no more than the bound + ctx := context.Background() + dir := t.TempDir() + + // two entries either side of a copy chunk, so the limit can bite mid-stream rather than refusing the + // first chunk and writing nothing + files := map[string]string{ + "a/first.txt": strings.Repeat("a", 40*1024), + "b/second.txt": strings.Repeat("b", 40*1024), + } + archivePath := createTestTarGz(t, dir, files) + overflow := storeFor(t) + + const budget = 64 * 1024 + charge := diskCharge(budget) + result, err := (&TarExtractor{}).Extract(ctx, fileContent(t, archivePath), overflow, charge) + require.NoError(t, err) + + assert.Equal(t, TruncatedByDiskLimit, result.Truncation) + assert.Greater(t, overflow.OnDisk(), int64(40*1024), + "the first entry landed whole and the second got part way in") + + _, disk := charge.held() + assert.LessOrEqual(t, disk, int64(budget), "the limiter never holds more than the bound") + assert.Greater(t, disk, overflow.OnDisk(), + "the charge also carries the index records, which are held in the store rather than written") + + info, err := os.Stat(filepath.Join(storeDir(t, overflow), overflowBlobName)) + require.NoError(t, err) + assert.Equal(t, overflow.OnDisk(), info.Size(), "the blob holds exactly what the store reports") +} + +func TestExtract_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) { + // an in-use limit rather than a counter: two archives of the same size both extract in full when the + // first is released between them, and the second is bounded when it is not + ctx := context.Background() + dir := t.TempDir() + zipPath := createTestZip(t, dir, map[string]string{"a.txt": strings.Repeat("a", 200)}) + + // one entry of 200 bytes costs its index record plus its content, there being no framing. A limit of + // one and a half archives leaves no room for a second while the first is held. + oneArchive := indexRecordCost(tar.Header{Name: "a.txt"}) + 200 + limiter := NewLimiter(Limits{MaxDiskBytes: oneArchive * 3 / 2}) + + first := limiter.Charge() + firstStore := storeFor(t) + result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), firstStore, first) + require.NoError(t, err) + require.False(t, result.Truncated()) + require.Equal(t, int64(200), firstStore.OnDisk()) + + t.Run("a second archive is bounded while the first is still held", func(t *testing.T) { + second := limiter.Charge() + result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), storeFor(t), second) + require.NoError(t, err) + assert.Equal(t, TruncatedByDiskLimit, result.Truncation) + second.Release() + }) + + t.Run("and extracts in full once the first is released", func(t *testing.T) { + first.Release() + _, disk := limiter.InUse() + require.Zero(t, disk, "releasing every charge must take the limiter back to nothing") + + third := limiter.Charge() + thirdStore := storeFor(t) + result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), thirdStore, third) + require.NoError(t, err) + assert.False(t, result.Truncated(), "a counter would have refused this; a limiter does not") + assert.Equal(t, int64(200), thirdStore.OnDisk()) + }) +} + +func TestExtract_directoryEntriesAreBoundedByDisk(t *testing.T) { + // an archive of directory entries alone carries no content, so what bounds it is the disk limit + // charging each entry's header and the tar's padding + ctx := context.Background() + + t.Run("zip", func(t *testing.T) { + dir := t.TempDir() + zipPath := createDirOnlyZip(t, dir, 50) + overflow := storeFor(t) + + // each directory entry is one header block; room for twelve admits ten entries plus the two + // end-of-archive blocks + result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, diskCharge(12*512)) + require.NoError(t, err, "a bound is a truncation, not a failure") + assert.Equal(t, TruncatedByDiskLimit, result.Truncation) + assert.Less(t, len(overflow.Entries()), 50, "the bound must stop the archive short") + assert.NotEmpty(t, storedNames(t, overflow), "what was written before the bound stays readable") + }) + + t.Run("tar.gz is copied whole, so the disk limit is what stops it", func(t *testing.T) { + // a tar is not walked entry by entry: the copy takes the whole archive, and the disk limit is + // charged as those bytes land + dir := t.TempDir() + tarPath := createDirOnlyTarGz(t, dir, 50) + overflow := storeFor(t) + + result, err := (&TarExtractor{}).Extract(ctx, fileContent(t, tarPath), overflow, diskCharge(-1)) + require.NoError(t, err) + assert.False(t, result.Truncated()) + assert.Len(t, storedNames(t, overflow), 50) + }) +} + +// symlinkEntry describes one archive entry for the escape tests: a symlink when linkTarget is set, +// a regular file otherwise. +type symlinkEntry struct { + name string + linkTarget string + body string +} + +func TestExtract_symlinkChainEscapeWritesNothing(t *testing.T) { + // two cooperating link entries that each pass a lexical check and together resolve outside the + // extraction directory: "d" -> "." lands on the root, so "d/up" -> ".." is created as "/up" + // -> ".." pointing above it, and a third entry is written through it. + // + // The chain needs real links on a real filesystem. Entries are headers, so no link is created and + // nothing can be written through one. Kept as the input that would violate "nothing outside the + // store is created". + chain := []symlinkEntry{ + {name: "d", linkTarget: "."}, + {name: "d/up", linkTarget: ".."}, + {name: "up/PWNED", body: "arbitrary write"}, + } + + // three link pairs, each climbing one more level: a check walking only one level would pass a + // one-level test and still let this through + escalating := []symlinkEntry{ + {name: "a", linkTarget: "."}, + {name: "a/b", linkTarget: ".."}, + {name: "b/c", linkTarget: "."}, + {name: "b/c/d", linkTarget: ".."}, + {name: "b/d/e", linkTarget: "."}, + {name: "b/d/e/f", linkTarget: ".."}, + {name: "b/d/f/PWNED", body: "arbitrary write, three levels up"}, + } + + for _, tc := range []struct { + name string + entries []symlinkEntry + }{ + {name: "one pair", entries: chain}, + {name: "three pairs escalating", entries: escalating}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Run("tar.gz", func(t *testing.T) { + assertNoEscape(t, createSymlinkChainTarGz, func() Extractor { return &TarExtractor{} }, "chain.tar.gz", tc.entries) + }) + t.Run("zip", func(t *testing.T) { + assertNoEscape(t, createSymlinkChainZip, func() Extractor { return &ZipExtractor{} }, "chain.zip", tc.entries) + }) + }) + } +} + +func TestTarExtractor_Extract_legitimateRelativeSymlinkIsCarriedThrough(t *testing.T) { + // containing an escape must not cost what real archives contain: a relative link into a sibling + // directory of the archive's own tree keeps its target verbatim so the filetree can resolve it. + // That it does resolve is asserted in the archive resolver's tests. + dir := t.TempDir() + + archivePath := createSymlinkChainTarGz(t, dir, "legit.tar.gz", []symlinkEntry{ + {name: "lib/real.txt", body: "real content"}, + {name: "bin/link.txt", linkTarget: "../lib/real.txt"}, + {name: "lib/sub/deep.txt", body: "deep"}, + {name: "bin/deep.txt", linkTarget: "../lib/sub/deep.txt"}, + }) + overflow := storeFor(t) + + _, err := (&TarExtractor{}).Extract(context.Background(), fileContent(t, archivePath), overflow, nil) + require.NoError(t, err) + + entries := readStore(t, overflow) + assert.Equal(t, "../lib/real.txt", entries["bin/link.txt"].linkTarget) + assert.Equal(t, "../lib/sub/deep.txt", entries["bin/deep.txt"].linkTarget) + assert.Equal(t, "real content", entries["lib/real.txt"].body) + assert.Equal(t, "deep", entries["lib/sub/deep.txt"].body) +} + +func TestExtract_siblingOfExtractionDirectoryWritesNothing(t *testing.T) { + // the realistic form of the string-prefix hole SafeJoin was fixed for: an entry naming + // "../contents-evil/..." shares the extraction directory's prefix and is outside it. + // + // Nothing joins an entry name to a filesystem path, so the entry is recorded as the name it claims + // and no path is derived from it. The assertion: nothing is written beside the extraction directory. + ctx := context.Background() + + for _, tc := range []struct { + name string + build func(t *testing.T, dir string, files map[string]string) string + extract func(path string, sink EntrySink) (ExtractionResult, error) + }{ + { + name: "zip", + build: createTestZip, + extract: func(path string, sink EntrySink) (ExtractionResult, error) { + return (&ZipExtractor{}).Extract(ctx, fileContent(t, path), sink, nil) + }, + }, + { + name: "tar.gz", + build: createTestTarGz, + extract: func(path string, sink EntrySink) (ExtractionResult, error) { + return (&TarExtractor{}).Extract(ctx, fileContent(t, path), sink, nil) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + base := t.TempDir() + destDir := filepath.Join(base, "contents") + require.NoError(t, os.MkdirAll(destDir, 0o755)) + + archivePath := tc.build(t, base, map[string]string{"../contents-evil/x.txt": "escaped"}) + + store := NewEntryStore(WorkDirAt(base), "evil", nil) + t.Cleanup(func() { _ = store.Close() }) + + _, err := tc.extract(archivePath, store) + require.NoError(t, err, "a hostile name is a header field, not a failure") + + _, statErr := os.Stat(filepath.Join(base, "contents-evil", "x.txt")) + assert.True(t, os.IsNotExist(statErr), + "nothing may be written to a sibling of the extraction directory") + + entries, err := os.ReadDir(destDir) + require.NoError(t, err) + assert.Empty(t, entries, "the extraction directory stays empty: entries live in the store") + + assert.Equal(t, "escaped", readStore(t, store)["../contents-evil/x.txt"].body) + }) + } +} + +func Test_HasZipEndOfCentralDirectory(t *testing.T) { + var plainZip bytes.Buffer + zw := zip.NewWriter(&plainZip) + f, err := zw.Create("a.txt") + require.NoError(t, err) + _, err = f.Write([]byte("hello")) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + var tarGz bytes.Buffer + gw := gzip.NewWriter(&tarGz) + tw := tar.NewWriter(gw) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "a.txt", Size: 5, Mode: 0o600})) + _, err = tw.Write([]byte("hello")) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + + tests := []struct { + name string + data []byte + want bool + }{ + {"a plain zip", plainZip.Bytes(), true}, + { + "a zip behind a launcher script - the case content sniffing types by its head", + scriptPrefixedZip(t, "#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n", + map[string]string{"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n"}), + true, + }, + {"prose, whatever it is named", []byte("this is not an archive, it is a sentence"), false}, + {"a tar.gz, which is an archive but not a zip", tarGz.Bytes(), false}, + {"empty", nil, false}, + {"shorter than the signature", []byte("PK"), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, HasZipEndOfCentralDirectory(bytes.NewReader(tt.data))) + }) + } +} + +func Test_ZipExtractor_canExtractAZipBehindALauncherScript(t *testing.T) { + // a Spring Boot executable jar sniffs as text/x-shellscript, which format identification refuses, + // losing every package inside + data := scriptPrefixedZip(t, "#!/bin/bash\necho launcher\nexit 0\n", map[string]string{ + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", + "BOOT-INF/lib/dep.jar": "not really a jar, just bytes", + }) + + extractor := &ZipExtractor{} + require.True(t, extractor.CanExtract(context.Background(), bytesContent("app.jar", data)), + "a zip behind a launcher script must be recognized") + + // and it extracts, because archive/zip finds the central directory from the end + store := storeFor(t) + result, err := extractor.Extract(context.Background(), bytesContent("app.jar", data), store, + diskCharge(-1)) + require.NoError(t, err) + assert.Len(t, store.Entries(), 2, "the entries behind the script prefix must actually be stored") + assert.Empty(t, string(result.Truncation)) + assert.Contains(t, readStore(t, store), "BOOT-INF/lib/dep.jar") +} + +func Test_ZipExtractor_refusesANonArchiveThatNothingIdentifies(t *testing.T) { + // the probe must not turn a file with no central directory into an archive. The name matters here in + // a way it does not in a scan, since identification matches on it and `notes.zip` is claimed by name + // before the probe is reached; the `misnamed non-archive is not extracted` scenario is therefore + // asserted at the task level - see Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError. + extractor := &ZipExtractor{} + for _, name := range []string{"app.jar", "bundle", "launcher"} { + t.Run(name, func(t *testing.T) { + content := bytesContent(name, []byte("plain text, no central directory anywhere in it")) + assert.False(t, extractor.CanExtract(context.Background(), content)) + }) + } +} + +// fileContent opens an archive already on disk as Content, which is what the extractors take. +// Registered for close so a test that opens several does not leak handles. func fileContent(t *testing.T, path string) Content { t.Helper() - c, err := OpenFileContent(path) + c, err := openFileContent(path) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, c.Close()) }) return c } -// diskCharge returns a charge against a disk limit of the given size, for a test that wants the byte +// diskCharge returns a charge against a disk limit of the given size, for a test wanting the byte // bound to bite without standing up a whole scan. func diskCharge(max int64) *Charge { return NewLimiter(Limits{MaxDiskBytes: max}).Charge() } -// sortedNames fixes the entry order of a test archive. Limits are enforced as the walk proceeds, so -// which entries land before a truncation is a function of entry order and a map's order is random. +// sortedNames fixes the entry order of a test archive: limits are enforced as the walk proceeds, so +// entry order decides which entries land before a truncation, and map order is random. func sortedNames(files map[string]string) []string { names := make([]string, 0, len(files)) for name := range files { @@ -50,60 +981,45 @@ func sortedNames(files map[string]string) []string { return names } -// overflowEntry is one entry read back out of an overflow tar. Assertions are made against these rather -// than against files in a directory, because an archive's entries are now headers in one file. -type overflowEntry struct { - name string - body string - linkTarget string - mode fs.FileMode - typeflag byte -} - -// overflowFor makes the one tar an archive's entries are written into, in the test's own temp space. -func overflowFor(t *testing.T) *OverflowTar { +// storeFor makes the sink an archive's entries are written into, in the test's own temp space. +func storeFor(t *testing.T) *EntryStore { t.Helper() - s, err := newOverflowTar(filepath.Join(t.TempDir(), overflowTarName)) - require.NoError(t, err) + s := NewEntryStore(WorkDirAt(t.TempDir()), "test-archive", nil) t.Cleanup(func() { _ = s.Close() }) return s } -// readOverflow reads an overflow tar back, keyed by entry name. It closes the tar first: the writer holds -// it open for the header rewrites that give each entry the size that actually arrived. -func readOverflow(t *testing.T, s *OverflowTar) map[string]overflowEntry { +// storeDir is the directory a store would spill into, for tests asserting what is and is not on disk. +func storeDir(t *testing.T, s *EntryStore) string { t.Helper() - require.NoError(t, s.Close()) - - f, err := os.Open(s.Path()) + dir, err := s.workDir.Path() require.NoError(t, err) - defer f.Close() + return dir +} - out := map[string]overflowEntry{} - tr := tar.NewReader(f) - for { - hdr, err := tr.Next() - if errors.Is(err, io.EOF) { - break - } +func readStore(t *testing.T, s *EntryStore) map[string]storedEntry { + t.Helper() + + out := map[string]storedEntry{} + for _, entry := range s.Entries() { + reader, err := s.Open(entry) require.NoError(t, err) - body, err := io.ReadAll(tr) + body, err := io.ReadAll(reader) require.NoError(t, err) - out[hdr.Name] = overflowEntry{ - name: hdr.Name, + out[entry.Header.Name] = storedEntry{ + name: entry.Header.Name, body: string(body), - linkTarget: hdr.Linkname, - mode: hdr.FileInfo().Mode(), - typeflag: hdr.Typeflag, + linkTarget: entry.Header.Linkname, + mode: entry.Header.FileInfo().Mode(), + typeflag: entry.Header.Typeflag, } } return out } -// overflowNames is the entry names an overflow tar holds, sorted. -func overflowNames(t *testing.T, s *OverflowTar) []string { +func storedNames(t *testing.T, s *EntryStore) []string { t.Helper() - entries := readOverflow(t, s) + entries := readStore(t, s) names := make([]string, 0, len(entries)) for name := range entries { names = append(names, name) @@ -156,706 +1072,6 @@ func createTestTarGz(t *testing.T, dir string, files map[string]string) string { return tarPath } -func TestZipExtractor_CanExtract(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) - - ext := &ZipExtractor{} - - assert.True(t, ext.CanExtract(ctx, fileContent(t, zipPath))) - - // non-zip file - nonZipPath := filepath.Join(dir, "notazip.txt") - require.NoError(t, os.WriteFile(nonZipPath, []byte("just text"), 0o644)) - - assert.False(t, ext.CanExtract(ctx, fileContent(t, nonZipPath))) -} - -func TestZipExtractor_Extract(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - files := map[string]string{ - "file1.txt": "content1", - "dir/file2.txt": "content2", - } - zipPath := createTestZip(t, dir, files) - - ext := &ZipExtractor{} - overflow := overflowFor(t) - - result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) - require.NoError(t, err) - assert.Equal(t, 2, result.FilesExtracted) - - // both entries are in the one tar, under their own names and with their own content - entries := readOverflow(t, overflow) - assert.Equal(t, "content1", entries["file1.txt"].body) - assert.Equal(t, "content2", entries["dir/file2.txt"].body) - - // and the archive occupies exactly one filesystem entry, which is the whole point - info, err := os.Stat(overflow.Path()) - require.NoError(t, err) - assert.True(t, info.Mode().IsRegular()) -} - -func TestZipExtractor_Extract_FileLimitReached(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - files := map[string]string{ - "file1.txt": "a", - "file2.txt": "b", - "file3.txt": "c", - } - zipPath := createTestZip(t, dir, files) - - ext := &ZipExtractor{} - overflow := overflowFor(t) - - // reaching a limit is a truncation, not a failure: what was written before the limit stays in the - // tar so the caller can still catalog it. Two entries of one byte each cost two header blocks and - // two data blocks, so four blocks of room admits two entries and refuses the third. - result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{ - Charge: diskCharge(4 * 512), - }) - require.NoError(t, err) - assert.True(t, result.Truncated()) - assert.Equal(t, TruncatedByDiskLimit, result.Truncation) - assert.Equal(t, 2, result.FilesExtracted) - - assert.Equal(t, []string{"file1.txt", "file2.txt"}, overflowNames(t, overflow), - "the two entries written before the limit must still be readable") -} - -func TestZipExtractor_Extract_SizeLimitReached(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - files := map[string]string{ - "file1.txt": strings.Repeat("x", 100), - } - zipPath := createTestZip(t, dir, files) - - ext := &ZipExtractor{} - overflow := overflowFor(t) - - result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{Charge: diskCharge(50)}) - require.NoError(t, err) - assert.True(t, result.Truncated()) - assert.Equal(t, TruncatedByDiskLimit, result.Truncation) - - // the single entry blew the budget on its own, so it is dropped rather than left half-written: - // a partial jar or pom that a cataloger parses anyway is worse than an absent one. The budget is - // spent on the entry's header before any of its content, because a header is bytes on disk too. - assert.Zero(t, result.FilesExtracted) - assert.Zero(t, result.BytesWritten) - assert.Empty(t, overflowNames(t, overflow), "the over-limit entry must not be left behind") -} - -func TestTarExtractor_CanExtract(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) - - ext := &TarExtractor{} - assert.True(t, ext.CanExtract(ctx, fileContent(t, tarPath))) -} - -func TestTarExtractor_Extract(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - files := map[string]string{ - "file1.txt": "content1", - "dir/file2.txt": "content2", - } - tarPath := createTestTarGz(t, dir, files) - - ext := &TarExtractor{} - overflow := overflowFor(t) - - result, err := ext.Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{}) - require.NoError(t, err) - - // a compressed tar is decompressed once into the overflow file and never walked, so the extractor - // counts nothing: how many entries this archive contributes is the index's answer, reported by - // ExtractToResolver. What the extractor promises is that the file it produced is that tar. - assert.Zero(t, result.FilesExtracted, "a tar is copied, not walked; the index does the counting") - assert.NotZero(t, result.BytesWritten) - - entries := readOverflow(t, overflow) - assert.Equal(t, "content1", entries["file1.txt"].body) - assert.Equal(t, "content2", entries["dir/file2.txt"].body) -} - -func TestFindExtractor(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - extractors := DefaultExtractors() - - // zip file should match - zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) - ext := FindExtractor(ctx, extractors, fileContent(t, zipPath)) - require.NotNil(t, ext) - assert.IsType(t, &ZipExtractor{}, ext) - - // tar.gz should match - tarPath := createTestTarGz(t, dir, map[string]string{"hello.txt": "world"}) - ext = FindExtractor(ctx, extractors, fileContent(t, tarPath)) - require.NotNil(t, ext) - assert.IsType(t, &TarExtractor{}, ext) - - // non-archive should return nil - textPath := filepath.Join(dir, "plain.txt") - require.NoError(t, os.WriteFile(textPath, []byte("just text"), 0o644)) - ext = FindExtractor(ctx, extractors, fileContent(t, textPath)) - assert.Nil(t, ext) -} - -func TestFindExtractor_NoReaders(t *testing.T) { - // content no extractor claims yields nil rather than an error, which is how a file that matched a - // broad mime filter but is not an archive gets skipped - ctx := context.Background() - dir := t.TempDir() - path := filepath.Join(dir, "plain.txt") - require.NoError(t, os.WriteFile(path, []byte("just text"), 0o644)) - - ext := FindExtractor(ctx, DefaultExtractors(), fileContent(t, path)) - assert.Nil(t, ext) -} - -func TestTarExtractor_CanExtract_NonTarFile(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - // create a plain text file - path := filepath.Join(dir, "plain.txt") - require.NoError(t, os.WriteFile(path, []byte("not a tar"), 0o644)) - - ext := &TarExtractor{} - assert.False(t, ext.CanExtract(ctx, fileContent(t, path))) -} - -func TestTarExtractor_CanExtract_ZipFileReturnsFalse(t *testing.T) { - dir := t.TempDir() - ctx := context.Background() - - zipPath := createTestZip(t, dir, map[string]string{"hello.txt": "world"}) - - ext := &TarExtractor{} - - // tar extractor should NOT claim to handle zip files - assert.False(t, ext.CanExtract(ctx, fileContent(t, zipPath))) -} - -func TestDefaultExtractionLimits(t *testing.T) { - limits := DefaultExtractionLimits(cataloging.ArchiveSearchConfig{}) - assert.Nil(t, limits.Charge, "the charge belongs to one archive, so the config cannot supply it") -} - -func TestDefaultLimits(t *testing.T) { - cfg := cataloging.ArchiveSearchConfig{ - MaxMemoryBytes: 111, - MaxDiskBytes: 222, - } - limits := DefaultLimits(cfg) - assert.Equal(t, int64(111), limits.MaxMemoryBytes) - assert.Equal(t, int64(222), limits.MaxDiskBytes) -} - -func TestZipExtractor_Extract_ZipSlipPrevented(t *testing.T) { - // An entry named "../../etc/passwd" no longer has anywhere to go: it becomes a field in a tar - // header, and writing a header traverses no path. So the extraction succeeds and nothing is - // created anywhere, where before the entry was refused outright and the archive failed with it. - // Where the name still matters is the path the SBOM reports, and that is clamped when the tar is - // indexed - see the archive resolver's own tests. - dir := t.TempDir() - ctx := context.Background() - - zipPath := filepath.Join(dir, "evil.zip") - f, err := os.Create(zipPath) - require.NoError(t, err) - - w := zip.NewWriter(f) - fw, err := w.Create("../../etc/passwd") - require.NoError(t, err) - _, err = fw.Write([]byte("evil")) - require.NoError(t, err) - require.NoError(t, w.Close()) - require.NoError(t, f.Close()) - - ext := &ZipExtractor{} - overflow := overflowFor(t) - - result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) - require.NoError(t, err) - assert.Equal(t, 1, result.FilesExtracted) - - // nothing is created outside the tar, at the traversed path or anywhere else - _, statErr := os.Stat(filepath.Join(dir, "etc", "passwd")) - assert.True(t, os.IsNotExist(statErr)) - _, statErr = os.Stat(filepath.Join(filepath.Dir(overflow.Path()), "etc")) - assert.True(t, os.IsNotExist(statErr)) - - // the name is carried through unsanitized, because this is the archive's own record of what it - // held; sanitizing happens once, where the logical filetree is built - assert.Equal(t, []string{"../../etc/passwd"}, overflowNames(t, overflow)) -} - -func TestTarExtractor_Extract_SymlinkIsRecordedAsALink(t *testing.T) { - // A symlink entry is recorded as a symlink header carrying its target, not as a regular file - // containing the target as bytes and not as a link on the real filesystem. Nothing is created, so - // there is nothing for a target to redirect; where the link points is decided inside the archive's - // own filetree when the tar is indexed. - dir := t.TempDir() - ctx := context.Background() - - tarPath := filepath.Join(dir, "with-link.tar.gz") - f, err := os.Create(tarPath) - require.NoError(t, err) - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - target := []byte("real content") - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "real.txt", - Mode: 0o644, - Size: int64(len(target)), - })) - _, err = tw.Write(target) - require.NoError(t, err) - - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "link.txt", - Linkname: "real.txt", - Typeflag: tar.TypeSymlink, - Mode: 0o777, - })) - - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - require.NoError(t, f.Close()) - - ext := &TarExtractor{} - overflow := overflowFor(t) - - _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{}) - require.NoError(t, err) - - entries := readOverflow(t, overflow) - link := entries["link.txt"] - assert.Equal(t, byte(tar.TypeSymlink), link.typeflag, "link.txt must stay a link, not become a file") - assert.Equal(t, "real.txt", link.linkTarget) - assert.Empty(t, link.body, "a link carries a target, not content") - assert.Equal(t, "real content", entries["real.txt"].body) - - // and no link exists on the real filesystem beside the tar - _, statErr := os.Lstat(filepath.Join(filepath.Dir(overflow.Path()), "link.txt")) - assert.True(t, os.IsNotExist(statErr)) -} - -func TestTarExtractor_Extract_SymlinkEscapingRootCreatesNothing(t *testing.T) { - // A link target climbing out of the archive used to be refused at extraction, because extraction - // created a real link that a later entry could be written through. Now the entry is a header: the - // target is recorded as the archive gave it and nothing is created, so there is no write to - // redirect. The target is clamped to the archive's own root when the filetree is built, which is - // what keeps it from naming the host - see the archive resolver's own tests. - dir := t.TempDir() - ctx := context.Background() - - tarPath := filepath.Join(dir, "evil-link.tar.gz") - f, err := os.Create(tarPath) - require.NoError(t, err) - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "passwd", - Linkname: "../../../../etc/passwd", - Typeflag: tar.TypeSymlink, - Mode: 0o777, - })) - - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - require.NoError(t, f.Close()) - - ext := &TarExtractor{} - overflow := overflowFor(t) - - _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{}) - require.NoError(t, err) - - // nothing on the real filesystem beside the tar - _, statErr := os.Lstat(filepath.Join(filepath.Dir(overflow.Path()), "passwd")) - assert.True(t, os.IsNotExist(statErr), "no link may be created for an escaping symlink") - - entries := readOverflow(t, overflow) - assert.Equal(t, byte(tar.TypeSymlink), entries["passwd"].typeflag) - assert.Equal(t, "../../../../etc/passwd", entries["passwd"].linkTarget) -} - -func TestTarExtractor_Extract_AbsoluteSymlinkTargetCreatesNothing(t *testing.T) { - // An absolute target had to be rejected while extraction created real links: os.Symlink writes the - // literal string, so reading the link resolved on the host filesystem whatever was checked at - // extraction time. A header write creates no link, so there is nothing that resolves anywhere; - // the target is re-rooted inside the archive when the filetree is built. - dir := t.TempDir() - ctx := context.Background() - - tarPath := filepath.Join(dir, "absolute-link.tar.gz") - f, err := os.Create(tarPath) - require.NoError(t, err) - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "shadow", - Linkname: "/etc/shadow", - Typeflag: tar.TypeSymlink, - Mode: 0o777, - })) - - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - require.NoError(t, f.Close()) - - ext := &TarExtractor{} - overflow := overflowFor(t) - - _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{}) - require.NoError(t, err) - - _, statErr := os.Lstat(filepath.Join(filepath.Dir(overflow.Path()), "shadow")) - assert.True(t, os.IsNotExist(statErr), "no link may be created for an absolute target") - - entries := readOverflow(t, overflow) - assert.Equal(t, "/etc/shadow", entries["shadow"].linkTarget) -} - -func TestZipExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { - // JARs may contain directory entries with read-only permissions (e.g., META-INF/ with mode 0o555). - // The extractor must still be able to write files into those directories. - dir := t.TempDir() - ctx := context.Background() - - zipPath := filepath.Join(dir, "readonly-dirs.zip") - f, err := os.Create(zipPath) - require.NoError(t, err) - - w := zip.NewWriter(f) - - // create a directory entry with read-only permissions (0o555) - dirHeader := &zip.FileHeader{ - Name: "META-INF/", - } - dirHeader.SetMode(0o555) - _, err = w.CreateHeader(dirHeader) - require.NoError(t, err) - - // create a file inside that directory - fileHeader := &zip.FileHeader{ - Name: "META-INF/MANIFEST.MF", - } - fileHeader.SetMode(0o644) - fw, err := w.CreateHeader(fileHeader) - require.NoError(t, err) - _, err = fw.Write([]byte("Manifest-Version: 1.0")) - require.NoError(t, err) - - require.NoError(t, w.Close()) - require.NoError(t, f.Close()) - - ext := &ZipExtractor{} - overflow := overflowFor(t) - - result, err := ext.Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) - require.NoError(t, err) - // two entries: the directory and the file. A directory carries no bytes of its own, but its header - // and padding are charged to the disk limit, which is what bounds an archive of nothing but - // directories. - assert.Equal(t, 2, result.FilesExtracted) - - // the directory's mode no longer decides whether the file beside it can be written - nothing is - // written into a directory - but both entries must be recorded, modes and all - entries := readOverflow(t, overflow) - assert.Equal(t, "Manifest-Version: 1.0", entries["META-INF/MANIFEST.MF"].body) - assert.Equal(t, fs.FileMode(0o555), entries["META-INF/"].mode.Perm()) - assert.Equal(t, byte(tar.TypeDir), entries["META-INF/"].typeflag) -} - -func TestTarExtractor_Extract_ReadOnlyDirectoryPermissions(t *testing.T) { - // Tar archives may contain directory entries with read-only permissions. - // The extractor must still be able to write files into those directories. - dir := t.TempDir() - ctx := context.Background() - - tarPath := filepath.Join(dir, "readonly-dirs.tar.gz") - f, err := os.Create(tarPath) - require.NoError(t, err) - - gw := gzip.NewWriter(f) - tw := tar.NewWriter(gw) - - // write a directory entry with read-only permissions - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "META-INF/", - Typeflag: tar.TypeDir, - Mode: 0o555, - })) - - // write a file inside that directory - content := []byte("Manifest-Version: 1.0") - require.NoError(t, tw.WriteHeader(&tar.Header{ - Name: "META-INF/MANIFEST.MF", - Mode: 0o644, - Size: int64(len(content)), - })) - _, err = tw.Write(content) - require.NoError(t, err) - - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - require.NoError(t, f.Close()) - - ext := &TarExtractor{} - overflow := overflowFor(t) - - _, err = ext.Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{}) - require.NoError(t, err) - - entries := readOverflow(t, overflow) - assert.Equal(t, "Manifest-Version: 1.0", entries["META-INF/MANIFEST.MF"].body) - assert.Equal(t, fs.FileMode(0o555), entries["META-INF/"].mode.Perm()) -} - -func TestZipExtractor_Extract_nonPositiveLimitDisablesOnlyThatLimit(t *testing.T) { - // <= 0 means "no limit" per limit, so a caller can opt out of one without opting out of all of - // them. Substituting a default here would silently re-impose a limit the user turned off. - dir := t.TempDir() - ctx := context.Background() - - files := map[string]string{} - for i := range 20 { - files[fmt.Sprintf("file%02d.txt", i)] = "x" - } - files["big.txt"] = strings.Repeat("y", 500) - zipPath := createTestZip(t, dir, files) - - t.Run("the disk limit bites", func(t *testing.T) { - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{ - Charge: diskCharge(100), - }) - require.NoError(t, err) - assert.Equal(t, TruncatedByDiskLimit, result.Truncation, "the disk limit must still bite") - }) - - t.Run("unbounded extracts everything", func(t *testing.T) { - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{}) - require.NoError(t, err) - assert.False(t, result.Truncated()) - assert.Equal(t, len(files), result.FilesExtracted) - }) -} - -func TestZipExtractor_Extract_dataEdgeCases(t *testing.T) { - ctx := context.Background() - - t.Run("empty archive is a non-event", func(t *testing.T) { - dir := t.TempDir() - zipPath := createTestZip(t, dir, map[string]string{}) - - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{}) - require.NoError(t, err, "a valid archive containing nothing is not a failure") - assert.Zero(t, result.FilesExtracted) - assert.False(t, result.Truncated()) - }) - - t.Run("zero-byte entry is recorded", func(t *testing.T) { - dir := t.TempDir() - zipPath := createTestZip(t, dir, map[string]string{"empty.txt": ""}) - overflow := overflowFor(t) - - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) - require.NoError(t, err) - assert.Equal(t, 1, result.FilesExtracted) - - // its header and the end-of-archive marker are bytes on disk even though the entry carries - // none, which is exactly why the disk limit counts what is written rather than entry sizes - assert.Equal(t, int64(3*tarBlockSize), result.BytesWritten) - - entries := readOverflow(t, overflow) - require.Contains(t, entries, "empty.txt") - assert.Empty(t, entries["empty.txt"].body) - }) - - t.Run("absolute entry path is contained", func(t *testing.T) { - // distinct from ../ traversal: an entry literally named /etc/hosts must land inside the - // extraction dir, never at the host path - dir := t.TempDir() - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - w, err := zw.Create("/etc/hosts") - require.NoError(t, err) - _, err = w.Write([]byte("pwned")) - require.NoError(t, err) - require.NoError(t, zw.Close()) - - zipPath := filepath.Join(dir, "abs.zip") - require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0o600)) - - overflow := overflowFor(t) - - _, err = (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) - require.NoError(t, err) - - // it is one entry in the tar and nothing at the host path; the leading slash is dropped when - // the filetree is built, so it reads as a path inside this archive - assert.Equal(t, "pwned", readOverflow(t, overflow)["/etc/hosts"].body) - _, statErr := os.Stat(filepath.Join(dir, "etc", "hosts")) - assert.True(t, os.IsNotExist(statErr)) - }) - - t.Run("unicode and non-ascii entry names survive", func(t *testing.T) { - dir := t.TempDir() - names := map[string]string{ - "café/naïve.txt": "accents", - "日本語/ファイル.txt": "japanese", - "emoji-🎉.txt": "emoji", - } - zipPath := createTestZip(t, dir, names) - overflow := overflowFor(t) - - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{}) - require.NoError(t, err) - assert.Equal(t, len(names), result.FilesExtracted) - - entries := readOverflow(t, overflow) - for name, want := range names { - require.Contains(t, entries, name, "entry %q must be recorded under its own name", name) - assert.Equal(t, want, entries[name].body) - } - }) -} - -func TestExtract_diskLimitTruncatesOneArchive(t *testing.T) { - // the disk limit is enforced as the entries land, not between archives: a bound checked only - // between archives lets one archive overshoot by everything it writes. - // - // The budget is expressed in tar blocks because that is now what an entry costs: a 512-byte header - // plus its content padded up to the next block. Two entries here cost one block of header and one - // of data each, so a budget of two and a half entries' worth admits the first whole and refuses - // the second at its header. - ctx := context.Background() - files := map[string]string{ - "a/small.txt": strings.Repeat("s", 50), - "b/big.txt": strings.Repeat("b", 500), - } - - dir := t.TempDir() - archivePath := createTestZip(t, dir, files) - overflow := overflowFor(t) - - charge := diskCharge(3 * tarBlockSize) - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, archivePath), overflow, ExtractionLimits{Charge: charge}) - require.NoError(t, err, "reaching a limit is a truncation, not a failure") - - assert.Equal(t, TruncatedByDiskLimit, result.Truncation, - "a limit breach reports its own reason, so the log can name which bound was reached") - assert.Equal(t, 1, result.FilesExtracted) - assert.Equal(t, int64(2*tarBlockSize), result.BytesWritten, "one header and one padded data block") - - _, disk := charge.Held() - assert.Equal(t, int64(2*tarBlockSize), disk, - "the limiter holds exactly what landed: the dropped entry's header is refunded") - - // the partial tar is still a readable tar: what was written before the limit filled stays, and the - // entry that did not fit is absent rather than short - entries := readOverflow(t, overflow) - assert.Len(t, entries["a/small.txt"].body, 50) - assert.NotContains(t, entries, "b/big.txt") -} - -func TestTarExtractor_Extract_diskLimitTruncatesTheCopy(t *testing.T) { - // a tar-family archive is copied rather than walked, so the disk limit stops it at a copy chunk - // rather than on an entry boundary. What is left is a tar that ends part way through an entry, and - // keeping the entries that precede that point is the indexer's job - see the archive resolver's - // tests. Here the assertion is that the copy stops, says why, and holds only what it wrote. - ctx := context.Background() - dir := t.TempDir() - - // two entries either side of a copy chunk, so the limit can bite in the middle of the stream - // rather than refusing the first chunk and writing nothing at all - files := map[string]string{ - "a/first.txt": strings.Repeat("a", 40*1024), - "b/second.txt": strings.Repeat("b", 40*1024), - } - archivePath := createTestTarGz(t, dir, files) - overflow := overflowFor(t) - - charge := diskCharge(64 * 1024) - result, err := (&TarExtractor{}).Extract(ctx, fileContent(t, archivePath), overflow, ExtractionLimits{Charge: charge}) - require.NoError(t, err) - - assert.Equal(t, TruncatedByDiskLimit, result.Truncation) - assert.Equal(t, int64(64*1024), result.BytesWritten, "every byte written was charged, and no more") - - _, disk := charge.Held() - assert.Equal(t, result.BytesWritten, disk) - - info, err := os.Stat(overflow.Path()) - require.NoError(t, err) - assert.Equal(t, result.BytesWritten, info.Size(), "the tar holds exactly what was charged for") -} - -func TestExtract_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) { - // the whole point of a limiter over a counter: two archives of the same size both extract in full - // when the first is released between them, and the second is bounded when it is not - ctx := context.Background() - dir := t.TempDir() - zipPath := createTestZip(t, dir, map[string]string{"a.txt": strings.Repeat("a", 200)}) - - // one entry of 200 bytes costs a header block, a padded data block and the end-of-archive marker: - // four blocks. A limit of five leaves no room for a second archive while the first is held. - const oneArchive = 4 * tarBlockSize - limiter := NewLimiter(Limits{MaxDiskBytes: 5 * tarBlockSize}) - - first := limiter.Charge() - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{Charge: first}) - require.NoError(t, err) - require.False(t, result.Truncated()) - require.Equal(t, int64(oneArchive), result.BytesWritten) - - t.Run("a second archive is bounded while the first is still held", func(t *testing.T) { - second := limiter.Charge() - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{Charge: second}) - require.NoError(t, err) - assert.Equal(t, TruncatedByDiskLimit, result.Truncation) - second.Release() - }) - - t.Run("and extracts in full once the first is released", func(t *testing.T) { - first.Release() - _, disk := limiter.InUse() - require.Zero(t, disk, "releasing every charge must take the limiter back to nothing") - - third := limiter.Charge() - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflowFor(t), ExtractionLimits{Charge: third}) - require.NoError(t, err) - assert.False(t, result.Truncated(), "a counter would have refused this; a limiter does not") - assert.Equal(t, int64(oneArchive), result.BytesWritten) - }) -} - -// createDirOnlyZip writes a zip of nothing but directory entries. func createDirOnlyZip(t *testing.T, dir string, count int) string { t.Helper() zipPath := filepath.Join(dir, "dirs.zip") @@ -874,7 +1090,6 @@ func createDirOnlyZip(t *testing.T, dir string, count int) string { return zipPath } -// createDirOnlyTarGz writes a gzipped tar of nothing but directory entries. func createDirOnlyTarGz(t *testing.T, dir string, count int) string { t.Helper() tarPath := filepath.Join(dir, "dirs.tar.gz") @@ -896,52 +1111,6 @@ func createDirOnlyTarGz(t *testing.T, dir string, count int) string { return tarPath } -func TestExtract_directoryEntriesAreBoundedByDisk(t *testing.T) { - // an archive of directory entries alone carries no content, so what bounds it is the disk limit - // charging each entry's header and the tar's padding. That is the resource such an archive - // actually consumes, and it is the only bound left now that the entry count is gone. - ctx := context.Background() - - t.Run("zip", func(t *testing.T) { - dir := t.TempDir() - zipPath := createDirOnlyZip(t, dir, 50) - overflow := overflowFor(t) - - // each directory entry is one header block; room for twelve blocks admits ten entries plus the - // two end-of-archive blocks - result, err := (&ZipExtractor{}).Extract(ctx, fileContent(t, zipPath), overflow, ExtractionLimits{ - Charge: diskCharge(12 * tarBlockSize), - }) - require.NoError(t, err, "a bound is a truncation, not a failure") - assert.Equal(t, TruncatedByDiskLimit, result.Truncation) - assert.Less(t, result.FilesExtracted, 50, "the bound must stop the archive short") - assert.NotEmpty(t, overflowNames(t, overflow), "what was written before the bound stays readable") - }) - - t.Run("tar.gz is copied whole, so the disk limit is what stops it", func(t *testing.T) { - // a tar is not walked entry by entry: the copy takes the whole archive, and the disk limit is - // charged as those bytes land - dir := t.TempDir() - tarPath := createDirOnlyTarGz(t, dir, 50) - overflow := overflowFor(t) - - result, err := (&TarExtractor{}).Extract(ctx, fileContent(t, tarPath), overflow, ExtractionLimits{ - Charge: diskCharge(-1), - }) - require.NoError(t, err) - assert.False(t, result.Truncated()) - assert.Len(t, overflowNames(t, overflow), 50) - }) -} - -// symlinkEntry describes one archive entry for the escape tests: a symlink when linkTarget is set, -// a regular file otherwise. -type symlinkEntry struct { - name string - linkTarget string - body string -} - func createSymlinkChainTarGz(t *testing.T, dir, name string, entries []symlinkEntry) string { t.Helper() path := filepath.Join(dir, name) @@ -986,8 +1155,8 @@ func createSymlinkChainZip(t *testing.T, dir, name string, entries []symlinkEntr hdr := &zip.FileHeader{Name: e.name} body := e.body if e.linkTarget != "" { - // a zip stores a symlink as an entry whose mode carries the symlink bit and whose - // content is the link target + // a zip stores a symlink as an entry whose mode carries the symlink bit and whose content + // is the link target hdr.SetMode(fs.ModeSymlink | 0o777) body = e.linkTarget } else { @@ -1002,7 +1171,6 @@ func createSymlinkChainZip(t *testing.T, dir, name string, entries []symlinkEntr return path } -// findEscapedFiles returns every path under base, outside destDir, whose basename is name. func findEscapedFiles(t *testing.T, base, destDir, name string) []string { t.Helper() var found []string @@ -1025,164 +1193,35 @@ func findEscapedFiles(t *testing.T, base, destDir, name string) []string { return found } -func TestExtract_symlinkChainEscapeWritesNothing(t *testing.T) { - // two cooperating link entries used to each pass a lexical check and together resolve outside the - // extraction directory: "d" -> "." landed on the root, so "d/up" -> ".." was created AS - // "/up" -> ".." and pointed above it, and a third entry was written through it. - // - // The chain needed real links on a real filesystem to work. Entries are headers now, so the whole - // class is gone rather than closed: no link is created, so there is nothing for a later entry to - // be written through, and no entry is written anywhere in the first place. The archive is kept in - // the suite because "nothing outside the tar is created" is the property to keep asserting, and - // this is the input that would have violated it. - chain := []symlinkEntry{ - {name: "d", linkTarget: "."}, - {name: "d/up", linkTarget: ".."}, - {name: "up/PWNED", body: "arbitrary write"}, - } - - // three link pairs, each climbing one more level. A fix that only walked one level would pass a - // one-level test and still let this through. - escalating := []symlinkEntry{ - {name: "a", linkTarget: "."}, - {name: "a/b", linkTarget: ".."}, - {name: "b/c", linkTarget: "."}, - {name: "b/c/d", linkTarget: ".."}, - {name: "b/d/e", linkTarget: "."}, - {name: "b/d/e/f", linkTarget: ".."}, - {name: "b/d/f/PWNED", body: "arbitrary write, three levels up"}, - } - - for _, tc := range []struct { - name string - entries []symlinkEntry - }{ - {name: "one pair", entries: chain}, - {name: "three pairs escalating", entries: escalating}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Run("tar.gz", func(t *testing.T) { - assertNoEscape(t, createSymlinkChainTarGz, func() Extractor { return &TarExtractor{} }, "chain.tar.gz", tc.entries) - }) - t.Run("zip", func(t *testing.T) { - assertNoEscape(t, createSymlinkChainZip, func() Extractor { return &ZipExtractor{} }, "chain.zip", tc.entries) - }) - }) - } -} - func assertNoEscape(t *testing.T, build func(*testing.T, string, string, []symlinkEntry) string, newExtractor func() Extractor, archiveName string, entries []symlinkEntry) { t.Helper() - // several levels of headroom above the extraction directory, all inside the test's own temp - // tree, so an escape has somewhere to land and is still cleaned up + // several levels of headroom above the extraction directory, all inside the test's own temp tree, so + // an escape has somewhere to land and is still cleaned up base := t.TempDir() destDir := filepath.Join(base, "one", "two", "three", "contents") require.NoError(t, os.MkdirAll(destDir, 0o755)) archivePath := build(t, base, archiveName, entries) - overflow, err := newOverflowTar(filepath.Join(destDir, overflowTarName)) - require.NoError(t, err) - t.Cleanup(func() { _ = overflow.Close() }) + store := NewEntryStore(WorkDirAt(destDir), archiveName, nil) + t.Cleanup(func() { _ = store.Close() }) - _, err = newExtractor().Extract(context.Background(), fileContent(t, archivePath), overflow, ExtractionLimits{}) + _, err := newExtractor().Extract(context.Background(), fileContent(t, archivePath), store, nil) require.NoError(t, err, "an unsafe entry is skipped, not a failure of the whole archive") assert.Empty(t, findEscapedFiles(t, base, destDir, "PWNED"), "nothing may be written outside the extraction directory") - // and nothing was created inside it either, other than the one tar + // and nothing was created inside it either: entries are headers and bytes in the store, and with no + // bound pushing them out not even the overflow blob exists created, err := os.ReadDir(destDir) require.NoError(t, err) - require.Len(t, created, 1) - assert.Equal(t, overflowTarName, created[0].Name()) + assert.Empty(t, created) } -func TestTarExtractor_Extract_legitimateRelativeSymlinkIsCarriedThrough(t *testing.T) { - // containing an escape must not cost what real archives contain: a relative link into a sibling - // directory of the archive's own tree keeps its target verbatim, so the filetree can resolve it. - // That it then resolves is asserted where resolution happens, in the archive resolver's tests. - dir := t.TempDir() - - archivePath := createSymlinkChainTarGz(t, dir, "legit.tar.gz", []symlinkEntry{ - {name: "lib/real.txt", body: "real content"}, - {name: "bin/link.txt", linkTarget: "../lib/real.txt"}, - {name: "lib/sub/deep.txt", body: "deep"}, - {name: "bin/deep.txt", linkTarget: "../lib/sub/deep.txt"}, - }) - overflow := overflowFor(t) - - _, err := (&TarExtractor{}).Extract(context.Background(), fileContent(t, archivePath), overflow, ExtractionLimits{}) - require.NoError(t, err) - - entries := readOverflow(t, overflow) - assert.Equal(t, "../lib/real.txt", entries["bin/link.txt"].linkTarget) - assert.Equal(t, "../lib/sub/deep.txt", entries["bin/deep.txt"].linkTarget) - assert.Equal(t, "real content", entries["lib/real.txt"].body) - assert.Equal(t, "deep", entries["lib/sub/deep.txt"].body) -} - -func TestExtract_siblingOfExtractionDirectoryWritesNothing(t *testing.T) { - // the realistic form of the string-prefix hole SafeJoin was fixed for: an entry naming - // "../contents-evil/..." shares the extraction directory's prefix and is outside it. - // - // Both extractors used to fail the archive here, because SafeJoin refused the entry. Nothing joins - // an entry name to a path any more, so the entry is recorded as the name it claims and no - // filesystem path is derived from it at all. The assertion that matters is unchanged: nothing is - // written beside the extraction directory. - ctx := context.Background() - - for _, tc := range []struct { - name string - build func(t *testing.T, dir string, files map[string]string) string - extract func(path string, overflow *OverflowTar) (ExtractionResult, error) - }{ - { - name: "zip", - build: createTestZip, - extract: func(path string, overflow *OverflowTar) (ExtractionResult, error) { - return (&ZipExtractor{}).Extract(ctx, fileContent(t, path), overflow, ExtractionLimits{}) - }, - }, - { - name: "tar.gz", - build: createTestTarGz, - extract: func(path string, overflow *OverflowTar) (ExtractionResult, error) { - return (&TarExtractor{}).Extract(ctx, fileContent(t, path), overflow, ExtractionLimits{}) - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - base := t.TempDir() - destDir := filepath.Join(base, "contents") - require.NoError(t, os.MkdirAll(destDir, 0o755)) - - archivePath := tc.build(t, base, map[string]string{"../contents-evil/x.txt": "escaped"}) - - overflow, err := newOverflowTar(filepath.Join(base, overflowTarName)) - require.NoError(t, err) - t.Cleanup(func() { _ = overflow.Close() }) - - _, err = tc.extract(archivePath, overflow) - require.NoError(t, err, "a hostile name is a header field, not a failure") - - _, statErr := os.Stat(filepath.Join(base, "contents-evil", "x.txt")) - assert.True(t, os.IsNotExist(statErr), - "nothing may be written to a sibling of the extraction directory") - - entries, err := os.ReadDir(destDir) - require.NoError(t, err) - assert.Empty(t, entries, "the extraction directory stays empty: entries live in the tar") - - assert.Equal(t, "escaped", readOverflow(t, overflow)["../contents-evil/x.txt"].body) - }) - } -} - -// scriptPrefixedZip is a self-extracting archive in the shape Spring Boot produces: a launcher -// script with a complete zip concatenated onto it. Built rather than committed, so what makes the -// case is visible in the test. +// scriptPrefixedZip is a self-extracting archive in the shape Spring Boot produces: a launcher script +// with a complete zip appended. func scriptPrefixedZip(t *testing.T, prefix string, files map[string]string) []byte { t.Helper() var zbuf bytes.Buffer @@ -1200,82 +1239,3 @@ func scriptPrefixedZip(t *testing.T, prefix string, files map[string]string) []b func bytesContent(name string, data []byte) Content { return Content{Name: name, Reader: bytes.NewReader(data)} } - -func Test_HasZipEndOfCentralDirectory(t *testing.T) { - var plainZip bytes.Buffer - zw := zip.NewWriter(&plainZip) - f, err := zw.Create("a.txt") - require.NoError(t, err) - _, err = f.Write([]byte("hello")) - require.NoError(t, err) - require.NoError(t, zw.Close()) - - var tarGz bytes.Buffer - gw := gzip.NewWriter(&tarGz) - tw := tar.NewWriter(gw) - require.NoError(t, tw.WriteHeader(&tar.Header{Name: "a.txt", Size: 5, Mode: 0o600})) - _, err = tw.Write([]byte("hello")) - require.NoError(t, err) - require.NoError(t, tw.Close()) - require.NoError(t, gw.Close()) - - tests := []struct { - name string - data []byte - want bool - }{ - {"a plain zip", plainZip.Bytes(), true}, - { - "a zip behind a launcher script - the case content sniffing types by its head", - scriptPrefixedZip(t, "#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n", - map[string]string{"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n"}), - true, - }, - {"prose, whatever it is named", []byte("this is not an archive, it is a sentence"), false}, - {"a tar.gz, which is an archive but not a zip", tarGz.Bytes(), false}, - {"empty", nil, false}, - {"shorter than the signature", []byte("PK"), false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, HasZipEndOfCentralDirectory(bytes.NewReader(tt.data))) - }) - } -} - -func Test_ZipExtractor_canExtractAZipBehindALauncherScript(t *testing.T) { - // the regression this closes: a Spring Boot executable jar sniffs as text/x-shellscript, so - // format identification refuses it and every package inside is lost - data := scriptPrefixedZip(t, "#!/bin/bash\necho launcher\nexit 0\n", map[string]string{ - "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", - "BOOT-INF/lib/dep.jar": "not really a jar, just bytes", - }) - - extractor := &ZipExtractor{} - require.True(t, extractor.CanExtract(context.Background(), bytesContent("app.jar", data)), - "a zip behind a launcher script must be recognized") - - // and it extracts, because archive/zip finds the central directory from the end - overflow, err := newOverflowTar(filepath.Join(t.TempDir(), "app.jar.tar")) - require.NoError(t, err) - result, err := extractor.Extract(context.Background(), bytesContent("app.jar", data), overflow, - ExtractionLimits{Charge: diskCharge(-1)}) - require.NoError(t, err) - assert.Positive(t, result.BytesWritten, "the entries behind the script prefix must actually be written") - assert.Empty(t, string(result.Truncation)) -} - -func Test_ZipExtractor_refusesANonArchiveThatNothingIdentifies(t *testing.T) { - // the probe must not turn a file with no central directory into an archive. Note the name matters - // here in a way it does not in a scan: identification matches on it, so `notes.zip` is claimed by - // name before the probe is reached. That is why the scenario `misnamed non-archive is not - // extracted` is asserted at the task level, where the candidate set decides - see - // Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError. - extractor := &ZipExtractor{} - for _, name := range []string{"app.jar", "bundle", "launcher"} { - t.Run(name, func(t *testing.T) { - content := bytesContent(name, []byte("plain text, no central directory anywhere in it")) - assert.False(t, extractor.CanExtract(context.Background(), content)) - }) - } -} diff --git a/internal/archive/helpers_test.go b/internal/archive/helpers_test.go new file mode 100644 index 000000000..3d6f99343 --- /dev/null +++ b/internal/archive/helpers_test.go @@ -0,0 +1,59 @@ +package archive + +import ( + "os" + "path/filepath" +) + +// Seams the package itself does not need, kept here so no production file carries a function only +// tests call. + +// WorkDirAt returns a WorkDir over a directory that already exists. Production always creates its own +// (NewWorkDir); a test lends one it can inspect afterwards, so Remove leaves it alone. +func WorkDirAt(dir string) *WorkDir { + return &WorkDir{path: dir, done: true} +} + +// heldInMemory reports the entry content this store is still holding in memory. Production never asks: +// the limiter is what bounds it, and the store charges as it goes rather than tracking a total. +func (s *EntryStore) heldInMemory() int64 { + s.mu.RLock() + defer s.mu.RUnlock() + var total int64 + for _, entry := range s.entries { + if entry.ref.mem != nil { + total += entry.ref.length + } + } + return total +} + +// created reports whether anything ever asked for the directory, and so whether there is anything to +// remove. +func (w *WorkDir) created() bool { + if w == nil { + return false + } + w.mu.Lock() + defer w.mu.Unlock() + return w.remove != nil +} + +// held reports what this charge is currently holding, in memory and on disk. +func (c *Charge) held() (memory, disk int64) { + if c == nil || c.limiter == nil { + return 0, 0 + } + c.limiter.mu.Lock() + defer c.limiter.mu.Unlock() + return c.memory, c.disk +} + +// openFileContent opens an archive already on disk as Content. The caller closes it. +func openFileContent(path string) (Content, error) { + f, err := os.Open(path) + if err != nil { + return Content{}, err + } + return Content{Name: filepath.Base(path), Reader: f, closer: f}, nil +} diff --git a/internal/archive/limits.go b/internal/archive/limits.go index 0b01adc9b..21733e468 100644 --- a/internal/archive/limits.go +++ b/internal/archive/limits.go @@ -5,57 +5,42 @@ import ( "sync" ) -// ErrDiskLimitReached reports that an archive's content could not be admitted because placing it -// would have taken disk usage past the disk limit. It is not a failure of the scan: the archive is -// skipped and the walk continues. +// ErrDiskLimitReached reports that placing an archive's content would exceed the disk limit. Not a +// scan failure: the archive is skipped and the walk continues. var ErrDiskLimitReached = errors.New("archive content would exceed the disk limit") -// Limits bounds how much archive content one scan may hold at once, and decides where an -// archive's content is held. +// Limits bounds how much archive content one scan may hold at once, and so where content is held. // -// These are in-use limits rather than counters. An in-use limit measures what is held right now and -// falls when an archive is released, so what it bounds is a scan's peak concurrent usage. A bound on -// bytes ever written measures no state that ever exists, because every archive's extraction directory -// is removed on the way out: it would restrict a long scan of small archives that never held much at -// once, and permit a deep scan that held far more. +// These are in-use limits, not counters: each measures what is held right now and falls when an +// archive is released, bounding peak concurrent usage. Work directories are removed on release, so a +// bound on bytes ever written would measure state that never exists at once. // -// Both fields read three states the same way: positive is the limit, zero means none of that -// resource may be used at all, and negative means that resource is not bounded. Unbounded is a real -// hazard on either - a crafted archive can exhaust memory or disk - so it is reachable only by asking -// for it explicitly with a negative value, never as the reading of an unset or zeroed field. +// Both fields read three states the same way: positive is the limit, zero forbids that resource, and +// negative is unbounded. Unbounded is a real hazard, so it must be asked for explicitly. type Limits struct { - // MaxMemoryBytes bounds the archive content held in memory at once. Content is held in memory - // while this admits it and overflows to disk when it does not - there is no separate configured size - // at which overflowing begins. Zero holds nothing, sending every archive's content to disk; - // negative holds content regardless of how much is already held. + // MaxMemoryBytes bounds archive content held in memory at once; content spills to disk when it does + // not admit more, so there is no separate spill threshold. Zero sends all content to disk; negative + // is unbounded. MaxMemoryBytes int64 - // MaxDiskBytes bounds the archive content placed on disk at once: content overflowed there instead - // of being held in memory, plus the bytes of the entries extracted from it. Zero writes nothing - // to disk, so content that does not fit in memory has nowhere to go and its archive is skipped; - // negative overflows as needed with no ceiling. + // MaxDiskBytes bounds archive content on disk at once: content spilled there plus the entry bytes + // extracted from it. Zero writes nothing, so content that does not fit in memory has nowhere to go + // and its archive is skipped; negative is unbounded. MaxDiskBytes int64 } -// Limiter measures how much archive content a scan is holding right now. One instance per task run, -// which is one scan - the only scoping under which "held at once" means anything. -// -// Guarded by a mutex even though the archive walk is sequential today: the archive task merges into -// a shared builder while top-level catalogers run, so this code already lives in a concurrent -// neighbourhood and an unguarded total here would be a data race the day someone parallelises the -// walk. -// -// A nil *Limiter enforces nothing, which is what an extraction with no configured bounds gets. +// Limiter measures how much archive content a scan is holding right now. One instance per task run +// (one scan), the only scope under which "held at once" means anything. A nil *Limiter enforces +// nothing. Mutex-guarded because the archive task merges into a shared builder while top-level +// catalogers run. type Limiter struct { limits Limits mu sync.Mutex memory int64 disk int64 - // peaks are the high-water marks of the two gauges above: the most this scan held at any one - // moment, rather than the sum of what every archive held. They only ever rise, which is the one - // place in this type where that is correct - a peak that fell on release would be measuring the - // same thing the gauge already measures. + // peaks are high-water marks of the two gauges above: the most held at any one moment, not the sum + // across archives. They only rise. peakMemory int64 peakDisk int64 } @@ -65,16 +50,7 @@ func NewLimiter(limits Limits) *Limiter { return &Limiter{limits: limits} } -// MaxDiskBytes returns the configured disk bound, for attributing a skip in a log line. -func (l *Limiter) MaxDiskBytes() int64 { - if l == nil { - return 0 - } - return l.limits.MaxDiskBytes -} - -// Peak reports the most archive content this scan held at any one moment, in memory and on disk. -// These are the same quantities the limits bound, so they say how close a scan came to them. +// Peak reports the most archive content held at any one moment, the quantities the limits bound. func (l *Limiter) Peak() (memory, disk int64) { if l == nil { return 0, 0 @@ -94,8 +70,7 @@ func (l *Limiter) InUse() (memory, disk int64) { return l.memory, l.disk } -// Charge starts one archive's accounting against this limiter. Releasing the returned charge gives -// back exactly what it took, which is what makes this a limiter rather than a counter. +// Charge starts one archive's accounting. Releasing the returned charge gives back what it took. func (l *Limiter) Charge() *Charge { if l == nil { return nil @@ -103,25 +78,18 @@ func (l *Limiter) Charge() *Charge { return &Charge{limiter: l} } -// Charge is one archive's accumulated draw on the limiter. +// Charge is one archive's accumulated draw on the limiter. A nil *Charge charges and refuses nothing. // -// Bytes are charged to it as they land, never from a size an entry declared: both tar and zip -// headers carry an entry size and a crafted archive is free to lie about it, so a reservation made -// against a declared size bounds what an attacker claims rather than what the machine does. The -// charge accumulates what was actually taken, so Release returns exactly that. -// -// A nil *Charge charges nothing and refuses nothing. +// Bytes are charged as they land, never from a declared size: tar and zip headers both carry a size a +// crafted archive can lie about, so reserving against it would bound the claim, not the usage. type Charge struct { limiter *Limiter memory int64 disk int64 } -// admitsThreeState reports whether n more bytes may be charged against a limit that is already -// holding held, reading the limit the same way on both bounds it governs: positive is the limit -// itself, zero refuses unconditionally (none of that resource may be used at all), and negative -// admits unconditionally (that resource is not bounded). One reading shared by Memory and Disk -// rather than the same three-way condition written twice, since the two have already drifted once. +// admitsThreeState reports whether n more bytes may be charged against a limit already holding held. +// Shared by Memory and Disk so the two readings of Limits cannot drift. func admitsThreeState(limit, held, n int64) bool { switch { case limit < 0: @@ -133,10 +101,8 @@ func admitsThreeState(limit, held, n int64) bool { } } -// Memory charges n bytes of content held in memory, reporting false when the memory limit does not -// admit it - which for a zero limit is every chunk, sending content to disk unconditionally, since -// there is no separate threshold naming a size at which content overflows. Nothing is charged when it -// reports false. +// Memory charges n bytes held in memory, reporting false when the memory limit does not admit it (for +// a zero limit, always). Nothing is charged when it reports false. func (c *Charge) Memory(n int64) bool { if c == nil || c.limiter == nil || n <= 0 { return true @@ -155,10 +121,8 @@ func (c *Charge) Memory(n int64) bool { return true } -// Disk charges n bytes of content placed on disk, reporting false when the disk limit does not admit -// it - which for a zero limit is every chunk, so content with nowhere further to go leaves its -// archive skipped exactly as content exceeding a positive disk limit does. Nothing is charged when it -// reports false. +// Disk charges n bytes placed on disk, reporting false when the disk limit does not admit it (for a +// zero limit, always, which skips the archive). Nothing is charged when it reports false. func (c *Charge) Disk(n int64) bool { if c == nil || c.limiter == nil || n <= 0 { return true @@ -177,18 +141,16 @@ func (c *Charge) Disk(n int64) bool { return true } -// IndexRecord charges the in-memory cost of holding one entry's index record - the *Entry, its -// header, and the resolver node built over it - preferring the memory budget and overflowing to the -// disk budget when memory will not admit it. +// IndexRecord charges the memory an archive's index records keep regardless of content weight, +// preferring the memory budget and falling back to disk. It reports false only when neither budget +// admits the record, truncating the archive. // -// It overflows rather than refusing because the index is not content: a resolver needs an entry's -// record to reach that entry's bytes at all, and the memory budget may legitimately be zero, so an -// index that could be refused would leave an archive whose content is entirely on disk with no way to -// read it back. What this bounds instead is entry count - the one dimension the byte budgets never -// bounded, so an archive of millions of empty entries could grow the index without limit. It reports -// false only when neither budget admits the record, which truncates the archive. What it charges is -// released by Release along with everything else this charge took, so the index accounting falls when -// the archive does. +// It bounds the dimension the byte budgets miss: an archive of many empty entries, or one entry named +// a thousand components deep, weighs almost nothing as content yet would grow the index without limit. +// +// The disk fallback exists because a resolver needs an entry's record to reach its bytes at all, and +// MaxMemoryBytes may legitimately be zero (the configuration that spills every archive). So records +// are bounded by the disk budget once memory is full, though they are only ever held in memory. func (c *Charge) IndexRecord(n int64) bool { if c == nil || c.limiter == nil || n <= 0 { return true @@ -199,8 +161,8 @@ func (c *Charge) IndexRecord(n int64) bool { return c.Disk(n) } -// RefundDisk gives back n bytes charged for content that was removed again, so a partial file -// dropped at a limit does not leave the limiter holding bytes that are no longer on disk. +// RefundDisk gives back n bytes charged for content since removed, so a partial file dropped at a +// limit does not leave the limiter holding bytes no longer on disk. func (c *Charge) RefundDisk(n int64) { if c == nil || c.limiter == nil || n <= 0 { return @@ -230,18 +192,7 @@ func (c *Charge) RefundMemory(n int64) { c.memory -= n } -// Held reports what this charge is currently holding, in memory and on disk. -func (c *Charge) Held() (memory, disk int64) { - if c == nil || c.limiter == nil { - return 0, 0 - } - c.limiter.mu.Lock() - defer c.limiter.mu.Unlock() - return c.memory, c.disk -} - -// Release returns everything this charge took, so both limits fall by what this archive held. Safe -// to call more than once. +// Release returns everything this charge took. Safe to call more than once. func (c *Charge) Release() { if c == nil || c.limiter == nil { return diff --git a/internal/archive/limits_test.go b/internal/archive/limits_test.go index 32e367146..4c7f2d94b 100644 --- a/internal/archive/limits_test.go +++ b/internal/archive/limits_test.go @@ -38,7 +38,7 @@ func TestLimiter_chargeAndRelease(t *testing.T) { } func TestLimiter_refund(t *testing.T) { - // a partial entry dropped at a bound is no longer on disk, so the limiter must not go on holding it + // a partial entry dropped at a bound is no longer on disk, so the limiter must not still hold it limiter := NewLimiter(Limits{MaxDiskBytes: 100}) charge := limiter.Charge() @@ -55,8 +55,8 @@ func TestLimiter_refund(t *testing.T) { } func TestLimiter_boundsAreIndependent(t *testing.T) { - // a caller may bound one limit without bounding the other: a limit on one side of the pair does - // not change how the other side reads its own value. + // a caller may bound one limit without the other: a limit on one side does not change how the other + // reads its own value memoryBounded := NewLimiter(Limits{MaxMemoryBytes: 10, MaxDiskBytes: -1}).Charge() assert.False(t, memoryBounded.Memory(50)) assert.True(t, memoryBounded.Disk(1_000_000), "a negative disk limit admits anything") @@ -67,9 +67,8 @@ func TestLimiter_boundsAreIndependent(t *testing.T) { } func TestLimiter_threeStateReadingIsUniform(t *testing.T) { - // Memory and Disk read their limit the same way: positive is the limit, zero refuses - // unconditionally, negative admits unconditionally. The two used to disagree in opposite - // directions - Memory got zero right and negative wrong, Disk got negative right and zero wrong. + // Memory and Disk read their limit the same way: positive is the limit, zero refuses, negative + // admits t.Run("positive is the limit", func(t *testing.T) { c := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 100}).Charge() assert.True(t, c.Memory(100)) @@ -93,8 +92,7 @@ func TestLimiter_threeStateReadingIsUniform(t *testing.T) { func TestLimiter_nilIsUnbounded(t *testing.T) { // what an extraction with no configured bounds gets: charges succeed and nothing is measured. - // This is distinct from a configured but non-positive MaxMemoryBytes, which refuses instead - - // there is no limiter here at all to refuse anything. + // Distinct from a configured but non-positive MaxMemoryBytes, which refuses instead. var limiter *Limiter charge := limiter.Charge() assert.Nil(t, charge) @@ -104,22 +102,19 @@ func TestLimiter_nilIsUnbounded(t *testing.T) { charge.RefundDisk(10) charge.Release() - mem, disk := charge.Held() + mem, disk := charge.held() assert.Zero(t, mem) assert.Zero(t, disk) mem, disk = limiter.InUse() assert.Zero(t, mem) assert.Zero(t, disk) - assert.Zero(t, limiter.MaxDiskBytes()) } func TestLimiter_concurrentChargesAgree(t *testing.T) { // the walk is sequential today, but the archive task merges into a shared builder while top-level - // catalogers run, so the accounting is guarded rather than left for whoever parallelises it. - // Both limits need an explicit value here: zero now refuses every charge on either bound, so - // MaxMemoryBytes is set comfortably above what this test charges and MaxDiskBytes is negative - // for unbounded, rather than leaving either at its zero value. + // catalogers run, so the accounting is guarded. Both limits need an explicit value, since zero + // refuses every charge: memory is set above what this test charges and disk is negative. limiter := NewLimiter(Limits{MaxMemoryBytes: 1_000_000, MaxDiskBytes: -1}) var wg sync.WaitGroup @@ -150,8 +145,8 @@ func TestLimiter_concurrentChargesAgree(t *testing.T) { } func Test_Limiter_peakIsAHighWaterMark(t *testing.T) { - // the peak is what the scan held at its worst moment, so it must survive the release that takes - // the in-use gauge back down - a peak that fell on release would measure what InUse already does + // the peak is what the scan held at its worst moment, so it must survive the release that takes the + // in-use gauge back down l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) first := l.Charge() @@ -173,8 +168,7 @@ func Test_Limiter_peakIsAHighWaterMark(t *testing.T) { } func Test_Limiter_peakCountsConcurrentHoldersTogether(t *testing.T) { - // two archives held at the same time peak at their sum, which is the number the limits are - // enforced against and therefore the number worth reporting + // two archives held at once peak at their sum, the number the limits are enforced against l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) outer := l.Charge() @@ -196,8 +190,8 @@ func Test_Limiter_peakOfANilLimiterIsZero(t *testing.T) { } func Test_Limiter_refusedChargeDoesNotMoveThePeak(t *testing.T) { - // nothing is charged when a limit refuses, so nothing may be recorded either: a peak that rose - // on refusal would report bytes the scan never held + // nothing is charged when a limit refuses, so nothing may be recorded: a peak that rose on refusal + // would report bytes the scan never held l := NewLimiter(Limits{MaxMemoryBytes: 50, MaxDiskBytes: 50}) c := l.Charge() diff --git a/internal/archive/notify.go b/internal/archive/notify.go index fd3246d5c..4ce5df59a 100644 --- a/internal/archive/notify.go +++ b/internal/archive/notify.go @@ -6,54 +6,15 @@ import ( // Notify receives structured events from archive extraction. A nil Notify is the no-op. // -// The signature is deliberately `func(any)` rather than a set of typed callbacks or an interface with -// a method per event. Extraction is a hot path - a scan of a big image enters thousands of archives - -// and what instrumentation costs there has to be nothing when it is off. A single func field is one -// nil check, and a handler that cares about one event type asserts for it and ignores the rest, so -// adding an event never changes a signature or breaks a handler that did not ask for it. -// -// Guard the call rather than relying on a nil check inside the callee: +// func(any) rather than typed callbacks so adding an event changes no signature: a handler asserts +// for the events it wants and ignores the rest. Boxing into an any allocates, so on this hot path +// guard the call rather than relying on a nil check inside the callee: // // if notify != nil { // notify(EntriesOverflowed{Archive: name, Bytes: n}) // } -// -// Building the event value is what costs - putting a struct in an `any` allocates - so the guard is -// what makes "off" free rather than merely cheap. Send exists for the paths where that does not -// matter. type Notify func(msg any) -// Send delivers one event, doing nothing when there is no handler. Convenient, but it builds the -// event whether or not anyone is listening: on a hot path, guard the call instead. -func (n Notify) Send(msg any) { - if n == nil { - return - } - n(msg) -} - -// Tee returns a Notify that delivers to each of the given handlers, skipping the nil ones, and nil -// when none of them are live - so a tee of nothing stays free. -func Tee(handlers ...Notify) Notify { - var live []Notify - for _, h := range handlers { - if h != nil { - live = append(live, h) - } - } - switch len(live) { - case 0: - return nil - case 1: - return live[0] - } - return func(msg any) { - for _, h := range live { - h(msg) - } - } -} - // OverflowReason says why content was written to disk rather than held. type OverflowReason string @@ -61,12 +22,12 @@ const ( // MemoryLimitReached means the scan was already holding as much as the memory limit admits. MemoryLimitReached OverflowReason = "memory limit reached" - // MemoryHoldsNothing means the memory limit is zero, so nothing is ever held and every archive's - // content goes to disk by policy rather than by pressure. + // MemoryHoldsNothing means the memory limit is zero, so content goes to disk by policy rather than + // under pressure. MemoryHoldsNothing OverflowReason = "memory limit is zero" ) -// ContentOverflowed reports that one archive's own bytes were written to disk instead of being held in +// ContentOverflowed reports that one archive's own bytes were written to disk rather than held in // memory. Bytes is the size of the archive. type ContentOverflowed struct { Archive string @@ -74,9 +35,9 @@ type ContentOverflowed struct { Reason OverflowReason } -// EntriesOverflowed reports that the entries an archive's store was holding have been written to its -// overflow blob. Entries and Bytes are what moved, not what the archive holds in total: a store spills -// everything it is holding at once, and may spill more than once as it fills again. +// EntriesOverflowed reports that the entries a store was holding were written to its overflow blob. +// Entries and Bytes are what moved in this spill, not the archive's totals: a store spills everything +// it holds at once, and may spill again as it refills. type EntriesOverflowed struct { Archive string Entries int @@ -84,23 +45,21 @@ type EntriesOverflowed struct { Reason OverflowReason } -// Skipped reports that an archive contributed nothing, and why. Not a failure - the scan -// continues - but the packages inside it are absent from the SBOM, which is worth being able to see. +// Skipped reports that an archive contributed nothing, and why. The scan continues, but any packages +// inside that archive are absent from the SBOM. type Skipped struct { Archive string Reason string } -// Truncated reports that an archive was cataloged from part of its contents, naming the bound -// that stopped it. +// Truncated reports that an archive was cataloged from part of its contents, naming the bound that +// stopped it. type Truncated struct { Archive string Reason TruncationReason } -// LogNotify is the default handler: it writes each event to the debug log. Anything it does not -// recognize is ignored rather than logged blindly, so an event added for one consumer does not turn -// into noise for everyone. +// LogNotify is the default handler: it writes each known event to the debug log and ignores the rest. func LogNotify(msg any) { switch e := msg.(type) { case ContentOverflowed: diff --git a/internal/archive/notify_test.go b/internal/archive/notify_test.go index 3013c8dfb..fa598f938 100644 --- a/internal/archive/notify_test.go +++ b/internal/archive/notify_test.go @@ -10,23 +10,15 @@ import ( ) // recorder is what a consumer of these events looks like: one func, a type assertion for what it -// cares about, and silence for everything else. +// cares about, silence for everything else. type recorder struct { overflows []EntriesOverflowed other int } -func (r *recorder) notify(msg any) { - if e, ok := msg.(EntriesOverflowed); ok { - r.overflows = append(r.overflows, e) - return - } - r.other++ -} - func TestNotify_reportsWhichArchiveSpilledAndWhy(t *testing.T) { rec := &recorder{} - s := NewEntryStore(t.TempDir(), "lib/app.jar", rec.notify) + s := NewEntryStore(WorkDirAt(t.TempDir()), "lib/app.jar", rec.notify) t.Cleanup(func() { require.NoError(t, s.Close()) }) charge := memCharge(8) @@ -44,9 +36,9 @@ func TestNotify_reportsWhichArchiveSpilledAndWhy(t *testing.T) { func TestNotify_distinguishesPressureFromPolicy(t *testing.T) { // a scan at its memory bound and a scan whose bound is zero are different findings: the first says - // the limit is biting, the second says nothing was ever going to be held + // the limit is biting, the second that nothing was ever going to be held rec := &recorder{} - s := NewEntryStore(t.TempDir(), "app.jar", rec.notify) + s := NewEntryStore(WorkDirAt(t.TempDir()), "app.jar", rec.notify) t.Cleanup(func() { require.NoError(t, s.Close()) }) _, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), memCharge(0)) @@ -57,42 +49,28 @@ func TestNotify_distinguishesPressureFromPolicy(t *testing.T) { } func TestNotify_nilHandlerIsTheNoOp(t *testing.T) { - s := NewEntryStore(t.TempDir(), "app.jar", nil) + s := NewEntryStore(WorkDirAt(t.TempDir()), "app.jar", nil) t.Cleanup(func() { require.NoError(t, s.Close()) }) _, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), memCharge(0)) assert.NoError(t, err, "no handler must not mean no extraction") } -func TestNotify_teeDeliversToEveryLiveHandler(t *testing.T) { - first, second := &recorder{}, &recorder{} - tee := Tee(nil, first.notify, nil, second.notify) - require.NotNil(t, tee) - - tee(EntriesOverflowed{Archive: "app.jar", Entries: 1}) - - assert.Len(t, first.overflows, 1) - assert.Len(t, second.overflows, 1) -} - -func TestNotify_teeOfNothingIsNil(t *testing.T) { - // a tee that would deliver to no one stays nil, so the guard at every call site keeps it free - assert.Nil(t, Tee()) - assert.Nil(t, Tee(nil, nil)) -} - -func TestNotify_sendIsNilSafe(t *testing.T) { - var n Notify - n.Send(EntriesOverflowed{}) -} - func TestNotify_directoryEntriesDoNotSpill(t *testing.T) { // a directory carries no content, so it can never be the thing that pushes a store to disk rec := &recorder{} - s := NewEntryStore(t.TempDir(), "app.jar", rec.notify) + s := NewEntryStore(WorkDirAt(t.TempDir()), "app.jar", rec.notify) t.Cleanup(func() { require.NoError(t, s.Close()) }) _, err := s.Add(tar.Header{Name: "lib/", Typeflag: tar.TypeDir, Mode: 0o755}, nil, memCharge(0)) require.NoError(t, err) assert.Empty(t, rec.overflows) } + +func (r *recorder) notify(msg any) { + if e, ok := msg.(EntriesOverflowed); ok { + r.overflows = append(r.overflows, e) + return + } + r.other++ +} diff --git a/internal/archive/overflow.go b/internal/archive/overflow.go deleted file mode 100644 index 929c43a78..000000000 --- a/internal/archive/overflow.go +++ /dev/null @@ -1,352 +0,0 @@ -package archive - -import ( - "archive/tar" - "bytes" - "fmt" - "io" - "io/fs" - "os" - "strings" - "time" - - "github.com/mholt/archives" - - "github.com/anchore/syft/internal/log" -) - -// tarBlockSize is the tar format's fixed block size. Every header occupies a whole number of blocks -// and every entry's data is padded up to one, which is why the bytes an archive puts on disk are not -// the sum of its entry sizes - see OverflowTar. -const tarBlockSize = 512 - -// overflowTarName is the basename of the one file an archive's entries are written into. It sits beside -// the archive's logical root rather than inside it, so nothing in the archive's own filesystem can -// name it. -const overflowTarName = "contents.tar" - -// OverflowTar is the single file one archive's entries are overflowed into. -// -// The point of the whole shape is that a scan's filesystem cost tracks the number of archives it -// finds rather than the number of entries those archives hold. Expanding every entry as its own file -// makes an archive of a hundred thousand entries a hundred thousand inodes - exhausting inodes on a -// filesystem with bytes to spare, and making cleanup proportional to content as well. One tar per -// archive is one inode per archive, and stereoscope's TarIndex then gives direct access to any entry -// by seek offset, so nothing has to be re-read or re-decompressed to reach the end of it. -// -// It is written by hand rather than through a tar.Writer because tar wants an entry's size in the -// header, before its data, and a size an archive declares is exactly what must not be trusted: both -// tar and zip headers carry a size and a crafted archive is free to lie about it. So each entry's -// header is reserved, its data is copied and charged as it lands, and the header is then written over -// the reservation with the size that actually arrived. The reservation is exact because the header is -// encoded as GNU, whose numeric fields are fixed width - the encoded length of a header therefore -// depends on its name and link target and not at all on its size. That is asserted per entry rather -// than assumed, and an entry whose two encodings disagree is dropped rather than corrupting the tar. -type OverflowTar struct { - path string - file *os.File - - // offset is where the next byte written will land, which is also the number of bytes this tar - // holds. Tracked rather than queried so a header can be written back over its reservation. - offset int64 - - // verbatim records that this tar was copied wholesale from an already-tar-shaped stream rather - // than assembled entry by entry, in which case it carries the source's own end-of-archive marker - // and must not be given a second one. - verbatim bool -} - -func newOverflowTar(path string) (*OverflowTar, error) { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) - if err != nil { - return nil, fmt.Errorf("unable to create overflow tar %q: %w", path, err) - } - return &OverflowTar{path: path, file: f}, nil -} - -// Path is where this tar lives, which is what stereoscope's TarIndex needs: it indexes by seek -// offset and therefore wants a seekable, uncompressed file rather than a stream. -func (s *OverflowTar) Path() string { - return s.path -} - -func (s *OverflowTar) Close() error { - if s == nil || s.file == nil { - return nil - } - err := s.file.Close() - s.file = nil - return err -} - -// put charges b to the disk limit and writes it, reporting whether the limit refused it. Nothing is -// written when the charge is refused, so the tar never holds bytes the limiter is not accounting for. -func (s *OverflowTar) put(b []byte, result *ExtractionResult, limits ExtractionLimits) (refused bool, err error) { - if len(b) == 0 { - return false, nil - } - if !limits.Charge.Disk(int64(len(b))) { - return true, nil - } - n, err := s.file.Write(b) - s.offset += int64(n) - result.BytesWritten += int64(n) - if n < len(b) { - limits.Charge.RefundDisk(int64(len(b) - n)) - } - return false, err -} - -// AddEntry writes one archive entry into the tar: a header, its data, and the padding that takes the -// entry up to a block boundary. -// -// Directory entries carry no data, so what bounds an archive of a million of them is the disk limit -// charging their headers and padding, which is the only resource they actually consume. -func (s *OverflowTar) AddEntry(f archives.FileInfo, result *ExtractionResult, limits ExtractionLimits) error { - hdr, ok := entryHeader(f) - if !ok { - return nil - } - - reserved, err := encodeHeader(hdr, 0) - if err != nil { - // GNU could not represent this entry; let the tar package pick a format it can, and rely on - // the length check below to drop the entry if that choice makes the length size-dependent - hdr.Format = tar.FormatUnknown - reserved, err = encodeHeader(hdr, 0) - if err != nil { - log.WithFields("entry", f.NameInArchive, "error", err). - Debug("skipping archive entry whose header cannot be encoded") - return nil - } - } - - start := s.offset - if refused, err := s.put(reserved, result, limits); err != nil { - return err - } else if refused { - return s.dropEntry(start, result, limits) - } - - written, refused, err := s.copyEntryData(f, hdr, result, limits) - if err != nil { - return err - } - if refused { - // this entry would have taken the scan's disk usage past the disk limit. Drop it rather than - // leaving a silently short one behind: a half-written jar or pom is worse than an absent one, - // because a cataloger will try to parse it and may believe what it finds. - return s.dropEntry(start, result, limits) - } - - final, err := encodeHeader(hdr, written) - if err != nil || len(final) != len(reserved) { - // the reservation and the real header disagree on length, so writing the real header would - // either overwrite the entry's first bytes or leave a gap - either way an unreadable tar. - // GNU's fixed-width numeric fields make this unreachable in practice; it is checked because - // the alternative to checking is silent corruption. - log.WithFields("entry", f.NameInArchive, "error", err). - Debug("skipping archive entry whose header length depends on its size") - return s.dropEntry(start, result, limits) - } - if _, err := s.file.WriteAt(final, start); err != nil { - return fmt.Errorf("unable to write header for archive entry %q: %w", f.NameInArchive, err) - } - - if pad := (tarBlockSize - written%tarBlockSize) % tarBlockSize; pad > 0 { - if refused, err := s.put(make([]byte, pad), result, limits); err != nil { - return err - } else if refused { - return s.dropEntry(start, result, limits) - } - } - - result.FilesExtracted++ - return nil -} - -// copyEntryData copies one entry's content into the tar, charging every chunk as it lands, and reports -// how much actually arrived. A size the archive declared is never used: both tar and zip headers carry -// one and a crafted archive is free to lie about it, so the header is written afterwards from what was -// counted here. An entry carrying no data - a directory, a link - copies nothing. -func (s *OverflowTar) copyEntryData(f archives.FileInfo, hdr *tar.Header, result *ExtractionResult, limits ExtractionLimits) (written int64, refused bool, err error) { - if hdr.Typeflag != tar.TypeReg { - return 0, false, nil - } - - src, err := f.Open() - if err != nil { - return 0, false, fmt.Errorf("unable to open archive entry %q: %w", f.NameInArchive, err) - } - defer func() { - if closeErr := src.Close(); closeErr != nil { - log.WithFields("entry", f.NameInArchive, "error", closeErr).Trace("unable to close archive entry") - } - }() - - written, refused, err = copyCharged(s.file, src, limits.Charge) - s.offset += written - result.BytesWritten += written - return written, refused, err -} - -// dropEntry rewinds the tar to where the entry being written began, refunding what that entry -// charged since those bytes are no longer on disk, and stops the extraction. Whatever was written -// before this entry is a complete, readable tar and is still cataloged; see ExtractionResult. -func (s *OverflowTar) dropEntry(start int64, result *ExtractionResult, limits ExtractionLimits) error { - spent := s.offset - start - if err := s.file.Truncate(start); err != nil { - log.WithFields("path", s.path, "error", err).Trace("unable to rewind overflow tar past an over-limit entry") - } - if _, err := s.file.Seek(start, io.SeekStart); err != nil { - log.WithFields("path", s.path, "error", err).Trace("unable to seek overflow tar past an over-limit entry") - } - s.offset = start - limits.Charge.RefundDisk(spent) - result.BytesWritten -= spent - if result.Truncation == "" { - result.Truncation = TruncatedByDiskLimit - } - return errTruncated -} - -// CopyVerbatim writes an already-tar-shaped stream into the overflow file as it stands, which is what a -// tar-family archive gets: a compressed tar is decompressed once into a plain tar rather than -// expanded entry by entry, and a plain tar is copied. Charging is per chunk as the bytes land, so a -// decompression bomb is bounded by the disk limit rather than by what the archive claimed. -// -// A refused chunk leaves the tar ending part way through an entry. That is deliberately not repaired -// here: indexing keeps every entry it could read and reports the tail as a truncation, which is the -// same outcome as stopping on a whole-entry boundary and needs no second mechanism. -func (s *OverflowTar) CopyVerbatim(r io.Reader, result *ExtractionResult, limits ExtractionLimits) error { - s.verbatim = true - - written, refused, err := copyCharged(s.file, r, limits.Charge) - s.offset += written - result.BytesWritten += written - if err != nil { - return err - } - if refused { - result.Truncation = TruncatedByDiskLimit - return errTruncated - } - return nil -} - -// Finish writes the tar's end-of-archive marker. -// -// A marker that does not fit within the disk limit is not an error: Go's tar reader stops at the -// first zero block or at the end of the file, so a tar that simply ends reads exactly the same. -func (s *OverflowTar) Finish(result *ExtractionResult, limits ExtractionLimits) { - if s.verbatim { - // the source stream carried its own marker - return - } - if refused, err := s.put(make([]byte, 2*tarBlockSize), result, limits); err != nil || refused { - log.WithFields("path", s.path, "error", err).Trace("overflow tar written without an end-of-archive marker") - } -} - -// entryHeader builds the tar header for one archive entry, or reports false when the entry is not -// something worth carrying. -// -// Symlinks are recorded as symlink headers rather than being created and checked as real links, -// because writing a header traverses no path: nothing is created, so there is nothing for a crafted -// target to redirect. Where a link points is decided when the archive's filetree is built, inside -// that tree, so a target naming the host filesystem resolves to nothing rather than to the host. -// -// Device nodes, fifos and sockets are skipped, as they were when entries were files: they cannot -// round-trip and no cataloger can read one. -func entryHeader(f archives.FileInfo) (*tar.Header, bool) { - mode := f.Mode() - if !mode.IsRegular() && !mode.IsDir() && mode.Type()&fs.ModeSymlink == 0 { - log.WithFields("entry", f.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry") - return nil, false - } - - hdr, err := tar.FileInfoHeader(f.FileInfo, f.LinkTarget) - if err != nil { - log.WithFields("entry", f.NameInArchive, "error", err).Debug("skipping archive entry with an unusable header") - return nil, false - } - - // the name is carried through as the archive gave it, unsanitized. Nothing is written through it - // - it is a field in a header - and the one place it can still mislead is the logical filetree, - // so it is sanitized there, once, rather than in each of the two paths that write a tar. - hdr.Name = f.NameInArchive - if f.IsDir() && !strings.HasSuffix(hdr.Name, "/") { - hdr.Name += "/" - } - hdr.Linkname = f.LinkTarget - - // GNU rather than the format the tar package would pick: its numeric fields are fixed width, so - // the encoded length of a header does not depend on the size recorded in it. That is what lets - // the header be reserved before the entry's real size is known and written afterwards. Fields GNU - // cannot carry are cleared instead of forcing the encoder into PAX, where a large size becomes an - // extra record and the length stops being predictable. - hdr.Format = tar.FormatGNU - hdr.PAXRecords = nil - hdr.Uname, hdr.Gname = "", "" - hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} - hdr.ModTime = hdr.ModTime.Truncate(time.Second) - - return hdr, true -} - -// encodeHeader returns the bytes one header occupies in a tar, for the given size. It includes -// whatever the format needs around it - a GNU long-name record for a name a header field cannot -// hold - because all of it is written before the entry's data and all of it has to be reserved. -func encodeHeader(hdr *tar.Header, size int64) ([]byte, error) { - h := *hdr - h.Size = size - - var buf bytes.Buffer - tw := tar.NewWriter(&buf) - // WriteHeader emits the header, and any record the format needs ahead of it, before it returns. - // Close is deliberately not called: it would append the end-of-archive marker, which belongs to - // the tar being assembled and not to one header in it. - if err := tw.WriteHeader(&h); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// tarStream returns the plain-tar byte stream for a tar-family archive, decompressing once where the -// archive is a compressed tar, and reports false for anything that is not a tar. -// -// This is what makes a compressed tar strictly cheaper than it was: it used to be expanded entry by -// entry into a tree of files, and here it is decompressed once into a file that is already the -// storage format, then indexed by seek offset. A plain tar is copied for the same reason - TarIndex -// needs a seekable uncompressed file, and the archive's own bytes may be held in memory. -func tarStream(format archives.Format, r io.Reader) (io.ReadCloser, bool) { - switch f := format.(type) { - case archives.Tar: - return io.NopCloser(r), true - case archives.CompressedArchive: - if !isTarFormat(f) { - return nil, false - } - if f.Compression == nil { - return io.NopCloser(r), true - } - rc, err := f.OpenReader(r) - if err != nil { - log.WithFields("error", err).Debug("unable to decompress tar-family archive") - return nil, false - } - return rc, true - } - return nil, false -} - -// isTarFormat reports whether the archival half of a compressed archive is a tar. Identify sets both -// halves for a compressed tar, so either answers; both are checked because only one is guaranteed to -// be non-nil in general. -func isTarFormat(f archives.CompressedArchive) bool { - if _, ok := f.Archival.(archives.Tar); ok { - return true - } - _, ok := f.Extraction.(archives.Tar) - return ok -} diff --git a/internal/archive/overflow_test.go b/internal/archive/overflow_test.go deleted file mode 100644 index d51a0959a..000000000 --- a/internal/archive/overflow_test.go +++ /dev/null @@ -1,221 +0,0 @@ -package archive - -import ( - "archive/zip" - "bytes" - "context" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - stereoscopeFile "github.com/anchore/stereoscope/pkg/file" - "github.com/anchore/syft/syft/file" -) - -// manyEntryZip builds a zip of count small entries, in sorted name order. -func manyEntryZip(t *testing.T, count int) *bytes.Buffer { - t.Helper() - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - for i := range count { - w, err := zw.Create(fmt.Sprintf("e%06d.txt", i)) - require.NoError(t, err) - _, err = w.Write([]byte("x")) - require.NoError(t, err) - } - require.NoError(t, zw.Close()) - return &buf -} - -// extractedOverflow extracts one archive and hands back where its content went, along with the work -// directory holding it - which is what a filesystem cost has to be measured against. -func extractedOverflow(t *testing.T, content io.Reader, limits ExtractionLimits) (*ExtractedArchive, Overflow, string) { - t.Helper() - - var got Overflow - factory := func(overflow Overflow) (file.Resolver, IndexResult, error) { - got = overflow - return nil, IndexResult{}, nil - } - - extracted, err := ExtractToResolver( - context.Background(), content, "app.zip", "", "app.zip", - DefaultExtractors(), NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), limits, factory, nil, nil, - ) - require.NoError(t, err) - require.NotNil(t, extracted) - t.Cleanup(extracted.Cleanup) - - return extracted, got, filepath.Dir(got.TarPath) -} - -// filesystemEntries is every directory entry under dir, path by path, which is the cost the -// requirement is about: an archive must not put one of these on the filesystem per entry it holds. -func filesystemEntries(t *testing.T, dir string) []string { - t.Helper() - var out []string - require.NoError(t, filepath.WalkDir(dir, func(p string, _ fs.DirEntry, err error) error { - if err != nil { - return err - } - rel, relErr := filepath.Rel(dir, p) - if relErr != nil { - return relErr - } - if rel == "." { - return nil - } - out = append(out, filepath.ToSlash(rel)) - return nil - })) - return out -} - -func TestOverflow_filesystemCostDoesNotTrackEntryCount(t *testing.T) { - // the requirement this whole shape exists for: an archive of a hundred thousand entries must not - // put a hundred thousand files on the filesystem. Asserted as an exact set of paths for two - // archives three orders of magnitude apart in entry count, rather than as a bound - a count that - // merely grows slowly would pass a bound and still be proportional to content. - small, smallOverflow, smallWork := extractedOverflow(t, manyEntryZip(t, 2), ExtractionLimits{}) - large, largeOverflow, largeWork := extractedOverflow(t, manyEntryZip(t, 5000), ExtractionLimits{}) - - want := []string{"contents", "contents.tar"} - assert.Equal(t, want, filesystemEntries(t, smallWork)) - assert.Equal(t, want, filesystemEntries(t, largeWork), - "5000 entries must cost the same filesystem entries as 2: one tar, plus the empty root their "+ - "paths are reported relative to") - - // and the entries really are all there, so this is not passing because nothing was extracted - assert.Equal(t, 2, countTarEntries(t, smallOverflow.TarPath)) - assert.Equal(t, 5000, countTarEntries(t, largeOverflow.TarPath)) - // what one archive costs on disk, exactly: a header block and a padded data block per entry, plus - // the two-block end-of-archive marker. This is the figure the disk limit is charged, which is why - // a limit expressed in entry bytes now under-counts. - assert.Equal(t, int64((5000*2+2)*tarBlockSize), large.Result.BytesWritten) - assert.Equal(t, int64((2*2+2)*tarBlockSize), small.Result.BytesWritten) -} - -func TestOverflow_cleanupRemovesOneFileAndDoesNotWalkPerEntry(t *testing.T) { - // cleanup was proportional to content too: releasing an archive meant removing one filesystem - // entry per archive entry. It is now bounded by the same constant as the extraction, so the count - // of things removed is asserted rather than just the fact that the directory is gone. - extracted, overflow, workDir := extractedOverflow(t, manyEntryZip(t, 5000), ExtractionLimits{}) - - before := filesystemEntries(t, workDir) - require.Len(t, before, 2, "the release has two things to remove whatever the archive held") - - extracted.Cleanup() - - _, err := os.Stat(overflow.TarPath) - assert.True(t, os.IsNotExist(err), "the one file holding every entry must be gone") - _, err = os.Stat(workDir) - assert.True(t, os.IsNotExist(err), "and so must the work directory") - - // safe to call more than once, since the cataloger defers it past the sub-pipeline and past - // recursion - extracted.Cleanup() -} - -func TestOverflow_anEntryAtTheEndIsReadBySeeking(t *testing.T) { - // "overflow content MUST remain directly readable without re-reading or re-decompressing what - // precedes it". A design that met the entry-count requirement by re-streaming the whole archive - // per read would not satisfy it, so what is asserted here is the mechanism: the index records - // where the last entry's content begins, near the end of a tar whose earlier entries are large, - // and the reader it hands back is a bounded window onto the file at that offset. - const filler = 64 * 1024 - - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - for i := range 8 { - w, err := zw.Create(fmt.Sprintf("filler%d.bin", i)) - require.NoError(t, err) - _, err = w.Write(bytes.Repeat([]byte{byte(i)}, filler)) - require.NoError(t, err) - } - last, err := zw.Create("zz-last.txt") - require.NoError(t, err) - wanted := strings.Repeat("last", 64) - _, err = last.Write([]byte(wanted)) - require.NoError(t, err) - require.NoError(t, zw.Close()) - - _, overflow, _ := extractedOverflow(t, &buf, ExtractionLimits{}) - - info, err := os.Stat(overflow.TarPath) - require.NoError(t, err) - total := info.Size() - require.Greater(t, total, int64(8*filler), "the fixture must put real bulk ahead of the last entry") - - // the last entry's content sits near the end of the tar, which is the only reason a seek to it is - // worth anything: everything before it is what a re-streaming design would have had to read - offset := int64(bytes.Index(readFile(t, overflow.TarPath), []byte(wanted))) - require.Positive(t, offset) - assert.Greater(t, offset, total*9/10, "the last entry must really be at the end of the tar") - - entry := tarEntryNamed(t, overflow.TarPath, "zz-last.txt") - - contents := entry.Open() - t.Cleanup(func() { require.NoError(t, contents.Close()) }) - - random, ok := contents.(interface { - io.ReaderAt - io.Seeker - }) - require.True(t, ok, "the entry reader must be random access, or a read cannot be a seek") - - // the reader is bounded to this entry rather than being a view of the whole tar: its end is the - // entry's own size, so reading it cannot have read what precedes it - end, err := random.Seek(0, io.SeekEnd) - require.NoError(t, err) - assert.Equal(t, int64(len(wanted)), end) - - // and offset zero of the reader is the last entry's first byte, not the tar's. A reader over the - // whole file would answer this with the first filler entry's content. - head := make([]byte, 8) - _, err = random.ReadAt(head, 0) - require.NoError(t, err) - assert.Equal(t, wanted[:8], string(head)) -} - -func readFile(t *testing.T, path string) []byte { - t.Helper() - body, err := os.ReadFile(path) - require.NoError(t, err) - return body -} - -// countTarEntries is how many entries a tar on disk holds, read through the same index the resolver -// builds its filesystem from. -func countTarEntries(t *testing.T, tarPath string) int { - t.Helper() - count := 0 - _, err := stereoscopeFile.NewTarIndex(tarPath, func(stereoscopeFile.TarIndexEntry) error { - count++ - return nil - }) - require.NoError(t, err) - return count -} - -// tarEntryNamed is the index record for one entry of a tar, which is what carries its seek offset. -func tarEntryNamed(t *testing.T, tarPath, name string) stereoscopeFile.TarIndexEntry { - t.Helper() - var found *stereoscopeFile.TarIndexEntry - _, err := stereoscopeFile.NewTarIndex(tarPath, func(entry stereoscopeFile.TarIndexEntry) error { - if entry.ToTarFileEntry().Header.Name == name { - e := entry - found = &e - } - return nil - }) - require.NoError(t, err) - require.NotNil(t, found, "no entry %q in %q", name, tarPath) - return *found -} diff --git a/internal/archive/path_guards_test.go b/internal/archive/path_guards_test.go index 3cf52ff86..dfffee94e 100644 --- a/internal/archive/path_guards_test.go +++ b/internal/archive/path_guards_test.go @@ -9,25 +9,21 @@ import ( "github.com/stretchr/testify/require" ) -// The path-safety guards used to be exercised through extraction, because extraction created a file -// or a link per archive entry and every one of them went through them. An entry is a tar header now, -// so those tests assert something else and these guards need their own. +// The path-safety guards are covered here directly, since extraction no longer writes a file or link +// per archive entry: // -// They are covered here rather than retired because their situations differ: +// resolvesInsideRoot, resolveLink and isWithin guard the one filesystem write left in this package - +// an archive's own bytes spilling into its work directory, under a name derived from where the +// archive was found, which inside another archive is attacker-supplied text. // -// - resolvesInsideRoot, resolveLink and isWithin guard the one filesystem write left in this -// package - an archive's own bytes overflowing into its work directory, under a name derived from -// where the archive was found, which inside another archive is text the archive supplied. -// - writeSafeSymlink has no caller at all. A symlink entry is a header write, which creates -// nothing, and where the link points is decided inside the archive's own filetree. It is kept -// with its reasoning intact, and whether it retires is a judgement about every path that could -// write a link rather than about one call site. +// A symlink entry creates nothing on disk: it is stored as a header, and where the link points is +// decided inside the archive's own filetree. func TestResolvesInsideRoot_multiStepSymlinkChainIsRefused(t *testing.T) { // the escape the guard exists for: two cooperating links each pass a lexical check and together - // resolve above the root. "d" -> "." lands on the root, so "d/up" -> ".." is created as - // "/up" -> ".." and points above it, and a write through "d/up/x" lands outside. A lexical - // check sees /d/../x and allows it; only resolving against the real filesystem catches it. + // resolve above the root. "d" -> "." lands on the root, so "d/up" -> ".." becomes "/up" -> + // ".." pointing above it, and a write through "d/up/x" lands outside. A lexical check sees + // /d/../x and allows it; only resolving against the real filesystem catches it. base := t.TempDir() root := filepath.Join(base, "contents") require.NoError(t, os.MkdirAll(root, 0o755)) @@ -67,104 +63,34 @@ func TestResolvesInsideRoot_siblingSharingThePrefixIsRefused(t *testing.T) { assert.False(t, inside) } -func TestWriteSafeSymlink(t *testing.T) { - // asserted with no production caller, deliberately: see the note at the top of this file. The - // reasoning in these cases is what makes the function worth keeping, so it is kept executable - // rather than left as prose about code nothing runs. - newRoot := func(t *testing.T) string { - t.Helper() - base := t.TempDir() - root := filepath.Join(base, "contents") - require.NoError(t, os.MkdirAll(filepath.Join(root, "bin"), 0o755)) - require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(root, "lib", "real.txt"), []byte("real"), 0o600)) - return root - } +func TestResolvesInsideRoot_KNOWN_LIMIT_aLinkWhoseOwnTargetPathLeavesTheRoot(t *testing.T) { + // KNOWN GAP, recorded as a finding rather than a passing assertion of safety. + // + // Two links then a write: "up" -> "..", then "bin/link.txt" -> "../up/PWNED", then a write to + // "bin/link.txt/through". resolvesInsideRoot walks the write path, meets bin/link.txt, and resolves + // its target lexically without walking the target's own path, never seeing that /up is itself + // a link out of the root. On the real filesystem the write lands above the root. + // + // Not a live exposure: resolvesInsideRoot's only caller writes one flat name into a work directory + // it just created, where no link can be planted. Anything that gives it a caller over + // archive-supplied paths must close this first. + base := t.TempDir() + root := filepath.Join(base, "contents") + require.NoError(t, os.MkdirAll(filepath.Join(root, "bin"), 0o755)) + require.NoError(t, os.Symlink("..", filepath.Join(root, "up"))) + require.NoError(t, os.Symlink("../up/PWNED", filepath.Join(root, "bin", "link.txt"))) - t.Run("a relative target inside the root is written", func(t *testing.T) { - root := newRoot(t) - dest := filepath.Join(root, "bin", "link.txt") - require.NoError(t, writeSafeSymlink("../lib/real.txt", dest, root)) - - body, err := os.ReadFile(dest) - require.NoError(t, err) - assert.Equal(t, "real", string(body)) - }) - - for _, tc := range []struct { - name string - target string - }{ - {name: "an empty target", target: ""}, - { - // os.Symlink writes the literal string, so reading the link resolves on the host whatever - // was checked when it was created - name: "an absolute target", - target: "/etc/passwd", - }, - {name: "a target climbing out of the root", target: "../../../../etc/passwd"}, - } { - t.Run(tc.name+" is refused", func(t *testing.T) { - root := newRoot(t) - dest := filepath.Join(root, "bin", "link.txt") - require.Error(t, writeSafeSymlink(tc.target, dest, root)) - - _, err := os.Lstat(dest) - assert.True(t, os.IsNotExist(err), "nothing may be created for a refused link") - }) - } - - t.Run("a target that resolves out of the root through an earlier link is refused", func(t *testing.T) { - // the target itself can run through a link an earlier entry left behind - root := newRoot(t) - base := filepath.Dir(root) - require.NoError(t, os.Symlink("..", filepath.Join(root, "up"))) - require.NoError(t, os.WriteFile(filepath.Join(base, "PWNED"), []byte("x"), 0o600)) - - dest := filepath.Join(root, "bin", "link.txt") - require.Error(t, writeSafeSymlink("../up/PWNED", dest, root)) - - _, err := os.Lstat(dest) - assert.True(t, os.IsNotExist(err)) - }) - - t.Run("KNOWN LIMIT: a target whose own path runs out of the root is not caught by either guard", func(t *testing.T) { - // Recorded as a finding rather than a passing assertion of safety, because it is one. - // - // Three entries: "up" -> "..", then "bin/link.txt" -> "../up/PWNED", then a write to - // "bin/link.txt/through". writeSafeSymlink computes /up/PWNED, which is lexically inside - // the root, and its last check - does the target already resolve outside? - does not fire, - // because /up/PWNED does not exist yet. So the link is created. resolvesInsideRoot then - // walks the write path, meets bin/link.txt, and resolves its target the same lexical way: the - // target's own path is not walked, so the fact that /up is itself a link out of the root - // is never seen, and the write is allowed. On the real filesystem it lands above the root. - // - // It is not a live exposure. The only caller of resolvesInsideRoot writes one flat name into a - // work directory it just created, where no link exists and none can be planted, and - // writeSafeSymlink has no caller at all - a symlink entry is a header write now. It matters - // because it is what the retirement question actually hangs on: these two are not a working - // defence being kept in reserve, they are a defence with a hole in it and no exposure, and - // anything that gives them a caller again has to close this first. - root := newRoot(t) - require.NoError(t, os.Symlink("..", filepath.Join(root, "up"))) - - dest := filepath.Join(root, "bin", "link.txt") - require.NoError(t, writeSafeSymlink("../up/PWNED", dest, root), - "the link is created: its target does not resolve outside the root YET") - - inside, err := resolvesInsideRoot(root, filepath.Join(root, "bin", "link.txt", "through")) - require.NoError(t, err) - assert.True(t, inside, - "the guard allows this write, and the real filesystem puts it above the root - the hole this "+ - "case exists to record") - }) + inside, err := resolvesInsideRoot(root, filepath.Join(root, "bin", "link.txt", "through")) + require.NoError(t, err) + assert.True(t, inside, + "the guard allows this write, and the real filesystem puts it above the root - the hole this "+ + "case exists to record") } func TestArchiveFileName_refusesNamesThatWouldLeaveTheWorkDirectory(t *testing.T) { - // the name an archive's own bytes are overflowed under is the basename of where the archive was - // found, and Base returns ".." for an access path ending in it - which joined to the work - // directory would name the directory above. overflowContent checks its destination as well, so this - // is the earlier of two refusals rather than the only one. + // an archive spills under the basename of where it was found, and Base returns ".." for an access + // path ending in it, which joined to the work directory would name the directory above. + // overflowContent checks its destination too, so this is the earlier of two refusals. for _, accessPath := range []string{"", ".", "..", "a/..", string(filepath.Separator)} { assert.Equal(t, "archive", archiveFileName(accessPath), "%q must not become a name joined to the work directory", accessPath) diff --git a/internal/archive/resolver.go b/internal/archive/resolver.go index a39634b80..0f3bc6226 100644 --- a/internal/archive/resolver.go +++ b/internal/archive/resolver.go @@ -6,77 +6,48 @@ import ( "errors" "fmt" "io" - "os" "path/filepath" intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/internal/tmpdir" "github.com/anchore/syft/syft/file" ) -// ResolverFactory builds an indexed resolver over one archive's overflow tar. It is injected (rather -// than calling syft/internal/fileresolver directly) so this module-root package does not depend on -// the syft-subtree internal fileresolver package. -// -// It reports what indexing found as well as the resolver, because a tar-family archive is copied -// wholesale and its entries are first counted where the tar is indexed. -type ResolverFactory func(Overflow) (file.Resolver, IndexResult, error) - -// StoreResolverFactory builds a resolver over an archive's entries rather than over a tar of them. -// -// Where a ResolverFactory can only index content already written to a file - its index IS that file's -// offsets - this one indexes the entries themselves, so an archive small enough to stay in memory -// never reaches the disk and one that outgrows memory moves its bytes without the index changing. -// Injected the same way and for the same reason: this package must not depend on the resolver -// packages it hands work to. +// StoreResolverFactory builds an indexed resolver over one archive's entries. Injected so this +// package does not depend on the internal fileresolver package. type StoreResolverFactory func(*EntryStore, Overflow) (file.Resolver, IndexResult, error) -// Overflow names what one archive's entries were written into, and what the resolver built over them -// must know. +// Overflow names what the resolver built over one archive's entries must know. type Overflow struct { - // RootDir is the archive's logical root: an empty directory that every entry's path is reported - // relative to. It exists on disk, and stays empty - nothing is written into it - because the - // resolver relativizes an entry's path against a real root the way a directory scan does, and - // that is what keeps a path in the SBOM relative to its own archive rather than naming the - // scan's scratch space. - RootDir string - - // TarPath is the one file holding every entry of this archive, indexed by seek offset. - TarPath string - // FileSystemID is the identifier of the filesystem the archive file itself lives on (a layer // digest, or blank), inherited unchanged down the nesting chain, stamped onto every Location. FileSystemID string // ArchivePath is the colon-delimited chain of archive paths from the scan root to this archive, - // stamped onto every Location. It is what keeps identically-named files in different archives - // from colliding in the coordinate-keyed SBOM tables. + // stamped onto every Location. It keeps identically-named files in different archives from + // colliding in the coordinate-keyed SBOM tables. ArchivePath string + + // Charge is this archive's draw on the resource limiter; nil enforces nothing. The index keeps more + // per entry than the store does (one node per path component), so it charges for what it keeps + // rather than letting the store estimate for it. + Charge *Charge } -// IndexResult reports what indexing one archive's overflow tar produced. +// IndexResult reports what indexing one archive's entries produced. type IndexResult struct { - // Records is how many entries the archive's filesystem holds. - Records int - - // Truncated reports that indexing stopped before the end of the tar, which now means only one - // thing: the tar itself ends part way through an entry because the disk limit stopped the write. - // What was indexed is complete and usable; there is simply less of it. + // Truncated reports that the budget refused an index record and the remaining entries were never + // indexed. What was indexed is complete and usable. Truncated bool } -// archiveDigestHashes are the digests taken of an archive file as a whole. SHA-1 alone, matching -// what the java archive cataloger has always reported for a jar, because these are the digests it -// reads when it can no longer open the archive itself. +// archiveDigestHashes are the digests taken of an archive as a whole: SHA-1 alone, matching what the +// java archive cataloger reports for a jar. var archiveDigestHashes = []crypto.Hash{crypto.SHA1} -// digestsOf takes the archive's own digests from the content already acquired. One pass over a -// seekable source - memory or an overflow file - and it replaces a read the java cataloger used to -// do for itself, so it is not new work so much as work moved to where the bytes already are. -// -// A failure is logged and returns nothing: a digest is metadata about a package, and an archive -// whose contents cataloged fine should not be discarded because its hash could not be taken. +// digestsOf takes the archive's own digests in one pass over the acquired content, which the java +// cataloger reads when it can no longer open the archive itself. A failure is logged and returns +// nothing: an archive that cataloged fine is not discarded for a missing hash. func digestsOf(ctx context.Context, content Content, accessPath string) []file.Digest { if _, err := content.Reader.Seek(0, io.SeekStart); err != nil { log.WithFields("archive", accessPath, "error", err).Trace("unable to rewind archive content for digests") @@ -92,40 +63,34 @@ func digestsOf(ctx context.Context, content Content, accessPath string) []file.D // ExtractedArchive is a standalone, fully-indexed filesystem built from an extracted archive. type ExtractedArchive struct { - // Resolver is an indexed resolver over the extracted contents; its Locations are stamped with FileSystemID. + // Resolver is an indexed resolver over the extracted contents, its Locations stamped with + // FileSystemID. Resolver file.Resolver // Digests are the digests of the archive file itself, taken while its bytes were in hand. Digests []file.Digest - // FileSystemID is the identifier of the filesystem the archive file itself lives on (inherited unchanged down the nesting chain). + // FileSystemID is the filesystem the archive file itself lives on, inherited unchanged down the + // nesting chain. FileSystemID string - // ArchivePath is the colon-delimited chain of archive paths from the scan root to this archive. - ArchivePath string - // Result reports how much was extracted and whether a configured bound truncated the - // extraction. A truncated archive still yields a usable Resolver over whatever was written - // before the bound; see ExtractionResult. + // Result reports how much was extracted and whether a bound truncated it. A truncated archive + // still yields a usable Resolver over what was written before the bound; see ExtractionResult. Result ExtractionResult - // content is this archive's own bytes, held in memory or overflowed to disk. It is kept here rather - // than dropped after extraction for two reasons: content held in memory must stay reachable for - // as long as the memory limit accounts for it as held, otherwise the limiter measures something - // the garbage collector has already taken back; and openers over the extracted tree are heading - // towards reading lazily, which cannot work if the backing bytes are gone. + // content is this archive's own bytes, in memory or spilled to disk. Retained until Cleanup so the + // bytes the limiter counts as held stay reachable and lazy readers keep their backing. content Content cleanup func() } -// Cleanup removes the temp directory backing this archive's filesystem and releases what this -// archive was holding against the scan's limiter, so both limits fall by exactly what it took. Safe -// to call more than once. +// Cleanup removes the temp directory backing this archive's filesystem and releases what it held +// against the limiter. Safe to call more than once. func (e *ExtractedArchive) Cleanup() { if e == nil || e.cleanup == nil { return } - // close the resolver before the work directory is removed, so the entry store's overflow blob - // handle is released while the file it points at still exists. A resolver that holds nothing open - // - the tar path opens its file per read - does not implement io.Closer and is skipped. + // close before removing the work directory, so the overflow blob handle is released while the file + // still exists. A resolver holding nothing open is not an io.Closer. if closer, ok := e.Resolver.(io.Closer); ok { if err := closer.Close(); err != nil { log.WithFields("fsid", e.FileSystemID, "error", err).Trace("unable to close archive resolver") @@ -133,38 +98,28 @@ func (e *ExtractedArchive) Cleanup() { } e.cleanup() e.cleanup = nil - // drop the last reference to content held in memory, so the limit falling and the bytes becoming - // collectable happen together + // drop the last reference so the limit falls as the bytes become collectable e.content = Content{} } -// ExtractToResolver holds the given archive content - in memory while the memory limit admits it, on -// disk once it does not - extracts it, and returns a standalone indexed resolver over its contents -// whose Locations are stamped with the given fileSystemID (the filesystem the archive file lives on) -// and archivePath (the colon-delimited archive nesting chain from the scan root to this archive). It -// returns (nil, nil) when the content is not a supported/extractable archive. The caller owns the -// returned value and must call Cleanup on it. +// ExtractToResolver holds the given archive content (in memory while the memory limit admits it, on +// disk once it does not), extracts it, and returns a standalone indexed resolver over its contents +// whose Locations are stamped with fileSystemID and archivePath. Returns (nil, nil) when the content +// is not an extractable archive. The caller owns the result and must call Cleanup on it. Everything +// the archive holds is charged to one limiter handle and released by Cleanup. // -// Everything this archive holds is charged to one handle against the scan's limiter and released by -// Cleanup, which processArchive already defers past the sub-pipeline and past recursion - so the -// limits fall exactly where the resources are freed, and nothing new has to remember to do it. +// Reaching a limit never waits for capacity: the walk descends before it unwinds, so a containing +// archive still holds its content while its children are cataloged, and nothing would be released +// while this one waited. Content that cannot be admitted yields ErrDiskLimitReached. // -// Reaching a limit never waits for capacity. The walk descends before it unwinds, so a containing -// archive is still holding its content while its children are cataloged and nothing will be released -// while this archive waits: waiting would be a deadlock dressed as backpressure. Content that cannot -// be admitted yields ErrDiskLimitReached and the caller skips the archive. -// -// Reaching a bound part way through is not an error either: the returned ExtractedArchive is indexed -// over the partial contents and Result.Truncated() reports it, so the caller can still catalog what -// was extracted. Any other non-nil error means nothing usable was produced. -func ExtractToResolver(ctx context.Context, content io.Reader, archiveAccessPath, fileSystemID, archivePath string, extractors []Extractor, limiter *Limiter, limits ExtractionLimits, newResolver ResolverFactory, newStoreResolver StoreResolverFactory, notify Notify) (*ExtractedArchive, error) { - workDir, removeWorkDir, err := newWorkDir(ctx) - if err != nil { - return nil, err - } +// Reaching a bound part way through is not an error: the ExtractedArchive is indexed over the partial +// contents and Result.Truncated() reports it. Any other non-nil error means nothing usable was +// produced. +func ExtractToResolver(ctx context.Context, content io.Reader, archiveAccessPath, fileSystemID, archivePath string, extractors []Extractor, limiter *Limiter, newStoreResolver StoreResolverFactory, notify Notify) (*ExtractedArchive, error) { + // created only if this archive writes something; most never do + workDir := NewWorkDir(ctx) charge := limiter.Charge() - limits.Charge = charge var held Content cleanup := func() { @@ -172,28 +127,28 @@ func ExtractToResolver(ctx context.Context, content io.Reader, archiveAccessPath log.WithFields("archive", archiveAccessPath, "error", closeErr).Trace("unable to close archive content") } held = Content{} - removeWorkDir() + workDir.Remove() charge.Release() } - held, err = acquireContent(content, workDir, archiveFileName(archiveAccessPath), charge, notify) + held, err := acquireContent(content, workDir, archiveFileName(archiveAccessPath), charge, notify) if err != nil { cleanup() if notify != nil && errors.Is(err, ErrDiskLimitReached) { notify(Skipped{Archive: archiveAccessPath, Reason: "content would exceed the disk limit"}) } - // ErrDiskLimitReached travels unwrapped so the caller can attribute the skip to this archive + // unwrapped so the caller can attribute an ErrDiskLimitReached skip to this archive return nil, err } extractor := FindExtractor(ctx, extractors, held) if extractor == nil { - // not an archive we can extract (e.g. an executable that happened to match a broad mime filter) + // not extractable (e.g. an executable that matched a broad mime filter) cleanup() return nil, nil } - resolver, result, err := overflowAndIndex(ctx, extractor, held, workDir, fileSystemID, archivePath, archiveAccessPath, limits, newResolver, newStoreResolver, notify) + resolver, result, err := storeAndIndex(ctx, extractor, held, workDir, fileSystemID, archivePath, archiveAccessPath, charge, newStoreResolver, notify) if err != nil { cleanup() return nil, err @@ -203,138 +158,31 @@ func ExtractToResolver(ctx context.Context, content io.Reader, archiveAccessPath Resolver: resolver, Digests: digestsOf(ctx, held, archiveAccessPath), FileSystemID: fileSystemID, - ArchivePath: archivePath, Result: result, content: held, cleanup: cleanup, }, nil } -// workDirName is the prefix given to the directory holding one archive's content and entries. Named -// so a scan's leftovers are recognizable, and matched by the tests that sample what is on disk. -const workDirName = "syft-archive" - -// newWorkDir creates the directory one archive's content and entries are written into, under the -// scan's own temp root when the context carries one. +// inPlaceContent is content already random-access and already charged for by the archive holding it. +// A nested archive arrives as a reader over one entry of its parent's EntryStore supporting Read, +// Seek and ReadAt, so it is read where it lies rather than copied or spilled again. // -// The root is where every other cataloger that spills to disk already writes (see internal/tmpdir), -// which is what makes a scan's temp files land where the caller configured them and get removed -// together if an archive's own cleanup is ever missed. A context with no root - a caller reaching -// this package directly - still gets a directory, from the same place it would have come from -// before. -func newWorkDir(ctx context.Context) (string, func(), error) { - if td := tmpdir.FromContext(ctx); td != nil { - dir, cleanup, err := td.NewChild(workDirName) //nolint:gocritic // cleanup is returned to the caller, not deferred here - if err != nil { - return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err) - } - return dir, cleanup, nil - } - - dir, err := os.MkdirTemp("", workDirName+"-") - if err != nil { - return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err) - } - return dir, func() { - if rmErr := os.RemoveAll(dir); rmErr != nil { - log.WithFields("dir", dir, "error", rmErr).Trace("unable to remove archive temp dir") - } - }, nil -} - -// overflowAndIndex writes one archive's entries into the tar in workDir and builds the resolver over it. -// -// Errors travel unwrapped where the caller has to tell them apart: ErrDiskLimitReached attributes a -// skip to this archive, and everything else is a failure of this archive alone. -func overflowAndIndex(ctx context.Context, extractor Extractor, held Content, workDir, fsID, archivePath, accessPath string, limits ExtractionLimits, newResolver ResolverFactory, newStoreResolver StoreResolverFactory, notify Notify) (file.Resolver, ExtractionResult, error) { - if newStoreResolver != nil { - return storeAndIndex(ctx, extractor, held, workDir, fsID, archivePath, accessPath, limits, newStoreResolver, notify) - } - var result ExtractionResult - - // the archive's logical root, which stays empty: entries live in the tar beside it, and this is - // only what their paths are reported relative to - contentsDir := filepath.Join(workDir, "contents") - if err := os.MkdirAll(contentsDir, 0o755); err != nil { - return nil, result, fmt.Errorf("unable to create contents dir for archive %q: %w", fsID, err) - } - - overflow, err := newOverflowTar(filepath.Join(workDir, overflowTarName)) - if err != nil { - return nil, result, err - } - - result, err = extractor.Extract(ctx, held, overflow, limits) - if closeErr := overflow.Close(); closeErr != nil && err == nil { - err = closeErr - } - if err != nil { - return nil, result, fmt.Errorf("unable to extract archive %q: %w", fsID, err) - } - - // nothing of this archive could be placed: the disk limit refused its first byte, so there is - // nowhere for its content to go and the archive is skipped rather than cataloged as empty. This is - // where the terminal reading of the disk limit lands now that a nested archive is read where it - // lies: its own bytes are never copied, so they are never the thing refused, and what is refused - // instead is the file its entries are written into. An archive that placed SOME of its entries is - // a truncation, not a skip - see ExtractionResult. - if result.Truncation == TruncatedByDiskLimit && result.BytesWritten == 0 { - return nil, result, ErrDiskLimitReached - } - - resolver, indexed, err := newResolver(Overflow{ - RootDir: contentsDir, - TarPath: overflow.Path(), - FileSystemID: fsID, - ArchivePath: archivePath, - }) - if err != nil { - return nil, result, fmt.Errorf("unable to index extracted archive %q: %w", fsID, err) - } - - // the count of entries the archive's filesystem holds is the index's to report, not the writer's: - // a tar-family archive is copied wholesale and never walked, and an exclusion pattern drops - // entries after they were written - result.FilesExtracted = indexed.Records - if indexed.Truncated && !result.Truncated() { - // the tar ends part way through an entry and nothing recorded why. The disk limit is the only - // bound that can end a write early and it names itself, so reaching here means a ragged tar - // with no attributed cause - reported as a truncation without claiming which bound did it. - result.Truncation = TruncatedByLimit - } - - return resolver, result, nil -} - -// inPlaceContent is content that is already random-access on a file this scan wrote and is already -// accounted for by the archive that wrote it. -// -// A nested archive arrives as a reader over one entry of its parent's overflow tar, and that reader is -// Read, Seek AND ReadAt - which is everything an archive format needs, a zip's central directory -// included. So it is read in place rather than copied into memory or overflowed again: the payoff for -// choosing tar as the storage format, and the reason an inner zip needs no special handling. -// -// Declared as a marker method rather than by asserting on the reader's type, because the type belongs -// to the resolver package that this one deliberately does not import. Nothing else in the process -// answers to it, so nothing else is read in place by accident. +// A marker method rather than a type assertion, because the concrete type belongs to the fileresolver +// package, which this one does not import. Nothing else satisfies it. type inPlaceContent interface { io.Reader io.ReaderAt io.Seeker - // OverflowArchiveEntry marks a reader over one entry of an archive's overflow tar. + // OverflowArchiveEntry marks a reader over one entry of an archive's EntryStore. OverflowArchiveEntry() } -// acquireContent gets one archive's bytes ready to be read at random, which is what every archive -// format needs: it reads content already sitting in a file this scan wrote in place, and otherwise -// routes the bytes through the limiter - held in memory while the memory limit admits them, overflowed -// to disk once it does not. -// -// Content read in place is charged nothing, because nothing new is held: the bytes are part of the -// parent archive's tar, which the parent already charged for and releases on its own Cleanup. Its -// handle is the caller's to close, as it was before this ever reached here. -func acquireContent(r io.Reader, workDir, name string, charge *Charge, notify Notify) (Content, error) { +// acquireContent gets one archive's bytes ready for random access: content already held by a parent +// archive is used in place, anything else goes through the limiter. In-place content is charged +// nothing, since the parent already charged for those bytes and releases them on its own Cleanup. +func acquireContent(r io.Reader, workDir *WorkDir, name string, charge *Charge, notify Notify) (Content, error) { if entry, ok := r.(inPlaceContent); ok { return Content{Name: name, Reader: entry}, nil } @@ -342,11 +190,9 @@ func acquireContent(r io.Reader, workDir, name string, charge *Charge, notify No } // archiveFileName derives a safe basename for the saved archive, preserving compound extensions -// (e.g. ".tar.gz") so format detection has the best chance to identify the archive. -// -// ".." is refused along with the empty and root cases: Base returns it for an access path ending in -// it, and joining it to the work directory would name the directory above. overflowContent checks its -// destination as well, so this is the earlier of two refusals rather than the only one. +// (e.g. ".tar.gz") so format detection can identify it. "..", empty and root are refused, since +// joining them to the work directory would name the directory above; the destination is rechecked +// before the write. func archiveFileName(accessPath string) string { name := filepath.Base(accessPath) if name == "" || name == "." || name == ".." || name == string(filepath.Separator) { @@ -355,25 +201,15 @@ func archiveFileName(accessPath string) string { return name } -// storeAndIndex is the entry-store path: the archive's entries go into an EntryStore, which holds -// them in memory while the memory limit admits them, and the resolver is built over the entries. +// storeAndIndex puts the archive's entries into an EntryStore and builds the resolver over them. // -// Compared to the tar path this writes nothing for an archive that fits in memory - no tar to encode, -// no file to index - and where the archive does not fit, the content moves without the index being -// rebuilt. -func storeAndIndex(ctx context.Context, extractor Extractor, held Content, workDir, fsID, archivePath, accessPath string, limits ExtractionLimits, newStoreResolver StoreResolverFactory, notify Notify) (file.Resolver, ExtractionResult, error) { +// ErrDiskLimitReached travels unwrapped so the caller can attribute the skip to this archive. +func storeAndIndex(ctx context.Context, extractor Extractor, held Content, workDir *WorkDir, fsID, archivePath, accessPath string, charge *Charge, newStoreResolver StoreResolverFactory, notify Notify) (file.Resolver, ExtractionResult, error) { var result ExtractionResult - // the archive's logical root, which stays empty: entries live in the store, and this is only what - // their paths are reported relative to - contentsDir := filepath.Join(workDir, "contents") - if err := os.MkdirAll(contentsDir, 0o755); err != nil { - return nil, result, fmt.Errorf("unable to create contents dir for archive %q: %w", fsID, err) - } - store := NewEntryStore(workDir, accessPath, notify) - result, err := extractor.Extract(ctx, held, store, limits) + result, err := extractor.Extract(ctx, held, store, charge) if err != nil { if closeErr := store.Close(); closeErr != nil { log.WithFields("archive", fsID, "error", closeErr).Trace("unable to close archive entry store") @@ -381,10 +217,8 @@ func storeAndIndex(ctx context.Context, extractor Extractor, held Content, workD return nil, result, fmt.Errorf("unable to extract archive %q: %w", fsID, err) } - // nothing of this archive could be placed: the disk limit refused its first byte, so the entries - // it holds are content-less husks and the archive is skipped rather than cataloged as empty. The - // same terminal reading of the disk limit the tar path takes, measured the way this path stores - // its content - an archive that placed SOME of its entries, in memory or on disk, is a truncation. + // the disk limit refused the first byte, so every entry is content-less: skip the archive rather + // than catalog it as empty. An archive that stored some entries is a truncation instead. if result.Truncation == TruncatedByDiskLimit && store.OnDisk() == 0 { if closeErr := store.Close(); closeErr != nil { log.WithFields("archive", fsID, "error", closeErr).Trace("unable to close archive entry store") @@ -393,9 +227,9 @@ func storeAndIndex(ctx context.Context, extractor Extractor, held Content, workD } resolver, indexed, err := newStoreResolver(store, Overflow{ - RootDir: contentsDir, FileSystemID: fsID, ArchivePath: archivePath, + Charge: charge, }) if err != nil { if closeErr := store.Close(); closeErr != nil { @@ -404,7 +238,11 @@ func storeAndIndex(ctx context.Context, extractor Extractor, held Content, workD return nil, result, fmt.Errorf("unable to index extracted archive %q: %w", fsID, err) } - result.FilesExtracted = indexed.Records + // an extraction already stopped at a bound keeps that reason: it names where the archive was cut + // short, and the index only ran out of budget behind it + if indexed.Truncated && !result.Truncated() { + result.Truncation = TruncatedByIndexLimit + } if notify != nil && result.Truncated() { notify(Truncated{Archive: accessPath, Reason: result.Truncation}) } diff --git a/internal/archive/resolver_test.go b/internal/archive/resolver_test.go index 854a5547c..9838ac531 100644 --- a/internal/archive/resolver_test.go +++ b/internal/archive/resolver_test.go @@ -1,13 +1,12 @@ package archive import ( - "archive/tar" "archive/zip" "bytes" "context" - "errors" "io" "os" + "path/filepath" "strings" "testing" @@ -19,48 +18,75 @@ import ( ) func TestExtractToResolver_writesUnderTheScansTempRoot(t *testing.T) { - // where an archive's scratch space lands is the scan's decision, not this package's: the root on - // the context is what a caller configured and what gets cleaned up if an archive's own cleanup is - // ever missed. Reaching for os.MkdirTemp instead put archive work outside all of that. + // where an archive's scratch space lands is the scan's decision: the root on the context is what the + // caller configured, and what gets cleaned up if an archive's own cleanup is missed root := t.TempDir() ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)) - var got Overflow - factory := func(overflow Overflow) (file.Resolver, IndexResult, error) { - got = overflow - return nil, IndexResult{Records: 1}, nil + factory := func(_ *EntryStore, _ Overflow) (file.Resolver, IndexResult, error) { + return nil, IndexResult{}, nil } + // a zero memory bound, so this archive really does write: a work directory exists only once + // something has spilled into it extracted, err := ExtractToResolver( ctx, newTestZip(t, map[string]string{"hello.txt": "hi"}), "app.zip", "", "app.zip", - DefaultExtractors(), nil, ExtractionLimits{}, factory, nil, nil, + DefaultExtractors(), NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}), factory, nil, ) require.NoError(t, err) require.NotNil(t, extracted) t.Cleanup(extracted.Cleanup) - assert.True(t, strings.HasPrefix(got.TarPath, root+string(os.PathSeparator)), - "the archive's entries must be written under the scan's temp root, got %q", got.TarPath) + // the work directory is not named by anything the resolver is handed, so it is observed where it + // lands: under the temp root the scan configured + workDirs := workDirsUnder(t, root) + require.Len(t, workDirs, 1, "expected one archive work directory under the scan's temp root") // and the archive still cleans up after itself, so the root is a safety net rather than the only // thing reclaiming the space extracted.Cleanup() - _, statErr := os.Stat(got.RootDir) + _, statErr := os.Stat(workDirs[0]) assert.True(t, os.IsNotExist(statErr), "expected the archive work directory to be removed by Cleanup") } +func TestExtractToResolver_anArchiveThatNeverWritesCreatesNoWorkDir(t *testing.T) { + // the scratch directory is created at the point of writing, so an archive held entirely in memory - + // which is most of them - costs no directory on the filesystem at all + root := t.TempDir() + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)) + + factory := func(_ *EntryStore, _ Overflow) (file.Resolver, IndexResult, error) { + return nil, IndexResult{}, nil + } + + extracted, err := ExtractToResolver( + ctx, newTestZip(t, map[string]string{"hello.txt": "hi"}), "app.zip", "", "app.zip", + DefaultExtractors(), NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), factory, nil, + ) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + assert.Empty(t, workDirsUnder(t, root), "nothing was written, so no work directory was created") + + // and cleanup of an archive that created nothing is still safe + extracted.Cleanup() + assert.Empty(t, workDirsUnder(t, root)) +} + func TestExtractToResolver(t *testing.T) { content := newTestZip(t, map[string]string{"dir/hello.txt": "hello world"}) var got Overflow - factory := func(overflow Overflow) (file.Resolver, IndexResult, error) { - got = overflow - return nil, IndexResult{Records: 1}, nil + var gotStore *EntryStore + factory := func(store *EntryStore, overflow Overflow) (file.Resolver, IndexResult, error) { + got, gotStore = overflow, store + return nil, IndexResult{}, nil } extracted, err := ExtractToResolver( context.Background(), content, "some/path/app.zip", "parentFS", "app.war:some/path/app.zip", - DefaultExtractors(), nil, ExtractionLimits{}, factory, nil, nil, + DefaultExtractors(), nil, factory, nil, ) require.NoError(t, err) require.NotNil(t, extracted) @@ -70,66 +96,108 @@ func TestExtractToResolver(t *testing.T) { // separately as the archive path, and both are handed to the resolver factory assert.Equal(t, "parentFS", extracted.FileSystemID) assert.Equal(t, "parentFS", got.FileSystemID) - assert.Equal(t, "app.war:some/path/app.zip", extracted.ArchivePath) assert.Equal(t, "app.war:some/path/app.zip", got.ArchivePath) - // the archive's entries are in the one tar handed to the factory, and its logical root is an - // empty directory beside it: an entry's path is reported relative to that root, and nothing is - // written into it - entries, err := os.ReadDir(got.RootDir) - require.NoError(t, err) - assert.Empty(t, entries, "the archive root holds no files: the entries are in the tar") + // the archive's entries are in the store handed to the factory, keyed by their archive-relative + // path: nothing was unpacked to a directory for the resolver to be pointed at + assert.Equal(t, "hello world", readStoreEntry(t, gotStore, "dir/hello.txt")) - body := readTarEntry(t, got.TarPath, "dir/hello.txt") - assert.Equal(t, "hello world", body) - - // the count of entries this archive contributes is the index's answer, not the extractor's - assert.Equal(t, 1, extracted.Result.FilesExtracted) - - // Cleanup removes the temp tree extracted.Cleanup() - _, statErr := os.Stat(got.TarPath) - assert.True(t, os.IsNotExist(statErr), "expected the overflow tar to be removed by Cleanup") - _, statErr = os.Stat(got.RootDir) - assert.True(t, os.IsNotExist(statErr), "expected temp dir to be removed by Cleanup") -} - -// readTarEntry returns the content of one entry of a tar on disk. -func readTarEntry(t *testing.T, tarPath, name string) string { - t.Helper() - f, err := os.Open(tarPath) - require.NoError(t, err) - defer f.Close() - - tr := tar.NewReader(f) - for { - hdr, err := tr.Next() - if errors.Is(err, io.EOF) { - break - } - require.NoError(t, err) - if hdr.Name != name { - continue - } - body, err := io.ReadAll(tr) - require.NoError(t, err) - return string(body) - } - t.Fatalf("no entry %q in %q", name, tarPath) - return "" } func TestExtractToResolver_notAnArchive(t *testing.T) { - factory := func(Overflow) (file.Resolver, IndexResult, error) { return nil, IndexResult{}, nil } + factory := func(*EntryStore, Overflow) (file.Resolver, IndexResult, error) { return nil, IndexResult{}, nil } extracted, err := ExtractToResolver( context.Background(), strings.NewReader("this is not an archive"), "notes.txt", "", "notes.txt", - DefaultExtractors(), nil, ExtractionLimits{}, factory, nil, nil, + DefaultExtractors(), nil, factory, nil, ) require.NoError(t, err) assert.Nil(t, extracted, "non-archive content should yield a nil ExtractedArchive") } +func TestExtractToResolver_passesTheArchivesChargeToTheIndex(t *testing.T) { + // the index keeps more per entry than the store can measure - one node per path component - so it + // charges the scan's limiter itself, against this archive's handle + limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: 1 << 20}) + + var got Overflow + factory := func(_ *EntryStore, overflow Overflow) (file.Resolver, IndexResult, error) { + got = overflow + return nil, IndexResult{}, nil + } + + extracted, err := ExtractToResolver( + context.Background(), newTestZip(t, map[string]string{"hello.txt": "hi"}), "app.zip", "", "app.zip", + DefaultExtractors(), limiter, factory, nil, + ) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + require.NotNil(t, got.Charge, "the index must be handed this archive's charge, not left unbounded") + + // and it is this archive's handle, so Cleanup gives back whatever the index charged through it + require.True(t, got.Charge.Memory(4096)) + before, _ := limiter.InUse() + require.Positive(t, before) + + extracted.Cleanup() + after, _ := limiter.InUse() + assert.Zero(t, after, "everything the archive charged, the index included, is released with it") +} + +func TestExtractToResolver_reportsAnIndexThatRanOutOfBudget(t *testing.T) { + // an index that stops part way is a truncation, not a failure: the archive is cataloged from what was + // indexed, and the reason names the index rather than the disk, which nothing was written to + factory := func(_ *EntryStore, _ Overflow) (file.Resolver, IndexResult, error) { + return nil, IndexResult{Truncated: true}, nil + } + + extracted, err := ExtractToResolver( + context.Background(), newTestZip(t, map[string]string{"a.txt": "a", "b.txt": "b"}), "app.zip", "", "app.zip", + DefaultExtractors(), nil, factory, nil, + ) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + assert.True(t, extracted.Result.Truncated()) + assert.Equal(t, TruncatedByIndexLimit, extracted.Result.Truncation) +} + +// workDirsUnder returns the archive work directories directly under the scan's temp root. +func workDirsUnder(t *testing.T, root string) []string { + t.Helper() + children, err := os.ReadDir(root) + require.NoError(t, err) + + var out []string + for _, child := range children { + if child.IsDir() && strings.HasPrefix(child.Name(), workDirName) { + out = append(out, filepath.Join(root, child.Name())) + } + } + return out +} + +func readStoreEntry(t *testing.T, store *EntryStore, name string) string { + t.Helper() + require.NotNil(t, store) + for _, entry := range store.Entries() { + if entry.Header.Name != name { + continue + } + reader, err := store.Open(entry) + require.NoError(t, err) + body, err := io.ReadAll(reader) + require.NoError(t, err) + return string(body) + } + t.Fatalf("no entry %q in the store", name) + return "" +} + func newTestZip(t *testing.T, files map[string]string) *bytes.Buffer { t.Helper() var buf bytes.Buffer diff --git a/internal/archive/storage_test.go b/internal/archive/storage_test.go new file mode 100644 index 000000000..c7eb50858 --- /dev/null +++ b/internal/archive/storage_test.go @@ -0,0 +1,215 @@ +package archive + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/syft/file" +) + +// unboundedLimits hold everything in memory, as an archive well inside the default bounds does. +var unboundedLimits = Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1} + +// spillingLimits hold nothing in memory, so every entry lands in the store's overflow blob. The +// filesystem-cost requirement must be measured in this shape: an archive that never touches disk +// satisfies it trivially. +var spillingLimits = Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1} + +func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) { + // an archive of a hundred thousand entries must not put a hundred thousand files on the filesystem. + // Asserted as an exact set of paths for two archives three orders of magnitude apart in entry count, + // since a count that merely grows slowly would pass a bound and still be proportional to content. + _, smallStore, smallRoot := extractedStore(t, manyEntryZip(t, 2), spillingLimits) + _, largeStore, largeRoot := extractedStore(t, manyEntryZip(t, 5000), spillingLimits) + smallWork, largeWork := workDirIn(t, smallRoot), workDirIn(t, largeRoot) + + // the archive's own bytes spill too under a zero memory bound, so the set is the spilled archive and + // the blob its entries went into - two, whatever it holds + want := []string{"app.zip", "contents.blob"} + assert.Equal(t, want, filesystemEntries(t, smallWork)) + assert.Equal(t, want, filesystemEntries(t, largeWork), + "5000 entries must cost the same filesystem entries as 2") + + // and the entries really are all there, so this is not passing because nothing was extracted + assert.Len(t, smallStore.Entries(), 2) + assert.Len(t, largeStore.Entries(), 5000) + + // an archive costs its entry content and nothing else on disk: no headers, no padding, so a one-byte + // entry costs one byte + assert.Equal(t, int64(5000), largeStore.OnDisk()) + assert.Equal(t, int64(2), smallStore.OnDisk()) +} + +func TestStorage_anArchiveWithinTheMemoryBoundTouchesNoFilesystemAtAll(t *testing.T) { + // an archive that fits in memory pays for no write at all - not even the directory it would have + // written into, which is created only when something actually spills + _, store, root := extractedStore(t, manyEntryZip(t, 5000), unboundedLimits) + + assert.Len(t, store.Entries(), 5000) + assert.Zero(t, store.OnDisk()) + assert.Empty(t, filesystemEntries(t, root), + "an archive held entirely in memory creates no work directory at all") +} + +func TestStorage_cleanupRemovesOneFileAndDoesNotWalkPerEntry(t *testing.T) { + // cleanup is bounded by the same constant as the extraction, so the count of things removed is + // asserted rather than only that the directory is gone + extracted, _, root := extractedStore(t, manyEntryZip(t, 5000), spillingLimits) + workDir := workDirIn(t, root) + + before := filesystemEntries(t, workDir) + require.Len(t, before, 2, "the release has two things to remove whatever the archive held") + + extracted.Cleanup() + + _, err := os.Stat(filepath.Join(workDir, overflowBlobName)) + assert.True(t, os.IsNotExist(err), "the one file holding every entry must be gone") + _, err = os.Stat(workDir) + assert.True(t, os.IsNotExist(err), "and so must the work directory") + + // safe to call more than once, since the cataloger defers it past the sub-pipeline and recursion + extracted.Cleanup() +} + +func TestStorage_anEntryAtTheEndIsReadBySeeking(t *testing.T) { + // "overflow content MUST remain directly readable without re-reading or re-decompressing what + // precedes it". Asserted through the mechanism: the store records where the last entry's content + // begins, near the end of a blob whose earlier entries are large, and hands back a bounded window + // onto the file at that offset. + const filler = 64 * 1024 + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for i := range 8 { + w, err := zw.Create(fmt.Sprintf("filler%d.bin", i)) + require.NoError(t, err) + _, err = w.Write(bytes.Repeat([]byte{byte(i)}, filler)) + require.NoError(t, err) + } + last, err := zw.Create("zz-last.txt") + require.NoError(t, err) + wanted := strings.Repeat("last", 64) + _, err = last.Write([]byte(wanted)) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + _, store, root := extractedStore(t, &buf, spillingLimits) + + blobPath := filepath.Join(workDirIn(t, root), overflowBlobName) + info, err := os.Stat(blobPath) + require.NoError(t, err) + total := info.Size() + require.Greater(t, total, int64(8*filler), "the fixture must put real bulk ahead of the last entry") + + // the last entry's content sits near the end of the blob, so everything before it is what a + // re-streaming design would have had to read + body, err := os.ReadFile(blobPath) + require.NoError(t, err) + offset := int64(bytes.Index(body, []byte(wanted))) + require.Positive(t, offset) + assert.Greater(t, offset, total*9/10, "the last entry must really be at the end of the blob") + + entries := store.Entries() + entry := entries[len(entries)-1] + require.Equal(t, "zz-last.txt", entry.Header.Name) + + // the reader is bounded to this entry rather than the whole blob: its end is the entry's own size + random, err := store.Open(entry) + require.NoError(t, err) + + end, err := random.Seek(0, io.SeekEnd) + require.NoError(t, err) + assert.Equal(t, int64(len(wanted)), end) + + // offset zero of the reader is the last entry's first byte, not the blob's; a reader over the whole + // file would answer with the first filler entry's content + head := make([]byte, 8) + _, err = random.ReadAt(head, 0) + require.NoError(t, err) + assert.Equal(t, wanted[:8], string(head)) +} + +func manyEntryZip(t *testing.T, count int) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for i := range count { + w, err := zw.Create(fmt.Sprintf("e%06d.txt", i)) + require.NoError(t, err) + _, err = w.Write([]byte("x")) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + return &buf +} + +// extractedStore extracts one archive and returns the store its entries went into, with the scan temp +// root the extraction was pointed at. +// +// The root is where a work directory would appear: nothing the resolver is handed names a path on the +// host, and an archive that never writes creates no directory to name. +func extractedStore(t *testing.T, content io.Reader, limits Limits) (*ExtractedArchive, *EntryStore, string) { + t.Helper() + + root := t.TempDir() + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)) + + var got *EntryStore + factory := func(store *EntryStore, _ Overflow) (file.Resolver, IndexResult, error) { + got = store + return nil, IndexResult{}, nil + } + + extracted, err := ExtractToResolver( + ctx, content, "app.zip", "", "app.zip", + DefaultExtractors(), NewLimiter(limits), factory, nil, + ) + require.NoError(t, err) + require.NotNil(t, extracted) + t.Cleanup(extracted.Cleanup) + + return extracted, got, root +} + +// workDirIn returns the one work directory an archive that spilled created under the scan temp root. +func workDirIn(t *testing.T, root string) string { + t.Helper() + children, err := os.ReadDir(root) + require.NoError(t, err) + require.Len(t, children, 1, "expected exactly one archive work directory under the temp root") + return filepath.Join(root, children[0].Name()) +} + +// filesystemEntries is every directory entry under dir: an archive must not put one of these on the +// filesystem per entry it holds. +func filesystemEntries(t *testing.T, dir string) []string { + t.Helper() + var out []string + require.NoError(t, filepath.WalkDir(dir, func(p string, _ fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, relErr := filepath.Rel(dir, p) + if relErr != nil { + return relErr + } + if rel == "." { + return nil + } + out = append(out, filepath.ToSlash(rel)) + return nil + })) + return out +} diff --git a/internal/archive/traversal.go b/internal/archive/traversal.go index 2db0d0542..3a6467908 100644 --- a/internal/archive/traversal.go +++ b/internal/archive/traversal.go @@ -7,11 +7,10 @@ import ( "github.com/anchore/syft/syft/file" ) -// Traversal describes one level of archive nesting during recursive archive cataloging. The archive -// cataloger task places one on the context before running the cataloger sub-pipeline against an -// archive's contents, so ecosystem catalogers (e.g. java) can reconstruct nesting-aware identity -// such as colon-delimited virtual paths (e.g. "app.war:WEB-INF/lib/dep.jar") without owning the -// recursion themselves. +// Traversal describes one level of archive nesting. The archive cataloger task places one on the +// context before running the sub-pipeline against an archive's contents, so ecosystem catalogers +// (e.g. java) can reconstruct nesting-aware identity like "app.war:WEB-INF/lib/dep.jar" without +// owning the recursion. type Traversal struct { // Location is the archive file's location within its parent filesystem. Location file.Location @@ -20,28 +19,14 @@ type Traversal struct { // e.g. "app.war:WEB-INF/lib/dep.jar". VirtualPath string - // FileSystemID is the identifier of the filesystem the archive file itself lives on (a container - // image layer digest, or empty for a directory source), inherited unchanged down the nesting - // chain. The nesting chain is carried by VirtualPath, not by this field. - FileSystemID string - - // Depth is 1 for an archive found directly in the scan source, incrementing per nesting level. - Depth int - - // Digests are the digests of the archive file itself. They are carried here because they are a - // property of the file and not of its contents: a cataloger running against an archive's extracted - // filesystem cannot compute them, since the archive is not inside itself. Taken once by the - // extraction, which already has the bytes in hand. + // Digests are of the archive file itself, taken once during extraction. A cataloger running against + // the extracted filesystem cannot compute them, since the archive is not inside itself. Digests []file.Digest - - // Parent is the traversal for the containing archive; nil at depth 1. - Parent *Traversal } -// VirtualPathOf returns the colon-delimited virtual path for an entry within this archive. It is -// nil-safe: with no traversal the entry path is returned unchanged, matching non-nested behavior. -// The entry path's leading slash is trimmed so chains match the historical java cataloger format, -// which joins slash-less zip entry names onto the containing archive's path. +// VirtualPathOf returns the colon-delimited virtual path for an entry within this archive, or the +// entry path unchanged on a nil traversal. The leading slash is trimmed to match the java cataloger +// format, which joins slash-less zip entry names onto the containing archive's path. func (t *Traversal) VirtualPathOf(entryPath string) string { if t == nil { return entryPath @@ -55,16 +40,14 @@ func (t *Traversal) VirtualPathOf(entryPath string) string { type limiterCtxKey struct{} -// WithLimiter returns a context carrying the scan's archive limiter, so a cataloger running within -// the recursive walk can read the scan's live draw on the memory and disk budgets (Limiter.InUse) - -// the same totals the limits are enforced against. Carried on the context rather than copied onto -// each Traversal so there is one source of truth for the charge, not a snapshot per level. +// WithLimiter returns a context carrying the scan's archive limiter, so a cataloger inside the walk +// can read the live draw on the budgets (Limiter.InUse). It rides the context rather than each +// Traversal so every level reads one shared total. func WithLimiter(ctx context.Context, l *Limiter) context.Context { return context.WithValue(ctx, limiterCtxKey{}, l) } -// LimiterFromContext returns the scan's archive limiter, or nil when the current pass is not running -// within the recursive archive walk. +// LimiterFromContext returns the scan's archive limiter, or nil outside the recursive archive walk. func LimiterFromContext(ctx context.Context) *Limiter { l, _ := ctx.Value(limiterCtxKey{}).(*Limiter) return l @@ -77,8 +60,7 @@ func WithTraversal(ctx context.Context, t *Traversal) context.Context { return context.WithValue(ctx, traversalCtxKey{}, t) } -// TraversalFromContext returns the archive traversal on the context, or nil when the current -// cataloging pass is not running within an archive. +// TraversalFromContext returns the archive traversal on the context, or nil outside an archive. func TraversalFromContext(ctx context.Context) *Traversal { t, _ := ctx.Value(traversalCtxKey{}).(*Traversal) return t diff --git a/internal/archive/traversal_test.go b/internal/archive/traversal_test.go index 5619308b6..772bc5f27 100644 --- a/internal/archive/traversal_test.go +++ b/internal/archive/traversal_test.go @@ -15,24 +15,18 @@ func TestTraversalContextRoundTrip(t *testing.T) { assert.Nil(t, TraversalFromContext(context.Background())) trav := &Traversal{ - Location: file.NewLocation("app.war"), - VirtualPath: "app.war", - FileSystemID: "", - Depth: 1, + Location: file.NewLocation("app.war"), + VirtualPath: "app.war", } ctx := WithTraversal(context.Background(), trav) require.Same(t, trav, TraversalFromContext(ctx)) child := &Traversal{ - Location: file.NewLocation("/WEB-INF/lib/dep.jar"), - VirtualPath: trav.VirtualPathOf("/WEB-INF/lib/dep.jar"), - FileSystemID: "", - Depth: 2, - Parent: trav, + Location: file.NewLocation("/WEB-INF/lib/dep.jar"), + VirtualPath: trav.VirtualPathOf("/WEB-INF/lib/dep.jar"), } ctx = WithTraversal(ctx, child) require.Same(t, child, TraversalFromContext(ctx)) - assert.Same(t, trav, TraversalFromContext(ctx).Parent) } func TestTraversal_VirtualPathOf(t *testing.T) { @@ -73,8 +67,8 @@ func TestTraversal_VirtualPathOf(t *testing.T) { expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar", }, { - // a colon in the entry name itself must not be mistaken for a chain delimiter - - // see decisions.md#colon-in-an-entry-path-is-escaped + // a colon in the entry name must not be mistaken for a chain delimiter - see + // decisions.md#colon-in-an-entry-path-is-escaped name: "colon in entry path is escaped", traversal: &Traversal{VirtualPath: "app.war"}, entryPath: "WEB-INF/lib/dep:special.jar", @@ -87,8 +81,8 @@ func TestTraversal_VirtualPathOf(t *testing.T) { expected: "app.war:lib/a%3Ab%3Ac.jar", }, { - // a traversal carrying no virtual path (as distinct from a nil traversal) still - // yields the entry path alone, never a bare leading colon + // a traversal carrying no virtual path (as distinct from a nil traversal) still yields + // the entry path alone, never a bare leading colon name: "empty virtual path yields the entry path alone, no leading colon", traversal: &Traversal{VirtualPath: ""}, entryPath: "WEB-INF/lib/dep.jar", @@ -102,13 +96,11 @@ func TestTraversal_VirtualPathOf(t *testing.T) { } } -// TestTraversal_VirtualPathOf_noColonIsByteIdenticalToPreEscaping is the assertion that carries -// the compatibility guarantee behind java-virtual-path-identity-preserved -// (decisions.md#colon-in-an-entry-path-is-escaped): escaping a colon in an entry path must not -// alter the result for any entry path that does not contain one. Every case below has no colon, -// and every expected value is exactly what VirtualPathOf produced before escaping existed - do -// not "fix" a failure here by updating the expected string, since that is precisely the -// regression this test exists to catch. +// TestTraversal_VirtualPathOf_noColonIsByteIdenticalToPreEscaping carries the compatibility guarantee +// behind java-virtual-path-identity-preserved (decisions.md#colon-in-an-entry-path-is-escaped): +// escaping a colon must not alter the result for any entry path without one. Every case below is +// colon-free, and every expected value is what VirtualPathOf produced before escaping existed. Do not +// fix a failure here by updating the expected string - that is the regression this test catches. func TestTraversal_VirtualPathOf_noColonIsByteIdenticalToPreEscaping(t *testing.T) { tests := []struct { name string @@ -154,19 +146,13 @@ func TestTraversal_VirtualPathOf_noColonIsByteIdenticalToPreEscaping(t *testing. } } -// TestTraversal_VirtualPathOf_escapingComposesAcrossNestingLevels asserts that escaping is -// applied at every level of a chain, not only the first: a colon in an entry name at a deeper -// nesting level is just as ambiguous as one at the outermost level, so it gets the same -// treatment. After composing, splitting the result on ':' must yield exactly one segment per -// real archive boundary - none of the escaped colons survive as a literal ':' byte, so they -// cannot be mistaken for delimiters by a naive splitter. +// TestTraversal_VirtualPathOf_escapingComposesAcrossNestingLevels asserts escaping applies at every +// level of a chain, not only the first: splitting the composed result on ':' yields exactly one +// segment per archive boundary, since no escaped colon survives as a literal ':' byte. func TestTraversal_VirtualPathOf_theEscapeCharacterIsNotItselfEscaped(t *testing.T) { - // a known and accepted limit, not a defect: an entry path holding the literal text %3A is - // indistinguishable from one holding a colon. Escaping % as %25 would close it and is - // deliberately not done, because % is far more common in real paths than : and closing this - // would move a large number of well-defined identities to resolve a collision between two - // inputs both already assumed absent. Asserted so the limit is a decision on the record - // rather than something a later reader discovers and "fixes". + // a known and accepted limit: an entry path holding the literal text %3A is indistinguishable from + // one holding a colon. Escaping % as %25 would close it and is deliberately not done, since % is far + // more common in real paths than : and doing so would move many well-defined identities. t.Parallel() outer := &Traversal{VirtualPath: "outer.zip"} diff --git a/internal/archive/workdir.go b/internal/archive/workdir.go new file mode 100644 index 000000000..21e93acbb --- /dev/null +++ b/internal/archive/workdir.go @@ -0,0 +1,90 @@ +package archive + +import ( + "context" + "fmt" + "os" + "sync" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/tmpdir" +) + +// workDirName prefixes the directory holding one archive's content and entries, so a scan's leftovers +// are recognizable. Tests that sample what is on disk match on it. +const workDirName = "syft-archive" + +// WorkDir is the scratch space one archive writes into, created lazily the first time something needs +// to write - an archive's own bytes spilling out of memory, or its entries spilling into the overflow +// blob. +// +// Eager creation would put one directory on disk per archive opened, whether or not it ever wrote a +// byte; most do not, and a scan walks far more archives than it spills. So the path is asked for at +// the point of writing. A nil *WorkDir has no path and fails the write. +type WorkDir struct { + mu sync.Mutex + + // create makes the directory and returns it with the func that removes it. Called at most once. + create func() (string, func(), error) + + path string + remove func() + err error + done bool +} + +// NewWorkDir returns the lazily-created work directory for one archive, rooted under the scan's temp +// root (internal/tmpdir) when the context carries one - so files land where the caller configured and +// are removed together if an archive's own cleanup is missed - otherwise the system temp dir. +func NewWorkDir(ctx context.Context) *WorkDir { + return &WorkDir{create: func() (string, func(), error) { + if td := tmpdir.FromContext(ctx); td != nil { + dir, cleanup, err := td.NewChild(workDirName) //nolint:gocritic // cleanup is returned, not deferred here + if err != nil { + return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err) + } + return dir, cleanup, nil + } + + dir, err := os.MkdirTemp("", workDirName+"-") + if err != nil { + return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err) + } + return dir, func() { + if rmErr := os.RemoveAll(dir); rmErr != nil { + log.WithFields("dir", dir, "error", rmErr).Trace("unable to remove archive temp dir") + } + }, nil + }} +} + +// Path returns the directory, creating it on the first call. Later calls return the same result, +// including the same failure. +func (w *WorkDir) Path() (string, error) { + if w == nil { + return "", fmt.Errorf("no work directory for archive content") + } + + w.mu.Lock() + defer w.mu.Unlock() + if !w.done { + w.path, w.remove, w.err = w.create() + w.done = true + } + return w.path, w.err +} + +// Remove deletes the directory and its contents, and does nothing if it was never created. Safe to +// call more than once. +func (w *WorkDir) Remove() { + if w == nil { + return + } + w.mu.Lock() + defer w.mu.Unlock() + if w.remove == nil { + return + } + w.remove() + w.remove = nil +} diff --git a/internal/archive/workdir_test.go b/internal/archive/workdir_test.go new file mode 100644 index 000000000..8052427a5 --- /dev/null +++ b/internal/archive/workdir_test.go @@ -0,0 +1,101 @@ +package archive + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/tmpdir" +) + +func TestWorkDir_isNotCreatedUntilAskedFor(t *testing.T) { + root := t.TempDir() + w := NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))) + + assert.False(t, w.created(), "constructing a work directory must touch nothing") + entries, err := os.ReadDir(root) + require.NoError(t, err) + assert.Empty(t, entries) + + dir, err := w.Path() + require.NoError(t, err) + assert.DirExists(t, dir) + assert.True(t, w.created()) +} + +func TestWorkDir_pathIsStableAcrossCalls(t *testing.T) { + w := NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) + + first, err := w.Path() + require.NoError(t, err) + second, err := w.Path() + require.NoError(t, err) + assert.Equal(t, first, second, "the directory is created once, not once per writer") +} + +func TestWorkDir_concurrentCallersShareOneDirectory(t *testing.T) { + // the store spills under its own lock while the archive's content may be spilling too, so the + // creation has to be safe to race + root := t.TempDir() + w := NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))) + + var wg sync.WaitGroup + paths := make([]string, 8) + for i := range paths { + wg.Add(1) + go func() { + defer wg.Done() + dir, err := w.Path() + assert.NoError(t, err) + paths[i] = dir + }() + } + wg.Wait() + + for _, p := range paths { + assert.Equal(t, paths[0], p) + } + entries, err := os.ReadDir(root) + require.NoError(t, err) + assert.Len(t, entries, 1, "racing callers must not each create a directory") +} + +func TestWorkDir_removeIsANoOpWhenNothingWasCreated(t *testing.T) { + root := t.TempDir() + w := NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))) + + w.Remove() + w.Remove() + + entries, err := os.ReadDir(root) + require.NoError(t, err) + assert.Empty(t, entries) +} + +func TestWorkDir_removeDeletesWhatWasCreated(t *testing.T) { + w := NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) + + dir, err := w.Path() + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "blob"), []byte("x"), 0o600)) + + w.Remove() + assert.NoDirExists(t, dir) + + // safe to call more than once, as the extraction's cleanup is + w.Remove() +} + +func TestWorkDir_nilHasNowhereToWrite(t *testing.T) { + var w *WorkDir + + _, err := w.Path() + assert.Error(t, err, "a caller with nowhere to put content must fail rather than pick a directory") + assert.False(t, w.created()) + w.Remove() +} diff --git a/internal/bus/helpers.go b/internal/bus/helpers.go index 24a925efa..1e4b45b5d 100644 --- a/internal/bus/helpers.go +++ b/internal/bus/helpers.go @@ -2,6 +2,7 @@ package bus import ( "context" + "sync" "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" @@ -38,39 +39,96 @@ func Notify(message string) { }) } -type catalogerTaskProgressKey struct{} +type catalogerTaskRegistryKey struct{} -// WithCatalogerTaskProgress returns a context whose cataloger tasks report into prog rather than -// starting a row of their own. +// catalogerTaskRegistry remembers the row each cataloger publishes, keyed by the publishing task's +// identity. // -// Nested cataloging runs under such a context: re-running the cataloger pipeline against the -// contents of an archive is work that belongs to the progress of the task driving the recursion, not -// a second start of every cataloger. Starting rows again per archive floods a consumer's display, -// and where the new row carries an ID the consumer has already seen (every package cataloger does - -// the ID is the cataloger name) it replaces the live row for that ID. The replaced row is then never -// rendered again, its completion is never observed, and a UI that waits on every row it started - as -// the syft CLI does - blocks forever on teardown. -func WithCatalogerTaskProgress(ctx context.Context, prog *monitor.TaskProgress) context.Context { - return context.WithValue(ctx, catalogerTaskProgressKey{}, prog) +// A cataloger runs more than once per scan: once over the scan root, then again inside every archive +// the archive cataloger walks. A row per run floods the display, and reusing a published ID replaces +// the live row - which is then never completed, hanging a UI (like the syft CLI) that waits on every +// row it started. So a re-run reports into the row it already owns, and its count covers the whole +// scan rather than the root alone. +type catalogerTaskRegistry struct { + mu sync.Mutex + rows map[string]*monitor.TaskProgress } -// catalogerTaskProgressFromContext returns the progress that cataloger tasks started under this -// context report into, or nil when they should start one of their own. -func catalogerTaskProgressFromContext(ctx context.Context) *monitor.TaskProgress { - prog, _ := ctx.Value(catalogerTaskProgressKey{}).(*monitor.TaskProgress) - return prog +// WithCatalogerTaskRegistry returns a context under which each cataloger publishes one row no matter +// how many times it runs; without it every run publishes, which suits a single-pass caller. +// +// Install once per scan before any task runs, so a re-run finds the first run's row. Idempotent: a +// second registry would hold none of the existing rows, so every re-run would publish again. +func WithCatalogerTaskRegistry(ctx context.Context) context.Context { + if catalogerTaskRegistryFromContext(ctx) != nil { + return ctx + } + return context.WithValue(ctx, catalogerTaskRegistryKey{}, &catalogerTaskRegistry{ + rows: make(map[string]*monitor.TaskProgress), + }) +} + +func catalogerTaskRegistryFromContext(ctx context.Context) *catalogerTaskRegistry { + reg, _ := ctx.Value(catalogerTaskRegistryKey{}).(*catalogerTaskRegistry) + return reg +} + +// catalogerTaskKey identifies the row a task owns: its ID where it has one (a package cataloger's ID +// is its name), otherwise its parent and title (as every file cataloger, which publishes no ID). +func catalogerTaskKey(info monitor.GenericTask) string { + if info.ID != "" { + return "id:" + info.ID + } + return "title:" + info.ParentID + "/" + info.Title.Default } func StartCatalogerTask(ctx context.Context, info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress { - if parent := catalogerTaskProgressFromContext(ctx); parent != nil { - // the stage is shared, so what this cataloger is working on shows up on the row already on - // screen; the count is not, so this task completing does not complete the row it reports into - return &monitor.TaskProgress{ - AtomicStage: parent.AtomicStage, - Manual: progress.NewManual(size), - } + reg := catalogerTaskRegistryFromContext(ctx) + if reg == nil { + return publishCatalogerTask(info, size, initialStage) } + reg.mu.Lock() + defer reg.mu.Unlock() + + key := catalogerTaskKey(info) + if row, ok := reg.rows[key]; ok { + resumeCatalogerTask(row, size, initialStage) + return row + } + + t := publishCatalogerTask(info, size, initialStage) + reg.rows[key] = t + + return t +} + +// resumeCatalogerTask reopens a row an earlier run completed and grows its total by what this run +// adds. +// +// Growing the total matters: a row completes once its count reaches its total, so incrementing a row +// whose total covers only the first run makes it read as finished mid-walk. An indeterminate total on +// either side stays indeterminate. +// +// Only a completed row is reopened; one carrying a real failure keeps it, since a later clean run +// does not mean the earlier one succeeded. +func resumeCatalogerTask(row *monitor.TaskProgress, size int64, initialStage string) { + if size < 0 || row.Size() < 0 { + row.SetTotal(-1) + } else { + row.SetTotal(row.Size() + size) + } + + if progress.IsErrCompleted(row.Error()) { + row.SetError(nil) + } + + if initialStage != "" { + row.AtomicStage.Set(initialStage) + } +} + +func publishCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress { t := &monitor.TaskProgress{ AtomicStage: progress.NewAtomicStage(initialStage), Manual: progress.NewManual(size), diff --git a/internal/bus/helpers_test.go b/internal/bus/helpers_test.go new file mode 100644 index 000000000..4df218c24 --- /dev/null +++ b/internal/bus/helpers_test.go @@ -0,0 +1,152 @@ +package bus + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-progress" + + "github.com/anchore/syft/syft/event/monitor" +) + +func digestsTask() monitor.GenericTask { + return monitor.GenericTask{ + Title: monitor.Title{Default: "File digests"}, + ParentID: monitor.TopLevelCatalogingTaskID, + } +} + +func TestStartCatalogerTask_withoutARegistryEveryRunPublishes(t *testing.T) { + // the right answer for a caller driving a single pass: nothing re-runs, so nothing should be reused + ctx := context.Background() + + first := StartCatalogerTask(ctx, digestsTask(), 2, "") + second := StartCatalogerTask(ctx, digestsTask(), 3, "") + + assert.NotSame(t, first, second) + assert.Equal(t, int64(2), first.Size()) + assert.Equal(t, int64(3), second.Size()) +} + +func TestStartCatalogerTask_reRunReportsIntoTheRowItAlreadyOwns(t *testing.T) { + ctx := WithCatalogerTaskRegistry(context.Background()) + + first := StartCatalogerTask(ctx, digestsTask(), 2, "") + first.Increment() + first.Increment() + first.SetCompleted() + + // the archive walk reaches the same cataloger again, over an archive's contents this time + second := StartCatalogerTask(ctx, digestsTask(), 3, "") + require.Same(t, first, second, "a re-run must find the row it already published") + + assert.Equal(t, int64(5), second.Size(), "the total must cover both runs, not just the first") + assert.False(t, progress.IsCompleted(second), "a row with work still to do must not read as complete") + + for i := 0; i < 3; i++ { + second.Increment() + } + second.SetCompleted() + + assert.Equal(t, int64(5), second.Current()) + assert.Equal(t, int64(5), second.Size(), "the count must not run past the total") + assert.True(t, progress.IsCompleted(second)) +} + +func TestStartCatalogerTask_aTotalResolvedOnCompletionIsAddable(t *testing.T) { + // a run that could not say how much it would do says so when it signs off (Manual.SetCompleted + // backfills the total from the count), so by the time a re-run arrives there is a real number to + // add to. This is the file metadata cataloger's shape: it publishes -1 every run. + ctx := WithCatalogerTaskRegistry(context.Background()) + + row := StartCatalogerTask(ctx, digestsTask(), -1, "") + row.Increment() + row.SetCompleted() + require.Equal(t, int64(1), row.Size(), "signing off must resolve the total to what the run did") + + require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), -1, "")) + assert.Equal(t, int64(-1), row.Size(), "a re-run that cannot say its own size makes the row indeterminate again") + + row.Increment() + row.SetCompleted() + assert.Equal(t, int64(2), row.Current()) + assert.Equal(t, int64(2), row.Size(), "and signing off resolves it again, over both runs") +} + +func TestStartCatalogerTask_indeterminateTotalsStayIndeterminate(t *testing.T) { + // a run whose size is unknown cannot be added to one that is known + tests := []struct { + name string + first, next int64 + complete func(row *monitor.TaskProgress) + }{ + { + name: "the re-run cannot say its own size", + first: 2, next: -1, + complete: func(row *monitor.TaskProgress) { row.Increment(); row.SetCompleted() }, + }, + { + name: "the first run did nothing, so completing resolved nothing", + first: -1, next: 3, + complete: func(row *monitor.TaskProgress) { row.SetCompleted() }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := WithCatalogerTaskRegistry(context.Background()) + + row := StartCatalogerTask(ctx, digestsTask(), tt.first, "") + tt.complete(row) + + require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), tt.next, "")) + assert.Equal(t, int64(-1), row.Size()) + assert.False(t, progress.IsCompleted(row), "an indeterminate row is not complete until it says so") + }) + } +} + +func TestStartCatalogerTask_aFailedRowKeepsItsFailure(t *testing.T) { + // a later run finding nothing wrong does not mean the earlier one succeeded + ctx := WithCatalogerTaskRegistry(context.Background()) + failure := errors.New("unable to read the thing") + + row := StartCatalogerTask(ctx, digestsTask(), 2, "") + row.SetError(failure) + + require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), 3, "")) + assert.ErrorIs(t, row.Error(), failure) +} + +func TestStartCatalogerTask_rowsAreKeyedByTaskIdentity(t *testing.T) { + ctx := WithCatalogerTaskRegistry(context.Background()) + + byID := func(id string) monitor.GenericTask { + return monitor.GenericTask{ID: id, Title: monitor.Title{Default: "shared title"}} + } + + assert.Same(t, StartCatalogerTask(ctx, byID("a"), -1, ""), StartCatalogerTask(ctx, byID("a"), -1, "")) + assert.NotSame(t, StartCatalogerTask(ctx, byID("a"), -1, ""), StartCatalogerTask(ctx, byID("b"), -1, ""), + "an ID is the identity where there is one, even when two tasks share a title") + + // a task publishing without an ID, as every file cataloger does, is keyed by where it sits and what + // it is called + underCataloging := monitor.GenericTask{Title: monitor.Title{Default: "Files"}, ParentID: "cataloging"} + underSomethingElse := monitor.GenericTask{Title: monitor.Title{Default: "Files"}, ParentID: "other"} + + assert.Same(t, StartCatalogerTask(ctx, underCataloging, -1, ""), StartCatalogerTask(ctx, underCataloging, -1, "")) + assert.NotSame(t, StartCatalogerTask(ctx, underCataloging, -1, ""), StartCatalogerTask(ctx, underSomethingElse, -1, "")) +} + +func TestWithCatalogerTaskRegistry_isIdempotent(t *testing.T) { + // anything that re-runs catalogers may ask for a registry; a second one would hold none of the rows + // already published and every re-run would publish again + ctx := WithCatalogerTaskRegistry(context.Background()) + row := StartCatalogerTask(ctx, digestsTask(), 2, "") + + nested := WithCatalogerTaskRegistry(ctx) + assert.Same(t, row, StartCatalogerTask(nested, digestsTask(), 2, "")) +} diff --git a/internal/file/copy.go b/internal/file/copy.go index c02c264bb..27c12ef93 100644 --- a/internal/file/copy.go +++ b/internal/file/copy.go @@ -8,8 +8,7 @@ import ( const perFileReadLimit = 2 * GB -// SafeCopy limits the copy from the reader. This is useful when extracting files from archives to -// protect against decompression bomb attacks. +// SafeCopy bounds the copy to guard against decompression bombs when extracting from archives. func SafeCopy(writer io.Writer, reader io.Reader) error { numBytes, err := io.Copy(writer, io.LimitReader(reader, perFileReadLimit)) if numBytes >= perFileReadLimit { diff --git a/internal/file/zip_file_traversal.go b/internal/file/zip_file_traversal.go index c3ea0c8e2..211527037 100644 --- a/internal/file/zip_file_traversal.go +++ b/internal/file/zip_file_traversal.go @@ -165,22 +165,10 @@ func UnzipToDir(ctx context.Context, archivePath, targetDir string) error { } // SafeJoin ensures that any destinations do not resolve to a path above the prefix path. -// -// The comparison is on a path boundary rather than on the string prefix. A sibling directory that -// happens to share the prefix as a string is not inside it: under "/tmp/x/contents" the destination -// "../contents-evil/y" cleans to "/tmp/x/contents-evil/y", which starts with the prefix and is -// outside it. func SafeJoin(prefix string, dest ...string) (string, error) { joinResult := filepath.Join(append([]string{prefix}, dest...)...) - cleanPrefix := filepath.Clean(prefix) cleanJoinResult := filepath.Clean(joinResult) - - boundary := cleanPrefix - if !strings.HasSuffix(boundary, string(filepath.Separator)) { - boundary += string(filepath.Separator) - } - - if cleanJoinResult != cleanPrefix && !strings.HasPrefix(cleanJoinResult, boundary) { + if !strings.HasPrefix(cleanJoinResult, filepath.Clean(prefix)) { return "", &errZipSlipDetected{ Prefix: prefix, JoinArgs: dest, diff --git a/internal/file/zip_file_traversal_test.go b/internal/file/zip_file_traversal_test.go index 5b7f73538..4fe960b66 100644 --- a/internal/file/zip_file_traversal_test.go +++ b/internal/file/zip_file_traversal_test.go @@ -305,45 +305,6 @@ func TestSafeJoin(t *testing.T) { expected: "", errAssertion: assertErrorAs(&errZipSlipDetected{}), }, - // a legitimate nested path, several components deep - { - prefix: "/a/place", - args: []string{ - "lib", - "sub", - "file.txt", - }, - expected: "/a/place/lib/sub/file.txt", - errAssertion: assert.NoError, - }, - // the prefix itself is inside the prefix - { - prefix: "/a/place", - args: []string{ - ".", - }, - expected: "/a/place", - errAssertion: assert.NoError, - }, - // siblings that share the prefix as a STRING are not inside it. A string-prefix comparison - // accepted these, so an archive entry could write into a directory next to the one it was - // being extracted into. - { - prefix: "/a/place", - args: []string{ - "../place-evil/x", - }, - expected: "", - errAssertion: assertErrorAs(&errZipSlipDetected{}), - }, - { - prefix: "/tmp/syft-archive-abc/contents", - args: []string{ - "../contents-evil/x", - }, - expected: "", - errAssertion: assertErrorAs(&errZipSlipDetected{}), - }, } for _, test := range tests { diff --git a/internal/index/key_split.go b/internal/index/key_split.go index f631f2a0c..27c9e6a24 100644 --- a/internal/index/key_split.go +++ b/internal/index/key_split.go @@ -1,12 +1,10 @@ // Package index provides a concurrent key-split (radix) index and a forward/reverse pair over it. // -// Ported from a prototype dir scan package whose search strategy this -// exists to serve: a path index that answers "which entries end in .jar" and "which entries live -// under this directory" without walking a tree, so a cataloger's `**/*.ext` glob is a lookup keyed -// on the extension rather than a scan of every path. +// It answers path queries like "which entries end in .jar" or "live under this directory" as keyed +// lookups rather than tree walks, so a cataloger's `**/*.ext` glob need not scan every path. // -// Kept close to the original so the two can be diffed. The lock type is local (see lock.go) rather -// than pulled in from the prototype's sync/atomic package, which carries unrelated helpers. +// Ported from a prototype dir-scan package and kept close to the original for diffing. The lock type +// is local (see lock.go) rather than the prototype's, which carries unrelated helpers. package index import ( @@ -81,11 +79,6 @@ func (n *Node[T]) Value() (v T) { return } -func (n *Node[T]) Contains(s string) bool { - _, equal := n._find(s) - return equal -} - func (n *Node[T]) Get(s string) (out T) { v, equal := n._find(s) if !equal || v == nil { @@ -108,6 +101,41 @@ func (n *Node[T]) ByPrefix(s string) []T { return v.Collect() } +// ByPrefixUpTo returns the values under s, or false once there are more than limit of them. It lets a +// caller choosing between two lookups take whichever is selective without paying for the other: a +// complete result costs at most limit. The values returned alongside false are partial; discard them. +func (n *Node[T]) ByPrefixUpTo(s string, limit int) (values []T, complete bool) { + v, _ := n._find(s) + if v == nil { + return nil, true + } + return v.CollectUpTo(limit) +} + +// CollectUpTo gathers this node's values, giving up as soon as there are more than limit. +func (n *Node[T]) CollectUpTo(limit int) (values []T, complete bool) { + complete = n._collectUpTo(&values, limit) + return values, complete +} + +func (n *Node[T]) _collectUpTo(values *[]T, limit int) bool { + defer n.RLock()() + + if n.set { + if len(*values) >= limit { + return false + } + *values = append(*values, n.value) + } + + for _, v := range n.keyMap { + if !v._collectUpTo(values, limit) { + return false + } + } + return true +} + func (n *Node[T]) Update(name string, f NodeUpdateFunc[T]) { unlock := n.Lock() node := n._makeNodeP(&unlock, name, nil) @@ -123,14 +151,6 @@ func (n *Node[T]) SetValue(value T) { unlock() } -func (n *Node[T]) UnsetValue() { - unlock := n.Lock() - var zero T - n.value = zero - n.set = false - unlock() -} - func (n *Node[T]) Collect() (values []T) { n._collect(&values) return values @@ -149,7 +169,7 @@ func (n *Node[T]) _collect(values *[]T) { unlock() } -// _startsWith returns the node starting with the given string +// _find returns the node for s and whether it matched exactly. func (n *Node[T]) _find(s string) (node *Node[T], equal bool) { defer n.RLock()() diff --git a/internal/index/lock.go b/internal/index/lock.go index 515821cd2..e399e9b0f 100644 --- a/internal/index/lock.go +++ b/internal/index/lock.go @@ -5,12 +5,10 @@ import ( "sync" ) -// lockable is a read-write lock that hands back its own unlock function. +// lockable is a read-write lock that returns its own unlock function. // -// Returning the unlocker rather than exposing Unlock is what makes the index's lock upgrades safe to -// write: a caller holds an opaque `unlock`, and the node it is walking can tell whether that unlock -// belongs to the write lock or the read lock and upgrade only when it has to. Carried over from the -// prototype this index came from, where the pattern is used throughout. +// Handing back an opaque unlocker (rather than exposing Unlock) lets a node tell whether the caller +// holds the read or write lock and upgrade only when needed. Carried over from the prototype index. type lockable struct { lock sync.RWMutex } diff --git a/internal/index/prefix_suffix.go b/internal/index/prefix_suffix.go index 7d4c453eb..fada252dd 100644 --- a/internal/index/prefix_suffix.go +++ b/internal/index/prefix_suffix.go @@ -28,6 +28,16 @@ func (f *PrefixSuffix[T]) BySuffix(suffix string) []T { return f.reverseIndex.ByPrefix(reverse(suffix)) } +// ByPrefixUpTo and BySuffixUpTo are the bounded forms, reporting false rather than returning more than +// limit values, so a caller can take whichever side is selective without paying for the other. +func (f *PrefixSuffix[T]) ByPrefixUpTo(prefix string, limit int) ([]T, bool) { + return f.forwardIndex.ByPrefixUpTo(prefix, limit) +} + +func (f *PrefixSuffix[T]) BySuffixUpTo(suffix string, limit int) ([]T, bool) { + return f.reverseIndex.ByPrefixUpTo(reverse(suffix), limit) +} + func reverse(s string) string { runes := []rune(s) for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { diff --git a/internal/task/archive_tasks.go b/internal/task/archive_tasks.go index c8fddfa6a..439d082b8 100644 --- a/internal/task/archive_tasks.go +++ b/internal/task/archive_tasks.go @@ -29,51 +29,47 @@ import ( const ArchiveCatalogerTaskName = "archive-cataloger" // NewArchiveCatalogerTask returns a task that recursively extracts archives (including JAR-family -// archives, whose cataloger skips its own unarchiving when this task is enabled), treats each as -// its own standalone indexed filesystem, runs the given cataloger sub-pipeline against each, and -// records file-level CONTAINS relationships from the archive to the packages and files discovered -// inside it. Returns nil when archive cataloging is disabled (MaxDepth <= 0), so a disabled feature -// registers no task at all rather than a task that runs and does nothing. +// ones, whose cataloger skips its own unarchiving when this task is enabled), treats each as its own +// standalone indexed filesystem, runs subPipeline against each, and records file-level CONTAINS +// relationships from the archive to what was found inside it. // -// subPipeline is the set of package/file cataloger tasks to run against each extracted archive; it -// must NOT include this task, so recursion is driven only by the explicit depth-limited walk here. -// newResolver builds an FSID-stamped indexed resolver over an extracted archive directory (injected -// so this package does not depend on the syft-subtree internal fileresolver package). -func NewArchiveCatalogerTask(cfg cataloging.ArchiveSearchConfig, subPipeline []Task, newResolver archive.ResolverFactory, newStoreResolver archive.StoreResolverFactory, notify archive.Notify) Task { - if cfg.MaxDepth == 0 || newResolver == nil { +// Returns nil when archive cataloging is disabled (MaxDepth == 0; negative means unbounded). +// +// subPipeline must not include this task, so recursion is driven only by the depth-limited walk here. +// newStoreResolver is injected so this package does not depend on the internal fileresolver package. +func NewArchiveCatalogerTask(cfg cataloging.ArchiveSearchConfig, subPipeline []Task, newStoreResolver archive.StoreResolverFactory, notify archive.Notify) Task { + if cfg.MaxDepth == 0 || newStoreResolver == nil { return nil } - // this task drives recursion itself via an explicit depth-bounded walk, so it must never appear - // in its own sub-pipeline: each nesting level would then be processed once by the walk and - // again by the nested copy. Enforced rather than left to a comment, since the caller assembles - // the sub-pipeline from whatever package and file tasks are selected. + // this task drives recursion itself: in its own sub-pipeline, each nesting level would be processed + // once by the walk and again by the nested copy subPipeline = withoutArchiveCataloger(subPipeline) if len(subPipeline) == 0 { return nil } fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error { - // one progress row for the whole recursive walk, updated as each archive is entered. The - // sub-pipeline underneath it reports into this same row rather than starting rows of its own: - // those catalogers have already started (and finished) a row for the scan root, and starting - // them again per archive is what both floods the display and strands the rows it replaces - // (see bus.WithCatalogerTaskProgress). + // this task re-runs catalogers, so it must ensure each keeps to one row; a scan installs the + // registry up front, this covers a pipeline assembled without one + ctx = bus.WithCatalogerTaskRegistry(ctx) + + // one progress row for the whole walk; the sub-pipeline's catalogers each report into the row + // they already own (see bus.WithCatalogerTaskRegistry), not this one prog := bus.StartCatalogerTask(ctx, archiveCatalogerProgressInfo(), -1, "") c := &archiveCataloger{ cfg: cfg, subPipeline: subPipeline, extractors: archive.DefaultExtractors(), - limits: archive.DefaultExtractionLimits(cfg), limiter: archive.NewLimiter(archive.DefaultLimits(cfg)), - newResolver: newResolver, newStoreResolver: newStoreResolver, notify: notify, prog: prog, } - err := c.catalog(archive.WithLimiter(bus.WithCatalogerTaskProgress(ctx, prog), c.limiter), resolver, nil, 0, builder) - c.left() + err := c.catalog(archive.WithLimiter(ctx, c.limiter), resolver, nil, 0, builder) + c.left(nil) c.logStats() prog.SetCompleted() + return err } return NewTask(ArchiveCatalogerTaskName, fn, pkgcataloging.PackageTag, "archive") @@ -91,7 +87,6 @@ func archiveCatalogerProgressInfo() monitor.GenericTask { } } -// withoutArchiveCataloger returns the given tasks with any archive-cataloger task removed. func withoutArchiveCataloger(tasks []Task) []Task { out := make([]Task, 0, len(tasks)) for _, t := range tasks { @@ -112,30 +107,22 @@ type archiveCataloger struct { cfg cataloging.ArchiveSearchConfig subPipeline []Task extractors []archive.Extractor - limits archive.ExtractionLimits - newResolver archive.ResolverFactory newStoreResolver archive.StoreResolverFactory - // notify carries structured extraction events - which archive overflowed to disk and why, which was - // skipped - to whoever asked for them. Nil is the no-op, and nil is what a scan that wants nothing - // instrumented passes. + // notify carries structured extraction events - which archive spilled to disk and why, which was + // skipped. Nil is the no-op. notify archive.Notify - // limiter measures the archive content this scan is holding right now, falling as each archive is - // released. One instance per task run, because one task run is one scan and that is the only - // scoping under which "held at once" means anything. It replaces a monotonic byte counter, which - // bounded a number matching no state that ever existed: every archive's extraction directory is - // removed on the way out while the counter only ever rose. + // limiter measures the archive content held right now, falling as each archive is released. One + // instance per task run, since one task run is one scan. limiter *archive.Limiter - // prog is the single progress row for the whole walk: its stage names the archive currently being - // cataloged and its count is the number of archives entered so far. The recursion reports through - // this one row rather than letting the sub-pipeline start rows of its own at every level. + // prog is the single progress row for the whole walk: its count and stage are how many archives have + // been entered, its context names the one being cataloged right now. prog *monitor.TaskProgress - // slowest is the archive that spent the most time on its own extraction and cataloging, named by - // its full virtual path. Guarded for the same reason the limiter is: the walk is sequential today - // and lives in a concurrent neighbourhood. + // slowest is the archive that spent the most time on its own extraction and cataloging, by virtual + // path. Guarded because the walk runs alongside the top-level catalogers. statsMu sync.Mutex slowest time.Duration slowestPath string @@ -143,8 +130,7 @@ type archiveCataloger struct { } // recordArchiveTime keeps the longest self time seen and the archive that spent it. Self time -// excludes the archives nested inside this one, which are timed against themselves - inclusive time -// would name the outermost archive every run and say nothing. +// excludes nested archives; inclusive time would name the outermost archive every run. func (c *archiveCataloger) recordArchiveTime(virtualPath string, took time.Duration) { c.statsMu.Lock() defer c.statsMu.Unlock() @@ -155,11 +141,9 @@ func (c *archiveCataloger) recordArchiveTime(virtualPath string, took time.Durat } } -// logStats reports what the walk cost, once, when it is done. The two peaks are the high-water marks -// of the gauges the limits are enforced against, so they are directly comparable to the configured -// bounds - they are not process memory and not filesystem usage. Reported at info rather than debug -// because it is one line per scan and it is what someone tuning the limits is looking for; the -// per-archive skip and truncation lines around it stay at debug. +// logStats reports what the walk cost, once, when it is done. The peaks are high-water marks of the +// gauges the limits are enforced against - comparable to the configured bounds, not process memory or +// filesystem usage. One line per scan, so it logs at info; per-archive lines stay at debug. func (c *archiveCataloger) logStats() { c.statsMu.Lock() slowest, slowestPath, seen := c.slowest, c.slowestPath, c.archivesSeen @@ -178,7 +162,7 @@ func (c *archiveCataloger) logStats() { } // catalog discovers archives in the given resolver and processes each, recursing up to MaxDepth. -// parent is the traversal for the archive whose contents are being searched (nil at the scan root). +// parent is the traversal for the archive being searched (nil at the scan root). func (c *archiveCataloger) catalog(ctx context.Context, resolver file.Resolver, parent *archive.Traversal, depth int, builder sbomsync.Builder) error { if c.cfg.MaxDepth >= 0 && depth >= c.cfg.MaxDepth { return nil @@ -186,9 +170,8 @@ func (c *archiveCataloger) catalog(ctx context.Context, resolver file.Resolver, var errs error for _, candidate := range c.discoverArchives(resolver) { - // there is no between-archives check: a limit falls when an archive is released, so an - // archive that does not fit says nothing about the next one, and reaching a bound skips only - // the archive that reached it. + // no between-archives check: a limit falls when an archive is released, so reaching a bound skips + // only the archive that reached it if err := c.processArchive(ctx, resolver, candidate, parent, depth, builder); err != nil { errs = unknown.Append(errs, candidate.location, err) } @@ -204,10 +187,8 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi } defer internal.CloseAndLogError(content, archiveLoc.AccessPath) - // a candidate that is only a candidate is checked cheaply before its bytes are read in full: an - // archive wearing a stub carries an entry signature near the front, and almost nothing else does. - // Reading every executable in a scan to its end to ask the authoritative question instead doubled - // a stock ruby image. + // check a non-sniffing candidate cheaply before reading it in full: an appended archive carries an + // entry signature near the front, and almost nothing else does var archiveContent io.Reader = content if !candidate.sniffedAsArchive { var mayHide bool @@ -217,23 +198,18 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi } } - // self time for this archive: its extraction, its sub-pipeline and its merge, stopped before the - // recursive descent so a containing archive is not charged for what it contains + // self time: extraction, sub-pipeline and merge, stopped before the recursive descent so a + // containing archive is not charged for what it contains started := time.Now() - // the extracted files inherit the filesystem id of the location the archive was found at, - // uniformly at every level: for an image source that is the layer digest, for a directory source - // it is empty, and inside another archive the enclosing resolver stamped it there already. The - // nesting chain is carried separately as the archive path (the colon-delimited chain of archive - // access paths from the scan root). Together these keep identically-named files in different - // archives - and the same archive path in different layers - apart in the coordinate-keyed tables. + // extracted files inherit the filesystem id of where the archive was found (a layer digest for an + // image source, empty for a directory source). The nesting chain rides separately as the archive + // path, so same-named files in different archives - and the same archive path in different layers - + // stay distinct in coordinate-keyed tables. archivePath := parent.VirtualPathOf(accessPath(archiveLoc)) - extracted, err := archive.ExtractToResolver(ctx, archiveContent, archiveLoc.AccessPath, archiveLoc.FileSystemID, archivePath, c.extractors, c.limiter, c.limits, c.newResolver, c.newStoreResolver, c.notify) + extracted, err := archive.ExtractToResolver(ctx, archiveContent, archiveLoc.AccessPath, archiveLoc.FileSystemID, archivePath, c.extractors, c.limiter, c.newStoreResolver, c.notify) if errors.Is(err, archive.ErrDiskLimitReached) { - // the disk limit is terminal and nothing will be released while this archive waits, since - // the walk descends before it unwinds - so the archive is skipped and the scan continues. - // Attributed to the archive, because a log line carrying only the bound says nothing about - // what was lost. + // terminal: nothing is released while this archive waits, so skip it and continue the scan log.WithFields("archive", accessPath(archiveLoc), "limit", c.cfg.MaxDiskBytes). Debug("skipping archive whose content would exceed the disk limit") return nil @@ -242,29 +218,20 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi return err } if extracted == nil { - // nothing could open it: a format with no extractor (an ar archive, a bare gzip that is not a - // tar), or a candidate whose head could have been hiding an archive and was not. Logged, not - // returned as an error. - // - // Returning one was measured at 3x the scan on a stock ruby image, whose 3,679 gzip files are - // mostly man pages with no extractor: every one produced an error, and appending thousands of - // them into one chain that is then walked for coordinate errors is quadratic. A file syft - // cannot open is also not the failure `#archive-failure-is-not-fatal` is about - that is a - // corrupt archive, which fails during extraction and still returns an error from there. + // nothing could open it: a format with no extractor (ar, a bare non-tar gzip), or a candidate + // whose head only looked like it hid an archive. Logged rather than returned: an image can hold + // thousands, and accumulating an error each into a chain later walked for coordinate errors is + // quadratic. A corrupt archive does return an error. log.WithFields("archive", accessPath(archiveLoc), "sniffed-as-archive", candidate.sniffedAsArchive). Trace("no extractor could read this candidate; skipping") return nil } - // releases the extraction directory and gives back everything this archive charged to the - // limiter, after the sub-pipeline and after recursion + // releases the extraction directory and this archive's charges, after the sub-pipeline and recursion defer extracted.Cleanup() if extracted.Result.Truncated() { - // a bound stopped the extraction early: catalog what was written rather than discarding the - // archive, but say so, since the packages found here are not the full set. Recorded in the - // SBOM's unknowns and not only in the logs, because "syft saw part of this archive" is a fact - // about the SBOM's completeness that outlives the run that produced it - which is exactly what - // unknowns are for. + // catalog what was stored rather than discarding the archive, and record it in the SBOM's + // unknowns as well as the log, since the packages found here are not the full set log.WithFields("archive", accessPath(archiveLoc), "limit", string(extracted.Result.Truncation)). Debug("archive extraction truncated by a configured limit; cataloging partial contents") @@ -275,58 +242,50 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi }}) } - // expose the nesting chain to catalogers in the sub-pipeline (and deeper recursion) so they can - // reconstruct nesting-aware identity (e.g. the java cataloger's colon-delimited virtual paths) + // expose the nesting chain to the sub-pipeline and deeper recursion, so catalogers can reconstruct + // nesting-aware identity such as the java cataloger's colon-delimited virtual paths trav := &archive.Traversal{ - Location: archiveLoc, - VirtualPath: archivePath, - FileSystemID: extracted.FileSystemID, - Depth: depth + 1, - Parent: parent, - Digests: extracted.Digests, + Location: archiveLoc, + VirtualPath: archivePath, + Digests: extracted.Digests, } subCtx := archive.WithTraversal(ctx, trav) - c.entering(trav.VirtualPath) + c.entering() - // run the package/file cataloger sub-pipeline against the archive's standalone filesystem scratch := c.runSubPipeline(subCtx, extracted.Resolver, archiveLoc.Coordinates) - // merge results into the shared SBOM and record file-level CONTAINS edges from this archive mergeArchiveResults(archiveLoc.Coordinates, scratch, builder) - c.left() + c.left(trav) c.recordArchiveTime(trav.VirtualPath, time.Since(started)) - // recurse into archives nested within this one return c.catalog(subCtx, extracted.Resolver, trav, depth+1, builder) } -// entering records on the walk's single progress row that this archive is now being cataloged, and -// left takes that row's stage back once it is done. Both are nil-safe, so a cataloger assembled -// without a progress row simply reports nothing. +// entering counts this archive onto the walk's single progress row; left takes the row's stage back +// once the archive is done, naming it unless the whole walk has finished. Both are nil-safe. // -// The row is taken back because the sub-pipeline shares its stage - that is what makes the nested -// catalogers' progress visible - and each of them signs off with a summary of its own ("0 -// executables"). Left to stand, that summary is what the row reads while the next archive is being -// extracted, which is both wrong and unchanging. The count of archives entered is the one thing the -// row can say that is always true of the walk as a whole. -func (c *archiveCataloger) entering(virtualPath string) { +// The stage is reset because each sub-pipeline signs off with its own summary, which would otherwise +// stand while the next archive is extracted. The count of archives entered is always true. +func (c *archiveCataloger) entering() { if c.prog == nil { return } c.prog.Increment() - c.prog.AtomicStage.Set(virtualPath) } -func (c *archiveCataloger) left() { +func (c *archiveCataloger) left(traversal *archive.Traversal) { if c.prog == nil { return } - c.prog.AtomicStage.Set(fmt.Sprintf("%s archives", humanize.Comma(c.prog.Current()))) + if traversal == nil { + c.prog.AtomicStage.Set(fmt.Sprintf("%s archives", humanize.Comma(c.prog.Current()))) + } else { + c.prog.AtomicStage.Set(fmt.Sprintf("%s archives (%s)", humanize.Comma(c.prog.Current()), traversal.VirtualPath)) + } } -// accessPath returns the location's access path, falling back to the real path when unset. func accessPath(loc file.Location) string { if loc.AccessPath != "" { return loc.AccessPath @@ -334,14 +293,12 @@ func accessPath(loc file.Location) string { return loc.RealPath } -// runSubPipeline runs every cataloger task against the given resolver into a throwaway SBOM so the -// results can be attributed to the containing archive before being merged into the shared SBOM. +// runSubPipeline runs every cataloger task against the given resolver into a throwaway SBOM, so +// results can be attributed to the containing archive before merging into the shared SBOM. // -// A task's failure is recorded the same way RunTask records one at the top level (executor.go), so a -// cataloger that fails inside an archive is visible in the SBOM's unknowns rather than only in the -// logs: errors carrying their own coordinates keep them, and anything left has no location of its -// own, so it is attributed to archiveCoord - the archive that was being cataloged. A failure here -// never fails the archive or the scan. +// Failures are recorded as RunTask records them at the top level (executor.go), so they land in the +// SBOM's unknowns rather than only the logs: errors carrying coordinates keep them, anything left is +// attributed to archiveCoord. A failure here never fails the archive or the scan. func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Resolver, archiveCoord file.Coordinates) *sbom.SBOM { scratch := &sbom.SBOM{ Artifacts: sbom.Artifacts{ @@ -376,50 +333,35 @@ type archiveCandidate struct { location file.Location // sniffedAsArchive is true when the file's own content types it as an archive. Such a file that - // then cannot be extracted is a broken archive and is reported; a candidate that got here only - // because its head could be hiding one is simply not an archive, and is passed over in silence. + // cannot be extracted is a broken archive and is reported; a candidate here only because its head + // might hide one is passed over silently. sniffedAsArchive bool } -// zipArchiveGlobs find the files worth a second look: those that claim to be zip archives by name -// and whose content did not say so. A self-extracting archive is one - a launcher stub with a zip -// concatenated onto it, typed by the stub - and the canonical case is a Spring Boot executable jar, -// which sniffs as a shell script. +// zipArchiveGlobs find files worth a second look: those named like zip archives whose content did not +// sniff as one, such as a Spring Boot executable jar - a launcher stub with a zip appended, which +// sniffs as a shell script. // -// This is not a detection rule. Whether a candidate is an archive is settled by the -// end-of-central-directory record and by nothing else, so a file named `.jar` that holds prose is -// refused exactly as it is today. The name decides only what gets looked at. +// Not a detection rule: the end-of-central-directory record settles whether a candidate is an +// archive, so a `.jar` holding prose is still refused. The name decides only what gets opened, which +// is why a self-extracting archive with no telling extension is not found. // -// It is a glob rather than a MIME lookup because the MIME lookup was the whole cost. Asking the -// resolver for every file that sniffs as executable, script, or unrecognized returns 3,274 locations -// on a stock ruby image - all of which this then discarded, since none of them were named like -// archives - and that one call took the scan from 15 seconds to 45. Asking for the names directly -// returns the handful that matter. -// -// The limit this accepts is that a self-extracting archive with no telling extension is not found. -// That is deliberate, and it is the one place here where a name decides whether syft looks. -// -// One pattern per extension, not one pattern with brace alternation: the alternation form matched -// nothing, and the cost of the extra patterns is not measurable - the whole lookup is about a -// millisecond and returns nothing on an image with no self-extracting archives in it. +// A glob rather than a MIME lookup because asking for every file sniffed as executable, script or +// unrecognized returns thousands of locations that are all discarded, which tripled the scan on a +// stock ruby image. One pattern per extension because brace alternation matched nothing. var zipArchiveGlobs = []string{ "**/*.jar", "**/*.war", "**/*.ear", "**/*.par", "**/*.sar", "**/*.nar", "**/*.kar", "**/*.hpi", "**/*.jpi", "**/*.far", "**/*.rar", "**/*.zip", "**/*.apk", "**/*.aar", "**/*.egg", "**/*.whl", "**/*.lpkg", "**/*.zap", "**/*.exe", } -// discoverArchives returns every location worth opening, in two groups. +// discoverArchives returns every location worth opening: files that are archives by their own +// content, plus files named like zip archives whose head says script or executable but whose tail may +// still be a whole zip. The second group is only opened; the extractor refuses anything with no +// end-of-central-directory record. // -// There is no exclusion filter here on purpose. The scan's exclusion patterns are applied where each -// filesystem is indexed - the source's own index for the scan root, and the extraction directory's -// index for an archive's contents - so an excluded archive is not in the resolver to be returned in -// the first place. Filtering again here would be a second mechanism answering the same question. -// -// The archive MIME types are archives by their own content. The appended set are files whose head -// says script or executable and whose tail may still be a whole zip: a self-extracting archive is -// typed by its stub, so nothing sniffed at offset zero can see it. Membership in the second set -// decides only that a file is worth opening - whether it is an archive is decided by the extractor, -// which refuses anything with no end-of-central-directory record. +// No exclusion filter here: exclusion patterns are applied where each filesystem is indexed, so an +// excluded archive is not in the resolver to begin with. func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCandidate { archives, err := resolver.FilesByMIMEType(mimetype.ArchiveMIMETypeSet.List()...) if err != nil { @@ -443,10 +385,7 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan seen[c.location.Coordinates] = struct{}{} } for _, loc := range appended { - // a location reached by both lookups is one file and must be opened once. The two sets are - // disjoint today, so this guards the invariant rather than a known overlap - and processing - // an archive twice would double every package it holds, which is the failure this whole - // design exists to prevent. + // one file reached by both lookups must be opened once, or every package it holds is doubled if _, dup := seen[loc.Coordinates]; dup { continue } @@ -456,9 +395,8 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan return candidates } -// mergeArchiveResults copies the throwaway SBOM's packages, files, and relationships into the shared -// SBOM, and adds a file-level CONTAINS relationship from the archive's coordinates to every package -// and file discovered inside it. +// mergeArchiveResults copies the throwaway SBOM's packages, files and relationships into the shared +// SBOM, adding a file-level CONTAINS relationship from the archive to everything found inside it. func mergeArchiveResults(archiveCoord file.Coordinates, scratch *sbom.SBOM, builder sbomsync.Builder) { pkgs := scratch.Artifacts.Packages.Sorted() if len(pkgs) > 0 { diff --git a/internal/task/archive_tasks_test.go b/internal/task/archive_tasks_test.go index 34b97ac7c..39a46927d 100644 --- a/internal/task/archive_tasks_test.go +++ b/internal/task/archive_tasks_test.go @@ -39,143 +39,33 @@ import ( "github.com/anchore/syft/syft/sbom" ) -// dirTestResolver is a minimal file.Resolver over a directory for exercising the archive cataloger -// task: FilesByMIMEType reports any zip-family file (ignoring the actual MIME type arguments) and -// FileContentsByLocation reads from disk. All other resolver methods are unimplemented. +// dirTestResolver is a minimal file.Resolver over a directory: FilesByMIMEType reports any zip-family +// file (ignoring the MIME arguments) and FileContentsByLocation reads from disk. Other methods are +// unimplemented. type dirTestResolver struct { file.Resolver dir string fsid string } -func (d dirTestResolver) FilesByMIMEType(types ...string) ([]file.Location, error) { - return filesByMIMEType(d.dir, d.fsid, types...) -} - -// filesByMIMEType is what a real resolver does: sniff each file's content at index time and return -// the ones whose type was asked for. The test resolvers used to ignore the argument and return -// everything, which made every caller look like it was being handed archives - and hid the fact -// that the task now asks two different questions with two different consequences. -func filesByMIMEType(dir, fsid string, types ...string) ([]file.Location, error) { - wanted := strset.New(types...) - var locs []file.Location - err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { - if err != nil || entry.IsDir() { - return err - } - f, err := os.Open(path) - if err != nil { - return err - } - mimeType := stereoscopeFile.MIMEType(f) - if closeErr := f.Close(); closeErr != nil { - return closeErr - } - if !wanted.Has(mimeType) { - return nil - } - rel, err := filepath.Rel(dir, path) - if err != nil { - return err - } - coords := file.Coordinates{RealPath: "/" + filepath.ToSlash(rel), FileSystemID: fsid} - locs = append(locs, file.NewLocationFromCoordinates(coords)) - return nil - }) - return locs, err -} - -func (d dirTestResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { - return os.Open(filepath.Join(d.dir, filepath.FromSlash(loc.RealPath))) -} - -// tarTestResolver is a minimal file.Resolver over one archive's overflow tar, standing in for the real -// tar-backed resolver: the archive cataloger's own tests should not need the syft-subtree -// fileresolver package, but they do need something shaped like what it produces. +// storeTestResolver is a minimal file.Resolver over one archive's entry store, standing in for the +// real one so these tests need not import the fileresolver package. // // FilesByMIMEType reports any zip-family entry (ignoring the MIME arguments) and -// FileContentsByLocation opens that entry at its offset in the tar. The reader it returns carries the -// marker the extraction path looks for, so a nested archive is exercised through the read-in-place -// route it takes in a real scan rather than through a copy. -type tarTestResolver struct { +// FileContentsByLocation opens it from the store. The reader carries the marker the extraction path +// looks for, so a nested archive is exercised through the read-in-place route rather than a copy. +type storeTestResolver struct { file.Resolver - entries map[string]stereoscopeFile.TarIndexEntry + store *archive.EntryStore + entries map[string]*archive.Entry fsid string archivePath string } -// newTarTestResolver is an archive.ResolverFactory over the overflow tar. -func newTarTestResolver(overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - r := &tarTestResolver{entries: map[string]stereoscopeFile.TarIndexEntry{}, fsid: overflow.FileSystemID, archivePath: overflow.ArchivePath} - if _, err := stereoscopeFile.NewTarIndex(overflow.TarPath, func(entry stereoscopeFile.TarIndexEntry) error { - name := entry.ToTarFileEntry().Header.Name - r.entries[path.Clean("/"+name)] = entry - return nil - }); err != nil { - return nil, archive.IndexResult{}, err - } - return r, archive.IndexResult{Records: len(r.entries)}, nil -} - -func (r *tarTestResolver) sortedNames() []string { - names := make([]string, 0, len(r.entries)) - for name := range r.entries { - names = append(names, name) - } - sort.Strings(names) - return names -} - -func (r *tarTestResolver) location(name string) file.Location { - return file.NewLocationFromCoordinates(file.Coordinates{RealPath: name, FileSystemID: r.fsid, ArchivePath: r.archivePath}) -} - -func (r *tarTestResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) { - var locs []file.Location - for _, name := range r.sortedNames() { - switch strings.ToLower(filepath.Ext(name)) { - case ".zip", ".jar", ".war": - locs = append(locs, r.location(name)) - } - } - return locs, nil -} - -func (r *tarTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { - var locs []file.Location - for _, name := range r.sortedNames() { - for _, pattern := range patterns { - if matched, err := doublestar.Match(pattern, strings.TrimPrefix(name, "/")); err == nil && matched { - locs = append(locs, r.location(name)) - break - } - } - } - return locs, nil -} - -func (r *tarTestResolver) FilesByPath(paths ...string) ([]file.Location, error) { - var locs []file.Location - for _, p := range paths { - name := path.Clean("/" + p) - if _, ok := r.entries[name]; ok { - locs = append(locs, r.location(name)) - } - } - return locs, nil -} - -func (r *tarTestResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { - entry, ok := r.entries[loc.RealPath] - if !ok { - return nil, fmt.Errorf("no entry %q", loc.RealPath) - } - contents := entry.Open() - seekable, ok := contents.(randomAccessReadCloser) - if !ok { - return contents, nil - } - return testOverflowEntry{seekable}, nil +// nopCloserAt gives a store reader the Close the resolver interface asks for; the store owns the +// bytes, so closing a reader over them must close nothing. +type nopCloserAt struct { + archive.ReaderAtSeeker } type randomAccessReadCloser interface { @@ -184,67 +74,12 @@ type randomAccessReadCloser interface { io.Seeker } -// testOverflowEntry carries the marker method that says "already random access on a file this scan +// testOverflowEntry carries the marker method saying "already random access on a file this scan // wrote", which is how a nested archive gets read where it lies. type testOverflowEntry struct { randomAccessReadCloser } -func (testOverflowEntry) OverflowArchiveEntry() {} - -// makeZip builds a zip in sorted entry-name order. The order matters and must not be the map's: -// extraction limits are enforced as the walk proceeds, so which entries land before a truncation is -// a function of entry order, and a randomized order makes any limit assertion flaky. -func makeZip(t *testing.T, entries map[string][]byte) []byte { - t.Helper() - names := make([]string, 0, len(entries)) - for name := range entries { - names = append(names, name) - } - sort.Strings(names) - - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - for _, name := range names { - w, err := zw.Create(name) - require.NoError(t, err) - _, err = w.Write(entries[name]) - require.NoError(t, err) - } - require.NoError(t, zw.Close()) - return buf.Bytes() -} - -// matchesAnyGlob reports whether the path matches one of the patterns, against the resolver-relative -// path with a leading slash, which is the form a real resolver globs over. No patterns means -// everything matches. -func matchesAnyGlob(root, path string, patterns []string) bool { - if len(patterns) == 0 { - return true - } - rel, err := filepath.Rel(root, path) - if err != nil { - return false - } - rel = "/" + filepath.ToSlash(rel) - for _, pattern := range patterns { - if ok, err := doublestar.Match(pattern, rel); err == nil && ok { - return true - } - } - return false -} - -// FilesByGlob for the resolvers that only ever stand in for a MIME lookup: the archive walk asks -// them for archive-named files too, and an unimplemented method on an embedded nil resolver panics. -func (a allFilesResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { - return dirTestResolver{dir: a.dir}.FilesByGlob(patterns...) -} - -func (r twoLayerResolver) FilesByGlob(_ ...string) ([]file.Location, error) { - return nil, nil -} - func Test_archiveCataloger_traversalThreading(t *testing.T) { manifest := []byte("Manifest-Version: 1.0\nImplementation-Title: example\nImplementation-Version: 1.0.0\n") innerJar := makeZip(t, map[string][]byte{"META-INF/MANIFEST.MF": manifest}) @@ -265,9 +100,9 @@ func Test_archiveCataloger_traversalThreading(t *testing.T) { cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3) - newResolver := newTarTestResolver + newResolver := newStoreTestResolver - tsk := NewArchiveCatalogerTask(cfg, []Task{captureTask}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, []Task{captureTask}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -285,23 +120,12 @@ func Test_archiveCataloger_traversalThreading(t *testing.T) { "/outer.zip:lib/inner.jar", "/outer.zip:nested/inner.zip", }, chains) - - // verify parent linkage and depth - for _, trav := range captured { - if trav.Depth == 1 { - assert.Nil(t, trav.Parent) - continue - } - require.NotNil(t, trav.Parent) - assert.Equal(t, trav.Parent.Depth+1, trav.Depth) - assert.True(t, strings.HasPrefix(trav.VirtualPath, trav.Parent.VirtualPath+":")) - } } func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) { - // a limit is a truncation, not a failure: the sub-pipeline must still run over whatever was - // written, so packages found before the limit survive. Discarding the whole archive was the - // original behavior and it silently lost every package in an oversized archive. + // a limit is a truncation, not a failure: the sub-pipeline still runs over what was written, so + // packages found before the limit survive. Discarding the whole archive silently lost every package + // in an oversized one. big := bytes.Repeat([]byte("x"), 4096) outerZip := makeZip(t, map[string][]byte{ "a/small.txt": []byte("small"), @@ -317,16 +141,16 @@ func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) { return nil }) - // small.txt costs a header block and a padded data block; big.txt's header does not fit after - // them. The archive's own bytes are held in memory at the default limit, so what is charged to - // disk here is only the tar its entries are written into. + // nothing is held in memory, so every entry is charged to disk as it lands: the archive's own bytes + // spill first, then small.txt, and big.txt is refused part way through cfg := cataloging.DefaultArchiveSearchConfig(). WithMaxDepth(1). - WithMaxDiskBytes(1500) + WithMaxMemoryBytes(0). + WithMaxDiskBytes(int64(len(outerZip)) + 1500) - newResolver := newTarTestResolver + newResolver := newStoreTestResolver - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -335,9 +159,8 @@ func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) { require.NoError(t, err, "a truncated extraction must not fail the scan") assert.Equal(t, 1, ran, "the sub-pipeline must run over the partial contents") - // and the SBOM says the archive was read in part, at the archive's own coordinates: a scan that - // finished cleanly but saw only some of an archive is not the same SBOM as one that saw all of - // it, and a debug log does not survive the run that wrote it + // and the SBOM records, at the archive's coordinates, that it was read in part: a scan that saw only + // some of an archive is not the same SBOM as one that saw all of it reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] require.True(t, ok, "the truncation must be recorded against the archive; got %v", s.Artifacts.Unknowns) require.Len(t, reasons, 1) @@ -346,14 +169,12 @@ func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) { } func Test_archiveCataloger_failureIsRecordedAndSkipsSubPipeline(t *testing.T) { - // a genuine failure is not a truncation: nothing usable was produced, so the sub-pipeline must - // not run over an empty tree, and the failure must be visible in the SBOM rather than only in - // the logs + // a genuine failure is not a truncation: nothing usable was produced, so the sub-pipeline must not + // run over an empty tree, and the failure must reach the SBOM rather than only the logs rootDir := t.TempDir() - // a zip whose entry data has been overwritten: its header still says zip and its - // end-of-central-directory record is intact, so it is detected as an archive and then fails when - // the entry is read. Prose named `.zip` would not do - that is not detected as an archive at all, - // which is a different requirement and is covered by + // a zip whose entry data has been overwritten: the header and end-of-central-directory record are + // intact, so it is detected as an archive and fails when the entry is read. Prose named `.zip` is a + // different case - not detected as an archive at all - see // Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError. corrupt := makeZip(t, map[string][]byte{"data.bin": incompressible(4096)}) require.Greater(t, len(corrupt), 1024) @@ -373,17 +194,17 @@ func Test_archiveCataloger_failureIsRecordedAndSkipsSubPipeline(t *testing.T) { }) cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1) - newResolver := newTarTestResolver + newResolver := newStoreTestResolver - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) - // the failure must be a CoordinateError, because that is what executor.go's - // unknown.ExtractCoordinateErrors pulls out and records in sbom.Artifacts.Unknowns. A plain - // error would propagate as a scan failure instead of being reported against the archive + // the failure must be a CoordinateError: that is what executor.go's + // unknown.ExtractCoordinateErrors pulls out into sbom.Artifacts.Unknowns. A plain error would + // propagate as a scan failure instead of being reported against the archive require.Error(t, err) coordErrs, remaining := unknown.ExtractCoordinateErrors(err) assert.NoError(t, remaining, "nothing may escape as a non-coordinate error") @@ -395,14 +216,14 @@ func Test_archiveCataloger_failureIsRecordedAndSkipsSubPipeline(t *testing.T) { } func Test_NewArchiveCatalogerTask_gating(t *testing.T) { - newResolver := newTarTestResolver + newResolver := newStoreTestResolver someTask := NewTask("noop", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { return nil }) tests := []struct { name string cfg cataloging.ArchiveSearchConfig subPipeline []Task - resolver archive.ResolverFactory + resolver archive.StoreResolverFactory wantTask bool }{ { @@ -441,7 +262,7 @@ func Test_NewArchiveCatalogerTask_gating(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := NewArchiveCatalogerTask(tt.cfg, tt.subPipeline, tt.resolver, nil, nil) + got := NewArchiveCatalogerTask(tt.cfg, tt.subPipeline, tt.resolver, nil) if tt.wantTask { require.NotNil(t, got) assert.Equal(t, ArchiveCatalogerTaskName, got.Name()) @@ -460,7 +281,7 @@ func Test_archiveCataloger_depthBoundIsExact(t *testing.T) { rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) - newResolver := newTarTestResolver + newResolver := newStoreTestResolver run := func(t *testing.T, depth int) []string { t.Helper() @@ -472,7 +293,7 @@ func Test_archiveCataloger_depthBoundIsExact(t *testing.T) { return nil }) - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth), []Task{captureTask}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth), []Task{captureTask}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -486,11 +307,10 @@ func Test_archiveCataloger_depthBoundIsExact(t *testing.T) { } func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) { - // the task drives recursion with its own depth-bounded walk, so a copy of itself in the - // sub-pipeline would process every nesting level twice. The depth bound still terminates the - // recursion, so the symptom is duplicated packages rather than a hang - which is exactly the - // kind of failure nobody notices in a large SBOM. - newResolver := newTarTestResolver + // the task drives recursion with its own depth-bounded walk, so a copy of itself in the sub-pipeline + // would process every nesting level twice. The depth bound still terminates, so the symptom is + // duplicated packages rather than a hang - the kind of failure nobody notices in a large SBOM. + newResolver := newStoreTestResolver cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2) innerZip := makeZip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) @@ -505,10 +325,10 @@ func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) { }) // a sub-pipeline that (incorrectly) carries an archive cataloger task alongside real work - poisoned := []Task{countTask, NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil, nil)} + poisoned := []Task{countTask, NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil)} require.NotNil(t, poisoned[1]) - tsk := NewArchiveCatalogerTask(cfg, poisoned, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, poisoned, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -519,39 +339,16 @@ func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) { assert.Equal(t, 2, ran) t.Run("a sub-pipeline of nothing but itself yields no task", func(t *testing.T) { - only := []Task{NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil, nil)} - assert.Nil(t, NewArchiveCatalogerTask(cfg, only, newResolver, nil, nil)) + only := []Task{NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil)} + assert.Nil(t, NewArchiveCatalogerTask(cfg, only, newResolver, nil)) }) } -// FilesByGlob lists the files under the resolver's directory that match, stamped with its -// filesystem id, so a sub-pipeline task can record file artifacts the way a real file cataloger -// does. With no patterns it lists everything, which is what the sub-pipeline tasks here want. -func (d dirTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { - var locs []file.Location - err := filepath.WalkDir(d.dir, func(path string, entry fs.DirEntry, err error) error { - if err != nil || entry.IsDir() { - return err - } - if !matchesAnyGlob(d.dir, path, patterns) { - return nil - } - rel, err := filepath.Rel(d.dir, path) - if err != nil { - return err - } - coords := file.Coordinates{RealPath: "/" + filepath.ToSlash(rel), FileSystemID: d.fsid} - locs = append(locs, file.NewLocationFromCoordinates(coords)) - return nil - }) - return locs, err -} - func Test_archiveCataloger_chainStartsAtTheArchivesOwnFileSystemID(t *testing.T) { // covers syft/archive-content-identity#nested-filesystem-id-chain, scenario "image source chain - // extends the layer digest": an image-source archive's extracted files keep the layer digest as - // their FileSystemID (inherited unchanged down the nesting chain), while the archive nesting chain - // is carried separately as the traversal's VirtualPath / the coordinate ArchivePath + // extends the layer digest": extracted files keep the layer digest as their FileSystemID at every + // level, while the nesting chain rides on the traversal's VirtualPath and the coordinate's + // ArchivePath innerZip := makeZip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip}) @@ -566,13 +363,13 @@ func Test_archiveCataloger_chainStartsAtTheArchivesOwnFileSystemID(t *testing.T) trav := archive.TraversalFromContext(ctx) require.NotNil(t, trav) chains = append(chains, trav.VirtualPath) - fsids = append(fsids, trav.FileSystemID) + fsids = append(fsids, trav.Location.FileSystemID) return nil }) - newResolver := newTarTestResolver + newResolver := newStoreTestResolver - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), []Task{captureTask}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), []Task{captureTask}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -581,43 +378,25 @@ func Test_archiveCataloger_chainStartsAtTheArchivesOwnFileSystemID(t *testing.T) // the layer digest is preserved unchanged at every level as the filesystem the archive lives on assert.Equal(t, []string{layerDigest, layerDigest}, fsids) - // exact strings: the archive path is serialized into the SBOM, so a changed shape is a changed - // output format, not an internal detail + // exact strings: the archive path is serialized into the SBOM, so its shape is output format assert.ElementsMatch(t, []string{ "/outer.zip", "/outer.zip:nested/inner.zip", }, chains) } -// twoLayerResolver reports the same archive path twice under two different filesystem ids, each -// holding different content — what an all-layers image scan yields for an archive rewritten in a -// later layer. FilesByMIMEType dedupes on the file reference rather than the path, so both survive. +// twoLayerResolver reports the same archive path twice under two different filesystem ids, each with +// different content - what an all-layers image scan yields for an archive rewritten in a later layer. +// FilesByMIMEType dedupes on the file reference rather than the path, so both survive. type twoLayerResolver struct { file.Resolver path string contentBy map[string][]byte } -func (r twoLayerResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) { - var locs []file.Location - for _, fsid := range []string{"layer-one", "layer-two"} { - locs = append(locs, file.NewLocationFromCoordinates(file.Coordinates{RealPath: r.path, FileSystemID: fsid})) - } - return locs, nil -} - -func (r twoLayerResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { - body, ok := r.contentBy[loc.FileSystemID] - if !ok { - return nil, os.ErrNotExist - } - return io.NopCloser(bytes.NewReader(body)), nil -} - func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T) { - // the collision the chain exists to prevent: two layers holding /app/bundle.zip at the same path - // composed the same identifier while the chain was seeded from the parent traversal, so the two - // extractions' contents overwrote each other in the coordinate-keyed file tables + // the collision the chain exists to prevent: two layers holding /app/bundle.zip would otherwise + // compose the same identifier and overwrite each other in the coordinate-keyed file tables resolver := twoLayerResolver{ path: "/app/bundle.zip", contentBy: map[string][]byte{ @@ -632,7 +411,7 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T fileTask := NewTask("record-files", func(ctx context.Context, r file.Resolver, builder sbomsync.Builder) error { trav := archive.TraversalFromContext(ctx) require.NotNil(t, trav) - chains = append(chains, trav.FileSystemID) + chains = append(chains, trav.Location.FileSystemID) locs, err := r.FilesByGlob("**") require.NoError(t, err) @@ -645,9 +424,9 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T return nil }) - newResolver := newTarTestResolver + newResolver := newStoreTestResolver - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{fileTask}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{fileTask}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{ @@ -659,28 +438,26 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T assert.ElementsMatch(t, []string{"layer-one", "layer-two"}, chains, "the two archives keep the distinct filesystem ids of the layers they were found in") - // two file-table entries, asserted as a count: the tables are keyed by Coordinates, so a - // collision is a silently missing entry rather than any kind of error. The same archive path in - // two layers stays distinct because the coordinates differ by FileSystemID (the layer), not by - // ArchivePath (which is identical here) + // asserted as a count: the tables are keyed by Coordinates, so a collision is a silently missing + // entry rather than an error. The two stay distinct by FileSystemID (the layer); their ArchivePath + // is identical here. assert.Len(t, s.Artifacts.FileMetadata, 2) } func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) { // syft/nested-archive-cataloging#archive-failure-is-not-fatal wants a catalog failure inside an - // archive "visible in output rather than only in logs". runSubPipeline bypasses RunTask, which is - // where the coordinate-error to unknown conversion lives, so the error used to be trace-logged - // and dropped. + // archive "visible in output rather than only in logs". runSubPipeline bypasses RunTask, where the + // coordinate-error to unknown conversion lives, so it does that conversion itself. outerZip := makeZip(t, map[string][]byte{"lib/broken.json": []byte("{")}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) - newResolver := newTarTestResolver + newResolver := newStoreTestResolver cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1) t.Run("a coordinate error keeps the coordinates the cataloger named", func(t *testing.T) { - // the cataloger attributed its failure to a file it was reading, which is the normal case and - // is more precise than the archive: it must not be flattened onto the archive's coordinates + // a failure attributed to a file the cataloger was reading is more precise than the archive, so + // it must not be flattened onto the archive's coordinates failing := NewTask("failing-cataloger", func(_ context.Context, r file.Resolver, _ sbomsync.Builder) error { locs, err := r.FilesByGlob("**") require.NoError(t, err) @@ -692,7 +469,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) return errs }) - tsk := NewArchiveCatalogerTask(cfg, []Task{failing}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, []Task{failing}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -712,7 +489,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) return errors.New("cataloger blew up") }) - tsk := NewArchiveCatalogerTask(cfg, []Task{failing}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, []Task{failing}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -730,7 +507,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) panic("nope") }) - tsk := NewArchiveCatalogerTask(cfg, []Task{panicking}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, []Task{panicking}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -745,8 +522,8 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) } func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) { - // mergeFileArtifacts appends to the shared SBOM's unknowns, so the ones a sub-cataloger wrote - // into the scratch SBOM itself must survive alongside the one the failure produced + // mergeFileArtifacts appends to the shared SBOM's unknowns, so the ones a sub-cataloger wrote into + // the scratch SBOM must survive alongside the one the failure produced outerZip := makeZip(t, map[string][]byte{"lib/thing.json": []byte("{}")}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) @@ -762,8 +539,8 @@ func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) return errors.New("boom") }) - newResolver := newTarTestResolver - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{writer, failing}, newResolver, nil, nil) + newResolver := newStoreTestResolver + tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{writer, failing}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -774,9 +551,8 @@ func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) } func Test_archiveCataloger_manySmallArchivesAreEachCataloged(t *testing.T) { - // the case the monotonic counter wrongly rejected: every archive is released before the next - // begins, so a scan whose archives sum well past the disk limit never holds more than one of - // them at a time and every one is cataloged + // the case a monotonic counter wrongly rejected: every archive is released before the next begins, + // so a scan whose archives sum well past the disk limit never holds more than one at a time entry := bytes.Repeat([]byte("z"), 400) entries := map[string][]byte{} for i := range 5 { @@ -795,15 +571,15 @@ func Test_archiveCataloger_manySmallArchivesAreEachCataloged(t *testing.T) { return nil }) - // one archive extracts 2000 bytes, so the limit holds one comfortably and the five together - // would be four times over a counter that never fell + // one archive extracts 2000 bytes, so the limit holds one comfortably and the five together would be + // four times over a counter that never fell cfg := cataloging.DefaultArchiveSearchConfig(). WithMaxDepth(1). WithMaxDiskBytes(4000) - newResolver := newTarTestResolver + newResolver := newStoreTestResolver - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -826,22 +602,21 @@ func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, entries), 0o600)) } - // each entry costs a header block plus a padded data block, and the tar carries an - // end-of-archive marker: five entries of 400 bytes is 11 blocks. Room for one archive's tar and - // not for two at once. + // each entry costs a header block plus a padded data block, plus the tar's end-of-archive marker: + // five entries of 400 bytes is 11 blocks. Room for one archive's tar, not two at once. cfg := cataloging.DefaultArchiveSearchConfig(). WithMaxDepth(1). WithMaxDiskBytes(15 * 512) limiter := archive.NewLimiter(archive.DefaultLimits(cfg)) - // measured while the archive is still held: the resolver is built straight after extraction and - // before anything is cleaned up + // measured while the archive is still held: the resolver is built straight after extraction, before + // anything is cleaned up var extracted []int var peakMemory int64 - newResolver := func(overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - resolver, indexed, err := newTarTestResolver(overflow) - extracted = append(extracted, indexed.Records) + newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { + resolver, indexed, err := newStoreTestResolver(store, overflow) + extracted = append(extracted, len(store.Entries())) if mem, _ := limiter.InUse(); mem > peakMemory { peakMemory = mem } @@ -849,12 +624,11 @@ func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) { } c := &archiveCataloger{ - cfg: cfg, - subPipeline: []Task{NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil })}, - extractors: archive.DefaultExtractors(), - limits: archive.DefaultExtractionLimits(cfg), - limiter: limiter, - newResolver: newResolver, + cfg: cfg, + subPipeline: []Task{NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil })}, + extractors: archive.DefaultExtractors(), + limiter: limiter, + newStoreResolver: newResolver, } s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -871,8 +645,8 @@ func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) { } func Test_archiveCataloger_archiveExceedingTheDiskLimitIsSkipped(t *testing.T) { - // the disk limit is terminal, so an archive whose own content will not fit is skipped rather - // than waited for, and a sibling that does fit is cataloged in full + // the disk limit is terminal, so an archive whose content will not fit is skipped rather than waited + // for, and a sibling that does fit is cataloged in full oversized := makeZip(t, map[string][]byte{"payload.bin": incompressible(8000)}) small := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")}) @@ -880,15 +654,15 @@ func Test_archiveCataloger_archiveExceedingTheDiskLimitIsSkipped(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "small.zip"), small, 0o600)) - // room for the small archive's own bytes and the tar its one entry is written into - a header - // block, a padded data block and an end-of-archive marker - and nowhere near the oversized one + // room for the small archive's own bytes and the tar its one entry is written into - a header block, + // a padded data block and an end-of-archive marker - and nowhere near the oversized one diskLimit := int64(len(small)) + 8*512 require.Greater(t, int64(len(oversized)), diskLimit, "the fixture must actually exceed the limit") var sawArchivePaths []string - newResolver := func(overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { + newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { sawArchivePaths = append(sawArchivePaths, overflow.ArchivePath) - return newTarTestResolver(overflow) + return newStoreTestResolver(store, overflow) } cfg := cataloging.DefaultArchiveSearchConfig(). @@ -902,7 +676,7 @@ func Test_archiveCataloger_archiveExceedingTheDiskLimitIsSkipped(t *testing.T) { return nil }) - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -914,16 +688,10 @@ func Test_archiveCataloger_archiveExceedingTheDiskLimitIsSkipped(t *testing.T) { assert.Equal(t, 1, ran) } -func Test_archiveCataloger_zeroDiskLimitAdmitsNoArchiveContent(t *testing.T) { - // This case used to say "a zero disk limit skips what will not fit in memory": an archive's own - // bytes were held in memory while the memory limit admitted them, and only what did not fit had - // nowhere to go. It now says something stronger, because an archive's ENTRIES live in a file - // whatever happened to the archive's own bytes: with no disk to write it, that file cannot be - // created, so no archive's content can be admitted and every archive is skipped - the memory limit - // notwithstanding. - // - // The scan still succeeds, and every skip is attributed to the archive that could not be placed, - // which is the part that has not changed. +func Test_archiveCataloger_zeroDiskLimitSkipsWhatWillNotFitInMemory(t *testing.T) { + // with a zero disk limit there is nowhere to overflow, so an archive is cataloged exactly when it + // and its entries fit in memory. The memory limit below admits the small archive and not the + // oversized one; the scan still succeeds and the skip names the archive that could not be placed. oversized := makeZip(t, map[string][]byte{"payload.bin": incompressible(8000)}) small := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")}) @@ -931,20 +699,19 @@ func Test_archiveCataloger_zeroDiskLimitAdmitsNoArchiveContent(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "small.zip"), small, 0o600)) - // generous: even the small archive, which fits in memory with room to spare, has nowhere to put - // its entries - memoryLimit := int64(len(oversized) + len(small) + 1000) + // room for the small archive and its entries, and not for the oversized one + memoryLimit := int64(len(small) + 1000) var sawArchivePaths []string - newResolver := func(overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { + newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { sawArchivePaths = append(sawArchivePaths, overflow.ArchivePath) - return newTarTestResolver(overflow) + return newStoreTestResolver(store, overflow) } cfg := cataloging.DefaultArchiveSearchConfig(). WithMaxDepth(1). WithMaxMemoryBytes(memoryLimit). - WithMaxDiskBytes(0) // nowhere to write the file an archive's entries live in + WithMaxDiskBytes(0) // nowhere to overflow what memory will not hold var ran int countTask := NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { @@ -952,30 +719,30 @@ func Test_archiveCataloger_zeroDiskLimitAdmitsNoArchiveContent(t *testing.T) { return nil }) - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), "reaching the disk limit must not fail the scan") - assert.Empty(t, sawArchivePaths, - "with no disk there is nowhere for an archive's entries to go, so no archive is descended into") - assert.Zero(t, ran) + assert.Equal(t, []string{"/small.zip"}, sawArchivePaths, + "the archive that fits in memory needs no disk at all and is still cataloged") + assert.Equal(t, 1, ran) } func Test_archiveCataloger_bothLimitsZeroSkipsEveryArchiveAndSucceeds(t *testing.T) { - // the degenerate configuration where nothing is admitted anywhere: every archive is skipped, and - // that is a well-defined success rather than a failure + // the degenerate configuration where nothing is admitted anywhere: every archive is skipped, which + // is a well-defined success rather than a failure entries := map[string][]byte{"ok.txt": []byte("ok")} rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "one.zip"), makeZip(t, entries), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "two.zip"), makeZip(t, entries), 0o600)) var sawArchivePaths []string - newResolver := func(overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { + newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { sawArchivePaths = append(sawArchivePaths, overflow.ArchivePath) - return newTarTestResolver(overflow) + return newStoreTestResolver(store, overflow) } cfg := cataloging.DefaultArchiveSearchConfig(). @@ -989,7 +756,7 @@ func Test_archiveCataloger_bothLimitsZeroSkipsEveryArchiveAndSucceeds(t *testing return nil }) - tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, []Task{countTask}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -1001,24 +768,24 @@ func Test_archiveCataloger_bothLimitsZeroSkipsEveryArchiveAndSucceeds(t *testing } func Test_archiveCataloger_nestedArchiveExceedingALimitIsSkippedNotBlocked(t *testing.T) { - // the walk descends before it unwinds, so a parent is still holding its content while its - // children are cataloged and nothing will be released while a child waits. Waiting for capacity - // would be a deadlock, so a child that does not fit is skipped and the scan finishes. + // the walk descends before it unwinds, so a parent still holds its content while its children are + // cataloged and nothing is released while a child waits. Waiting for capacity would deadlock, so a + // child that does not fit is skipped and the scan finishes. inner := makeZip(t, map[string][]byte{"payload.bin": incompressible(3000)}) outer := makeZip(t, map[string][]byte{"inner.zip": inner}) rootDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outer, 0o600)) - // enough for the outer archive's own bytes plus the inner.zip it extracts, and not enough for - // the inner archive's bytes on top of them + // enough for the outer archive's own bytes plus the inner.zip it extracts, and not enough for the + // inner archive's bytes on top of them diskLimit := int64(len(outer)+len(inner)) + 500 require.Greater(t, int64(len(inner)), int64(500), "the inner archive must not fit in the slack") var sawArchivePaths []string - newResolver := func(overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { + newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { sawArchivePaths = append(sawArchivePaths, overflow.ArchivePath) - return newTarTestResolver(overflow) + return newStoreTestResolver(store, overflow) } cfg := cataloging.DefaultArchiveSearchConfig(). @@ -1027,7 +794,7 @@ func Test_archiveCataloger_nestedArchiveExceedingALimitIsSkippedNotBlocked(t *te WithMaxDiskBytes(diskLimit) noop := NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil }) - tsk := NewArchiveCatalogerTask(cfg, []Task{noop}, newResolver, nil, nil) + tsk := NewArchiveCatalogerTask(cfg, []Task{noop}, newResolver, nil) require.NotNil(t, tsk) s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} @@ -1037,6 +804,469 @@ func Test_archiveCataloger_nestedArchiveExceedingALimitIsSkippedNotBlocked(t *te "the parent is cataloged and the child it cannot afford is skipped rather than blocking") } +// allFilesResolver is dirTestResolver without a filesystem id. Both sniff content the way a real +// resolver does, so which files reach the task is decided by what they are. +type allFilesResolver struct { + file.Resolver + dir string +} + +func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) { + // every other task-level fixture is a zip, so the tar branch of FindExtractor had never been reached + // through the task's own discovery + marker := map[string][]byte{"nested/marker.txt": []byte("hello")} + + scanDir := t.TempDir() + for name, body := range map[string][]byte{ + "bundle.tar.gz": makeTarGz(t, marker), + "bundle.tgz": makeTarGz(t, marker), + "bundle.tar": makeTar(t, marker), + "bundle.zip": makeZip(t, marker), + "notes.txt": []byte("not an archive at all"), + } { + require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), body, 0o600)) + } + + var seen []string + captureTask := NewTask("capture", func(ctx context.Context, r file.Resolver, _ sbomsync.Builder) error { + trav := archive.TraversalFromContext(ctx) + require.NotNil(t, trav) + + // the contents are there, so extraction really happened rather than the traversal being built + // over an empty tree + locs, err := r.FilesByGlob("**") + require.NoError(t, err) + assert.NotEmpty(t, locs, "expected contents inside %s", trav.VirtualPath) + + seen = append(seen, trav.VirtualPath) + return nil + }) + + newResolver := newStoreTestResolver + + tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{captureTask}, newResolver, nil) + require.NotNil(t, tsk) + + s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + require.NoError(t, tsk.Execute(context.Background(), allFilesResolver{dir: scanDir}, sbomsync.NewBuilder(s))) + + assert.ElementsMatch(t, []string{"/bundle.tar.gz", "/bundle.tgz", "/bundle.tar", "/bundle.zip"}, seen, + "every tar-family archive must be reached, and the plain text file must not be") + + t.Run("the MIME types the tar family is sniffed as are in the set discovery asks for", func(t *testing.T) { + // discoverArchives passes mimetype.ArchiveMIMETypeSet to FilesByMIMEType, so a tar type missing + // from it makes the whole tar path unreachable whatever the extractor can do + for _, mt := range []string{"application/x-tar", "application/gzip", "application/x-gtar", "application/zip"} { + assert.True(t, mimetype.ArchiveMIMETypeSet.Has(mt), "%s must be a candidate archive type", mt) + } + }) +} + +type recordingPublisher struct { + events []partybus.Event +} + +func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) { + // the sub-pipeline re-runs catalogers that start progress rows keyed by cataloger name. Publishing + // those again per archive floods the UI, and a consumer that keys rows by ID (as the syft CLI does) + // replaces the live row, never renders the replaced one again, and so never observes its + // completion - hanging teardown. Each cataloger keeps to the one row it already owns instead, and + // what it finds inside archives is counted onto that row. + inner := makeZip(t, map[string][]byte{"inner/file.txt": []byte("hello")}) + scanDir := t.TempDir() + for _, name := range []string{"one.zip", "two.zip"} { + require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), inner, 0o600)) + } + + publisher := &recordingPublisher{} + bus.Set(publisher) + t.Cleanup(func() { bus.Set(nil) }) + + // a task that starts a progress row and counts what it found, the way a file cataloger does + var rows []*monitor.TaskProgress + noisy := NewTask("noisy", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { + p := bus.StartCatalogerTask(ctx, monitor.GenericTask{ID: "noisy"}, 3, "") + rows = append(rows, p) + p.AtomicStage.Set("working on something") + p.Add(3) + p.SetCompleted() + return nil + }) + + // the archive row's stage as each archive is about to be extracted: the previous archive's + // sub-pipeline has finished by then, so the row must read the walk's own count and the archive it + // last finished, rather than anything a nested cataloger left behind + var stageAtExtraction []string + newResolver := func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { + row := publisher.events[0].Value.(progress.StagedProgressable) + stageAtExtraction = append(stageAtExtraction, row.Stage()) + return newStoreTestResolver(store, overflow) + } + + tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{noisy}, newResolver, nil) + require.NotNil(t, tsk) + + s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(s))) + + var started []string + for _, e := range publisher.events { + if e.Type != event.CatalogerTaskStarted { + continue + } + info, ok := e.Source.(monitor.GenericTask) + require.True(t, ok, "expected a generic task as the event source") + started = append(started, info.ID) + } + + assert.Equal(t, []string{ArchiveCatalogerTaskName, "noisy"}, started, + "each cataloger publishes one row, however many archives the walk enters") + + require.Len(t, rows, 2, "the sub-pipeline ran once per archive") + assert.Same(t, rows[0], rows[1], "both runs must report into the one row that cataloger owns") + assert.Equal(t, int64(6), rows[0].Current(), "what every run found is counted onto that row") + + require.Len(t, publisher.events, 2) + prog, ok := publisher.events[0].Value.(progress.StagedProgressable) + require.True(t, ok) + assert.ErrorIs(t, prog.Error(), progress.ErrCompleted, "the row must be completed, or a waiting consumer never finishes") + assert.Equal(t, int64(2), prog.Current(), "the archive row counts every archive the walk entered") + assert.Equal(t, "2 archives", prog.Stage(), "and says so once the walk is done") + + // one.zip sorts before two.zip, and the resolver walks the scan directory in lexical order + assert.Equal(t, []string{"", "1 archives (/one.zip)"}, stageAtExtraction, + "the row must take its stage back from the sub-pipeline once an archive is done") +} + +func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) { + // the slowest archive is the one that spent the time itself. Inclusive time would name the outer + // archive every scan, so here the outer's own sub-pipeline is fast and the inner's slow: the report + // must name the inner. + innerJar := makeZip(t, map[string][]byte{"README.txt": []byte("inner")}) + outerZip := makeZip(t, map[string][]byte{"lib/inner.jar": innerJar}) + + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) + + const innerWork = 40 * time.Millisecond + slow := NewTask("slow-inside-the-inner-archive", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { + if trav := archive.TraversalFromContext(ctx); trav != nil && strings.HasSuffix(trav.VirtualPath, "inner.jar") { + time.Sleep(innerWork) + } + return nil + }) + + cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3) + limiter := archive.NewLimiter(archive.DefaultLimits(cfg)) + c := &archiveCataloger{ + cfg: cfg, + subPipeline: []Task{slow}, + extractors: archive.DefaultExtractors(), + limiter: limiter, + newStoreResolver: newStoreTestResolver, + } + + s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, nil, 0, sbomsync.NewBuilder(s))) + + assert.Equal(t, "/outer.zip:lib/inner.jar", c.slowestPath, + "the slowest archive is named by its full chain, not by its base name, and self time must exclude the archives nested inside") + assert.GreaterOrEqual(t, c.slowest, innerWork) + assert.Equal(t, int64(2), c.archivesSeen) + + // the peaks outlive the walk that set them, so they are still readable once everything is released + peakMemory, _ := limiter.Peak() + assert.Positive(t, peakMemory, "content held in memory must show on the peak") + memory, disk := limiter.InUse() + assert.Zero(t, memory) + assert.Zero(t, disk) +} + +func Test_archiveCataloger_slowestIsUnsetWhenNoArchiveIsCataloged(t *testing.T) { + // a directory with nothing extractable in it: there is no archive to name + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.txt"), []byte("not an archive"), 0o600)) + + cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2) + c := &archiveCataloger{ + cfg: cfg, + subPipeline: []Task{NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil })}, + extractors: archive.DefaultExtractors(), + limiter: archive.NewLimiter(archive.DefaultLimits(cfg)), + newStoreResolver: newStoreTestResolver, + } + + s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, nil, 0, sbomsync.NewBuilder(s))) + + assert.Empty(t, c.slowestPath) + assert.Zero(t, c.archivesSeen) + c.logStats() // must not panic with nothing to report +} + +func Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError(t *testing.T) { + // `syft/nested-archive-cataloging#content-based-archive-detection`, scenario "misnamed non-archive + // is not extracted": the name says archive, the content does not, and it never becomes a candidate + // because its content types it as text + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.zip"), + []byte("this is prose, not an archive, whatever the extension claims\n"), 0o600)) + + var ran int + countTask := NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { + ran++ + return nil + }) + + tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), + []Task{countTask}, newStoreTestResolver, nil) + require.NotNil(t, tsk) + + s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) + + assert.NoError(t, err, "a file that is not an archive is not a failure to report") + assert.Zero(t, ran, "and nothing is cataloged inside it") +} + +func Test_archiveCataloger_catalogsAZipBehindALauncherScript(t *testing.T) { + // end to end for the Spring Boot executable jar shape: a launcher script with a zip appended. + // Content sniffing types it text/x-shellscript, so it reaches the walk only through the + // appended-archive candidate set, and qualifies only by its end-of-central-directory record. + inner := makeZip(t, map[string][]byte{"nested/marker.txt": []byte("found me")}) + prefixed := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"), + makeZip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...) + + rootDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "app.jar"), prefixed, 0o600)) + + var seen []string + capture := NewTask("capture", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { + if trav := archive.TraversalFromContext(ctx); trav != nil { + seen = append(seen, trav.VirtualPath) + } + return nil + }) + + tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3), + []Task{capture}, newStoreTestResolver, nil) + require.NotNil(t, tsk) + + s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} + require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) + + assert.Equal(t, []string{"/app.jar", "/app.jar:BOOT-INF/lib/inner.jar"}, seen, + "the stub must not hide the archive, and the walk must carry on into what it holds") +} + +func (d dirTestResolver) FilesByMIMEType(types ...string) ([]file.Location, error) { + return filesByMIMEType(d.dir, d.fsid, types...) +} + +// filesByMIMEType sniffs each file's content and returns the ones whose type was asked for, as a real +// resolver does. Returning everything would make every caller look like it was handed archives, +// hiding that the task asks two different questions with two different consequences. +func filesByMIMEType(dir, fsid string, types ...string) ([]file.Location, error) { + wanted := strset.New(types...) + var locs []file.Location + err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + f, err := os.Open(path) + if err != nil { + return err + } + mimeType := stereoscopeFile.MIMEType(f) + if closeErr := f.Close(); closeErr != nil { + return closeErr + } + if !wanted.Has(mimeType) { + return nil + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + coords := file.Coordinates{RealPath: "/" + filepath.ToSlash(rel), FileSystemID: fsid} + locs = append(locs, file.NewLocationFromCoordinates(coords)) + return nil + }) + return locs, err +} + +func (d dirTestResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { + return os.Open(filepath.Join(d.dir, filepath.FromSlash(loc.RealPath))) +} + +// newStoreTestResolver is an archive.StoreResolverFactory over the archive's entries. +func newStoreTestResolver(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { + r := &storeTestResolver{ + store: store, + entries: map[string]*archive.Entry{}, + fsid: overflow.FileSystemID, + archivePath: overflow.ArchivePath, + } + for _, entry := range store.Entries() { + r.entries[path.Clean("/"+entry.Header.Name)] = entry + } + return r, archive.IndexResult{}, nil +} + +func (r *storeTestResolver) sortedNames() []string { + names := make([]string, 0, len(r.entries)) + for name := range r.entries { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func (r *storeTestResolver) location(name string) file.Location { + return file.NewLocationFromCoordinates(file.Coordinates{RealPath: name, FileSystemID: r.fsid, ArchivePath: r.archivePath}) +} + +func (r *storeTestResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) { + var locs []file.Location + for _, name := range r.sortedNames() { + switch strings.ToLower(filepath.Ext(name)) { + case ".zip", ".jar", ".war": + locs = append(locs, r.location(name)) + } + } + return locs, nil +} + +func (r *storeTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + var locs []file.Location + for _, name := range r.sortedNames() { + for _, pattern := range patterns { + if matched, err := doublestar.Match(pattern, strings.TrimPrefix(name, "/")); err == nil && matched { + locs = append(locs, r.location(name)) + break + } + } + } + return locs, nil +} + +func (r *storeTestResolver) FilesByPath(paths ...string) ([]file.Location, error) { + var locs []file.Location + for _, p := range paths { + name := path.Clean("/" + p) + if _, ok := r.entries[name]; ok { + locs = append(locs, r.location(name)) + } + } + return locs, nil +} + +func (r *storeTestResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { + entry, ok := r.entries[loc.RealPath] + if !ok { + return nil, fmt.Errorf("no entry %q", loc.RealPath) + } + contents, err := r.store.Open(entry) + if err != nil { + return nil, err + } + return testOverflowEntry{nopCloserAt{contents}}, nil +} + +func (nopCloserAt) Close() error { return nil } + +func (testOverflowEntry) OverflowArchiveEntry() {} + +// makeZip builds a zip in sorted entry-name order rather than map order: limits are enforced as the +// walk proceeds, so entry order decides which entries land before a truncation. +func makeZip(t *testing.T, entries map[string][]byte) []byte { + t.Helper() + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, name := range names { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write(entries[name]) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +// matchesAnyGlob reports whether the path matches one of the patterns, against the resolver-relative +// path with a leading slash, the form a real resolver globs over. No patterns matches everything. +func matchesAnyGlob(root, path string, patterns []string) bool { + if len(patterns) == 0 { + return true + } + rel, err := filepath.Rel(root, path) + if err != nil { + return false + } + rel = "/" + filepath.ToSlash(rel) + for _, pattern := range patterns { + if ok, err := doublestar.Match(pattern, rel); err == nil && ok { + return true + } + } + return false +} + +// FilesByGlob for the resolvers that otherwise only stand in for a MIME lookup: the archive walk also +// asks them for archive-named files, and an unimplemented method on an embedded nil resolver panics. +func (a allFilesResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + return dirTestResolver{dir: a.dir}.FilesByGlob(patterns...) +} + +func (r twoLayerResolver) FilesByGlob(_ ...string) ([]file.Location, error) { + return nil, nil +} + +// FilesByGlob lists the matching files under the resolver's directory, stamped with its filesystem id, +// so a sub-pipeline task can record file artifacts the way a real file cataloger does. No patterns +// lists everything, which is what the sub-pipeline tasks here want. +func (d dirTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error) { + var locs []file.Location + err := filepath.WalkDir(d.dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + if !matchesAnyGlob(d.dir, path, patterns) { + return nil + } + rel, err := filepath.Rel(d.dir, path) + if err != nil { + return err + } + coords := file.Coordinates{RealPath: "/" + filepath.ToSlash(rel), FileSystemID: d.fsid} + locs = append(locs, file.NewLocationFromCoordinates(coords)) + return nil + }) + return locs, err +} + +func (r twoLayerResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) { + var locs []file.Location + for _, fsid := range []string{"layer-one", "layer-two"} { + locs = append(locs, file.NewLocationFromCoordinates(file.Coordinates{RealPath: r.path, FileSystemID: fsid})) + } + return locs, nil +} + +func (r twoLayerResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) { + body, ok := r.contentBy[loc.FileSystemID] + if !ok { + return nil, os.ErrNotExist + } + return io.NopCloser(bytes.NewReader(body)), nil +} + // incompressible returns n bytes deflate cannot shrink, so an archive built from them has a size on // disk predictable enough to size a limit against. func incompressible(n int) []byte { @@ -1059,7 +1289,6 @@ func makeTarGz(t *testing.T, entries map[string][]byte) []byte { return buf.Bytes() } -// makeTar builds an uncompressed tar in memory. func makeTar(t *testing.T, entries map[string][]byte) []byte { t.Helper() var buf bytes.Buffer @@ -1088,14 +1317,6 @@ func writeTar(t *testing.T, w io.Writer, entries map[string][]byte) error { return tw.Close() } -// allFilesResolver is dirTestResolver without a filesystem id. Both sniff content the way a real -// resolver does, so which files reach the task is decided by what they are rather than by what the -// test wishes they were. -type allFilesResolver struct { - file.Resolver - dir string -} - func (a allFilesResolver) FilesByMIMEType(types ...string) ([]file.Location, error) { return filesByMIMEType(a.dir, "", types...) } @@ -1104,262 +1325,6 @@ func (a allFilesResolver) FileContentsByLocation(loc file.Location) (io.ReadClos return os.Open(filepath.Join(a.dir, filepath.FromSlash(loc.RealPath))) } -func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) { - // every other task-level fixture is a zip, so the tar branch of FindExtractor had never been - // reached through the task's own discovery - marker := map[string][]byte{"nested/marker.txt": []byte("hello")} - - scanDir := t.TempDir() - for name, body := range map[string][]byte{ - "bundle.tar.gz": makeTarGz(t, marker), - "bundle.tgz": makeTarGz(t, marker), - "bundle.tar": makeTar(t, marker), - "bundle.zip": makeZip(t, marker), - "notes.txt": []byte("not an archive at all"), - } { - require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), body, 0o600)) - } - - var seen []string - captureTask := NewTask("capture", func(ctx context.Context, r file.Resolver, _ sbomsync.Builder) error { - trav := archive.TraversalFromContext(ctx) - require.NotNil(t, trav) - - // the contents are actually there, so the extraction really happened rather than the - // traversal being built over an empty tree - locs, err := r.FilesByGlob("**") - require.NoError(t, err) - assert.NotEmpty(t, locs, "expected contents inside %s", trav.VirtualPath) - - seen = append(seen, trav.VirtualPath) - return nil - }) - - newResolver := newTarTestResolver - - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{captureTask}, newResolver, nil, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, tsk.Execute(context.Background(), allFilesResolver{dir: scanDir}, sbomsync.NewBuilder(s))) - - assert.ElementsMatch(t, []string{"/bundle.tar.gz", "/bundle.tgz", "/bundle.tar", "/bundle.zip"}, seen, - "every tar-family archive must be reached, and the plain text file must not be") - - t.Run("the MIME types the tar family is sniffed as are in the set discovery asks for", func(t *testing.T) { - // discoverArchives passes mimetype.ArchiveMIMETypeSet to FilesByMIMEType, so a tar type - // missing from it makes the whole tar path unreachable no matter what the extractor can do - for _, mt := range []string{"application/x-tar", "application/gzip", "application/x-gtar", "application/zip"} { - assert.True(t, mimetype.ArchiveMIMETypeSet.Has(mt), "%s must be a candidate archive type", mt) - } - }) -} - -// recordingPublisher captures every event published to the bus during a test. -type recordingPublisher struct { - events []partybus.Event -} - func (p *recordingPublisher) Publish(e partybus.Event) { p.events = append(p.events, e) } - -func Test_archiveCatalogerTask_publishesOneProgressRowForTheWholeWalk(t *testing.T) { - // the sub-pipeline re-runs catalogers that start a progress row of their own, keyed by cataloger - // name. Publishing those again per archive floods a consumer's UI with rows, and a consumer that - // keys rows by ID (as the syft CLI does) replaces the live row for that ID, never renders the - // replaced one again, and so never observes its completion - which hangs UI teardown. The whole - // walk therefore reports through the single row this task starts for itself. - inner := makeZip(t, map[string][]byte{"inner/file.txt": []byte("hello")}) - scanDir := t.TempDir() - for _, name := range []string{"one.zip", "two.zip"} { - require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), inner, 0o600)) - } - - publisher := &recordingPublisher{} - bus.Set(publisher) - t.Cleanup(func() { bus.Set(nil) }) - - // a task that starts a progress row the same way every package cataloger does - var reported []string - noisy := NewTask("noisy", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { - p := bus.StartCatalogerTask(ctx, monitor.GenericTask{ID: "noisy"}, -1, "") - p.AtomicStage.Set("working on something") - - // what the nested cataloger is doing reaches the row that is already on screen - row := publisher.events[0].Value.(progress.StagedProgressable) - reported = append(reported, row.Stage()) - - // and the nested task finishing does not finish the row it reports into - p.SetCompleted() - assert.NotErrorIs(t, row.Error(), progress.ErrCompleted, - "a sub-task completing must not complete the row it reports into") - return nil - }) - - // the row's stage as each archive is about to be extracted: by then the previous archive's - // sub-pipeline has finished, so the row must read the walk's own count rather than whatever the - // last nested cataloger signed off with - var stageAtExtraction []string - newResolver := func(overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - stageAtExtraction = append(stageAtExtraction, publisher.events[0].Value.(progress.StagedProgressable).Stage()) - return newTarTestResolver(overflow) - } - - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), []Task{noisy}, newResolver, nil, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(s))) - - var started []string - for _, e := range publisher.events { - if e.Type != event.CatalogerTaskStarted { - continue - } - info, ok := e.Source.(monitor.GenericTask) - require.True(t, ok, "expected a generic task as the event source") - started = append(started, info.ID) - } - - assert.Equal(t, []string{ArchiveCatalogerTaskName}, started, - "only the archive cataloger's own row may be published, however many archives the walk enters") - - require.Len(t, publisher.events, 1) - prog, ok := publisher.events[0].Value.(progress.StagedProgressable) - require.True(t, ok) - assert.ErrorIs(t, prog.Error(), progress.ErrCompleted, "the row must be completed, or a waiting consumer never finishes") - assert.Equal(t, int64(2), prog.Current(), "the row counts every archive the walk entered") - assert.Equal(t, "2 archives", prog.Stage(), "and says so once the walk is done") - - assert.Equal(t, []string{"working on something", "working on something"}, reported, - "every nested run must report through the existing row") - - assert.Equal(t, []string{"", "1 archives"}, stageAtExtraction, - "the row must take its stage back from the sub-pipeline once an archive is done") -} - -func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) { - // the slowest archive is the one that spent the time itself. Inclusive time would name the outer - // archive on every scan, since it contains everything below it, so the outer's own sub-pipeline - // is made fast here and the inner's slow: the report must name the inner. - innerJar := makeZip(t, map[string][]byte{"README.txt": []byte("inner")}) - outerZip := makeZip(t, map[string][]byte{"lib/inner.jar": innerJar}) - - rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) - - const innerWork = 40 * time.Millisecond - slow := NewTask("slow-inside-the-inner-archive", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { - if trav := archive.TraversalFromContext(ctx); trav != nil && strings.HasSuffix(trav.VirtualPath, "inner.jar") { - time.Sleep(innerWork) - } - return nil - }) - - cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3) - limiter := archive.NewLimiter(archive.DefaultLimits(cfg)) - c := &archiveCataloger{ - cfg: cfg, - subPipeline: []Task{slow}, - extractors: archive.DefaultExtractors(), - limits: archive.DefaultExtractionLimits(cfg), - limiter: limiter, - newResolver: newTarTestResolver, - } - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, nil, 0, sbomsync.NewBuilder(s))) - - assert.Equal(t, "/outer.zip:lib/inner.jar", c.slowestPath, - "the slowest archive is named by its full chain, not by its base name, and self time must exclude the archives nested inside") - assert.GreaterOrEqual(t, c.slowest, innerWork) - assert.Equal(t, int64(2), c.archivesSeen) - - // the peaks outlive the walk that set them, so they are still readable once everything is released - peakMemory, _ := limiter.Peak() - assert.Positive(t, peakMemory, "content held in memory must show on the peak") - memory, disk := limiter.InUse() - assert.Zero(t, memory) - assert.Zero(t, disk) -} - -func Test_archiveCataloger_slowestIsUnsetWhenNoArchiveIsCataloged(t *testing.T) { - // a directory with nothing extractable in it: there is no archive to name, and the report must - // not invent one - rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.txt"), []byte("not an archive"), 0o600)) - - cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2) - c := &archiveCataloger{ - cfg: cfg, - subPipeline: []Task{NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil })}, - extractors: archive.DefaultExtractors(), - limits: archive.DefaultExtractionLimits(cfg), - limiter: archive.NewLimiter(archive.DefaultLimits(cfg)), - newResolver: newTarTestResolver, - } - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, nil, 0, sbomsync.NewBuilder(s))) - - assert.Empty(t, c.slowestPath) - assert.Zero(t, c.archivesSeen) - c.logStats() // must not panic with nothing to report -} - -func Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError(t *testing.T) { - // `syft/nested-archive-cataloging#content-based-archive-detection`, scenario "misnamed - // non-archive is not extracted": the name says archive, the content does not, and the scan says - // nothing about it. It never becomes a candidate, because its content types it as text. - rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.zip"), - []byte("this is prose, not an archive, whatever the extension claims\n"), 0o600)) - - var ran int - countTask := NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { - ran++ - return nil - }) - - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), - []Task{countTask}, newTarTestResolver, nil, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) - - assert.NoError(t, err, "a file that is not an archive is not a failure to report") - assert.Zero(t, ran, "and nothing is cataloged inside it") -} - -func Test_archiveCataloger_catalogsAZipBehindALauncherScript(t *testing.T) { - // end to end for the Spring Boot executable jar shape: a launcher script with a zip concatenated - // onto it. Content sniffing types it text/x-shellscript, so it reaches the walk only through the - // appended-archive candidate set, and it is an archive only because it ends in an - // end-of-central-directory record. - inner := makeZip(t, map[string][]byte{"nested/marker.txt": []byte("found me")}) - prefixed := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"), - makeZip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...) - - rootDir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(rootDir, "app.jar"), prefixed, 0o600)) - - var seen []string - capture := NewTask("capture", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error { - if trav := archive.TraversalFromContext(ctx); trav != nil { - seen = append(seen, trav.VirtualPath) - } - return nil - }) - - tsk := NewArchiveCatalogerTask(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3), - []Task{capture}, newTarTestResolver, nil, nil) - require.NotNil(t, tsk) - - s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} - require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) - - assert.Equal(t, []string{"/app.jar", "/app.jar:BOOT-INF/lib/inner.jar"}, seen, - "the stub must not hide the archive, and the walk must carry on into what it holds") -} diff --git a/internal/task/package_tasks.go b/internal/task/package_tasks.go index 142f483c6..89b5bd201 100644 --- a/internal/task/package_tasks.go +++ b/internal/task/package_tasks.go @@ -150,13 +150,6 @@ func DefaultPackageTaskFactories() Factories { ), newPackageTaskFactory( func(cfg CatalogingFactoryConfig) pkg.Cataloger { - // once the archive cataloger task owns extraction, java archives arrive as extracted - // filesystems rather than as files, and the cataloger that reads them is a different - // one. The archive-reading cataloger is what runs when nothing else extracts, which is - // the default; it is deprecated in the sense that it has no role in the enabled path. - if cfg.PackagesConfig.JavaArchive.NestedArchivesHandledExternally { - return java.NewArchiveContentsCataloger(cfg.PackagesConfig.JavaArchive) - } return java.NewArchiveCataloger(cfg.PackagesConfig.JavaArchive) }, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, Java, Maven, diff --git a/syft/archive_cataloger_test.go b/syft/archive_cataloger_test.go index dc5074824..cdff1ac9e 100644 --- a/syft/archive_cataloger_test.go +++ b/syft/archive_cataloger_test.go @@ -12,17 +12,23 @@ import ( "path/filepath" "sort" "strings" + "sync" "testing" "unicode/utf8" "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/cataloging/filecataloging" "github.com/anchore/syft/syft/cataloging/pkgcataloging" + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/java" @@ -31,36 +37,13 @@ import ( "github.com/anchore/syft/syft/source/directorysource" ) -// markerCataloger is a stub package cataloger that emits one package for every "marker.txt" file it -// finds. It lets the archive-cataloger end-to-end test assert behavior without depending on the -// fixture format of any real language cataloger. +// markerCataloger is a stub package cataloger emitting one package per "marker.txt" it finds, so the +// end-to-end tests assert behavior without depending on any real cataloger's fixture format. type markerCataloger struct{} -func (markerCataloger) Name() string { return "marker-cataloger" } - -func (markerCataloger) Catalog(_ context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { - locations, err := resolver.FilesByGlob("**/marker.txt") - if err != nil { - return nil, nil, err - } - var pkgs []pkg.Package - for _, loc := range locations { - p := pkg.Package{ - Name: "marker-pkg", - Version: "1.0.0", - Type: pkg.BinaryPkg, - Locations: file.NewLocationSet(loc), - } - p.SetID() - pkgs = append(pkgs, p) - } - return pkgs, nil, nil -} - -// TestArchiveCataloger_endToEnd scans a directory containing a zip that holds a marker file, with -// recursive archive cataloging enabled, and asserts that: (1) the package discovered inside the -// archive is stamped with the archive's nesting-chain FileSystemID, and (2) a file-level CONTAINS -// relationship links the archive to the discovered package. +// TestArchiveCataloger_endToEnd scans a directory holding a zip with a marker file inside: the +// package found there must carry the archive's nesting-chain FileSystemID, and a file-level CONTAINS +// relationship must link the archive to it. func TestArchiveCataloger_endToEnd(t *testing.T) { scanDir := t.TempDir() writeTestZip(t, filepath.Join(scanDir, "app.zip"), map[string]string{"nested/marker.txt": "hello"}) @@ -84,10 +67,10 @@ func TestArchiveCataloger_endToEnd(t *testing.T) { locs := found.Locations.ToSlice() require.NotEmpty(t, locs) - // root directory FSID is empty, so the chain starts with the relative archive path + // the root directory's FSID is empty, so the chain starts with the relative archive path assert.Equal(t, "app.zip", locs[0].ArchivePath, "package location should carry the archive's FileSystemID") - // a file-level CONTAINS relationship should link the archive file to the discovered package + // a file-level CONTAINS relationship links the archive file to the discovered package var hasContains bool for _, rel := range s.Relationships { if rel.Type != artifact.ContainsRelationship || rel.To.ID() != found.ID() { @@ -101,8 +84,8 @@ func TestArchiveCataloger_endToEnd(t *testing.T) { assert.True(t, hasContains, "expected a CONTAINS relationship from app.zip to the discovered package") } -// TestArchiveCataloger_nestedRecursion validates depth-limited recursion and the FileSystemID -// nesting chain: a marker inside inner.zip inside outer.zip should be stamped with the full chain. +// TestArchiveCataloger_nestedRecursion covers depth-limited recursion and the FileSystemID chain: a +// marker inside inner.zip inside outer.zip carries the full chain. func TestArchiveCataloger_nestedRecursion(t *testing.T) { innerZip := buildZipBytes(t, map[string]string{"nested/marker.txt": "hello"}) scanDir := t.TempDir() @@ -130,126 +113,9 @@ func TestArchiveCataloger_nestedRecursion(t *testing.T) { "nested package location should carry the full archive nesting chain as its FileSystemID") } -func writeTestZip(t *testing.T, path string, files map[string]string) { - t.Helper() - raw := make(map[string][]byte, len(files)) - for k, v := range files { - raw[k] = []byte(v) - } - writeTestZipRaw(t, path, raw) -} - -func writeTestZipRaw(t *testing.T, path string, files map[string][]byte) { - t.Helper() - require.NoError(t, os.WriteFile(path, buildZipBytesRaw(t, files), 0o644)) -} - -func buildZipBytes(t *testing.T, files map[string]string) []byte { - t.Helper() - raw := make(map[string][]byte, len(files)) - for k, v := range files { - raw[k] = []byte(v) - } - return buildZipBytesRaw(t, raw) -} - -// buildZipBytesRaw writes entries in sorted name order. Extraction limits are enforced as the walk -// proceeds, so which entries land before a truncation depends on entry order; taking the order from -// the map would make any limit assertion flaky. -func buildZipBytesRaw(t *testing.T, files map[string][]byte) []byte { - t.Helper() - names := make([]string, 0, len(files)) - for name := range files { - names = append(names, name) - } - sort.Strings(names) - - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - for _, name := range names { - w, err := zw.Create(name) - require.NoError(t, err) - _, err = w.Write(files[name]) - require.NoError(t, err) - } - require.NoError(t, zw.Close()) - return buf.Bytes() -} - -// archiveScanConfig builds a CreateSBOMConfig limited to the given catalogers, with nested archive -// cataloging at the given depth (0 disables it). -func archiveScanConfig(depth int, catalogers ...pkg.Cataloger) *CreateSBOMConfig { - refs := make([]pkgcataloging.CatalogerReference, 0, len(catalogers)) - for _, c := range catalogers { - refs = append(refs, pkgcataloging.NewCatalogerReference(c, []string{"directory"})) - } - cfg := DefaultCreateSBOMConfig().WithoutCatalogers().WithCatalogers(refs...) - cfg.Archive.MaxDepth = depth - return cfg -} - -// javaScanConfig builds a config whose java cataloger comes from the cataloging FACTORY rather than -// being handed in pre-built. That distinction matters: the yield flag that hands recursion to the -// archive task is derived onto the factory's PackagesConfig, so a pre-built cataloger passed via -// WithCatalogers keeps whatever config it was constructed with and never learns to yield. -// -// It selects only the java-tagged catalogers rather than taking the whole default set, because the -// default set includes the RPM cataloger, which needs a sqlite driver this test binary does not -// register. -func javaScanConfig(depth int) *CreateSBOMConfig { - cfg := DefaultCreateSBOMConfig(). - WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")) - if depth > 0 { - cfg = cfg.WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth)) - } - return cfg -} - -func scanDirWith(t *testing.T, dir string, cfg *CreateSBOMConfig) *sbom.SBOM { - t.Helper() - src, err := directorysource.New(directorysource.Config{Path: dir}) - require.NoError(t, err) - t.Cleanup(func() { _ = src.Close() }) - - s, err := cfg.Create(context.Background(), src) - require.NoError(t, err) - return s -} - -// scanDirWithExclusions scans with the given exclusion patterns configured on the SOURCE, which is -// where exclusions live: this capability has no setting of its own, so this is the only way to -// configure them and the same call a real consumer makes. -func scanDirWithExclusions(t *testing.T, dir string, cfg *CreateSBOMConfig, exclude ...string) *sbom.SBOM { - t.Helper() - src, err := directorysource.New(directorysource.Config{ - Path: dir, - Exclude: source.ExcludeConfig{Paths: exclude}, - }) - require.NoError(t, err) - t.Cleanup(func() { _ = src.Close() }) - - s, err := cfg.Create(context.Background(), src) - require.NoError(t, err) - return s -} - -func jarBytes(t *testing.T, title, version string, extra map[string][]byte) []byte { - t.Helper() - entries := map[string][]byte{ - "META-INF/MANIFEST.MF": []byte( - "Manifest-Version: 1.0\nImplementation-Title: " + title + "\nImplementation-Version: " + version + "\n", - ), - } - for k, v := range extra { - entries[k] = v - } - return buildZipBytesRaw(t, entries) -} - func TestArchiveCataloger_jarInsideArchiveCatalogedOnce(t *testing.T) { - // the failure this guards against is a DUPLICATE, not an error: before the archive task took - // ownership of recursion, a jar inside a zip was reached both by the task's walk and by the java - // cataloger's own unarchiving. Count, do not assert presence. + // the failure mode is a duplicate, not an error: a jar inside a zip can be reached by the task's + // walk and by the java cataloger's own unarchiving, so count rather than assert presence dep := jarBytes(t, "nested-lib", "2.0", nil) tests := []struct { @@ -280,8 +146,8 @@ func TestArchiveCataloger_jarInsideArchiveCatalogedOnce(t *testing.T) { } func TestArchiveCataloger_javaVirtualPathUnchangedByMechanism(t *testing.T) { - // these colon-delimited strings appear in already-published SBOMs, so the package must be - // reported under the same virtual path whichever mechanism reached it + // these colon-delimited strings appear in already-published SBOMs, so the package must be reported + // under the same virtual path whichever mechanism reached it dep := jarBytes(t, "nested-lib", "2.0", nil) scanDir := t.TempDir() writeTestZipRaw(t, filepath.Join(scanDir, "app.war"), map[string][]byte{ @@ -306,14 +172,13 @@ func TestArchiveCataloger_javaVirtualPathUnchangedByMechanism(t *testing.T) { require.Contains(t, off, "nested-lib", "the java cataloger's own recursion must find it when the feature is off") require.Contains(t, on, "nested-lib", "the archive cataloger must find it when the feature is on") - // this equality IS the compatibility guarantee: the string is in already-published SBOMs, so a - // package reached through the archive cataloger must not be reported under a different virtual - // path than the java cataloger's own recursion reported for it + // the compatibility guarantee: the string is in published SBOMs, so a package reached through the + // archive cataloger must carry the same virtual path java's own recursion gave it assert.Equal(t, off["nested-lib"], on["nested-lib"], "the nested jar's virtual path must be byte-identical between the two mechanisms") - // and the format itself is the historical colon-delimited one. For a directory source the - // archive's access path is relative, so there is no leading separator on the chain. + // and in colon-delimited form; for a directory source the archive's access path is relative, so the + // chain has no leading separator assert.Equal(t, "app.war:WEB-INF/lib/nested-lib-2.0.jar", on["nested-lib"]) } @@ -326,9 +191,9 @@ func TestArchiveCataloger_archiveContentsDoNotLeakIntoParentTree(t *testing.T) { for _, p := range s.Artifacts.Packages.Sorted() { for _, loc := range p.Locations.ToSlice() { - // the nested file's path is relative to the archive, and the FileSystemID says which - // archive. A joined or flattened path would mean the archive's contents were smeared - // into the parent's tree, which is what the rejected resolver-recursion approach did. + // the nested file's path is relative to the archive, and the FileSystemID says which archive. + // A joined or flattened path would smear the archive's contents into the parent's tree, which + // is what the rejected resolver-recursion approach did. assert.NotContains(t, loc.RealPath, "data.zip/") assert.NotEqual(t, "/app/nested/marker.txt", loc.RealPath) assert.Equal(t, "app/data.zip", loc.ArchivePath) @@ -337,19 +202,17 @@ func TestArchiveCataloger_archiveContentsDoNotLeakIntoParentTree(t *testing.T) { } func TestArchiveCataloger_siblingArchivesDoNotCollide(t *testing.T) { - // two archives containing the same internal path: the file tables are map[file.Coordinates]..., - // so without a distinct FileSystemID the second write silently overwrites the first and one - // package disappears with no error anywhere + // two archives holding the same internal path: the file tables are keyed by Coordinates, so without + // a distinct FileSystemID the second write overwrites the first and a package disappears silently scanDir := t.TempDir() writeTestZip(t, filepath.Join(scanDir, "one.zip"), map[string]string{"nested/marker.txt": "from one"}) writeTestZip(t, filepath.Join(scanDir, "two.zip"), map[string]string{"nested/marker.txt": "from two"}) s := scanDirWith(t, scanDir, archiveScanConfig(1, markerCataloger{})) - // note: the two markers are the same package by syft's identity rules (same name, version and - // type), so they merge into one package carrying two locations. That merge is pre-existing - // behavior and is not what this test is about - what matters is that the two LOCATIONS survive - // as distinct coordinates instead of one overwriting the other. + // the two markers are the same package by syft's identity rules (same name, version and type), so + // they merge into one carrying two locations; what matters is that both locations survive as + // distinct coordinates fsIDs := strset.New() var locCount int for _, p := range s.Artifacts.Packages.Sorted() { @@ -374,9 +237,8 @@ func TestArchiveCataloger_containsEdgesChainAcrossLevels(t *testing.T) { scanDir := t.TempDir() writeTestZipRaw(t, filepath.Join(scanDir, "outer.zip"), map[string][]byte{"deeper/inner.zip": innerZip}) - // file cataloging is on deliberately: the archive-to-file edges are recorded for the files the - // FILE catalogers found inside an archive, so without them there is no coordinate for inner.zip - // to hang an edge on and only the archive-to-package edges exist + // file cataloging is on because archive-to-file edges are recorded for the files the file catalogers + // found inside an archive; without them inner.zip has no coordinate to hang an edge on cfg := archiveScanConfig(2, markerCataloger{}). WithFilesConfig(filecataloging.Config{Selection: file.AllFilesSelection}) s := scanDirWith(t, scanDir, cfg) @@ -409,9 +271,8 @@ func TestArchiveCataloger_containsEdgesChainAcrossLevels(t *testing.T) { } func TestArchiveCataloger_truncationRetainsWhatWasFound(t *testing.T) { - // a bound must truncate, not discard: the marker extracted before the disk limit filled stays in - // the SBOM, and a small sibling archive is cataloged in full because the first archive was - // released before it started + // a bound truncates rather than discards: the marker extracted before the disk limit filled stays in + // the SBOM, and a small sibling archive is cataloged in full, the first having been released big := bytes.Repeat([]byte("x"), 8192) scanDir := t.TempDir() writeTestZipRaw(t, filepath.Join(scanDir, "oversized.zip"), map[string][]byte{ @@ -468,15 +329,15 @@ func TestArchiveCataloger_detectionIsContentBased(t *testing.T) { }) } -// TestArchiveCataloger_exclusionsAreTheScansOwn covers the whole of -// exclusions-apply-by-pattern-scope from the outside: there is no archive-specific exclusion -// setting, and the scan's own patterns decide, by their shape, whether they reach inside an archive. +// TestArchiveCataloger_exclusionsAreTheScansOwn covers exclusions-apply-by-pattern-scope from the +// outside: there is no archive-specific setting, and a pattern's shape decides whether it reaches +// inside an archive. func TestArchiveCataloger_exclusionsAreTheScansOwn(t *testing.T) { marker := map[string]string{"nested/marker.txt": "hello"} t.Run("a recursive pattern reaches inside archives", func(t *testing.T) { - // the marker is INSIDE the archive, so nothing but an exclusion applied to the archive's own - // index can remove it: a filter over the scan root never sees this path at all + // the marker is inside the archive, so only an exclusion applied to the archive's own index + // removes it; a filter over the scan root never sees this path scanDir := t.TempDir() writeTestZip(t, filepath.Join(scanDir, "app.zip"), marker) @@ -486,8 +347,8 @@ func TestArchiveCataloger_exclusionsAreTheScansOwn(t *testing.T) { }) t.Run("a root-anchored pattern does not reach inside archives", func(t *testing.T) { - // the same relative path exists twice: once at the scan root, once inside the archive. An - // anchored pattern names the first and says nothing about the second + // the same relative path exists twice, at the scan root and inside the archive; an anchored + // pattern names the first and says nothing about the second scanDir := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(scanDir, "nested"), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(scanDir, "nested", "marker.txt"), []byte("at the root"), 0o644)) @@ -506,8 +367,7 @@ func TestArchiveCataloger_exclusionsAreTheScansOwn(t *testing.T) { }) t.Run("excluding an archive's own extension stops it being descended into", func(t *testing.T) { - // this is what replaces the removed extension list: the archive is not extracted, it is still - // an ordinary file, and nothing about it is an error + // the archive is not extracted; it is still an ordinary file, and nothing about it is an error scanDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(scanDir, "pkg.rpm"), buildZipBytes(t, marker), 0o644)) @@ -517,10 +377,9 @@ func TestArchiveCataloger_exclusionsAreTheScansOwn(t *testing.T) { }) t.Run("a nested archive is excluded at its own level", func(t *testing.T) { - // outer.zip is not excluded and is cataloged; inner.zip is, so it is absent from outer's own - // filesystem, which is what makes it neither extracted nor recordable as a file. Asserted - // through the resolver rather than through the file catalogers: absence from the index is - // upstream of every consumer, so it settles both halves at once + // outer.zip is cataloged; inner.zip is excluded, so it is absent from outer's own filesystem + // and is neither extracted nor recordable as a file. Asserted through the resolver, since + // absence from the index is upstream of every consumer. innerZip := buildZipBytes(t, marker) scanDir := t.TempDir() writeTestZipRaw(t, filepath.Join(scanDir, "outer.zip"), map[string][]byte{ @@ -554,10 +413,8 @@ func TestArchiveCataloger_exclusionsAreTheScansOwn(t *testing.T) { }) t.Run("matching follows the scan's own case sensitivity", func(t *testing.T) { - // worth pinning because it differs from the removed extension list, which lower-cased both - // sides. An exclusion now behaves the same inside an archive as it does anywhere else in a - // scan, which is the point of using the scan's own patterns; a caller who wants both cases - // says so in the pattern, the way they would for any other exclusion + // an exclusion behaves the same inside an archive as anywhere else; a caller wanting both + // cases says so in the pattern, as for any other exclusion scanDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(scanDir, "PKG.RPM"), buildZipBytes(t, marker), 0o644)) @@ -580,8 +437,8 @@ func TestArchiveCataloger_exclusionsAreTheScansOwn(t *testing.T) { }) t.Run("an excluded file is absent from the filesystem, not present and skipped", func(t *testing.T) { - // the distinction the whole mechanism rests on: an excluded entry is not in the archive's - // index, so nothing downstream can resolve it by path or by glob + // an excluded entry is not in the archive's index at all, so nothing downstream resolves it by + // path or by glob scanDir := t.TempDir() writeTestZip(t, filepath.Join(scanDir, "app.zip"), map[string]string{ "nested/marker.txt": "hello", @@ -603,14 +460,11 @@ func TestArchiveCataloger_exclusionsAreTheScansOwn(t *testing.T) { }) } -// TestArchiveCataloger_exclusionPatternPrecedence covers every-boundary-populates-the-patterns, which -// superseded the single-writer clause of patterns-ride-the-archive-config: -// cataloging.ArchiveSearchConfig.ExclusionPatterns is still a derived field (no yaml/json/mapstructure -// tag - there is still no key to supply a second, competing list through), but it is no longer written -// only by CreateSBOM. cmd/syft/internal/options.Catalog.ToArchiveConfig is the other boundary, filling -// it from the CLI's --exclude flag, and precedence between the two is defined rather than left to -// ordering: a value already present is never overwritten, and CreateSBOM consults the source only to -// fill an empty field. +// TestArchiveCataloger_exclusionPatternPrecedence covers every-boundary-populates-the-patterns. +// cataloging.ArchiveSearchConfig.ExclusionPatterns is a derived field with no yaml/json/mapstructure +// tag, written by two boundaries: cmd/syft/internal/options.Catalog.ToArchiveConfig from --exclude, +// and CreateSBOM from the source. Precedence is defined rather than left to ordering: a value already +// present is never overwritten, and CreateSBOM only fills an empty field. func TestArchiveCataloger_exclusionPatternPrecedence(t *testing.T) { marker := map[string]string{"nested/marker.txt": "hello"} @@ -621,16 +475,16 @@ func TestArchiveCataloger_exclusionPatternPrecedence(t *testing.T) { cfg := archiveScanConfig(1, markerCataloger{}) cfg.Archive.ExclusionPatterns = []string{"**/marker.txt"} - // the source publishes a different, non-matching pattern - if the source won, this would - // have no effect on the caller-supplied value + // the source publishes a different, non-matching pattern: if it won, the caller-supplied value + // would be replaced s := scanDirWithExclusions(t, scanDir, cfg, "**/does-not-exist.txt") assert.Empty(t, s.Artifacts.Packages.Sorted(), "a value already present on the config must be what is applied, not overwritten by what the source publishes") }) t.Run("an empty field is filled from the source", func(t *testing.T) { - // the configure-only-your-source path: a consumer who never touches ExclusionPatterns - // directly still gets the source's own patterns applied inside archives + // the configure-only-your-source path: a consumer who never touches ExclusionPatterns still + // gets the source's own patterns applied inside archives scanDir := t.TempDir() writeTestZip(t, filepath.Join(scanDir, "app.zip"), marker) @@ -653,28 +507,13 @@ func TestArchiveCataloger_exclusionPatternPrecedence(t *testing.T) { }) } -// resolverProbeCataloger reports what the resolver it was handed can actually see, which is how the -// difference between "absent from the index" and "present and skipped" is observed from a scan. +// resolverProbeCataloger reports what the resolver it was handed can see, which is how "absent from +// the index" is told from "present and skipped" within a scan. type resolverProbeCataloger struct { glob string record func([]string) } -func (resolverProbeCataloger) Name() string { return "resolver-probe-cataloger" } - -func (c resolverProbeCataloger) Catalog(_ context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { - locations, err := resolver.FilesByGlob(c.glob) - if err != nil { - return nil, nil, err - } - var paths []string - for _, loc := range locations { - paths = append(paths, loc.RealPath) - } - c.record(paths) - return nil, nil, nil -} - func TestArchiveCataloger_leavesNothingBehind(t *testing.T) { // extracted content is written outside the scanned source and must not survive the scan, so a // long-running process scanning many archives does not accumulate temp trees @@ -708,20 +547,6 @@ func TestArchiveCataloger_leavesNothingBehind(t *testing.T) { "nothing new may appear under the scanned directory") } -func archiveTempDirs(t *testing.T) []string { - t.Helper() - entries, err := os.ReadDir(os.TempDir()) - require.NoError(t, err) - - var out []string - for _, entry := range entries { - if entry.IsDir() && strings.HasPrefix(entry.Name(), "syft-archive-") { - out = append(out, entry.Name()) - } - } - return out -} - func TestArchiveCataloger_unicodeEntryNamesSurviveToOutput(t *testing.T) { // entry names reach the FileSystemID and then the encoded SBOM; a mangled one is a corrupt // identifier rather than a visible failure @@ -742,8 +567,8 @@ func TestArchiveCataloger_unicodeEntryNamesSurviveToOutput(t *testing.T) { } func TestArchiveCataloger_libraryParity(t *testing.T) { - // the library entry point must give a Go consumer the same behavior as a CLI user, and must not - // silently replace anything the consumer set on the package cataloger config + // the library entry point must behave as the CLI does, and must not silently replace anything the + // consumer set on the package cataloger config dep := jarBytes(t, "nested-lib", "2.0", nil) scanDir := t.TempDir() writeTestZipRaw(t, filepath.Join(scanDir, "bundle.zip"), map[string][]byte{"lib/nested-lib-2.0.jar": dep}) @@ -765,33 +590,33 @@ func TestArchiveCataloger_libraryParity(t *testing.T) { }) t.Run("setting only java's own depth neither enables the feature nor breaks java", func(t *testing.T) { - // java's squash-inlined MaxDepth is a config field a consumer can set; it must have no - // effect on recursion ownership, and must not be overwritten either + // java's squash-inlined MaxDepth is a config field a consumer can set: it must not affect + // recursion ownership, and must not be overwritten supplied := java.DefaultArchiveCatalogerConfig() supplied.ArchiveSearchConfig = supplied.ArchiveSearchConfig.WithMaxDepth(3) cfg := javaScanConfig(0).WithPackagesConfig(pkgcataloging.Config{JavaArchive: supplied}) s := scanDirWith(t, scanDir, cfg) - // the archive task never ran, so java's own recursion is what found it - once + // the archive task never ran, so java's own recursion found it - once assert.Equal(t, 1, countNestedLib(t, s), "java must keep recursing itself") assert.Equal(t, 3, cfg.Packages.JavaArchive.MaxDepth, "the consumer's value must survive") }) } func TestArchiveCataloger_noExtractionPathReachesTheSBOM(t *testing.T) { - // covers syft/archive-content-identity#nested-paths-are-archive-relative. AllLocations emitted - // the resolver's own root unrelativized, and filemetadata.Cataloger records every location it is - // handed, so a coordinate whose RealPath was the extraction temp dir reached FileMetadata, was - // merged into the shared SBOM, and gained a CONTAINS edge. File cataloging must be on: that is - // the only path that enumerates all locations rather than searching for them. + // covers syft/archive-content-identity#nested-paths-are-archive-relative. AllLocations emitted the + // resolver's own root unrelativized, and filemetadata.Cataloger records every location it is handed, + // so a coordinate whose RealPath was the extraction temp dir reached FileMetadata, merged into the + // shared SBOM, and gained a CONTAINS edge. File cataloging must be on: it is the only path that + // enumerates all locations rather than searching for them. innerZip := buildZipBytes(t, map[string]string{"nested/marker.txt": "hello"}) scanDir := t.TempDir() writeTestZipRaw(t, filepath.Join(scanDir, "outer.zip"), map[string][]byte{"deeper/inner.zip": innerZip}) - // the resolver normalizes its root with EvalSymlinks, so the scan root it reports is the - // resolved path. On macOS t.TempDir() sits under /var, a symlink to /private/var, so comparing - // against the unresolved value fails there while passing on linux. + // the resolver normalizes its root with EvalSymlinks, so the scan root it reports is the resolved + // path. On macOS t.TempDir() sits under /var, a symlink to /private/var, so comparing against the + // unresolved value fails there while passing on linux. resolvedScanDir, err := filepath.EvalSymlinks(scanDir) require.NoError(t, err) @@ -821,23 +646,22 @@ func TestArchiveCataloger_noExtractionPathReachesTheSBOM(t *testing.T) { for _, p := range paths { assert.NotContains(t, p, "syft-archive-", "an extraction directory reached the SBOM at %q", p) - // the storage shape is an implementation detail of the scan, so the file an archive's entries - // were overflowed into must not name itself either - in a path, or as a segment of one + // the storage shape is an implementation detail, so the file an archive's entries were + // overflowed into must not name itself either - in a path, or as a segment of one assert.NotContains(t, p, "contents.tar", "the file holding an archive's entries reached the SBOM at %q", p) if filepath.IsAbs(p) { - // the scan root itself is reported absolute by the directory resolver, and has been - // since long before this feature: it is the path the user asked for. Anything else - // absolute is a filesystem this feature invented leaking its own root. + // the directory resolver has always reported the scan root absolute: it is the path the + // user asked for. Anything else absolute is a filesystem this feature invented leaking its + // own root. assert.Equal(t, resolvedScanDir, p, "only the scanned directory itself may be reported absolute") } } } func TestArchiveCataloger_repeatedScansAgreeOnNestedPaths(t *testing.T) { - // the reproducibility property the requirement exists for: extraction directories are created - // fresh per run, so anything of theirs that reaches output makes two scans of identical input - // disagree + // the reproducibility property the requirement exists for: extraction directories are created fresh + // per run, so anything of theirs reaching output makes two scans of identical input disagree innerZip := buildZipBytes(t, map[string]string{"nested/marker.txt": "hello"}) scanDir := t.TempDir() writeTestZipRaw(t, filepath.Join(scanDir, "outer.zip"), map[string][]byte{"deeper/inner.zip": innerZip}) @@ -861,47 +685,11 @@ func TestArchiveCataloger_repeatedScansAgreeOnNestedPaths(t *testing.T) { assert.Equal(t, first, scan(t)) } -// buildTarGzBytes builds a gzipped tar in memory, in sorted entry order. -func buildTarGzBytes(t *testing.T, files map[string][]byte) []byte { - t.Helper() - var buf bytes.Buffer - gw := gzip.NewWriter(&buf) - writeTarEntries(t, gw, files) - require.NoError(t, gw.Close()) - return buf.Bytes() -} - -// buildTarBytes builds an uncompressed tar in memory. -func buildTarBytes(t *testing.T, files map[string][]byte) []byte { - t.Helper() - var buf bytes.Buffer - writeTarEntries(t, &buf, files) - return buf.Bytes() -} - -func writeTarEntries(t *testing.T, w io.Writer, files map[string][]byte) { - t.Helper() - names := make([]string, 0, len(files)) - for name := range files { - names = append(names, name) - } - sort.Strings(names) - - tw := tar.NewWriter(w) - for _, name := range names { - body := files[name] - require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))})) - _, err := tw.Write(body) - require.NoError(t, err) - } - require.NoError(t, tw.Close()) -} - func TestArchiveCataloger_tarFamilyIsCatalogedEndToEnd(t *testing.T) { - // nothing asserted the tar path end to end: the extensionless-archive case builds a ZIP named - // "bundle", so TarExtractor was only ever reached by its own unit tests. The compound extension - // is the interesting one - the archive is saved to a temp file before detection and the basename - // is preserved specifically so ".tar.gz" survives that round trip. + // nothing asserted the tar path end to end: the extensionless-archive case builds a zip named + // "bundle", so TarExtractor was only ever reached by its own unit tests. The compound extension is + // the interesting one - the archive is saved to a temp file before detection, and the basename is + // preserved specifically so ".tar.gz" survives that round trip. dep := jarBytes(t, "tar-nested-lib", "3.1", nil) entry := map[string][]byte{"lib/tar-nested-lib-3.1.jar": dep} @@ -934,9 +722,9 @@ func TestArchiveCataloger_tarFamilyIsCatalogedEndToEnd(t *testing.T) { } func TestArchiveCataloger_legacySearchSettingsAreSuperseded(t *testing.T) { - // syft/nested-archive-cataloging#legacy-archive-search-settings-are-superseded. The row that - // matters is nested-on with both settings false: it looks like a bug and is not, so a future - // change that "fixes" it is changing a requirement. + // syft/nested-archive-cataloging#legacy-archive-search-settings-are-superseded. The row that matters + // is nested-on with both settings false: it looks like a bug and is not, so a future change that + // "fixes" it is changing a requirement. dep := jarBytes(t, "legacy-lib", "1.2", nil) entry := map[string][]byte{"lib/legacy-lib-1.2.jar": dep} @@ -945,8 +733,8 @@ func TestArchiveCataloger_legacySearchSettingsAreSuperseded(t *testing.T) { "bundle.tar.gz": buildTarGzBytes(t, entry), } - // with the feature off, java's own wrapped-archive parsers are what reach the jar, and those are - // gated by these two settings: indexed covers the zip, unindexed covers the tar family + // with the feature off, java's own wrapped-archive parsers reach the jar, gated by these two + // settings: indexed covers the zip, unindexed covers the tar family found := func(t *testing.T, container string, depth int, searchSettings *bool) bool { t.Helper() scanDir := t.TempDir() @@ -955,8 +743,8 @@ func TestArchiveCataloger_legacySearchSettingsAreSuperseded(t *testing.T) { cfg := javaScanConfig(depth) if searchSettings != nil { // these two settings only ever reach the java cataloger's wrapped-archive parser - // registrations, which is where the CLI's search-indexed-archives and - // search-unindexed-archives land as well + // registrations, where the CLI's search-indexed-archives and search-unindexed-archives + // land as well packages := cfg.Packages packages.JavaArchive.IncludeIndexedArchives = *searchSettings packages.JavaArchive.IncludeUnindexedArchives = *searchSettings @@ -996,3 +784,273 @@ func TestArchiveCataloger_legacySearchSettingsAreSuperseded(t *testing.T) { }) } } + +func (markerCataloger) Name() string { return "marker-cataloger" } + +func (markerCataloger) Catalog(_ context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + locations, err := resolver.FilesByGlob("**/marker.txt") + if err != nil { + return nil, nil, err + } + var pkgs []pkg.Package + for _, loc := range locations { + p := pkg.Package{ + Name: "marker-pkg", + Version: "1.0.0", + Type: pkg.BinaryPkg, + Locations: file.NewLocationSet(loc), + } + p.SetID() + pkgs = append(pkgs, p) + } + return pkgs, nil, nil +} + +func writeTestZip(t *testing.T, path string, files map[string]string) { + t.Helper() + raw := make(map[string][]byte, len(files)) + for k, v := range files { + raw[k] = []byte(v) + } + writeTestZipRaw(t, path, raw) +} + +func writeTestZipRaw(t *testing.T, path string, files map[string][]byte) { + t.Helper() + require.NoError(t, os.WriteFile(path, buildZipBytesRaw(t, files), 0o644)) +} + +func buildZipBytes(t *testing.T, files map[string]string) []byte { + t.Helper() + raw := make(map[string][]byte, len(files)) + for k, v := range files { + raw[k] = []byte(v) + } + return buildZipBytesRaw(t, raw) +} + +// buildZipBytesRaw writes entries in sorted name order rather than map order: limits are enforced as +// the walk proceeds, so entry order decides which entries land before a truncation. +func buildZipBytesRaw(t *testing.T, files map[string][]byte) []byte { + t.Helper() + names := make([]string, 0, len(files)) + for name := range files { + names = append(names, name) + } + sort.Strings(names) + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, name := range names { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write(files[name]) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +// archiveScanConfig builds a CreateSBOMConfig limited to the given catalogers, with nested archive +// cataloging at the given depth (0 disables it). +func archiveScanConfig(depth int, catalogers ...pkg.Cataloger) *CreateSBOMConfig { + refs := make([]pkgcataloging.CatalogerReference, 0, len(catalogers)) + for _, c := range catalogers { + refs = append(refs, pkgcataloging.NewCatalogerReference(c, []string{"directory"})) + } + cfg := DefaultCreateSBOMConfig().WithoutCatalogers().WithCatalogers(refs...) + cfg.Archive.MaxDepth = depth + return cfg +} + +// javaScanConfig builds a config whose java cataloger comes from the cataloging factory rather than +// pre-built: the flag handing recursion to the archive task is derived onto the factory's +// PackagesConfig, so a cataloger passed via WithCatalogers keeps its own config and never yields. +// +// It selects only the java-tagged catalogers, not the whole default set, which includes the RPM +// cataloger and its sqlite driver this test binary does not register. +func javaScanConfig(depth int) *CreateSBOMConfig { + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")) + if depth > 0 { + cfg = cfg.WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth)) + } + return cfg +} + +func scanDirWith(t *testing.T, dir string, cfg *CreateSBOMConfig) *sbom.SBOM { + t.Helper() + src, err := directorysource.New(directorysource.Config{Path: dir}) + require.NoError(t, err) + t.Cleanup(func() { _ = src.Close() }) + + s, err := cfg.Create(context.Background(), src) + require.NoError(t, err) + return s +} + +// scanDirWithExclusions scans with the given exclusion patterns configured on the source, where +// exclusions live; archive cataloging has no exclusion setting of its own. +func scanDirWithExclusions(t *testing.T, dir string, cfg *CreateSBOMConfig, exclude ...string) *sbom.SBOM { + t.Helper() + src, err := directorysource.New(directorysource.Config{ + Path: dir, + Exclude: source.ExcludeConfig{Paths: exclude}, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = src.Close() }) + + s, err := cfg.Create(context.Background(), src) + require.NoError(t, err) + return s +} + +func jarBytes(t *testing.T, title, version string, extra map[string][]byte) []byte { + t.Helper() + entries := map[string][]byte{ + "META-INF/MANIFEST.MF": []byte( + "Manifest-Version: 1.0\nImplementation-Title: " + title + "\nImplementation-Version: " + version + "\n", + ), + } + for k, v := range extra { + entries[k] = v + } + return buildZipBytesRaw(t, entries) +} + +func (resolverProbeCataloger) Name() string { return "resolver-probe-cataloger" } + +func (c resolverProbeCataloger) Catalog(_ context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + locations, err := resolver.FilesByGlob(c.glob) + if err != nil { + return nil, nil, err + } + var paths []string + for _, loc := range locations { + paths = append(paths, loc.RealPath) + } + c.record(paths) + return nil, nil, nil +} + +func archiveTempDirs(t *testing.T) []string { + t.Helper() + entries, err := os.ReadDir(os.TempDir()) + require.NoError(t, err) + + var out []string + for _, entry := range entries { + if entry.IsDir() && strings.HasPrefix(entry.Name(), "syft-archive-") { + out = append(out, entry.Name()) + } + } + return out +} + +func buildTarGzBytes(t *testing.T, files map[string][]byte) []byte { + t.Helper() + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + writeTarEntries(t, gw, files) + require.NoError(t, gw.Close()) + return buf.Bytes() +} + +func buildTarBytes(t *testing.T, files map[string][]byte) []byte { + t.Helper() + var buf bytes.Buffer + writeTarEntries(t, &buf, files) + return buf.Bytes() +} + +func writeTarEntries(t *testing.T, w io.Writer, files map[string][]byte) { + t.Helper() + names := make([]string, 0, len(files)) + for name := range files { + names = append(names, name) + } + sort.Strings(names) + + tw := tar.NewWriter(w) + for _, name := range names { + body := files[name] + require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))})) + _, err := tw.Write(body) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) +} + +// recordingBus captures the progress rows a scan publishes. +type recordingBus struct { + mu sync.Mutex + events []partybus.Event +} + +func (p *recordingBus) Publish(e partybus.Event) { + p.mu.Lock() + defer p.mu.Unlock() + p.events = append(p.events, e) +} + +// TestArchiveCataloger_fileCatalogerProgressCountsArchiveContents scans a directory holding one file +// and a zip holding two more. The file catalogers run once over the scan root and again over the +// archive, and the row each of them publishes must be one row carrying the total of both - a row that +// stops at the scan root reports a number that is wrong by everything inside every archive. +func TestArchiveCataloger_fileCatalogerProgressCountsArchiveContents(t *testing.T) { + scanDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(scanDir, "loose.txt"), []byte("outside"), 0o600)) + writeTestZip(t, filepath.Join(scanDir, "app.zip"), map[string]string{ + "nested/one.txt": "hello", + "nested/two.txt": "there", + }) + + recorder := &recordingBus{} + bus.Set(recorder) + t.Cleanup(func() { bus.Set(nil) }) + + src, err := directorysource.New(directorysource.Config{Path: scanDir}) + require.NoError(t, err) + t.Cleanup(func() { _ = src.Close() }) + + cfg := DefaultCreateSBOMConfig(). + WithoutCatalogers(). + WithCatalogers(pkgcataloging.NewCatalogerReference(markerCataloger{}, []string{"directory"})). + WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)) + cfg.Archive.MaxDepth = 1 + + _, err = cfg.Create(context.Background(), src) + require.NoError(t, err) + + rows := map[string][]*progress.Manual{} + for _, e := range recorder.events { + if e.Type != event.CatalogerTaskStarted { + continue + } + info, ok := e.Source.(monitor.GenericTask) + require.True(t, ok) + prog, ok := e.Value.(progress.StagedProgressable) + require.True(t, ok) + tp, ok := prog.(*monitor.TaskProgress) + require.True(t, ok) + rows[info.Title.Default] = append(rows[info.Title.Default], tp.Manual) + } + + for _, title := range []string{"File digests", "File metadata"} { + published := rows[title] + require.Len(t, published, 1, "%s must publish exactly one row for the whole scan", title) + row := published[0] + assert.GreaterOrEqual(t, row.Current(), int64(4), + "%s must count the two files inside the archive as well as the two beside it, got %d", + title, row.Current()) + + // a re-run reports into the row it already owns, so the row's total has to grow with it. A total + // describing only the scan root leaves the count running past it, which renders as an overshot + // bar and reads as complete (progress.IsCompleted) while the walk is still going. + if row.Size() >= 0 { + assert.Equal(t, row.Current(), row.Size(), + "%s must carry a total covering every run that reported into it, got %d/%d", + title, row.Current(), row.Size()) + } + } +} diff --git a/syft/archive_exclusions.go b/syft/archive_exclusions.go index d8e706e66..f575fb08f 100644 --- a/syft/archive_exclusions.go +++ b/syft/archive_exclusions.go @@ -1,10 +1,8 @@ package syft import ( - "fmt" "os" "path/filepath" - "strings" "github.com/bmatcuk/doublestar/v4" @@ -12,52 +10,28 @@ import ( "github.com/anchore/syft/syft/source" ) -// newArchiveExclusionVisitor builds an index visitor that drops the given patterns from the tree -// rooted at dir, which is one archive's extraction directory. Returns a nil visitor when there is -// nothing to exclude, so the resolver is built exactly as it was before when no pattern applies. +// newArchiveExclusionVisitor builds an index visitor that drops the given patterns from one archive's +// contents, or nil when there is nothing to exclude. // -// Matching is done against each entry's path RELATIVE to the extraction directory. The directory -// source instead rewrites every pattern to be absolute against the scan root, and reusing that here -// would be wrong twice over: the root would be the wrong one, and an absolute pattern is matched -// against the absolute path of the entry, so a segment of the temp directory holding the extraction -// could satisfy a pattern the archive's own contents never do. Relative matching also keeps the -// patterns intact, which is what lets `**/x` match x at the archive's own root as well as below it. -func newArchiveExclusionVisitor(dir string, patterns []string) (fileresolver.PathIndexVisitor, error) { +// Patterns match against each entry's archive-relative path, unlike the directory source, which +// rewrites patterns absolute against the scan root. An archive is indexed in memory with no host +// location to rewrite against, and matching absolute would let a segment of the scan's temp directory +// satisfy a pattern the contents never do. Relative matching also keeps patterns intact, so `**/x` +// matches x at the archive root as well as below it. +// +// Only `**/`-prefixed patterns reach here (see cataloging.ArchiveExclusionPatterns): those mean the +// same thing at any depth, which is what makes an archive-relative match well defined. +func newArchiveExclusionVisitor(patterns []string) fileresolver.PathIndexVisitor { if len(patterns) == 0 { - return nil, nil + return nil } - // the indexer reports an absolute, symlink-resolved path to every visitor: the resolver - // normalizes its root with EvalSymlinks (fileresolver.NormalizeRootDirectory, via the chroot - // context) and the walk then takes filepath.Abs of it. The prefix to strip has to be derived the - // same way or it strips nothing - on macOS an extraction directory lives under /var, a symlink - // to /private/var, and every pattern would quietly match nothing. - resolved, err := filepath.EvalSymlinks(dir) - if err != nil { - return nil, fmt.Errorf("unable to resolve archive extraction directory %q: %w", dir, err) - } - absolute, err := filepath.Abs(resolved) - if err != nil { - return nil, fmt.Errorf("unable to make archive extraction directory %q absolute: %w", dir, err) - } - - // slash-separated, to match doublestar's syntax on Windows too - root := filepath.ToSlash(absolute) - prefix := strings.TrimSuffix(root, "/") + "/" - - return func(_, path string, info os.FileInfo, _ error) error { - slashed := filepath.ToSlash(path) - if slashed == root || slashed == prefix { - // the extraction directory itself is not a candidate: it is the archive, not something in it - return nil - } - relative := strings.TrimPrefix(slashed, prefix) - + return func(_, entryPath string, info os.FileInfo, _ error) error { for _, pattern := range patterns { - matches, err := doublestar.Match(pattern, relative) + matches, err := doublestar.Match(pattern, entryPath) if err != nil { // a malformed pattern excludes nothing rather than failing the archive; the source - // rejected anything it does not accept long before the scan reached here + // already validated its patterns continue } if !matches { @@ -69,13 +43,12 @@ func newArchiveExclusionVisitor(dir string, patterns []string) (fileresolver.Pat return fileresolver.ErrSkipPath } return nil - }, nil + } } -// sourceExclusions returns the exclusion patterns the given source was configured with, or nil when -// it publishes none. Exclusions are the source's own configuration and appear nowhere in -// CreateSBOMConfig, so this is the only way the cataloging side learns them; a source that does not -// implement source.PathExcluder excludes nothing anywhere, archives included. +// sourceExclusions returns the exclusion patterns the source was configured with, or nil. Exclusions +// belong to the source and appear nowhere in CreateSBOMConfig, so this is how the cataloging side +// learns them; a source that is not a source.PathExcluder excludes nothing. func sourceExclusions(src source.Source) []string { excluder, ok := src.(source.PathExcluder) if !ok { diff --git a/syft/archive_exclusions_test.go b/syft/archive_exclusions_test.go index 23a2ac99b..6403ec7b5 100644 --- a/syft/archive_exclusions_test.go +++ b/syft/archive_exclusions_test.go @@ -1,6 +1,7 @@ package syft import ( + "archive/tar" "os" "path/filepath" "testing" @@ -14,80 +15,59 @@ import ( ) func Test_newArchiveExclusionVisitor(t *testing.T) { + // entry paths are archive-relative, the frame the index hands every filter; nothing here is a path + // on the host, so the fixtures are plain strings rather than files in a temp directory + dir := entryInfo(tar.TypeDir) + regular := entryInfo(tar.TypeReg) + t.Run("no patterns means no visitor", func(t *testing.T) { // so the archive resolver is built exactly as it was before this existed - visitor, err := newArchiveExclusionVisitor(t.TempDir(), nil) - require.NoError(t, err) - assert.Nil(t, visitor) + assert.Nil(t, newArchiveExclusionVisitor(nil)) }) - t.Run("matching is relative to the extraction directory", func(t *testing.T) { - // and NOT against a pattern rewritten to be absolute, which is what the directory source does - // with the scan root. The two are only distinguishable when the temp directory holding the - // extraction has a segment the pattern would match, which is why the check below builds one. - root := filepath.Join(t.TempDir(), "vendor") - require.NoError(t, os.MkdirAll(filepath.Join(root, "keep"), 0o755)) - - // the paths handed to a visitor are the symlink-resolved ones the indexer walks, so the test - // has to speak in those too: on macOS t.TempDir sits under /var, a symlink to /private/var, - // and an unresolved path would strip no prefix and match the temp directory's own segments - root, err := filepath.EvalSymlinks(root) - require.NoError(t, err) - - visitor, err := newArchiveExclusionVisitor(root, []string{"**/vendor"}) - require.NoError(t, err) + t.Run("matching is relative to the archive, not to any host path", func(t *testing.T) { + // an archive is indexed in memory, so a pattern can only be satisfied by what the archive holds; + // there is no scratch directory whose own segments could match + visitor := newArchiveExclusionVisitor([]string{"**/vendor"}) require.NotNil(t, visitor) - assert.NoError(t, visitor("", root, dirInfo(t, root), nil), - "the extraction directory itself is the archive, not something in it") - assert.NoError(t, visitor("", filepath.Join(root, "keep"), dirInfo(t, filepath.Join(root, "keep")), nil), - "a path under a matching temp directory must not inherit the match") + assert.ErrorIs(t, visitor("/", "vendor", dir, nil), filepath.SkipDir) + assert.NoError(t, visitor("/", "keep/vendored.txt", regular, nil), + "a path that merely contains the word must not match") + assert.NoError(t, visitor("/", "vendor-ish", regular, nil)) }) t.Run("a matching file is skipped and a matching directory is pruned", func(t *testing.T) { - root := t.TempDir() - dir := filepath.Join(root, "a", "vendor") - require.NoError(t, os.MkdirAll(dir, 0o755)) - file := filepath.Join(root, "a", "pkg.rpm") - require.NoError(t, os.WriteFile(file, []byte("x"), 0o644)) - - visitor, err := newArchiveExclusionVisitor(root, []string{"**/vendor", "**/*.rpm"}) - require.NoError(t, err) + visitor := newArchiveExclusionVisitor([]string{"**/vendor", "**/*.rpm"}) require.NotNil(t, visitor) - assert.ErrorIs(t, visitor("", file, fileInfo(t, file), nil), fileresolver.ErrSkipPath) - assert.ErrorIs(t, visitor("", dir, dirInfo(t, dir), nil), filepath.SkipDir) + assert.ErrorIs(t, visitor("/", "a/pkg.rpm", regular, nil), fileresolver.ErrSkipPath) + assert.ErrorIs(t, visitor("/", "a/vendor", dir, nil), filepath.SkipDir) }) t.Run("an any-depth pattern reaches the archive's own root", func(t *testing.T) { - // `**/x` matches x at zero depth as well as below it, which is what makes "exclude .rpm - // everywhere" stop an .rpm sitting directly inside another archive - root := t.TempDir() - file := filepath.Join(root, "pkg.rpm") - require.NoError(t, os.WriteFile(file, []byte("x"), 0o644)) + // `**/x` matches x at zero depth as well as below it, so "exclude .rpm everywhere" stops an .rpm + // sitting directly inside another archive + visitor := newArchiveExclusionVisitor([]string{"**/*.rpm"}) + require.NotNil(t, visitor) - visitor, err := newArchiveExclusionVisitor(root, []string{"**/*.rpm"}) - require.NoError(t, err) - - assert.ErrorIs(t, visitor("", file, fileInfo(t, file), nil), fileresolver.ErrSkipPath) + assert.ErrorIs(t, visitor("/", "pkg.rpm", regular, nil), fileresolver.ErrSkipPath) + assert.ErrorIs(t, visitor("/", "deep/down/pkg.rpm", regular, nil), fileresolver.ErrSkipPath) }) t.Run("a non-matching path is kept", func(t *testing.T) { - root := t.TempDir() - file := filepath.Join(root, "keep.txt") - require.NoError(t, os.WriteFile(file, []byte("x"), 0o644)) + visitor := newArchiveExclusionVisitor([]string{"**/*.rpm"}) + require.NotNil(t, visitor) - visitor, err := newArchiveExclusionVisitor(root, []string{"**/*.rpm"}) - require.NoError(t, err) - - assert.NoError(t, visitor("", file, fileInfo(t, file), nil)) + assert.NoError(t, visitor("/", "keep.txt", regular, nil)) }) - t.Run("an unresolvable directory is an error rather than a silent no-match", func(t *testing.T) { - // the failure mode this guards against is invisible: a prefix that cannot be resolved strips - // nothing, every pattern then matches nothing, and the scan looks like it had no exclusions - _, err := newArchiveExclusionVisitor(filepath.Join(t.TempDir(), "does-not-exist"), []string{"**/x"}) - assert.Error(t, err) + t.Run("a malformed pattern excludes nothing rather than failing the archive", func(t *testing.T) { + visitor := newArchiveExclusionVisitor([]string{"[", "**/*.rpm"}) + require.NotNil(t, visitor) + + assert.NoError(t, visitor("/", "keep.txt", regular, nil)) + assert.ErrorIs(t, visitor("/", "pkg.rpm", regular, nil), fileresolver.ErrSkipPath) }) } @@ -104,10 +84,9 @@ func Test_sourceExclusions(t *testing.T) { }) t.Run("building the source's own resolver does not rewrite them", func(t *testing.T) { - // the source rewrites exclusions to be absolute against the scan root while building its - // resolver. If that rewriting reached the configured patterns, none would begin "**/" any - // more and an archive's exclusions would degrade to "none configured" - which reads as a - // feature that was never switched on rather than as a bug + // the source rewrites exclusions absolute against the scan root while building its resolver. If that + // reached the configured patterns, none would begin "**/" any more and an archive's exclusions would + // silently degrade to none. src, err := directorysource.New(directorysource.Config{ Path: t.TempDir(), Exclude: source.ExcludeConfig{Paths: []string{"**/*.rpm"}}, @@ -126,18 +105,12 @@ func Test_sourceExclusions(t *testing.T) { }) } -func dirInfo(t *testing.T, path string) os.FileInfo { - t.Helper() - info, err := os.Stat(path) - require.NoError(t, err) - require.True(t, info.IsDir()) - return info -} - -func fileInfo(t *testing.T, path string) os.FileInfo { - t.Helper() - info, err := os.Stat(path) - require.NoError(t, err) - require.False(t, info.IsDir()) - return info +// entryInfo is the FileInfo an archive entry carries, which is what the index hands every filter - +// derived from the entry's header, not from anything on the filesystem. +func entryInfo(typeflag byte) os.FileInfo { + mode := int64(0o600) + if typeflag == tar.TypeDir { + mode = 0o755 + } + return (&tar.Header{Typeflag: typeflag, Mode: mode}).FileInfo() } diff --git a/syft/archive_nesting_test.go b/syft/archive_nesting_test.go index e82350736..676ccdb2e 100644 --- a/syft/archive_nesting_test.go +++ b/syft/archive_nesting_test.go @@ -31,22 +31,19 @@ import ( "github.com/anchore/syft/syft/source/directorysource" ) -// This file covers nesting ACROSS archive families. Every other fixture in the nested-archive tests -// is a zip, so a mechanism that happens to be zip-specific would pass all of them; the requirement -// is that an archive of one family inside another behaves exactly as it would inside its own family, -// with the same packages, the same filesystem-id chain, the same CONTAINS edges and the same depth -// accounting. +// This file covers nesting across archive families. The other nested-archive fixtures are all zips, +// so a zip-specific mechanism would pass them; the requirement here is that an archive of one family +// inside another behaves as it would inside its own family - same packages, same filesystem-id chain, +// same CONTAINS edges, same depth accounting. // -// The chains are built in the test and nothing is committed as a binary. A committed nested archive -// is opaque in review, cannot be varied per case, and hides which family sits at which level - which -// is the one thing every case here turns on. +// Chains are built in the test rather than committed as binaries, so each case can vary which family +// sits at which level. // archiveFamily is one archive format buildNestedArchive can produce. // -// ".jar" is its own family rather than a flavour of zip: the java cataloger meets it as a jar -// wherever it sits, while the generic archive task descends into it like any other zip. That is the -// single-owner coupling, and it behaves differently in the middle of a chain than at the leaf, which -// is why the matrix below has a jar as a container as well as a payload. +// ".jar" is its own family rather than a flavour of zip: the java cataloger meets it as a jar wherever +// it sits, while the archive task descends into it like any other zip. That differs mid-chain from at +// the leaf, so the matrix below uses a jar as container and as payload. type archiveFamily string const ( @@ -57,29 +54,9 @@ const ( familyTgz archiveFamily = ".tgz" ) -// pack builds one archive of this family holding the given entries. -func (f archiveFamily) pack(t *testing.T, base string, entries map[string][]byte) []byte { - t.Helper() - switch f { - case familyZip: - return buildZipBytesRaw(t, entries) - case familyJar: - // a jar carries a manifest, so it is a real jar to the java cataloger at any level. The - // package java reports for it is named after the file, not the manifest title. - return jarBytes(t, base, "1.0", entries) - case familyTar: - return buildTarBytes(t, entries) - case familyTarGz, familyTgz: - return buildTarGzBytes(t, entries) - } - t.Fatalf("unknown archive family %q", f) - return nil -} - -// nestPlan describes one chain to build: a family per level (outermost first), the entries to put -// inside the innermost archive, and any extra entries to add beside the nested archive at a given -// level. The extras are what let one helper serve the limit cases too: grading the levels' sizes or -// their entry counts is the only way to put a bound between two levels of a chain. +// nestPlan describes one chain to build: a family per level (outermost first), the entries for the +// innermost archive, and extras to add beside the nested archive at a given level. The extras grade +// the levels' sizes or entry counts, which is how the limit cases place a bound between two levels. type nestPlan struct { families []archiveFamily leaf map[string][]byte @@ -93,9 +70,8 @@ type nestedArchive struct { name string // bytes is the outermost archive. bytes []byte - // sizes is each archive's size in bytes, outermost first. A container is never smaller than what - // it holds, so these decrease, and a bound placed between two of them is what the limit cases - // need. + // sizes is each archive's size in bytes, outermost first. A container is never smaller than what it + // holds, so these decrease, and the limit cases place a bound between two of them. sizes []int // paths is each archive's path within its parent, outermost first; paths[0] is name. paths []string @@ -105,98 +81,6 @@ type nestedArchive struct { virtualPaths []string } -// leafFileSystemID is the chain the innermost archive's contents carry: every archive in the chain, -// outer to inner. It is asserted as an exact string throughout - a partially composed chain passes -// a substring check, which is how a real bug in this work went unnoticed for a while. -func (n nestedArchive) leafFileSystemID() string { - return n.fileSystemIDs[len(n.fileSystemIDs)-1] -} - -// buildNestedArchive builds one archive per family given, each holding the next, with leaf inside -// the innermost. A case is then one line rather than one fixture. -func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive { - t.Helper() - require.NotEmpty(t, plan.families, "a chain needs at least one level") - - out := nestedArchive{ - sizes: make([]int, len(plan.families)), - paths: make([]string, len(plan.families)), - fileSystemIDs: make([]string, len(plan.families)), - virtualPaths: make([]string, len(plan.families)), - } - - // the level index is in every name and every path segment, so a failed assertion says which - // level it was, and no two levels share a path - a chain assertion cannot pass by accident on a - // value composed from the wrong level - for i, family := range plan.families { - base := fmt.Sprintf("level%d", i) - if i == 0 { - out.paths[i] = base + string(family) - continue - } - out.paths[i] = fmt.Sprintf("nest%d/%s%s", i, base, family) - } - - out.name = out.paths[0] - for i := range plan.families { - if i == 0 { - out.fileSystemIDs[i] = out.paths[0] - out.virtualPaths[i] = out.paths[0] - continue - } - out.fileSystemIDs[i] = out.fileSystemIDs[i-1] + ":" + out.paths[i] - out.virtualPaths[i] = out.virtualPaths[i-1] + ":" + out.paths[i] - } - - // built innermost first, because each level's bytes are an entry of the level above it - var packed []byte - for i := len(plan.families) - 1; i >= 0; i-- { - entries := map[string][]byte{} - if i == len(plan.families)-1 { - for name, body := range plan.leaf { - entries[name] = body - } - } else { - entries[out.paths[i+1]] = packed - } - for name, body := range plan.extra[i] { - entries[name] = body - } - packed = plan.families[i].pack(t, fmt.Sprintf("level%d", i), entries) - out.sizes[i] = len(packed) - } - out.bytes = packed - - return out -} - -// writeNestedArchive writes the chain's outermost archive into dir and returns the chain. -func writeNestedArchive(t *testing.T, dir string, plan nestPlan) nestedArchive { - t.Helper() - nested := buildNestedArchive(t, plan) - require.NoError(t, os.WriteFile(filepath.Join(dir, nested.name), nested.bytes, 0o644)) - return nested -} - -// markerLeaf is the family-agnostic leaf payload: one file no archive format has any opinion about, -// found by markerCataloger. Using it rather than an ecosystem fixture is what makes the rows -// comparable - the same package must come out whichever families the chain is made of. -func markerLeaf() map[string][]byte { - return map[string][]byte{"nested/marker.txt": []byte("leaf")} -} - -// incompressibleBytes returns n bytes deflate and gzip cannot shrink, so a chain built from them has -// sizes per level predictable enough to place a bound between two of them. -func incompressibleBytes(n int) []byte { - b := make([]byte, n) - x := uint32(0x9e3779b9) - for i := range b { - x = x*1664525 + 1013904223 - b[i] = byte(x >> 24) - } - return b -} - // nestingRow is one line of the depth-3 matrix. type nestingRow struct { name string @@ -204,57 +88,22 @@ type nestingRow struct { why string } -// nestingRows is the depth-3 matrix. Each row exists for a reason the others do not cover, so a row -// that stops being interesting should be removed rather than quietly kept. -func nestingRows() []nestingRow { - return []nestingRow{ - { - name: "tar.gz/zip/jar", - families: []archiveFamily{familyTarGz, familyZip, familyJar}, - why: "the headline case: compressed tar outermost, jar innermost", - }, - { - name: "zip/tar.gz/jar", - families: []archiveFamily{familyZip, familyTarGz, familyJar}, - why: "compressed tar in the middle, where the stream-only format is not the entry point", - }, - { - name: "jar/tar/zip", - families: []archiveFamily{familyJar, familyTar, familyZip}, - why: "jar as a container rather than a payload", - }, - { - name: "tgz/tar/jar", - families: []archiveFamily{familyTgz, familyTar, familyJar}, - why: "the .tgz alias resolving at a non-outermost boundary", - }, - { - name: "tar/jar/tar.gz", - families: []archiveFamily{familyTar, familyJar, familyTarGz}, - why: "compressed tar as the innermost, which nothing else covers", - }, - } -} - // nestingVisit is one filesystem the archive cataloger's sub-pipeline was run against. type nestingVisit struct { virtualPath string depth int - // overflow names the archives whose content was on disk at this moment, sampled only when the + // overflowed names the archives whose content was on disk at this moment, sampled only when the // probe was given a temp dir to look in. overflowed []string - // diskBytes is the scan's total draw on the disk budget at this moment, read from the limiter on - // the context (Limiter.InUse): the archives' own overflow bytes plus the blob each archive's - // entries were written into plus the index records that overflowed to disk. It is what the disk - // limit is measured against, which is more than the bytes in the work directories once an index - // record is charged to the disk budget. + // diskBytes is the scan's draw on the disk budget at this moment, read from the limiter on the + // context (Limiter.InUse): spilled archive content, entry blobs, and index records charged to disk. + // It exceeds the bytes in the work directories, since index records are charged too. diskBytes int64 } -// nestingProbe is a stub cataloger that records one visit per filesystem the archive cataloger runs -// its sub-pipeline against, reading the chain off the traversal on the context. It is how a case -// asserts WHICH archives were cataloged, rather than inferring it from what the SBOM happens to -// contain. +// nestingProbe is a stub cataloger recording one visit per filesystem the archive cataloger runs its +// sub-pipeline against, reading the chain off the traversal on the context. It lets a case assert +// which archives were cataloged rather than inferring it from the SBOM. type nestingProbe struct { // tempDir, when set, is where extraction work directories live, so each visit can record which // archives were overflowed to disk while they were all still held. @@ -264,236 +113,13 @@ type nestingProbe struct { visits []nestingVisit } -func (p *nestingProbe) Name() string { return "nesting-probe" } - -func (p *nestingProbe) Catalog(ctx context.Context, _ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { - trav := archive.TraversalFromContext(ctx) - visit := nestingVisit{} - if trav != nil { - visit.virtualPath = trav.VirtualPath - visit.depth = trav.Depth - } - if lim := archive.LimiterFromContext(ctx); lim != nil { - _, visit.diskBytes = lim.InUse() - } - if p.tempDir != "" { - visit.overflowed = overflowArchives(p.tempDir) - } - - p.mu.Lock() - defer p.mu.Unlock() - p.visits = append(p.visits, visit) - return nil, nil, nil -} - -// archiveFileSystemIDs is the chain of every archive whose contents were cataloged, in visit order, -// with the scan root itself dropped. -func (p *nestingProbe) archiveFileSystemIDs() []string { - p.mu.Lock() - defer p.mu.Unlock() - var out []string - for _, v := range p.visits { - if v.virtualPath == "" { - continue - } - out = append(out, v.virtualPath) - } - return out -} - -// deepestVisit is the visit at the innermost level reached, which is the only moment at which every -// level of a chain is still held and its routing therefore observable. It is returned whole rather -// than just its overflow sample so a case can assert it got as deep as it meant to - an empty overflow -// sample from a visit that never left the scan root would otherwise read as "nothing overflowed". -// visitAtDepth is the visit at the given nesting depth, for a case that wants to measure what a run -// held while a particular level was being cataloged. -func (p *nestingProbe) visitAtDepth(depth int) nestingVisit { - p.mu.Lock() - defer p.mu.Unlock() - for _, v := range p.visits { - if v.depth == depth { - return v - } - } - return nestingVisit{} -} - -func (p *nestingProbe) deepestVisit() nestingVisit { - p.mu.Lock() - defer p.mu.Unlock() - var best nestingVisit - for _, v := range p.visits { - if v.depth > best.depth { - best = v - } - } - return best -} - -// overflowArchives names the archive files sitting in the live extraction work directories under -// tempDir. Content held in memory writes nothing there; content overflowed to disk is written beside -// the archive's logical root under the archive's own name. That makes the routing observable from -// outside without a hook into the limiter, and it is only meaningful while the archives are still -// held - so it is sampled from inside the sub-pipeline rather than after the scan. -// -// Whatever file the resolver in use stores an archive's entries in is skipped: it says nothing about -// where an archive's OWN bytes were routed, which is what this reports. Its bytes are counted by -// overflowBytes instead, which is about how much is on disk rather than about which archive is where. -func overflowArchives(tempDir string) []string { - var out []string - forEachHeldFile(tempDir, func(name string, _ os.FileInfo) { - if isEntryStorageFile(name) { - return - } - out = append(out, name) - }) - sort.Strings(out) - return out -} - // entryStorageFiles are the basenames internal/archive gives the file one archive's entries are stored -// in - a tar where the resolver indexes by seek offset, a blob where it indexes the entries -// themselves. Named here rather than reached for, since the probe reads a directory the implementation -// owns, and listed rather than hard-coded to one because which of them appears is a property of the -// resolver under test and not of the routing this probe is about. +// in. Which appears depends on the resolver under test, not on the routing this probe reports. var entryStorageFiles = []string{"contents.tar", "contents.blob"} -func isEntryStorageFile(name string) bool { - for _, storage := range entryStorageFiles { - if name == storage { - return true - } - } - return false -} - -// forEachHeldFile visits every file an archive work directory is holding, wherever under tempDir -// those directories were created. The tree is walked rather than read one level deep because an -// archive's work directory is a child of the scan's own temp root (internal/tmpdir), which is itself -// a directory of a name this probe does not choose. -func forEachHeldFile(tempDir string, visit func(name string, info os.FileInfo)) { - _ = filepath.WalkDir(tempDir, func(path string, d fs.DirEntry, err error) error { - if err != nil || !d.IsDir() || !strings.HasPrefix(d.Name(), "syft-archive-") { - return nil //nolint:nilerr // an unreadable directory holds nothing this probe can report - } - held, err := os.ReadDir(path) - if err != nil { - return filepath.SkipDir - } - for _, entry := range held { - if entry.IsDir() { - continue - } - info, err := entry.Info() - if err != nil { - continue - } - visit(entry.Name(), info) - } - return filepath.SkipDir - }) -} - -// isolatedTempDir points archive extraction at a directory of this test's own, so an overflow sample -// sees only this scan's work directories and not those of any other package's tests running beside -// it. -func isolatedTempDir(t *testing.T) string { - t.Helper() - dir := t.TempDir() - t.Setenv("TMPDIR", dir) - require.Equal(t, dir, os.TempDir(), "extraction must be writing where the probe is looking") - return dir -} - -// nestingScanConfig selects the java catalogers - so a jar is met as a jar wherever it sits in the -// chain - and always runs the given catalogers, whatever the selection. The java selection is by tag -// rather than the whole default set because the default set includes the RPM cataloger, which needs -// a sqlite driver this test binary does not register. -func nestingScanConfig(depth int, catalogers ...pkg.Cataloger) *CreateSBOMConfig { - cfg := DefaultCreateSBOMConfig(). - WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")) - for _, c := range catalogers { - cfg = cfg.WithCatalogers(pkgcataloging.NewAlwaysEnabledCatalogerReference(c)) - } - return cfg.WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth)) -} - -// withFileCataloging turns on file cataloging, which the CONTAINS chain needs: the archive-to-file -// edges key on the coordinates the FILE catalogers recorded, so without them an inner archive has no -// coordinate to hang an edge on and the chain has a hole. -func withFileCataloging(cfg *CreateSBOMConfig) *CreateSBOMConfig { - return cfg.WithFilesConfig(filecataloging.Config{Selection: file.AllFilesSelection}) -} - -// leafLocations returns the locations of the leaf marker package. -func leafLocations(s *sbom.SBOM) []file.Location { - var out []file.Location - for _, p := range s.Artifacts.Packages.Sorted() { - if p.Name != "marker-pkg" { - continue - } - out = append(out, p.Locations.ToSlice()...) - } - return out -} - -// javaVirtualPaths maps each java package's name to its virtual path. -func javaVirtualPaths(s *sbom.SBOM) map[string]string { - out := map[string]string{} - for _, p := range s.Artifacts.Packages.Sorted() { - if metadata, ok := p.Metadata.(pkg.JavaArchive); ok { - out[p.Name] = metadata.VirtualPath - } - } - return out -} - -func packageCount(s *sbom.SBOM, name string) int { - var n int - for _, p := range s.Artifacts.Packages.Sorted() { - if p.Name == name { - n++ - } - } - return n -} - -// scanWithinTimeout runs the scan on its own goroutine and fails if it has not finished in time. -// -// It exists for the depth-3 limit cases. The walk descends before it unwinds, so the outer and -// middle archives are still holding their content when the inner one is refused: an implementation -// that waited for capacity would deadlock there rather than fail, and a test asserting only that the -// result is right would HANG instead of reporting. The timeout is what turns that into a failure. -func scanWithinTimeout(t *testing.T, dir string, cfg *CreateSBOMConfig, timeout time.Duration) *sbom.SBOM { - t.Helper() - src, err := directorysource.New(directorysource.Config{Path: dir}) - require.NoError(t, err) - t.Cleanup(func() { _ = src.Close() }) - - type outcome struct { - s *sbom.SBOM - err error - } - done := make(chan outcome, 1) - go func() { - s, err := cfg.Create(context.Background(), src) - done <- outcome{s: s, err: err} - }() - - select { - case got := <-done: - require.NoError(t, got.err, "reaching a bound must not fail the scan") - return got.s - case <-time.After(timeout): - t.Fatalf("the scan did not finish within %s: reaching a limit must skip the archive that "+ - "reached it, never wait for capacity that nothing in the chain will release", timeout) - return nil - } -} - func Test_mixedFamilyNesting_leafIsCatalogedOnceWithTheFullChain(t *testing.T) { - // the requirement is that family order does not change the result: the same package, addressed - // by a chain naming its own containers in its own order + // family order must not change the result: the same package, addressed by a chain naming its own + // containers in its own order for _, row := range nestingRows() { t.Run(row.name, func(t *testing.T) { scanDir := t.TempDir() @@ -519,8 +145,8 @@ func Test_mixedFamilyNesting_leafIsCatalogedOnceWithTheFullChain(t *testing.T) { } func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) { - // the edges are what makes the chain navigable in the SBOM, and they key on coordinates the file - // catalogers recorded, so file cataloging has to be on for the inner archives to have any + // the edges make the chain navigable in the SBOM and key on coordinates the file catalogers + // recorded, so file cataloging has to be on for the inner archives to have any for _, row := range nestingRows() { t.Run(row.name, func(t *testing.T) { scanDir := t.TempDir() @@ -529,8 +155,8 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) { cfg := withFileCataloging(nestingScanConfig(3, markerCataloger{})) s := scanDirWith(t, scanDir, cfg) - // the coordinates of each archive AS SEEN IN ITS PARENT: the outermost sits in the - // scanned directory, so it carries no filesystem id of its own + // each archive's coordinates as seen in its parent; the outermost sits in the scanned + // directory, so it carries no filesystem id of its own archiveCoords := make([]file.Coordinates, len(row.families)) for i := range row.families { var parentFSID string @@ -574,10 +200,6 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) { } } -func coordKey(c file.Coordinates) string { - return c.ArchivePath + "|" + c.RealPath -} - func Test_mixedFamilyNesting_javaLeafVirtualPathIsTheColonJoinedChain(t *testing.T) { // these colon-delimited strings are in already-published SBOMs, so a jar reached through three // levels of mixed families must still be reported under the chain of all three @@ -591,7 +213,7 @@ func Test_mixedFamilyNesting_javaLeafVirtualPathIsTheColonJoinedChain(t *testing s := scanDirWith(t, scanDir, nestingScanConfig(3, markerCataloger{})) - // the innermost archive IS the jar, so its own package is the java leaf and the chain of + // the innermost archive is the jar, so its own package is the java leaf and the chain of // all three archives is exactly its virtual path leaf := fmt.Sprintf("level%d", len(row.families)-1) assert.Equal(t, 1, packageCount(s, leaf), "the leaf jar must be cataloged exactly once") @@ -601,9 +223,9 @@ func Test_mixedFamilyNesting_javaLeafVirtualPathIsTheColonJoinedChain(t *testing } func Test_mixedFamilyNesting_jarIsExercisedAsAContainer(t *testing.T) { - // the row that separates the two mechanisms that meet a jar: the java cataloger catalogs it as a - // package, and the generic archive task descends into it. A jar in the middle of a chain has to - // do both, exactly once each - the failure mode is a duplicate, not an error. + // two mechanisms meet a jar: the java cataloger catalogs it as a package, the archive task descends + // into it. A jar mid-chain must do both, exactly once each; the failure mode is a duplicate, not an + // error. for _, row := range nestingRows() { var containerLevels []int for i, family := range row.families { @@ -656,7 +278,7 @@ func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) { assert.Zero(t, packageCount(s, "marker-pkg"), "the leaf is one level past the bound, so its package must be absent") - // the innermost archive is still cataloged as a FILE, inside the middle archive + // the innermost archive is still cataloged as a file, inside the middle archive innermost := file.Coordinates{ RealPath: nested.paths[2], ArchivePath: nested.fileSystemIDs[1], @@ -668,8 +290,8 @@ func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) { } func Test_mixedFamilyNesting_defaultBoundsCatalogEveryRow(t *testing.T) { - // what a real user gets. The bounds are measured against a corpus rather than argued, so this - // also pins that a three-level chain of any family mix sits well inside them. + // the default bounds, pinned along with the fact that a three-level chain of any family mix sits + // well inside them bounds := cataloging.DefaultArchiveSearchConfig() require.Equal(t, int64(2*1024*1024*1024), bounds.MaxMemoryBytes) require.Equal(t, int64(100*1024*1024*1024), bounds.MaxDiskBytes) @@ -691,26 +313,9 @@ func Test_mixedFamilyNesting_defaultBoundsCatalogEveryRow(t *testing.T) { } } -// gradedChain is the headline row with each level padded so the three archives have clearly -// different sizes: a container is never smaller than what it holds, so a bound can only be placed -// between two levels if the levels are graded on purpose. -func gradedChain() nestPlan { - return nestPlan{ - families: []archiveFamily{familyTarGz, familyZip, familyJar}, - leaf: markerLeaf(), - extra: map[int]map[string][]byte{ - 0: {"pad0.bin": incompressibleBytes(24 * 1024)}, - 1: {"pad1.bin": incompressibleBytes(24 * 1024)}, - 2: {"pad2.bin": incompressibleBytes(8 * 1024)}, - }, - } -} - func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing.T) { - // memory pressure degrades to disk usage: there is somewhere further to put the bytes, so an - // archive that will not fit in memory overflows instead of being refused. Where the boundary falls - // is decided entirely by the memory limit - there is no separate configured size at which content - // overflows. + // memory pressure degrades to disk: an archive that will not fit spills rather than being refused, + // and the memory limit alone decides where that boundary falls scanDir := t.TempDir() tempDir := isolatedTempDir(t) nested := writeNestedArchive(t, scanDir, gradedChain()) @@ -729,12 +334,10 @@ func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing }) t.Run("a memory limit the outermost archive does not fit in", func(t *testing.T) { - // only the outermost archive is routed by the memory limit at all: it is the one whose bytes - // have to be copied out of the scanned filesystem. Every archive below it is already a region - // of its parent's tar, on disk and charged to that parent, so it is read where it lies - it - // neither takes memory nor writes a second copy. What this case pins is the half that is still - // the memory limit's: an archive too large for it overflows instead of failing, and the outcome - // is identical to a run with room for it. + // only the outermost archive is routed by the memory limit, being the one whose bytes are copied + // out of the scanned filesystem. Every archive below it is already an entry of its parent's store + // and is read where it lies, taking neither memory nor a second copy. This pins the remaining + // half: too large for memory means spilled, not failed, with the same outcome as a run with room. probe := &nestingProbe{tempDir: tempDir} cfg := nestingScanConfig(3, markerCataloger{}, probe) cfg = cfg.WithArchiveConfig(cfg.Archive. @@ -747,7 +350,7 @@ func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing require.Less(t, int64(nested.sizes[2]), int64(nested.sizes[0]), "the outermost archive must be the one that does not fit, or this case asserts nothing") assert.Equal(t, []string{"level0.tar.gz"}, deepest.overflowed, - "the archive that does not fit in memory overflows rather than failing, and the levels inside "+ + "the archive that does not fit in memory spills rather than failing, and the levels inside "+ "it are read where they lie rather than being copied anywhere") // and the outcome is identical to the default-bounds run @@ -759,18 +362,12 @@ func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing } func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) { - // This case replaces one that asserted the first-come consequence of having no per-archive memory - // cap: once the outermost archive had taken the whole memory limit, the archives nested inside it - // overflowed even though each was small enough to have fit had it arrived first. That property is - // gone, and not because the limit changed - because there is no longer anything to route. A nested - // archive is a region of its parent's tar, already on disk and already charged to the parent, and - // the tar entry reader is Read, Seek AND ReadAt - everything a zip's central directory needs. So it - // is handed to the archive format where it lies. Nothing is copied, so nothing competes for the - // memory limit, and the first-come question no longer has two archives to ask it about. + // a nested archive is an entry of its parent's store, already charged to that parent, and its reader + // supports Read, Seek and ReadAt - everything a zip's central directory needs - so it is handed to + // the archive format where it lies. Nothing is copied, so nothing competes for the memory limit. // - // What is asserted instead is that consequence directly, at both ends of the memory limit: whether - // the limit is generous or zero, the nested levels neither hold memory nor put a second copy of - // themselves on disk, and the whole chain is cataloged either way. + // Asserted at both ends of that limit: generous or zero, the nested levels neither hold memory nor + // put a second copy of themselves on disk, and the whole chain is cataloged either way. scanDir := t.TempDir() tempDir := isolatedTempDir(t) nested := writeNestedArchive(t, scanDir, gradedChain()) @@ -819,48 +416,30 @@ func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) { } } -// deepChain is the headline row with a single large, incompressible payload at the leaf and no -// padding anywhere else. A container is never smaller than what it holds, so with the payload -// dominating every level the three archives are all about the same size - which is what makes the -// disk limit arithmetic below robust: the slack is half an archive rather than a few hundred bytes. -func deepChain() nestPlan { - return nestPlan{ - families: []archiveFamily{familyTarGz, familyZip, familyJar}, - leaf: map[string][]byte{ - "nested/marker.txt": []byte("leaf"), - "payload.bin": incompressibleBytes(64 * 1024), - }, - } -} - func Test_mixedFamilyNesting_diskLimitSkipsTheLevelItCannotAdmit(t *testing.T) { - // the disk limit is terminal, and the walk descends before it unwinds: when the innermost level - // cannot be admitted, the outer and middle archives are still holding everything they took. So - // the level that would exceed it is skipped, the shallower levels keep what they found, and the - // scan finishes. An implementation that waited for capacity would deadlock instead - which is why - // every scan here is run under a timeout rather than merely checked for the right answer. + // the disk limit is terminal and the walk descends before it unwinds, so when the innermost level + // cannot be admitted the outer and middle archives still hold everything they took: that level is + // skipped, the shallower ones keep what they found, and the scan finishes. An implementation that + // waited for capacity would deadlock, hence the timeout. scanDir := t.TempDir() tempDir := isolatedTempDir(t) nested := writeNestedArchive(t, scanDir, deepChain()) - // a sibling of a different family, to show that reaching the bound is per archive: it is - // processed once the chain has been released, so it is cataloged in full + // a sibling of a different family, to show the bound is per archive: it is processed once the chain + // has been released, so it is cataloged in full require.NoError(t, os.WriteFile( filepath.Join(scanDir, "sibling.tgz"), buildTarGzBytes(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}), 0o644, )) - // The limit is MEASURED rather than computed: what the chain holds on disk at a given level is the - // outermost archive's own bytes plus every entry the levels above it wrote out, which is a fact - // about this corpus rather than arithmetic anyone should reproduce. So the chain is run once with - // no disk ceiling and the probe reports exactly what was on disk while two levels were held. + // the limit is measured rather than computed: what the chain holds on disk at a given level is a + // fact about this corpus, so the chain runs once with no disk ceiling and the probe reports what was + // on disk while two levels were held. // - // The real run then gets that figure with NO slack, which is what makes this a skip rather than a - // truncation. Entries are stored without framing, so any slack at all is enough to admit some of - // the innermost level's content - and an archive that placed some of its entries is truncated, a - // different outcome tested elsewhere. With none, the first byte of the first entry is refused and - // nothing of that level is placed at all. + // The real run gets that figure with no slack, making this a skip rather than a truncation. Entries + // are stored without framing, so any slack would admit some of the innermost level's content, and an + // archive that stored some entries is truncated instead - covered elsewhere. measuring := &nestingProbe{tempDir: tempDir} measureCfg := nestingScanConfig(3, markerCataloger{}, measuring) measureCfg = measureCfg.WithArchiveConfig(measureCfg.Archive.WithMaxMemoryBytes(0).WithMaxDiskBytes(-1)) @@ -873,7 +452,7 @@ func Test_mixedFamilyNesting_diskLimitSkipsTheLevelItCannotAdmit(t *testing.T) { probe := &nestingProbe{} cfg := nestingScanConfig(3, markerCataloger{}, probe) cfg = cfg.WithArchiveConfig(cfg.Archive. - WithMaxMemoryBytes(0). // every archive overflows, so its own bytes are charged to disk + WithMaxMemoryBytes(0). // every archive spills, so its own bytes are charged to disk WithMaxDiskBytes(diskLimit)) logs := captureLogs(t) @@ -884,8 +463,7 @@ func Test_mixedFamilyNesting_diskLimitSkipsTheLevelItCannotAdmit(t *testing.T) { "the level that would exceed the limit is skipped, the shallower ones are still cataloged, "+ "and a sibling of another family is unaffected") - // the skip is attributed to the archive that triggered it: a log line naming only the bound says - // nothing about what was lost + // the skip names the archive that triggered it, not just the bound assert.True(t, logs.sawFields(map[string]any{ "archive": nested.paths[2], "limit": diskLimit, @@ -900,9 +478,8 @@ func Test_mixedFamilyNesting_diskLimitSkipsTheLevelItCannotAdmit(t *testing.T) { } func Test_mixedFamilyNesting_unboundedLimitsEnforceNothing(t *testing.T) { - // a caller can opt out of one bound without opting out of the other, and opting out of both must - // still find the deepest leaf. Memory and disk opt out with a negative value, since zero means - // "none of that resource". + // a caller can opt out of one bound without the other, and opting out of both must still find the + // deepest leaf. Both opt out with a negative value, since zero means "none of that resource". for _, row := range nestingRows() { t.Run(row.name, func(t *testing.T) { scanDir := t.TempDir() @@ -925,9 +502,9 @@ func Test_mixedFamilyNesting_unboundedLimitsEnforceNothing(t *testing.T) { } func Test_mixedFamilyNesting_bothLimitsZeroSkipsEveryArchive(t *testing.T) { - // the degenerate but well-defined configuration where nothing is admitted anywhere: unlike - // negative (unbounded), zero on both in-use limits skips every archive, and the scan still - // succeeds rather than failing or finding anything inside the archives + // the degenerate but well-defined configuration where nothing is admitted anywhere: unlike negative + // (unbounded), zero on both limits skips every archive, and the scan still succeeds rather than + // failing or finding anything inside the archives for _, row := range nestingRows() { t.Run(row.name, func(t *testing.T) { scanDir := t.TempDir() @@ -947,10 +524,9 @@ func Test_mixedFamilyNesting_bothLimitsZeroSkipsEveryArchive(t *testing.T) { } } -// The attribution in a skip is a log line and nothing else: reaching a bound must not fail the scan, -// so there is no return value that says which archive was refused. Asserting it therefore means -// capturing what was logged. cmd/syft/internal/ui/log_writer_test.go does the same thing with the -// same log.Set/log.Get pair; this adds the field-carrying half, which is where the attribution lives. +// A skip is reported by a log line alone - reaching a bound must not fail the scan - so asserting the +// attribution means capturing what was logged. cmd/syft/internal/ui/log_writer_test.go uses the same +// log.Set/log.Get pair; this adds the field-carrying half, where the attribution lives. // logCapture collects the field-carrying log events emitted while it is installed. type logCapture struct { @@ -958,8 +534,7 @@ type logCapture struct { collected []map[string]any } -// captureLogs installs a capturing logger for the duration of the test and restores the previous one -// afterwards. +// captureLogs installs a capturing logger for the test and restores the previous one afterwards. func captureLogs(t *testing.T) *logCapture { t.Helper() capture := &logCapture{} @@ -981,8 +556,8 @@ func (c *logCapture) events() []map[string]any { return append([]map[string]any(nil), c.collected...) } -// eventsWith is the captured events carrying the given field, which is what a failure wants to show: -// a scan logs thousands of lines and the whole set is unreadable in a test failure. +// eventsWith is the captured events carrying the given field. A scan logs thousands of lines, so a +// failure message shows this subset rather than all of them. func (c *logCapture) eventsWith(field string) []map[string]any { var out []map[string]any for _, event := range c.events() { @@ -993,8 +568,8 @@ func (c *logCapture) eventsWith(field string) []map[string]any { return out } -// sawFields reports whether any captured event carried every one of the given fields with exactly -// the given values. +// sawFields reports whether any captured event carried every given field with exactly the given +// value. func (c *logCapture) sawFields(want map[string]any) bool { for _, got := range c.events() { matched := true @@ -1011,9 +586,8 @@ func (c *logCapture) sawFields(want map[string]any) bool { return false } -// capturingLogger discards every message but records the fields attached to the ones that carry any. -// Nested is deliberately left to the embedded discard logger: nothing under test uses it, and -// merging nested fields into an event would be untested machinery in a test helper. +// capturingLogger discards every message but records the fields attached to those that carry any. +// Nested is left to the embedded discard logger, since nothing under test uses it. type capturingLogger struct { logger.Logger capture *logCapture @@ -1063,7 +637,7 @@ func (l *capturingMessageLogger) Tracef(format string, args ...any) { l.emit(fmt.Sprintf(format, args...)) } -// fieldMap turns the alternating key/value form log.WithFields takes into a map, and folds in any +// fieldMap turns the alternating key/value form log.WithFields takes into a map, folding in any // logger.Fields passed as a whole. func fieldMap(fields []any) map[string]any { out := map[string]any{} @@ -1083,3 +657,389 @@ func fieldMap(fields []any) map[string]any { } return out } + +// pack builds one archive of this family holding the given entries. +func (f archiveFamily) pack(t *testing.T, base string, entries map[string][]byte) []byte { + t.Helper() + switch f { + case familyZip: + return buildZipBytesRaw(t, entries) + case familyJar: + // a manifest makes it a real jar to the java cataloger at any level; the package java reports is + // named after the file, not the manifest title + return jarBytes(t, base, "1.0", entries) + case familyTar: + return buildTarBytes(t, entries) + case familyTarGz, familyTgz: + return buildTarGzBytes(t, entries) + } + t.Fatalf("unknown archive family %q", f) + return nil +} + +// leafFileSystemID is the chain the innermost archive's contents carry: every archive, outer to +// inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check. +func (n nestedArchive) leafFileSystemID() string { + return n.fileSystemIDs[len(n.fileSystemIDs)-1] +} + +// buildNestedArchive builds one archive per family given, each holding the next, with leaf inside +// the innermost. +func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive { + t.Helper() + require.NotEmpty(t, plan.families, "a chain needs at least one level") + + out := nestedArchive{ + sizes: make([]int, len(plan.families)), + paths: make([]string, len(plan.families)), + fileSystemIDs: make([]string, len(plan.families)), + virtualPaths: make([]string, len(plan.families)), + } + + // the level index goes in every name and path segment, so no two levels share a path and a chain + // assertion cannot pass on a value composed from the wrong level + for i, family := range plan.families { + base := fmt.Sprintf("level%d", i) + if i == 0 { + out.paths[i] = base + string(family) + continue + } + out.paths[i] = fmt.Sprintf("nest%d/%s%s", i, base, family) + } + + out.name = out.paths[0] + for i := range plan.families { + if i == 0 { + out.fileSystemIDs[i] = out.paths[0] + out.virtualPaths[i] = out.paths[0] + continue + } + out.fileSystemIDs[i] = out.fileSystemIDs[i-1] + ":" + out.paths[i] + out.virtualPaths[i] = out.virtualPaths[i-1] + ":" + out.paths[i] + } + + // built innermost first, because each level's bytes are an entry of the level above it + var packed []byte + for i := len(plan.families) - 1; i >= 0; i-- { + entries := map[string][]byte{} + if i == len(plan.families)-1 { + for name, body := range plan.leaf { + entries[name] = body + } + } else { + entries[out.paths[i+1]] = packed + } + for name, body := range plan.extra[i] { + entries[name] = body + } + packed = plan.families[i].pack(t, fmt.Sprintf("level%d", i), entries) + out.sizes[i] = len(packed) + } + out.bytes = packed + + return out +} + +// writeNestedArchive writes the chain's outermost archive into dir and returns the chain. +func writeNestedArchive(t *testing.T, dir string, plan nestPlan) nestedArchive { + t.Helper() + nested := buildNestedArchive(t, plan) + require.NoError(t, os.WriteFile(filepath.Join(dir, nested.name), nested.bytes, 0o644)) + return nested +} + +// markerLeaf is the family-agnostic leaf payload: one file no archive format has an opinion about, +// found by markerCataloger. It keeps the rows comparable - the same package must come out whichever +// families the chain is made of. +func markerLeaf() map[string][]byte { + return map[string][]byte{"nested/marker.txt": []byte("leaf")} +} + +// incompressibleBytes returns n bytes deflate and gzip cannot shrink, so per-level sizes are +// predictable enough to place a bound between two of them. +func incompressibleBytes(n int) []byte { + b := make([]byte, n) + x := uint32(0x9e3779b9) + for i := range b { + x = x*1664525 + 1013904223 + b[i] = byte(x >> 24) + } + return b +} + +// nestingRows is the depth-3 matrix. Each row covers a case the others do not. +func nestingRows() []nestingRow { + return []nestingRow{ + { + name: "tar.gz/zip/jar", + families: []archiveFamily{familyTarGz, familyZip, familyJar}, + why: "the headline case: compressed tar outermost, jar innermost", + }, + { + name: "zip/tar.gz/jar", + families: []archiveFamily{familyZip, familyTarGz, familyJar}, + why: "compressed tar in the middle, where the stream-only format is not the entry point", + }, + { + name: "jar/tar/zip", + families: []archiveFamily{familyJar, familyTar, familyZip}, + why: "jar as a container rather than a payload", + }, + { + name: "tgz/tar/jar", + families: []archiveFamily{familyTgz, familyTar, familyJar}, + why: "the .tgz alias resolving at a non-outermost boundary", + }, + { + name: "tar/jar/tar.gz", + families: []archiveFamily{familyTar, familyJar, familyTarGz}, + why: "compressed tar as the innermost, which nothing else covers", + }, + } +} + +func (p *nestingProbe) Name() string { return "nesting-probe" } + +func (p *nestingProbe) Catalog(ctx context.Context, _ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + trav := archive.TraversalFromContext(ctx) + visit := nestingVisit{} + if trav != nil { + visit.virtualPath = trav.VirtualPath + visit.depth = strings.Count(trav.VirtualPath, ":") + 1 + } + if lim := archive.LimiterFromContext(ctx); lim != nil { + _, visit.diskBytes = lim.InUse() + } + if p.tempDir != "" { + visit.overflowed = overflowArchives(p.tempDir) + } + + p.mu.Lock() + defer p.mu.Unlock() + p.visits = append(p.visits, visit) + return nil, nil, nil +} + +// archiveFileSystemIDs is the chain of every archive whose contents were cataloged, in visit order, +// with the scan root dropped. +func (p *nestingProbe) archiveFileSystemIDs() []string { + p.mu.Lock() + defer p.mu.Unlock() + var out []string + for _, v := range p.visits { + if v.virtualPath == "" { + continue + } + out = append(out, v.virtualPath) + } + return out +} + +// visitAtDepth is the visit at the given nesting depth, for a case measuring what a run held while a +// particular level was being cataloged. +func (p *nestingProbe) visitAtDepth(depth int) nestingVisit { + p.mu.Lock() + defer p.mu.Unlock() + for _, v := range p.visits { + if v.depth == depth { + return v + } + } + return nestingVisit{} +} + +// deepestVisit is the visit at the innermost level reached, the only moment at which every level of a +// chain is still held and its routing observable. It returns the whole visit so a case can also assert +// it got as deep as it meant to: an empty overflow sample from a visit that never left the scan root +// would otherwise read as "nothing overflowed". +func (p *nestingProbe) deepestVisit() nestingVisit { + p.mu.Lock() + defer p.mu.Unlock() + var best nestingVisit + for _, v := range p.visits { + if v.depth > best.depth { + best = v + } + } + return best +} + +// overflowArchives names the archive files in the live extraction work directories under tempDir. +// Content held in memory writes nothing there; content spilled to disk is written beside the archive's +// logical root under the archive's own name, which makes the routing observable without a hook into +// the limiter. Meaningful only while the archives are still held, so it is sampled from inside the +// sub-pipeline rather than after the scan. +// +// The file an archive's entries are stored in is skipped: it says nothing about where the archive's +// own bytes were routed. +func overflowArchives(tempDir string) []string { + var out []string + forEachHeldFile(tempDir, func(name string, _ os.FileInfo) { + if isEntryStorageFile(name) { + return + } + out = append(out, name) + }) + sort.Strings(out) + return out +} + +func isEntryStorageFile(name string) bool { + for _, storage := range entryStorageFiles { + if name == storage { + return true + } + } + return false +} + +// forEachHeldFile visits every file an archive work directory is holding, wherever under tempDir those +// directories were created. It walks the tree rather than reading one level, since an archive's work +// directory sits under the scan's own temp root (internal/tmpdir), whose name this probe does not +// choose. +func forEachHeldFile(tempDir string, visit func(name string, info os.FileInfo)) { + _ = filepath.WalkDir(tempDir, func(path string, d fs.DirEntry, err error) error { + if err != nil || !d.IsDir() || !strings.HasPrefix(d.Name(), "syft-archive-") { + return nil //nolint:nilerr // an unreadable directory holds nothing this probe can report + } + held, err := os.ReadDir(path) + if err != nil { + return filepath.SkipDir + } + for _, entry := range held { + if entry.IsDir() { + continue + } + info, err := entry.Info() + if err != nil { + continue + } + visit(entry.Name(), info) + } + return filepath.SkipDir + }) +} + +// isolatedTempDir points archive extraction at a directory of this test's own, so an overflow sample +// sees only this scan's work directories. +func isolatedTempDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + t.Setenv("TMPDIR", dir) + require.Equal(t, dir, os.TempDir(), "extraction must be writing where the probe is looking") + return dir +} + +// nestingScanConfig selects the java catalogers - so a jar is met as a jar wherever it sits - and +// always runs the given catalogers. Selected by tag rather than the whole default set, which includes +// the RPM cataloger and its sqlite driver this test binary does not register. +func nestingScanConfig(depth int, catalogers ...pkg.Cataloger) *CreateSBOMConfig { + cfg := DefaultCreateSBOMConfig(). + WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")) + for _, c := range catalogers { + cfg = cfg.WithCatalogers(pkgcataloging.NewAlwaysEnabledCatalogerReference(c)) + } + return cfg.WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth)) +} + +// withFileCataloging turns on file cataloging, which the CONTAINS chain needs: archive-to-file edges +// key on coordinates the file catalogers recorded, so without them an inner archive has no coordinate +// to hang an edge on. +func withFileCataloging(cfg *CreateSBOMConfig) *CreateSBOMConfig { + return cfg.WithFilesConfig(filecataloging.Config{Selection: file.AllFilesSelection}) +} + +func leafLocations(s *sbom.SBOM) []file.Location { + var out []file.Location + for _, p := range s.Artifacts.Packages.Sorted() { + if p.Name != "marker-pkg" { + continue + } + out = append(out, p.Locations.ToSlice()...) + } + return out +} + +func javaVirtualPaths(s *sbom.SBOM) map[string]string { + out := map[string]string{} + for _, p := range s.Artifacts.Packages.Sorted() { + if metadata, ok := p.Metadata.(pkg.JavaArchive); ok { + out[p.Name] = metadata.VirtualPath + } + } + return out +} + +func packageCount(s *sbom.SBOM, name string) int { + var n int + for _, p := range s.Artifacts.Packages.Sorted() { + if p.Name == name { + n++ + } + } + return n +} + +// scanWithinTimeout runs the scan on its own goroutine and fails if it has not finished in time. +// +// The depth-3 limit cases need it: the walk descends before it unwinds, so the outer and middle +// archives still hold their content when the inner one is refused. An implementation that waited for +// capacity would deadlock, and a test asserting only the result would hang rather than fail. +func scanWithinTimeout(t *testing.T, dir string, cfg *CreateSBOMConfig, timeout time.Duration) *sbom.SBOM { + t.Helper() + src, err := directorysource.New(directorysource.Config{Path: dir}) + require.NoError(t, err) + t.Cleanup(func() { _ = src.Close() }) + + type outcome struct { + s *sbom.SBOM + err error + } + done := make(chan outcome, 1) + go func() { + s, err := cfg.Create(context.Background(), src) + done <- outcome{s: s, err: err} + }() + + select { + case got := <-done: + require.NoError(t, got.err, "reaching a bound must not fail the scan") + return got.s + case <-time.After(timeout): + t.Fatalf("the scan did not finish within %s: reaching a limit must skip the archive that "+ + "reached it, never wait for capacity that nothing in the chain will release", timeout) + return nil + } +} + +func coordKey(c file.Coordinates) string { + return c.ArchivePath + "|" + c.RealPath +} + +// gradedChain is the headline row with each level padded so the three archives have clearly different +// sizes: a container is never smaller than what it holds, so a bound can only be placed between two +// levels if they are graded on purpose. +func gradedChain() nestPlan { + return nestPlan{ + families: []archiveFamily{familyTarGz, familyZip, familyJar}, + leaf: markerLeaf(), + extra: map[int]map[string][]byte{ + 0: {"pad0.bin": incompressibleBytes(24 * 1024)}, + 1: {"pad1.bin": incompressibleBytes(24 * 1024)}, + 2: {"pad2.bin": incompressibleBytes(8 * 1024)}, + }, + } +} + +// deepChain is the headline row with one large, incompressible payload at the leaf and no other +// padding. The payload dominates every level, so the three archives are about the same size, giving +// the disk limit arithmetic below slack of half an archive rather than a few hundred bytes. +func deepChain() nestPlan { + return nestPlan{ + families: []archiveFamily{familyTarGz, familyZip, familyJar}, + leaf: map[string][]byte{ + "nested/marker.txt": []byte("leaf"), + "payload.bin": incompressibleBytes(64 * 1024), + }, + } +} diff --git a/syft/cataloging/archive_exclusions.go b/syft/cataloging/archive_exclusions.go index 83e0e71be..205d98900 100644 --- a/syft/cataloging/archive_exclusions.go +++ b/syft/cataloging/archive_exclusions.go @@ -4,31 +4,22 @@ import "strings" // anyDepthPrefix marks the one pattern shape that reaches inside an archive. // -// A directory source accepts exactly three shapes and rejects everything else outright -// (directorysource.GetDirectoryExclusionFunctions), and only this one is depth-independent: +// A directory source accepts three shapes (directorysource.GetDirectoryExclusionFunctions): // -// ./x names a path at the scan root -// */x names a path exactly one level below the scan root - `*` matches one segment and no -// separator, so it matches a/x but neither x nor a/b/x -// **/x names x at any depth, the scan root included +// ./x a path at the scan root +// */x a path exactly one level below the scan root - `*` matches one segment and no separator, +// so it matches a/x but neither x nor a/b/x +// **/x x at any depth, the scan root included // -// The first two are measured from the scan root and describe a layout that exists only there. The -// third means the same thing wherever it is rooted, and that is what makes re-rooting it at an -// archive's extraction directory a faithful reading of it rather than a coincidence: a user who -// wrote `*/vendor` was pointing at a directory they can see, while one who wrote `**/*.rpm` was -// saying something about every tree. -// -// Image sources do not validate their patterns against these three shapes at all - that check lives -// only in the directory-source helper - but the rule needs no adjustment for them, since a pattern -// either commits to a depth or does not. +// Only the third is depth-independent, so only it means the same thing re-rooted at an archive's +// extraction directory. The rule holds for image sources too, which do not validate these shapes. const anyDepthPrefix = "**/" // ArchiveExclusionPatterns returns the subset of the scan's exclusion patterns that reach inside an // archive, normalized for matching. A pattern's own shape decides: see anyDepthPrefix. // -// The trailing slash is trimmed for the same reason the directory source trims it - it signals a -// directory to a reader but is discarded by doublestar.Match, so a pattern keeping it silently -// matches nothing (issue #4839). +// The trailing slash is trimmed as the directory source trims it: it reads as "a directory" but +// doublestar.Match discards it, so a pattern keeping it matches nothing (issue #4839). func ArchiveExclusionPatterns(exclusions []string) []string { var out []string for _, exclusion := range exclusions { diff --git a/syft/cataloging/archive_exclusions_test.go b/syft/cataloging/archive_exclusions_test.go index b23dcba80..cb1ad44f4 100644 --- a/syft/cataloging/archive_exclusions_test.go +++ b/syft/cataloging/archive_exclusions_test.go @@ -7,11 +7,9 @@ import ( ) func Test_ArchiveExclusionPatterns(t *testing.T) { - // the pattern's own shape is the whole rule, so every accepted shape is stated here rather than - // only the two obvious ones. `*/x` is the interesting one: it names a definite depth measured - // from the scan root, so it describes a layout that exists only there and is anchored, exactly - // like `./x`. Only `**/x` means the same thing wherever it is rooted, which is what makes - // re-rooting it at an extraction directory a reading of it rather than a coincidence. + // the pattern's shape is the whole rule, so every accepted shape is listed. `*/x` names a definite + // depth from the scan root, anchoring it there exactly as `./x` is; only `**/x` means the same thing + // wherever it is rooted. tests := []struct { name string exclusions []string @@ -45,7 +43,7 @@ func Test_ArchiveExclusionPatterns(t *testing.T) { { name: "a trailing slash is trimmed", // it reads as "a directory" but doublestar.Match discards it, so a pattern keeping it - // matches nothing at all (issue #4839) + // matches nothing (issue #4839) exclusions: []string{"**/vendor/"}, want: []string{"**/vendor"}, }, diff --git a/syft/cataloging/archive_search.go b/syft/cataloging/archive_search.go index bd07d4ee5..4dc274ba6 100644 --- a/syft/cataloging/archive_search.go +++ b/syft/cataloging/archive_search.go @@ -3,30 +3,19 @@ package cataloging import "github.com/anchore/stereoscope/pkg/file" const ( - DefaultArchiveMaxDepth = 0 // disabled by default for backwards compatibility + DefaultArchiveMaxDepth = 0 // off by default, so old scans work the same way - // DefaultArchiveMaxMemoryBytes bounds the archive content held in memory at once. - // - // Both bounds are in-use gauges rather than cumulative totals: they measure what the scan is - // holding right now and fall as each archive is released, so what they cap is a scan's peak. - // That is what lets them be set well above the largest archive anyone is likely to meet - the - // number that matters is how much is held at the deepest point of a nesting chain, not the sum - // of everything the scan ever opened. - // - // Set generously on purpose. An archive's entries are held here entry by entry and moved out to - // the overflow blob the first time this bound says no, so the bound does not decide whether an - // archive can be cataloged - only whether it costs a write. Making room for the common case is - // therefore a straight speed win with a bounded cost, and the cost is bounded because what is - // held is released as soon as the archive is. + // DefaultArchiveMaxMemoryBytes bounds the archive content held in memory at once. Both bounds are + // in-use gauges: each measures what is held at a moment and falls as the scan releases an archive, + // so the peak they cap is the deepest point of a nesting chain, not the sum of everything opened. + // Set generously: it decides only whether an archive costs a write. Once refused, an archive's + // entries spill to the overflow blob and syft catalogs it either way. DefaultArchiveMaxMemoryBytes = 2 * file.GB - // DefaultArchiveMaxDiskBytes bounds the archive content on disk at once. - // - // Unlike the memory bound this one is terminal: there is nowhere further to overflow, so an - // archive whose first byte it refuses is skipped and its packages are absent from the SBOM. It - // is set high for that reason - it is a backstop against a pathological archive filling the - // disk, not a working budget - and, being an in-use gauge, it is released along with each - // archive rather than accumulating over a scan. + // DefaultArchiveMaxDiskBytes bounds the archive content on disk at once. This bound is terminal: + // spilled content has nowhere further to go, so syft skips an archive whose first byte it refuses, + // leaving that archive's packages out of the SBOM. Set high, as a backstop against a pathological + // archive filling the disk; it is a poor working budget. DefaultArchiveMaxDiskBytes = 100 * file.GB ) @@ -37,36 +26,29 @@ type ArchiveSearchConfig struct { // IncludeUnindexedArchives indicates whether to search within unindexed archive files (e.g., .tar*). IncludeUnindexedArchives bool `yaml:"include-unindexed-archives" json:"include-unindexed-archives" mapstructure:"include-unindexed-archives"` - // MaxDepth is the maximum depth of recursive archive extraction (0 = disabled, no recursive extraction). + // MaxDepth is the maximum depth of recursive archive extraction (0 disables it). MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"` - // MaxMemoryBytes is the maximum bytes of archive content held in memory at once. It is an in-use - // limit: it measures what is held right now and falls when an archive is released, so what it - // bounds is a scan's peak rather than its cumulative total. Content is held in memory while this - // limit admits it and overflows to disk when it does not - there is no separate configured size at - // which overflowing begins. Zero means content is never held in memory and everything overflows; a - // negative value means memory is not bounded at all. + // MaxMemoryBytes is the max archive content held in memory at once, an in-use gauge that falls as + // the scan releases an archive, so it bounds the peak. Content spills to disk once this stops + // admitting it, the only spill threshold. Zero spills everything; negative is unbounded. MaxMemoryBytes int64 `yaml:"max-memory-bytes" json:"max-memory-bytes" mapstructure:"max-memory-bytes"` - // MaxDiskBytes is the maximum bytes of archive content on disk at once - content overflowed there - // plus the entries extracted from it. Also an in-use limit, and terminal: there is nowhere - // further to overflow, so an archive that would exceed it is skipped. Zero admits no archive - // content at all, since an archive's entries are stored whatever happened to its own bytes; a - // negative value means disk is not bounded. + // MaxDiskBytes is the max archive content on disk at once: spilled content plus the entries + // extracted from it. Also an in-use gauge, and terminal, since spilled content has nowhere further + // to go; syft skips an archive that would exceed it. Zero keeps every archive out of the work + // directory; negative is unbounded. MaxDiskBytes int64 `yaml:"max-disk-bytes" json:"max-disk-bytes" mapstructure:"max-disk-bytes"` - // ExclusionPatterns carries the scan's own file exclusion patterns so the code that indexes an - // archive's contents does not have to interrogate the source for them. This capability has no - // exclusion setting of its own: it is deliberately not a yaml/json/mapstructure key, so no config - // file, flag, or environment variable reaches it, and `syft config` never names it. + // ExclusionPatterns carries the scan's file exclusion patterns so the code indexing an archive's + // contents need not interrogate the source for them. It has no config key, so no config file, flag + // or environment variable reaches it and `syft config` never names it. // - // Each boundary that already holds exclusion patterns populates this field from its own: the CLI - // fills it in cmd/syft/internal/options.Catalog.ToArchiveConfig from the --exclude flag, and - // syft.CreateSBOM fills it from the source's own published exclusions (source.PathExcluder). A - // value already present is never overwritten - CreateSBOM consults the source only to fill an - // empty field - so the boundary closest to the user wins and a library consumer who configures - // only their source still gets those patterns applied inside archives. Callers building this - // config directly should leave it alone; the zero value means nothing has populated it yet. + // Each boundary holding exclusion patterns populates it: the CLI from `--exclude` + // (options.Catalog.ToArchiveConfig), and syft.CreateSBOM from the source's published exclusions + // (source.PathExcluder). CreateSBOM fills only an empty field, so the boundary closest to the user + // wins, and a consumer who configures only their source still gets those patterns applied inside + // archives. Callers building this config directly should leave it alone. ExclusionPatterns []string `yaml:"-" json:"-" mapstructure:"-"` } @@ -107,12 +89,7 @@ func (c ArchiveSearchConfig) WithMaxDiskBytes(size int64) ArchiveSearchConfig { // WithExclusionPatterns sets the scan's exclusion patterns for the code that indexes an archive's // contents. Only patterns whose shape reaches inside an archive belong here; ArchiveExclusionPatterns -// selects them. -// -// This is how the two boundaries that populate the field do it, and it is not a way to configure a -// second exclusion list: the field carries no yaml, json or mapstructure tag, so nothing a user -// writes reaches it. A caller who sets it here is the boundary closest to the user, and -// syft.CreateSBOM will not overwrite a value that is already present. +// selects them. syft.CreateSBOM will not overwrite a value already set here. func (c ArchiveSearchConfig) WithExclusionPatterns(patterns []string) ArchiveSearchConfig { c.ExclusionPatterns = patterns return c diff --git a/syft/cataloging/archive_search_test.go b/syft/cataloging/archive_search_test.go index 6182876b7..d5c1e88db 100644 --- a/syft/cataloging/archive_search_test.go +++ b/syft/cataloging/archive_search_test.go @@ -18,15 +18,14 @@ func TestDefaultArchiveSearchConfig(t *testing.T) { } func TestArchiveSearchConfig_namesNoSeparateOverflowSize(t *testing.T) { - // two things are asserted by one list. The boundary between memory and disk is decided by the - // memory limit alone, so the surface must never grow a field naming a separate size at which - // content overflows. And nested archive cataloging honors the scan's own exclusion patterns, so the - // surface must name no exclusion setting of its own - an archive extension list here would be a - // second mechanism for a question the scan already answers. - // ExclusionPatterns is on the struct but is NOT config surface: it is derived, written only by - // CreateSBOM from what the source published, and reachable from no file, flag or variable. So it - // is listed here and separately asserted to carry no tags - which is what keeps this test able to - // catch a SETTABLE addition rather than merely any addition. + // one list, two assertions. The memory limit alone decides the boundary between memory and disk, so + // the surface must never grow a field naming a separate overflow size. And nested archive cataloging + // honors the scan's own exclusion patterns, so the surface must name no exclusion setting of its own + // - an archive extension list here would be a second mechanism for a question the scan answers. + // ExclusionPatterns is on the struct but is not config surface: derived, written only by CreateSBOM + // from what the source published, reachable from no file, flag or variable. It is listed here and + // separately asserted to carry no tags, which keeps this test catching a settable addition rather + // than any addition. var settable, derived []string typ := reflect.TypeOf(ArchiveSearchConfig{}) for i := 0; i < typ.NumField(); i++ { diff --git a/syft/create_sbom.go b/syft/create_sbom.go index a06b65a59..ff65162b5 100644 --- a/syft/create_sbom.go +++ b/syft/create_sbom.go @@ -38,15 +38,10 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) ( srcMetadata := src.Describe() - // read before the source builds its own resolver: exclusions belong to the source, and the - // archive cataloger needs them to index each archive it finds the way the scan indexes itself. - // This is the one type assertion (source.PathExcluder) left in the whole path. A local copy, so - // the CreateSBOMConfig the caller supplied is never mutated by deriving it. - // - // A value already present is never overwritten: cmd/syft/internal/options.ToArchiveConfig is the - // other boundary that populates this field, from the CLI's --exclude flag, and its value wins - // when set. The source is consulted only to fill an empty field, which is what makes - // "configure only your source" keep working for a library consumer who never touches this field. + // the archive cataloger needs the source's exclusions to index each archive the way the scan + // indexes itself. Copied locally so the caller's config is not mutated, and only when empty: + // options.ToArchiveConfig fills the same field from --exclude and wins when set, while a consumer + // who configures only their source still gets theirs. archiveCfg := cfg.Archive if len(archiveCfg.ExclusionPatterns) == 0 { archiveCfg = archiveCfg.WithExclusionPatterns(sourceExclusions(src)) @@ -126,6 +121,11 @@ func setupContext(ctx context.Context, cfg *CreateSBOMConfig) (context.Context, // configure parallel executors ctx = setContextExecutors(ctx, cfg) + // one progress row per cataloger for the whole scan, however many times it runs: the archive + // cataloger re-runs the pipeline against every archive it walks, and those findings belong on the + // row that cataloger already owns + ctx = bus.WithCatalogerTaskRegistry(ctx) + // configure temp dir factory for catalogers (if not already set) if tmpdir.FromContext(ctx) == nil { ctx, _ = tmpdir.Root(ctx, "syft-cataloger") diff --git a/syft/create_sbom_config.go b/syft/create_sbom_config.go index 8b97f0327..5579da2ae 100644 --- a/syft/create_sbom_config.go +++ b/syft/create_sbom_config.go @@ -19,64 +19,28 @@ import ( "github.com/anchore/syft/syft/source" ) -// archiveResolverFactory returns a factory that builds an indexed resolver over an extracted-archive -// directory whose Locations are stamped with the given (nesting-chain) fileSystemID. It is passed to -// the archive cataloger task so the internal/task and internal/archive packages do not need to -// import the syft-subtree internal fileresolver package directly. +// archiveEntryResolverFactory returns a factory that builds an indexed resolver over an extracted +// archive's entries. Passed to the archive cataloger task so internal/task and internal/archive need +// not import the internal fileresolver package directly. // -// exclusions are the scan's own exclusion patterns, carried here on cataloging.ArchiveSearchConfig's -// derived ExclusionPatterns field rather than passed in from the source directly; this capability has -// no exclusion setting of its own. Those whose shape reaches inside an archive are applied as index -// visitors, which is why an excluded file ends up absent from the archive's filesystem rather than -// present and skipped: an excluded archive is then never found to extract in the first place. -func archiveResolverFactory(exclusions []string) archive.ResolverFactory { - patterns := cataloging.ArchiveExclusionPatterns(exclusions) - - return func(overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - visitor, err := newArchiveExclusionVisitor(overflow.RootDir, patterns) - if err != nil { - return nil, archive.IndexResult{}, err - } - - var filters []fileresolver.PathIndexVisitor - if visitor != nil { - filters = append(filters, visitor) - } - - resolver, err := fileresolver.NewFromArchiveTar(overflow.RootDir, overflow.TarPath, overflow.FileSystemID, overflow.ArchivePath, filters...) - if err != nil { - return nil, archive.IndexResult{}, err - } - return resolver, archive.IndexResult{Records: resolver.Records(), Truncated: resolver.IndexTruncated()}, nil - } -} - -// archiveEntryResolverFactory returns a factory that builds a resolver over an archive's entries -// rather than over a tar of them. -// -// This is the store-backed path: an archive small enough to stay in memory never reaches the disk, and -// one that outgrows memory moves its bytes without the index being rebuilt. It stands beside the -// tar-backed factory rather than replacing it so the two can be measured against each other on the -// same corpus. +// exclusions reach here through ArchiveSearchConfig.ExclusionPatterns rather than from the source. +// Those whose shape reaches inside an archive become index visitors, so an excluded file is absent +// from the archive's filesystem rather than present and skipped, and an excluded archive is never +// found to extract. func archiveEntryResolverFactory(exclusions []string) archive.StoreResolverFactory { - patterns := cataloging.ArchiveExclusionPatterns(exclusions) + // built once for the scan, not per archive: the visitor matches against an entry's archive-relative + // path, so it carries nothing specific to the archive it is asked about + var filters []fileresolver.PathIndexVisitor + if visitor := newArchiveExclusionVisitor(cataloging.ArchiveExclusionPatterns(exclusions)); visitor != nil { + filters = append(filters, visitor) + } return func(store *archive.EntryStore, overflow archive.Overflow) (file.Resolver, archive.IndexResult, error) { - visitor, err := newArchiveExclusionVisitor(overflow.RootDir, patterns) + resolver, err := fileresolver.NewFromArchiveEntries(overflow.FileSystemID, overflow.ArchivePath, store, overflow.Charge, filters...) if err != nil { return nil, archive.IndexResult{}, err } - - var filters []fileresolver.PathIndexVisitor - if visitor != nil { - filters = append(filters, visitor) - } - - resolver, err := fileresolver.NewFromArchiveEntries(overflow.RootDir, overflow.FileSystemID, overflow.ArchivePath, store, false, filters...) - if err != nil { - return nil, archive.IndexResult{}, err - } - return resolver, archive.IndexResult{Records: resolver.Records(), Truncated: resolver.IndexTruncated()}, nil + return resolver, archive.IndexResult{Truncated: resolver.Truncated()}, nil } } @@ -210,8 +174,7 @@ func (c *CreateSBOMConfig) WithFilesConfig(cfg filecataloging.Config) *CreateSBO } // WithArchiveConfig allows for defining recursive archive cataloging parameters (e.g. max depth and -// extraction limits). When the configured MaxDepth is 0 (the default), recursive archive cataloging -// is disabled. +// extraction limits). MaxDepth 0, the default, disables recursive archive cataloging. func (c *CreateSBOMConfig) WithArchiveConfig(cfg cataloging.ArchiveSearchConfig) *CreateSBOMConfig { c.Archive = cfg return c @@ -257,10 +220,9 @@ func (c *CreateSBOMConfig) WithCatalogers(catalogerRefs ...pkgcataloging.Catalog // The final set of task groups is returned along with a cataloger manifest that describes the catalogers that were // selected and the tokens that were sensitive to this selection (both for adding and removing from the final set). // -// archiveCfg is c.Archive with ExclusionPatterns derived and populated by the caller (CreateSBOM, -// the only place that holds the source and can read source.PathExcluder off it). It is passed in -// rather than read off c so that deriving the field never mutates the CreateSBOMConfig the library -// consumer supplied. +// archiveCfg is c.Archive with ExclusionPatterns populated by the caller (CreateSBOM reads +// source.PathExcluder off the source). Passed in rather than read off c so deriving the field never +// mutates the consumer's CreateSBOMConfig. func (c *CreateSBOMConfig) makeTaskGroups(src source.Description, archiveCfg cataloging.ArchiveSearchConfig) ([][]task.Task, *catalogerManifest, error) { var taskGroups [][]task.Task @@ -284,13 +246,15 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description, archiveCfg cat taskGroups = append(taskGroups, append(pkgTasks, fileTasks...)) } - // recursively catalog archives as standalone indexed filesystems (opt-in via Archive.MaxDepth). - // the sub-pipeline is the package/file cataloger set so the same catalogers run against archive contents; - // it must run after the base scan group and before relationship/unknowns post-processing. + // recursively catalog archives as standalone indexed filesystems (opt-in via Archive.MaxDepth). the + // sub-pipeline is the package/file cataloger set, so the same catalogers run against archive + // contents; it runs after the base scan group and before relationship/unknowns post-processing + var subPipeline []task.Task subPipeline = append(subPipeline, pkgTasks...) subPipeline = append(subPipeline, fileTasks...) - if archiveTask := task.NewArchiveCatalogerTask(archiveCfg, subPipeline, archiveResolverFactory(archiveCfg.ExclusionPatterns), archiveEntryResolverFactory(archiveCfg.ExclusionPatterns), archive.LogNotify); archiveTask != nil { + + if archiveTask := task.NewArchiveCatalogerTask(archiveCfg, subPipeline, archiveEntryResolverFactory(archiveCfg.ExclusionPatterns), archive.LogNotify); archiveTask != nil { taskGroups = append(taskGroups, []task.Task{archiveTask}) } @@ -354,12 +318,10 @@ func (c *CreateSBOMConfig) selectTasks(src source.Description) ([]task.Task, []t FilesConfig: c.Files, } - // nested archive cataloging is controlled by a single knob (Archive.MaxDepth): when enabled, the - // archive cataloger task recurses into all archives (including JAR-family), and the java - // cataloger must not unarchive nested archives itself. This is the ONE place the two sides are - // coupled, and it writes a field that exists only for this purpose, so no value a caller - // supplied through WithPackagesConfig is overwritten. - cfg.PackagesConfig.JavaArchive.NestedArchivesHandledExternally = c.Archive.MaxDepth > 0 + // when enabled, the archive cataloger task recurses into all archives (JAR-family included), so the + // java cataloger must not unarchive them itself. This field exists only for that; derive it here so + // no value supplied through WithPackagesConfig is lost. + cfg.PackagesConfig.JavaArchive.NestedArchivesHandledExternally = c.Archive.MaxDepth != 0 persistentPkgTasks, selectablePkgTasks, err := c.allPackageTasks(cfg) if err != nil { diff --git a/syft/create_sbom_config_test.go b/syft/create_sbom_config_test.go index 14c339048..f26672263 100644 --- a/syft/create_sbom_config_test.go +++ b/syft/create_sbom_config_test.go @@ -605,10 +605,9 @@ func TestCreateSBOMConfig_archiveTaskGroup(t *testing.T) { }) t.Run("enabling the feature does not mutate the caller's package config", func(t *testing.T) { - // the java side's yield flag is derived from the archive depth, but it must be derived into - // the cataloging factory's own copy: a consumer who handed us a package config gets it back - // exactly as they supplied it. Nothing user-settable is overwritten, and nothing the - // framework sets leaks back out. + // the java side's yield flag is derived from the archive depth, into the cataloging factory's + // own copy: a consumer who handed us a package config gets it back exactly as supplied. + // Nothing user-settable is overwritten, and nothing the framework sets leaks back out. supplied := java.DefaultArchiveCatalogerConfig().WithUseNetwork(true) require.False(t, supplied.NestedArchivesHandledExternally) diff --git a/syft/file/coordinates.go b/syft/file/coordinates.go index a5ba50be4..985b4c30f 100644 --- a/syft/file/coordinates.go +++ b/syft/file/coordinates.go @@ -12,10 +12,10 @@ type Coordinates struct { // RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). RealPath string `json:"path" cyclonedx:"path"` - // FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file found inside an archive carries the FileSystemID of the filesystem the archive itself was found in (a layer digest, or blank), inherited unchanged down the nesting chain; the nesting chain is carried by ArchivePath instead. + // FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in; the nesting chain itself is carried by ArchivePath. FileSystemID string `json:"layerID,omitempty" cyclonedx:"layerID"` - // ArchivePath is the colon-delimited chain of archive paths, from the scan root, of every archive traversed to reach this file (e.g. "app.war:WEB-INF/lib/dep.jar"). It is blank for files not found within an archive. It disambiguates identically-named files residing in different archives on the same filesystem. + // ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. "app.war:WEB-INF/lib/dep.jar"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. ArchivePath string `json:"archivePath,omitempty" cyclonedx:"archivePath"` } @@ -53,11 +53,10 @@ func (c Coordinates) GetCoordinates() Coordinates { return c } -// HashInclude controls which fields participate in the artifact ID hash (see artifact.IDByHash, -// which uses hashstructure). ArchivePath is excluded from the hash when empty so that ordinary -// (non-archive) coordinates keep the same identity they had before ArchivePath existed; a non-empty -// ArchivePath is included, which is what keeps identically-named files in different archives from -// sharing an ID. +// HashInclude controls which fields participate in the artifact ID hash (see artifact.IDByHash). An +// empty ArchivePath is excluded, so non-archive coordinates keep the identity they had before the +// field existed; a non-empty one is included, keeping identically-named files in different archives +// from sharing an ID. func (c Coordinates) HashInclude(field string, _ any) (bool, error) { if field == "ArchivePath" && c.ArchivePath == "" { return false, nil diff --git a/syft/internal/fileresolver/archive_entry.go b/syft/internal/fileresolver/archive_entry.go new file mode 100644 index 000000000..549849a2e --- /dev/null +++ b/syft/internal/fileresolver/archive_entry.go @@ -0,0 +1,110 @@ +package fileresolver + +import ( + "io" + "os" + "path" + "path/filepath" + "strings" + + "github.com/anchore/syft/internal/log" +) + +// overflowEntryReader is a reader over one entry of an archive's store. Its marker method lets the +// archive cataloger recognize a nested archive as already-random-access, already-charged content, +// without either package importing the other. +type overflowEntryReader struct { + entryReadCloser +} + +// OverflowArchiveEntry marks this reader as content that needs no copy to be read at random. +func (overflowEntryReader) OverflowArchiveEntry() {} + +// entryReadCloser is a bounded view onto the bytes the store holds for one entry. +type entryReadCloser interface { + io.ReadCloser + io.ReaderAt + io.Seeker +} + +// archiveRoot is the root handed to a path filter. Archive entries are relative to the archive, which +// has no host location, so the root is the archive itself. +const archiveRoot = "/" + +type filterOutcome int + +const ( + keepEntry filterOutcome = iota + skipEntry + skipTree +) + +// runPathFilters asks each filter about one entry, mapping a walk's answers (skip path, skip dir) +// onto an unordered pass over archive entries. +// +// entryPath is archive-relative and the root is the archive's own root, not a host path: the archive +// is indexed in memory, and matching against the host temp path could match segments the archive's +// contents never carry. +func runPathFilters(filters []PathIndexVisitor, entryPath string, info os.FileInfo) filterOutcome { + for _, filter := range filters { + if filter == nil { + continue + } + switch err := filter(archiveRoot, entryPath, info, nil); err { + case nil: + continue + case filepath.SkipDir, filepath.SkipAll: + return skipTree + default: + return skipEntry + } + } + return keepEntry +} + +// isPruned reports whether an entry sits inside an already-pruned directory. An archive is a +// sequence, not a walk, so SkipDir cannot end a subtree; each later entry is checked instead. +func isPruned(entryPath string, pruned []string) bool { + for _, dir := range pruned { + if entryPath == dir || strings.HasPrefix(entryPath, dir+"/") { + return true + } + } + return false +} + +// maxEntryPathBytes is the longest entry path this package will index: PATH_MAX. +// +// Each path component becomes a tree node, so a long name like "a/a/.../a/f" expands a few hundred +// compressed bytes into a huge directory chain (a 352-byte tar.gz produced 200,000 nodes and 44 MB of +// index). This caps the per-entry cost; the index's per-node charge caps how many such entries a scan +// admits. +// +// A longer path is one no filesystem could hold, so refusing it drops nothing a real extraction or +// directory scan would have found. Go's tar reader alone allows names far longer. +const maxEntryPathBytes = 4096 + +// sanitizeEntryName converts an archive entry's name into a path inside that archive, reporting false +// when the name is empty or too long to be a path. +// +// Nothing is written through the name; this guards the path the SBOM reports. Cleaning against the +// root clamps "../" climbs, so an attacker-supplied name yields at worst a wrong path inside the right +// archive, never a claim about the host filesystem. +func sanitizeEntryName(name string) (string, bool) { + if strings.ContainsRune(name, 0) { + return "", false + } + if len(name) > maxEntryPathBytes { + // tested before cleaning: cleaning only shortens, and building the cleaned copy of a + // multi-megabyte name is itself part of what this refuses + log.WithFields("entry-name-bytes", len(name)). + Trace("skipping archive entry whose name is longer than any path a filesystem could hold") + return "", false + } + cleaned := strings.TrimPrefix(path.Clean("/"+name), "/") + if cleaned == "" || cleaned == "." { + // the archive's own root, which is not an entry in it + return "", false + } + return cleaned, true +} diff --git a/syft/internal/fileresolver/archive_entry_test.go b/syft/internal/fileresolver/archive_entry_test.go new file mode 100644 index 000000000..1614f1960 --- /dev/null +++ b/syft/internal/fileresolver/archive_entry_test.go @@ -0,0 +1,83 @@ +package fileresolver + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Test_resolveLinkWithinArchive pins the frame a link entry is resolved in: the archive's own root, +// never the host filesystem. An archive's link targets are attacker-supplied text, so a target +// naming the host must land inside this archive or nowhere. +func Test_resolveLinkWithinArchive(t *testing.T) { + tests := []struct { + name string + linkPath string + linkname string + hardLink bool + want string + }{ + { + name: "a relative target resolves against the link's own directory", + linkPath: "/a/b/link", + linkname: "sibling.txt", + want: "/a/b/sibling.txt", + }, + { + name: "a relative target may climb within the archive", + linkPath: "/a/b/link", + linkname: "../c/target.txt", + want: "/a/c/target.txt", + }, + { + name: "a climb past the archive root is clamped at it", + linkPath: "/a/b/link", + linkname: "../../../../etc/passwd", + want: "/etc/passwd", + }, + { + name: "an absolute target is archive-absolute, not host-absolute", + linkPath: "/a/b/link", + linkname: "/etc/passwd", + want: "/etc/passwd", + }, + { + name: "a link at the archive root resolves against the root", + linkPath: "/link", + linkname: "target.txt", + want: "/target.txt", + }, + { + name: "a target that is only a climb lands on the root itself", + linkPath: "/a/link", + linkname: "..", + want: "/", + }, + { + name: "a hard link names its target from the archive root, not from its own directory", + linkPath: "/opt/hard.jar", + linkname: "opt/real.jar", + hardLink: true, + want: "/opt/real.jar", + }, + { + name: "a symlink with the same target text is relative to its own directory", + linkPath: "/opt/sym.jar", + linkname: "opt/real.jar", + want: "/opt/opt/real.jar", + }, + { + name: "a hard link target is clamped at the root like any other", + linkPath: "/opt/hard.jar", + linkname: "../../etc/passwd", + hardLink: true, + want: "/etc/passwd", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, resolveLinkWithinArchive(tt.linkPath, tt.linkname, tt.hardLink)) + }) + } +} diff --git a/syft/internal/fileresolver/archive_glob_bounds.go b/syft/internal/fileresolver/archive_glob_bounds.go new file mode 100644 index 000000000..8aa91457b --- /dev/null +++ b/syft/internal/fileresolver/archive_glob_bounds.go @@ -0,0 +1,401 @@ +package fileresolver + +import "strings" + +// globMeta are the characters that end a run of literal text in a glob segment. A lone `]` or `}` is +// literal to doublestar but is treated as a stop here anyway, which only shortens a bound - a loss of +// selectivity, never of correctness. See literalBounds. +const globMeta = `*?[]{}\` + +// minLiteralBound is the shortest bound worth a lookup: any non-empty fixed text, since a lookup is a +// binary search plus a walk of one contiguous run. +const minLiteralBound = 1 + +// maxAlternatives bounds how far brace groups and character classes are enumerated. Past it, a +// segment reports nothing to look up rather than producing exponentially many lookups. +const maxAlternatives = 32 + +// maxClassRange is the widest `a-z` span enumerated into members. Wider is more lookups than the scan +// it replaces. +const maxClassRange = 64 + +// literalBounds returns the leading and trailing literal runs any matching name must carry: +// `libstd-????????????????.so` gives "libstd-" and ".so"; `*.[jw]ar` gives "" and "ar". A side +// starting with a metacharacter fixes nothing. +// +// Bounds only filter the index; doublestar still decides. A short bound merely widens the candidate +// set, so every globMeta character stops a run to avoid dropping real matches. +func literalBounds(segment string) (prefix, suffix string) { + first := strings.IndexAny(segment, globMeta) + if first < 0 { + return segment, segment + } + last := strings.LastIndexAny(segment, globMeta) + return segment[:first], segment[last+1:] +} + +// segmentBounds returns literal prefixes and suffixes covering every name this segment can match; a +// list is nil when the segment fixes too little on that side. +// +// Braces are expanded first, since an alternative can fix text the raw segment does not (`{go,go.exe}` +// prefixes "go" only once expanded). A side is usable only when every alternative carries a bound, +// else that alternative's matches would be lost. +func segmentBounds(segment string) (prefixes, suffixes []string) { + alternatives, ok := expandBraces(segment) + if !ok { + return nil, nil + } + + prefixes = make([]string, 0, len(alternatives)) + suffixes = make([]string, 0, len(alternatives)) + for _, alternative := range alternatives { + prefix, suffix := literalBounds(alternative) + if len(prefix) < minLiteralBound { + prefixes = nil + } + if len(suffix) < minLiteralBound { + suffixes = nil + } + if prefixes != nil { + prefixes = append(prefixes, prefix) + } + if suffixes != nil { + suffixes = append(suffixes, suffix) + } + } + return dedupe(prefixes), dedupe(suffixes) +} + +func dedupe(values []string) []string { + if len(values) < 2 { + return values + } + seen := make(map[string]struct{}, len(values)) + out := values[:0] + for _, v := range values { + if _, dup := seen[v]; dup { + continue + } + seen[v] = struct{}{} + out = append(out, v) + } + return out +} + +// expandBraces enumerates the alternatives a segment's `{a,b}` groups stand for, reporting false past +// maxAlternatives. +// +// It errs toward expanding: over-expanding costs extra lookups, under-expanding costs matches. An +// unmatched `{` is left for literalBounds to stop on. +func expandBraces(segment string) ([]string, bool) { + open, closing := braceGroup(segment) + if open < 0 { + return []string{segment}, true + } + + var out []string + for _, choice := range splitAlternatives(segment[open+1 : closing]) { + expanded, ok := expandBraces(segment[:open] + choice + segment[closing+1:]) + if !ok || len(out)+len(expanded) > maxAlternatives { + return nil, false + } + out = append(out, expanded...) + } + return out, true +} + +// braceGroup returns the bounds of the first top-level `{...}` group, or (-1, -1) when there is none. +// Escapes and character classes are skipped, so `\{` and `[{]` are not groups. +func braceGroup(segment string) (open, closing int) { + depth := 0 + open = -1 + for i := 0; i < len(segment); i++ { + switch segment[i] { + case '\\': + i++ + case '[': + if end := classEnd(segment, i); end >= 0 { + i = end + } + case '{': + if depth == 0 { + open = i + } + depth++ + case '}': + if depth > 0 { + depth-- + if depth == 0 { + return open, i + } + } + } + } + return -1, -1 +} + +// splitAlternatives splits the inside of a brace group on its top-level commas, leaving the commas +// of any nested group or character class alone. +func splitAlternatives(inner string) []string { + var out []string + depth, start := 0, 0 + for i := 0; i < len(inner); i++ { + switch inner[i] { + case '\\': + i++ + case '[': + if end := classEnd(inner, i); end >= 0 { + i = end + } + case '{': + depth++ + case '}': + depth-- + case ',': + if depth == 0 { + out = append(out, inner[start:i]) + start = i + 1 + } + } + } + return append(out, inner[start:]) +} + +// classEnd returns the index of the `]` closing the class opened at open, or -1 when it is never +// closed. A `]` first in the class, or directly after a leading negation, is a literal member. +func classEnd(segment string, open int) int { + i := open + 1 + if i < len(segment) && (segment[i] == '!' || segment[i] == '^') { + i++ + } + if i < len(segment) && segment[i] == ']' { + i++ + } + for ; i < len(segment); i++ { + switch segment[i] { + case '\\': + i++ + case ']': + return i + } + } + return -1 +} + +// segmentKind is what a name index can be asked for on behalf of one glob segment. +type segmentKind int + +const ( + // segExact is a literal name. + segExact segmentKind = iota + // segPrefix is a name with a fixed head, what the forward index answers. + segPrefix + // segSuffix is a name with a fixed tail, what the reverse index answers. + segSuffix +) + +// nameLookup is one index lookup: an exact name, or a name with a fixed head or tail. +type nameLookup struct { + kind segmentKind + literal string +} + +// enumerateLookups returns index lookups covering every name this segment can match, or nil when it +// cannot be reduced to a finite set. +// +// A segment whose only metacharacters are alternations and classes denotes a finite set: `{go,go.exe}` +// is two names, `*.[jw]ar` two suffixes, `[lm]*` two prefixes - a few lookups rather than a scan. +// Bounds serve these poorly: `*.[jw]ar` bounds to suffix "ar" (catches `calendar`), `[lm]*` bounds to +// nothing. +// +// Over-broad enumeration only costs filtering; under-broad loses files, so anything not exactly +// enumerable - a negated class, a POSIX class, a metacharacter member - abandons the attempt. +func enumerateLookups(segment string) []nameLookup { + alternatives, ok := expandBraces(segment) + if !ok { + return nil + } + + var expanded []string + for _, alternative := range alternatives { + members, ok := expandClasses(alternative) + if !ok || len(expanded)+len(members) > maxAlternatives { + return nil + } + expanded = append(expanded, members...) + } + + lookups := make([]nameLookup, 0, len(expanded)) + seen := make(map[nameLookup]struct{}, len(expanded)) + for _, alternative := range dedupe(expanded) { + lookup, ok := lookupFor(alternative) + if !ok { + // one alternative without a lookup would leave its matches out + return nil + } + if _, dup := seen[lookup]; dup { + continue + } + seen[lookup] = struct{}{} + lookups = append(lookups, lookup) + } + return lookups +} + +// lookupFor classifies one fully-expanded alternative (no alternation, no class) as an index lookup, +// or reports false when the index cannot answer it usefully. The answerable shapes are an exact name, +// `*tail`, `head*`; a `?`, an escape, or a second `*` leaves text the index cannot key on. +func lookupFor(alternative string) (nameLookup, bool) { + if strings.ContainsAny(alternative, nonWildcardMeta) { + return nameLookup{}, false + } + first := strings.IndexByte(alternative, '*') + last := strings.LastIndexByte(alternative, '*') + switch { + case first < 0: + return nameLookup{kind: segExact, literal: alternative}, true + case first != last || len(alternative) == 1: + // `*a*` needs every name anyway, and a bare `*` takes them all + return nameLookup{}, false + case first == 0: + return worthLooking(nameLookup{kind: segSuffix, literal: alternative[1:]}) + case last == len(alternative)-1: + return worthLooking(nameLookup{kind: segPrefix, literal: alternative[:len(alternative)-1]}) + } + // `head*tail`, which neither index answers on its own + return nameLookup{}, false +} + +func worthLooking(lookup nameLookup) (nameLookup, bool) { + if len(lookup.literal) < minLiteralBound { + return nameLookup{}, false + } + return lookup, true +} + +// expandClasses enumerates the names a segment's `[...]` classes stand for, reporting false when a +// class cannot be enumerated exactly. +// +// Members are substituted back as themselves, so a metacharacter member would change the pattern's +// meaning (`[*]` matches one asterisk, not anything). Such a class is refused rather than escaped, +// since the caller's fallback is correct. +func expandClasses(segment string) ([]string, bool) { + open := firstClass(segment) + if open < 0 { + return []string{segment}, true + } + closing := classEnd(segment, open) + if closing < 0 { + // unterminated, so doublestar reads the `[` as literal; lookupFor will refuse it + return []string{segment}, true + } + + members, ok := classMembers(segment[open : closing+1]) + if !ok { + return nil, false + } + + var out []string + for _, member := range members { + expanded, ok := expandClasses(segment[:open] + member + segment[closing+1:]) + if !ok || len(out)+len(expanded) > maxAlternatives { + return nil, false + } + out = append(out, expanded...) + } + return out, true +} + +// firstClass returns the index of the first unescaped `[`, or -1. +func firstClass(segment string) int { + for i := 0; i < len(segment); i++ { + switch segment[i] { + case '\\': + i++ + case '[': + return i + } + } + return -1 +} + +// classMembers returns the characters the given `[...]` matches, or false when the class is not +// enumerable. +func classMembers(class string) ([]string, bool) { + runes := []rune(class[1 : len(class)-1]) + if !enumerableClass(runes) { + return nil, false + } + + var out []string + for i := 0; i < len(runes); i++ { + members, width, ok := classItemAt(runes, i) + if !ok { + return nil, false + } + out = append(out, members...) + if len(out) > maxAlternatives { + return nil, false + } + i += width + } + if len(out) == 0 { + return nil, false + } + return out, true +} + +// enumerableClass reports whether the inside of a `[...]` describes a listable finite set. A negated +// class stands for all but a few characters, and a POSIX class for a set this does not model. +func enumerableClass(inner []rune) bool { + switch { + case len(inner) == 0: + return false + case inner[0] == '!' || inner[0] == '^': + return false + case strings.Contains(string(inner), "[:"): + return false + } + return true +} + +// classItemAt returns the members of the class item starting at i and how many runes past i it +// consumed. An item is an escape, an `a-z` range, or a single character. +func classItemAt(runes []rune, i int) (members []string, width int, ok bool) { + switch { + case runes[i] == '\\': + if i+1 >= len(runes) || !enumerableMember(runes[i+1]) { + return nil, 0, false + } + return []string{string(runes[i+1])}, 1, true + + case i+2 < len(runes) && runes[i+1] == '-': + return classRange(runes[i], runes[i+2]) + + default: + if !enumerableMember(runes[i]) { + return nil, 0, false + } + return []string{string(runes[i])}, 0, true + } +} + +// classRange expands an `a-z` range into its members, refusing one wider than maxClassRange. +func classRange(low, high rune) (members []string, width int, ok bool) { + if high < low || high-low > maxClassRange { + return nil, 0, false + } + for r := low; r <= high; r++ { + if !enumerableMember(r) { + return nil, 0, false + } + members = append(members, string(r)) + } + return members, 2, true +} + +// enumerableMember reports whether a class member can stand in the pattern as itself, which a +// metacharacter cannot: see expandClasses. +func enumerableMember(r rune) bool { + return !strings.ContainsRune(globMeta, r) +} diff --git a/syft/internal/fileresolver/archive_glob_bounds_test.go b/syft/internal/fileresolver/archive_glob_bounds_test.go new file mode 100644 index 000000000..0a22767f9 --- /dev/null +++ b/syft/internal/fileresolver/archive_glob_bounds_test.go @@ -0,0 +1,462 @@ +package fileresolver + +import ( + "fmt" + "strings" + "testing" + + "github.com/bmatcuk/doublestar/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syftindex "github.com/anchore/syft/internal/index" +) + +func Test_literalBounds(t *testing.T) { + tests := []struct { + segment string + prefix string + suffix string + }{ + {"pom.properties", "pom.properties", "pom.properties"}, + {"libstd-????????????????.so", "libstd-", ".so"}, + {"*.[jw]ar", "", "ar"}, + {"*.jar", "", ".jar"}, + {"pom*", "pom", ""}, + {"*", "", ""}, + {"{go,go.exe}", "", ""}, + {"lib{a,b}.so", "lib", ".so"}, + {"a?b", "a", "b"}, + {`esc\*ape`, "esc", "ape"}, + } + for _, tt := range tests { + t.Run(tt.segment, func(t *testing.T) { + prefix, suffix := literalBounds(tt.segment) + assert.Equal(t, tt.prefix, prefix, "prefix") + assert.Equal(t, tt.suffix, suffix, "suffix") + }) + } +} + +func Test_expandBraces(t *testing.T) { + tests := []struct { + segment string + want []string + wantOK bool + }{ + {segment: "plain.txt", want: []string{"plain.txt"}, wantOK: true}, + {segment: "{go,go.exe}", want: []string{"go", "go.exe"}, wantOK: true}, + {segment: "lib{a,b}.so", want: []string{"liba.so", "libb.so"}, wantOK: true}, + {segment: "{a,{b,c}}x", want: []string{"ax", "bx", "cx"}, wantOK: true}, + {segment: "{a,b}{c,d}", want: []string{"ac", "ad", "bc", "bd"}, wantOK: true}, + // a comma inside a character class is not a separator + {segment: "{x[a,b]y,z}", want: []string{"x[a,b]y", "z"}, wantOK: true}, + // an escaped brace is not a group + {segment: `\{a,b\}`, want: []string{`\{a,b\}`}, wantOK: true}, + // unmatched, so left alone for literalBounds to stop at + {segment: "{unclosed", want: []string{"{unclosed"}, wantOK: true}, + } + for _, tt := range tests { + t.Run(tt.segment, func(t *testing.T) { + got, ok := expandBraces(tt.segment) + require.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) + } + + t.Run("an exponential expansion is refused rather than enumerated", func(t *testing.T) { + _, ok := expandBraces(strings.Repeat("{a,b}", 10)) + assert.False(t, ok, "2^10 alternatives is not less work than a scan") + }) +} + +func Test_segmentBounds(t *testing.T) { + tests := []struct { + name string + segment string + wantPrefixes []string + wantSuffixes []string + }{ + { + name: "both sides fixed", + segment: "libstd-????????????????.so", + wantPrefixes: []string{"libstd-"}, + wantSuffixes: []string{".so"}, + }, + { + name: "only the tail is fixed", + segment: "*.[jw]ar", + wantSuffixes: []string{"ar"}, + }, + { + name: "only the head is fixed", + segment: "pom?.xml*", + wantPrefixes: []string{"pom"}, + }, + { + name: "alternation fixes each branch", + segment: "{go,go.exe}", + wantPrefixes: []string{"go", "go.exe"}, + wantSuffixes: []string{"go", "go.exe"}, + }, + { + name: "alternation around fixed text", + segment: "{mariadb,mysql}", + wantPrefixes: []string{"mariadb", "mysql"}, + wantSuffixes: []string{"mariadb", "mysql"}, + }, + { + name: "a branch with nothing fixed disqualifies the whole side", + segment: "{go,*}", + }, + { + name: "a one-character bound is still a lookup", + segment: "a?", + wantPrefixes: []string{"a"}, + }, + { + name: "nothing fixed at all", + segment: "*", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prefixes, suffixes := segmentBounds(tt.segment) + assert.Equal(t, tt.wantPrefixes, prefixes, "prefixes") + assert.Equal(t, tt.wantSuffixes, suffixes, "suffixes") + }) + } +} + +// Test_segmentBounds_neverExcludesAMatch is the invariant the optimization rests on: the bounds pick +// which names doublestar is shown, so a name the pattern matches must carry one. A bound that is too +// short only widens the candidate set; one that is too long silently loses files. +func Test_segmentBounds_neverExcludesAMatch(t *testing.T) { + segments := []string{ + "libstd-????????????????.so", "*.[jw]ar", "{go,go.exe}", "{mariadb,mysql}", + "lib{a,b}.so", "pom?.xml", "*.jar", "pom*", "*", "a?b", "{a,{b,c}}x", + `esc\*ape`, "[!x]yz", "{firefox,firefox.exe}", "**", + } + names := []string{ + "go", "go.exe", "gopher", "a.jar", "b.war", "b.rar", "libstd-0123456789abcdef.so", + "libstd-short.so", "liba.so", "libb.so", "libc.so", "pom1.xml", "pom.xml", "pom", + "mariadb", "mysql", "postgres", "axb", "ax", "bx", "cx", "esc*ape", "escape", + "xyz", "ayz", "firefox", "firefox.exe", "", "{go,go.exe}", + } + + for _, segment := range segments { + t.Run(segment, func(t *testing.T) { + prefixes, suffixes := segmentBounds(segment) + for _, name := range names { + matched, err := doublestar.Match(segment, name) + if err != nil || !matched { + continue + } + if len(prefixes) > 0 { + assert.True(t, anyHasPrefix(name, prefixes), + "%q matches %q but carries none of the prefixes %q", name, segment, prefixes) + } + if len(suffixes) > 0 { + assert.True(t, anyHasSuffix(name, suffixes), + "%q matches %q but carries none of the suffixes %q", name, segment, suffixes) + } + } + }) + } +} + +func anyHasPrefix(name string, prefixes []string) bool { + for _, p := range prefixes { + if strings.HasPrefix(name, p) { + return true + } + } + return false +} + +func anyHasSuffix(name string, suffixes []string) bool { + for _, s := range suffixes { + if strings.HasSuffix(name, s) { + return true + } + } + return false +} + +func Test_classMembers(t *testing.T) { + tests := []struct { + class string + want []string + wantOK bool + }{ + {class: "[jw]", want: []string{"j", "w"}, wantOK: true}, + {class: "[a-c]", want: []string{"a", "b", "c"}, wantOK: true}, + {class: "[a-cx]", want: []string{"a", "b", "c", "x"}, wantOK: true}, + {class: "[-a]", want: []string{"-", "a"}, wantOK: true}, + {class: "[a-]", want: []string{"a", "-"}, wantOK: true}, + // a negated class stands for every character but a few, which is not a finite lookup + {class: "[!x]"}, + {class: "[^x]"}, + // a POSIX class is a set this does not model + {class: "[[:alpha:]]"}, + // a member that is itself a metacharacter would change meaning on substitution + {class: "[*]"}, + {class: "[?]"}, + // wider than maxClassRange + {class: "[\x20-\x7e]"}, + {class: "[]"}, + } + for _, tt := range tests { + t.Run(tt.class, func(t *testing.T) { + got, ok := classMembers(tt.class) + require.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_enumerateLookups(t *testing.T) { + tests := []struct { + name string + segment string + want []nameLookup + }{ + { + name: "alternation of exact names", + segment: "{go,go.exe}", + want: []nameLookup{{segExact, "go"}, {segExact, "go.exe"}}, + }, + { + name: "class inside an extension becomes exact suffixes", + segment: "*.[jw]ar", + want: []nameLookup{{segSuffix, ".jar"}, {segSuffix, ".war"}}, + }, + { + name: "leading class becomes prefix lookups, where bounds fix nothing at all", + segment: "[lm]*", + want: []nameLookup{{segPrefix, "l"}, {segPrefix, "m"}}, + }, + { + name: "multi-character alternation likewise", + segment: "{lib,mod}*", + want: []nameLookup{{segPrefix, "lib"}, {segPrefix, "mod"}}, + }, + { + name: "alternation and class together", + segment: "{lib,bin}[0-2].so", + want: []nameLookup{ + {segExact, "lib0.so"}, {segExact, "lib1.so"}, {segExact, "lib2.so"}, + {segExact, "bin0.so"}, {segExact, "bin1.so"}, {segExact, "bin2.so"}, + }, + }, + { + name: "duplicate alternatives collapse to one lookup", + segment: "{go,go}", + want: []nameLookup{{segExact, "go"}}, + }, + // a `?` is not a finite set of names + {name: "question mark", segment: "libstd-?.so"}, + // a negated class is not either + {name: "negated class", segment: "[!x]yz"}, + // nor is a wildcard in the middle + {name: "interior wildcard", segment: "{a,b}*.{c,d}"}, + // nor a bare wildcard + {name: "bare wildcard", segment: "{a,*}"}, + // and an exponential expansion is refused rather than enumerated + {name: "too many alternatives", segment: "[a-z][a-z]"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, enumerateLookups(tt.segment)) + }) + } +} + +// Test_enumerateLookups_neverExcludesAMatch is the same invariant for the stronger claim this pass +// makes: the lookups are what the index is asked for, so a name the pattern matches must be found by +// at least one of them. Over-broad only costs filtering. +func Test_enumerateLookups_neverExcludesAMatch(t *testing.T) { + segments := []string{ + "{go,go.exe}", "*.[jw]ar", "[lm]*", "{lib,mod}*", "{mariadb,mysql}", "{lib,bin}[0-2].so", + "lib{a,b}.so", "[abc]", "*.[jw]ar", "{firefox,firefox.exe}", "x[!y]z", "[a-c]*.so", + "pom.{xml,properties}", "*.{jar,war}", "{a,b}{c,d}", + // only some branches reduce to a lookup + "{go,*}", "{lib,l}*", "{lo,m}*", "{*.jar,pom.xml}", + } + names := []string{ + "go", "go.exe", "gopher", "a.jar", "b.war", "c.rar", "calendar", "lib0.so", "lib1.so", + "lib3.so", "bin2.so", "liba.so", "libb.so", "libc.so", "mariadb", "mysql", "a", "b", "c", "d", + "lmn", "mno", "nop", "xyz", "xaz", "ayz", "firefox", "firefox.exe", "pom.xml", + "log.txt", "mn.txt", "lib.so", + "pom.properties", "pom.txt", "ac", "ad", "bc", "bd", "ab", "", "az.so", + } + + for _, segment := range segments { + t.Run(segment, func(t *testing.T) { + lookups := enumerateLookups(segment) + if len(lookups) == 0 { + return + } + for _, name := range names { + matched, err := doublestar.Match(segment, name) + if err != nil || !matched { + continue + } + assert.True(t, anyLookupFinds(name, lookups), + "%q matches %q but no lookup in %v would find it", name, segment, lookups) + } + }) + } +} + +func anyLookupFinds(name string, lookups []nameLookup) bool { + for _, l := range lookups { + switch l.kind { + case segExact: + if name == l.literal { + return true + } + case segPrefix: + if strings.HasPrefix(name, l.literal) { + return true + } + case segSuffix: + if strings.HasSuffix(name, l.literal) { + return true + } + } + } + return false +} + +// Test_lookupsAreExactlyTheirAlternative is what lets byLookups return the index's answer without +// asking doublestar to confirm it. +// +// The bounds path only narrows, so an imprecise bound there costs filtering. A lookup is the answer +// itself, so anything byPrefix("l") selected that `l*` does not match would be reported as a match. +// This pins the three shapes to doublestar's verdict. +func Test_lookupsAreExactlyTheirAlternative(t *testing.T) { + alternatives := []string{ + "go", "go.exe", "l", "lib", "*.jar", "*ar", "*", "l*", "lib*", "libstd-*", + "*.so", "a", "pom.xml", ".hidden", "*.", ".*", + } + names := []string{ + "go", "go.exe", "gopher", "l", "lib", "lib.so", "libstd-0.a", "libz.so", "a.jar", + "calendar", "ar", ".jar", "", "a", "pom.xml", ".hidden", "x.", ".x", "日本.jar", "日本", + } + + for _, alternative := range alternatives { + t.Run(alternative, func(t *testing.T) { + lookup, ok := lookupFor(alternative) + if !ok { + return + } + for _, name := range names { + want, err := doublestar.Match(alternative, name) + require.NoError(t, err) + assert.Equal(t, want, anyLookupFinds(name, []nameLookup{lookup}), + "pattern %q, name %q: the lookup and doublestar must agree exactly", alternative, name) + } + }) + } +} + +// Test_boundedLookupMatchesTheFullOne holds the probe to the lookup it stands in for: a bounded +// lookup that completes has to return exactly what the unbounded one would, or the side it picks is +// answering a different question than the side it rejects. +func Test_boundedLookupMatchesTheFullOne(t *testing.T) { + var x syftindex.PrefixSuffix[[]*indexNode] + names := []string{ + "libstd-a.so", "libstd-b.a", "libstd-c.a", "lib.so", "libz.so", + "go", "go.exe", "gopher", "a.jar", "b.war", "\xff\xfe", + } + for _, name := range names { + node := &indexNode{name: name} + x.Update(name, appendNode(node)) + } + + for _, probe := range []string{"lib", "libstd-", "go", "z", "", "a", "\xff", "libstd-a.so"} { + t.Run("prefix "+probe, func(t *testing.T) { + full := flattenNodes(x.ByPrefix(probe)) + got, complete := x.ByPrefixUpTo(probe, len(names)) + require.True(t, complete, "the budget covers every name, so nothing may give up") + assert.ElementsMatch(t, full, flattenNodes(got)) + }) + t.Run("suffix "+probe, func(t *testing.T) { + full := flattenNodes(x.BySuffix(probe)) + got, complete := x.BySuffixUpTo(probe, len(names)) + require.True(t, complete) + assert.ElementsMatch(t, full, flattenNodes(got)) + }) + } +} + +// Test_boundedLookupGivesUpPastTheBudget covers the other half: past the budget the lookup must report +// failure rather than a short answer, which a caller would read as a narrow side. +func Test_boundedLookupGivesUpPastTheBudget(t *testing.T) { + var x syftindex.PrefixSuffix[[]*indexNode] + for i := range 100 { + name := fmt.Sprintf("lib%03d.so", i) + x.Update(name, appendNode(&indexNode{name: name})) + } + + _, complete := x.ByPrefixUpTo("lib", 10) + assert.False(t, complete, "100 names under a budget of 10 must give up") + + _, complete = x.ByPrefixUpTo("lib000", 10) + assert.True(t, complete, "one name under a budget of 10 must not") +} + +// Test_narrowByBounds_takesTheNarrowerSide covers what probing the index buys. Both ends of +// `libstd-????????????????.so` can answer it, and which should is a property of the archive: here the +// seven-character head is the broad side and the three-character tail the narrow one, so a rule +// preferring the longer literal would take the worse lookup. +func Test_narrowByBounds_takesTheNarrowerSide(t *testing.T) { + entries := map[string]string{"usr/lib/libstd-0123456789abcdef.so": "real"} + for i := range 200 { + entries[fmt.Sprintf("usr/lib/libstd-%04d.a", i)] = "decoy" + } + r := indexOver(t, 1<<20, entries) + + const segment = "libstd-????????????????.so" + prefixes, suffixes := segmentBounds(segment) + require.Equal(t, []string{"libstd-"}, prefixes) + require.Equal(t, []string{".so"}, suffixes) + + // the head reaches every decoy and so must never be the side that runs; the tail reaches one + _, headComplete := r.fileNames.ByPrefixUpTo("libstd-", boundProbe) + require.False(t, headComplete, "the longer bound reaches past the probe") + tail, tailComplete := r.fileNames.BySuffixUpTo(".so", boundProbe) + require.True(t, tailComplete) + require.Len(t, flattenNodes(tail), 1, "the shorter bound reaches only the real file") + + candidates, ok := r.narrowByBounds(&r.fileNames, segment) + require.True(t, ok) + assert.Len(t, candidates, 1, "the narrower side must be the one that ran") + + // and the answer is the same either way, so this is only a question of cost + locations, err := r.FilesByGlob("**/" + segment) + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, "usr/lib/libstd-0123456789abcdef.so", locations[0].RealPath) +} + +// Test_narrowByBounds_fallsBackWhenNeitherSideIsNarrow covers the case the probe cannot decide: both +// ends reach past it, so the answer still has to be complete. +func Test_narrowByBounds_fallsBackWhenNeitherSideIsNarrow(t *testing.T) { + entries := map[string]string{} + for i := range boundProbe * 3 { + entries[fmt.Sprintf("usr/lib/libstd-%04d.so", i)] = "decoy" + } + entries["usr/lib/libstd-0123456789abcdef.so"] = "real" + r := indexOver(t, 1<<20, entries) + + candidates, ok := r.narrowByBounds(&r.fileNames, "libstd-????????????????.so") + require.True(t, ok) + assert.Len(t, candidates, boundProbe*3+1, "a probe that decides nothing must still gather everything") + + locations, err := r.FilesByGlob("**/libstd-????????????????.so") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, "usr/lib/libstd-0123456789abcdef.so", locations[0].RealPath) +} diff --git a/syft/internal/fileresolver/archive_index.go b/syft/internal/fileresolver/archive_index.go index 99b93f494..555deebd2 100644 --- a/syft/internal/fileresolver/archive_index.go +++ b/syft/internal/fileresolver/archive_index.go @@ -1,10 +1,12 @@ package fileresolver import ( + "archive/tar" "context" "fmt" "io" "path" + "reflect" "sort" "strings" @@ -14,33 +16,70 @@ import ( "github.com/anchore/syft/syft/file" ) -// ArchiveIndex is a resolver over the entries of one extracted archive, indexed by the entries -// themselves rather than by offsets into a file. +// The index is the largest per-archive structure a scan keeps, and the only archive cost the entry +// store cannot measure: one entry is one store record but one node per path component here. So the +// index charges the scan's budget node by node and stops when refused - see ArchiveIndex.Truncated. // -// It exists because the resolver it stands beside, ArchiveTar, can only index content that is already -// on disk: its index IS the tar's seek offsets, so every archive paid for a tar write and a tar walk -// before a cataloger could read a byte. Here the index is over archive.Entry values whose bytes may be -// in memory or in an overflow blob, and which may move between the two without the index noticing. +// The constants below are what one node and one file keep, deliberately over-estimated: an under-count +// lets the budget admit several times its setting. +// Test_indexChargeCoversWhatTheIndexKeeps holds them above the heap an index actually retains. +const ( + // mapSlotCost is what one string-keyed, pointer-valued pair costs a Go map beyond the pair itself: + // control bytes, load-factor headroom, and the doubling it grows by. + mapSlotCost = 96 + + // sliceSlotCost is one pointer held in a slice grown by appending, including the headroom append + // leaves behind it. + sliceSlotCost = 24 + + // keySplitNodeCost is one node of a key-split index - its lock, its value, and the two map headers + // it carries whether or not it has children - rounded up. The rounding is the headroom for a key + // that splits an existing one and so allocates a second node. + keySplitNodeCost = 112 + + // keySplitEntryCost is what one key costs the index that holds it - a node of its own, plus the + // slot it takes in its parent's two maps. A key that splits an existing one costs a second node, + // which is why this is charged per key rather than per distinct path through the trie. + keySplitEntryCost = keySplitNodeCost + 2*mapSlotCost + + // nameIndexCost is what one name keeps in a base-name index on top of its node: an entry in each of + // the forward and reverse tries, and a slot in the node slice each of those holds. The reverse + // trie's reversed name copy is charged by length alongside this, as is the ordered view + // AllLocations and FilesByMIMEType read. + nameIndexCost = 2*(keySplitEntryCost+sliceSlotCost) + sliceSlotCost +) + +// indexNodeCost is what one tree node keeps: the node itself - which carries a key-split index of its +// children by value - its slot in the by-path map, its slot in its parent's child list, and the slot +// it takes in its parent's child index. Name and path are substrings of the entry path, which is +// charged once per entry, so neither is counted again here. +var indexNodeCost = int64(reflect.TypeFor[indexNode]().Size()) + mapSlotCost + sliceSlotCost + keySplitEntryCost + +// ArchiveIndex is a resolver over the entries of one extracted archive. It indexes archive.Entry +// values rather than file offsets, so an entry's bytes can move between memory and the overflow blob +// without the index changing. // -// Lookups go through a key-split index rather than a walk. A cataloger's globs are overwhelmingly -// `**/*.ext` and `**/name`, and both are answered by a lookup keyed on the base name - by suffix for -// an extension, exactly for a name - instead of by matching every path in the archive. +// Glob lookups go through a base-name index rather than a walk: cataloger globs are overwhelmingly +// `**/*.ext` and `**/name`, answered by suffix and exact lookup respectively. type ArchiveIndex struct { fileSystemID string archivePath string root *indexNode - // byPath is every node by its full archive-relative path, which is what a path lookup wants and - // what HasPath answers from. + // byPath is every node by its full archive-relative path; path lookups and HasPath answer from it. byPath map[string]*indexNode - // fileNames and dirNames are the global base-name indexes the glob fast paths use. The values are - // slices because a name repeats across directories. + // fileNames and dirNames are the global base-name indexes the search answers from: files for a + // pattern's last segment, directories for the segment after a `**`. Values are slices because one + // name repeats across directories. fileNames syftindex.PrefixSuffix[[]*indexNode] dirNames syftindex.PrefixSuffix[[]*indexNode] - store *archive.EntryStore - records int + store *archive.EntryStore + records int + + // truncated reports that the budget refused a node and the remaining entries were never indexed. + // What was indexed before that is complete and usable. truncated bool // sorted is every file in a stable order, built once. See ordered. @@ -49,45 +88,47 @@ type ArchiveIndex struct { // indexNode is one file or directory in an archive's filesystem. // -// A directory may be synthesized: archives routinely name `a/b/c.txt` without ever naming `a/` or -// `a/b/`, and a resolver that only knew the named entries could not answer a glob whose middle -// segment is a directory. A synthesized node has no entry and no content. +// A directory may be synthesized: archives routinely name `a/b/c.txt` without naming `a/` or `a/b/`, +// and a glob whose middle segment is a directory needs those nodes. A synthesized node has no entry. type indexNode struct { name string path string isDir bool entry *archive.Entry metadata file.Metadata - parent *indexNode + // target is the node whose entry holds this node's content: itself for an ordinary file, what it + // points at for a link, and nil when the link dangles. Resolved once by resolveLinks. + target *indexNode + + // children indexes this directory's children by name, which is what an exact or `name*` segment + // looks up. childList is the same set in sorted order, for the segments that have to be matched + // one by one and for a stable result order. children syftindex.KeySplitIndex[*indexNode] childList []*indexNode } // NewFromArchiveEntries builds a resolver over the entries of one archive. // -// rootPath is what every entry's path is reported relative to, matching what the tar-backed resolver -// reports, so a location's path is archive-relative rather than naming the scan's scratch space. -// pathFilters are applied per entry exactly as they are for a directory walk, including the -// prune-a-subtree answer. -func NewFromArchiveEntries(rootDir, fileSystemID, archivePath string, store *archive.EntryStore, truncated bool, pathFilters ...PathIndexVisitor) (*ArchiveIndex, error) { +// An archive is its own root: entries are keyed, reported and filtered by their archive-relative +// path, and nothing here names a directory on the host. pathFilters are applied per entry as in a +// directory walk, including the prune-a-subtree answer, and are handed that same archive-relative +// path - see runPathFilters. +// +// charge is this archive's draw on the scan's resource limiter, charged per node kept. A refused +// charge truncates rather than fails the archive, as a refused entry does during extraction; a nil +// charge enforces nothing. +func NewFromArchiveEntries(fileSystemID, archivePath string, store *archive.EntryStore, charge *archive.Charge, pathFilters ...PathIndexVisitor) (*ArchiveIndex, error) { if store == nil { return nil, fmt.Errorf("no entry store for archive %q", fileSystemID) } - // Two frames, and they are not interchangeable. Nodes are keyed and reported by their - // archive-relative path, because that is what belongs in an SBOM - a location naming the scan's - // scratch directory would leak a temp path into published output, and the filesystem id chain is - // composed from these paths. Path filters, on the other hand, are built against the real - // directory, so they are handed the absolute path. - rootPath := treeRoot(rootDir) r := &ArchiveIndex{ fileSystemID: fileSystemID, archivePath: archivePath, root: &indexNode{name: "", path: "/", isDir: true}, byPath: map[string]*indexNode{}, store: store, - truncated: truncated, } r.byPath["/"] = r.root @@ -97,38 +138,58 @@ func NewFromArchiveEntries(rootDir, fileSystemID, archivePath string, store *arc if !ok { continue } - absPath := path.Join(rootPath, name) - - if isPruned(absPath, pruned) { + if isPruned(name, pruned) { continue } - switch runPathFilters(pathFilters, rootDir, absPath, entry.Header.FileInfo()) { + switch runPathFilters(pathFilters, name, entry.Header.FileInfo()) { case skipEntry: continue case skipTree: - pruned = append(pruned, absPath) + pruned = append(pruned, name) continue case keepEntry: } - r.add(entry, "/"+name) + if !r.add(entry, "/"+name, charge) { + // stop rather than carrying on and admitting later entries that need fewer nodes + r.truncated = true + break + } } + r.finalize() return r, nil } -// add puts one entry into the tree and the indexes, creating whatever directories its path implies. -func (r *ArchiveIndex) add(entry *archive.Entry, entryPath string) { +// add puts one entry into the tree and the indexes, creating the directories its path implies. It +// reports false when the budget refused a node, which truncates the index. +func (r *ArchiveIndex) add(entry *archive.Entry, entryPath string, charge *archive.Charge) bool { isDir := entry.Header.FileInfo().IsDir() - node := r.node(entryPath, isDir) + + // every node and by-path key is a substring of entryPath, so the one string backs the whole chain + // and is charged here rather than once per node + if !charge.IndexRecord(int64(len(entryPath))) { + return false + } + + node, ok := r.node(entryPath, isDir, charge) + if !ok { + return false + } if node == nil { - return + // names nothing inside this archive; not a budget refusal, so the walk carries on + return true } if node.entry != nil { - // a crafted archive naming the same path twice: the first entry is the one this filesystem - // has, matching the tar-backed resolver, which cannot add a second node at one path either - return + // an archive naming the same path twice: the first entry wins + return true + } + + // charged before committing the entry, so a refusal never leaves a file counted but unsearchable. + // The reverse index's key is the one string the name index allocates per name. + if !isDir && !charge.IndexRecord(nameIndexCost+int64(len(node.name))) { + return false } node.entry = entry @@ -136,55 +197,135 @@ func (r *ArchiveIndex) add(entry *archive.Entry, entryPath string) { r.records++ if isDir { - r.dirNames.Update(node.name, appendNode(node)) - return + // already placed in dirNames when the node was created; an archive naming a directory it also + // implies must not index it twice + return true } + r.fileNames.Update(node.name, appendNode(node)) + return true } +// appendNode adds one node to whatever a name already holds. The index stores a slice per key because +// base names repeat across directories. func appendNode(node *indexNode) syftindex.NodeUpdateFunc[[]*indexNode] { return func(current *syftindex.Node[[]*indexNode]) []*indexNode { return append(current.Value(), node) } } -// node returns the node at the given path, creating it and any missing parent directories. -func (r *ArchiveIndex) node(entryPath string, isDir bool) *indexNode { - if existing, ok := r.byPath[entryPath]; ok { - return existing +// node returns the node at the given path, creating any missing parents and charging the budget for +// each. It reports false when the budget refused a node, and a nil node for a path that names nothing +// inside the archive. +// +// Two passes: climb to the deepest ancestor already in the tree, then build down from there. The climb +// keeps the common case cheap - an entry whose parent is already placed costs one lookup however deep. +// +// Neither pass recurses: every key and name is a substring of entryPath, so a name 2,000 components +// deep costs 2,000 iterations, not 2,000 stack frames. Placing a node is still proportional to path +// length, which is why sanitizeEntryName bounds it. +func (r *ArchiveIndex) node(entryPath string, isDir bool, charge *archive.Charge) (*indexNode, bool) { + if existing, found := r.byPath[entryPath]; found { + return existing, true } - parentPath := path.Dir(entryPath) - if parentPath == entryPath { - // cannot climb any further without leaving the archive - return nil - } - parent := r.node(parentPath, true) - if parent == nil { - return nil + // climb: find the deepest ancestor already in the tree, and where in entryPath it ends + anchor, anchorEnd := r.root, 0 + for end := len(entryPath); ; { + slash := strings.LastIndexByte(entryPath[:end], '/') + if slash <= 0 { + // the parent is the archive root, which is always in the tree + break + } + if existing, found := r.byPath[entryPath[:slash]]; found { + anchor, anchorEnd = existing, slash + break + } + end = slash } - node := &indexNode{ - name: path.Base(entryPath), - path: entryPath, - isDir: isDir, - parent: parent, - } - parent.children.Set(node.name, node) - parent.childList = append(parent.childList, node) - r.byPath[entryPath] = node + // descend: create what is missing below the anchor. entryPath is rooted, cleaned and free of + // trailing separators (see sanitizeEntryName), so every stretch between separators is a component. + for start := anchorEnd + 1; start < len(entryPath); { + end := len(entryPath) + if i := strings.IndexByte(entryPath[start:], '/'); i >= 0 { + end = start + i + } - if isDir && node.entry == nil { - // a directory the archive never named still has to be findable by name, since a glob's middle - // segment may be exactly it - r.dirNames.Update(node.name, appendNode(node)) + if !charge.IndexRecord(indexNodeCost) { + return nil, false + } + child := &indexNode{ + name: entryPath[start:end], + path: entryPath[:end], + // anything with more path below it is a directory whatever the entry itself is + isDir: end < len(entryPath) || isDir, + } + anchor.children.Set(child.name, child) + anchor.childList = append(anchor.childList, child) + r.byPath[child.path] = child + + if child.isDir { + // a directory the archive never named still has to be findable by name, since the segment + // after a `**` is matched against directories - see searchFromRoot + if !charge.IndexRecord(nameIndexCost + int64(len(child.name))) { + return nil, false + } + r.dirNames.Update(child.name, appendNode(child)) + } + + anchor = child + start = end + 1 } - return node + + if anchor == r.root { + return nil, true + } + return anchor, true } -// metadataOf describes one entry as a file in the archive's filesystem, including the MIME type -// sniffed from its content - which is what FilesByMIMEType answers from, and what archive detection -// one level down depends on. +// finalize sorts each directory's child list by name. The key-split indexes need no finalizing - they +// are searchable as they are built - but childList is what the segments that cannot be looked up are +// matched against, and a map's iteration order there would make results unstable. +func (r *ArchiveIndex) finalize() { + for _, node := range r.byPath { + sort.Slice(node.childList, func(i, j int) bool { return node.childList[i].name < node.childList[j].name }) + node.target = r.linkTarget(node) + } +} + +// maxLinkHops bounds a chain of links pointing at links. No real archive is shaped that way, and the +// bound is also what stops a cycle from being followed forever. +const maxLinkHops = 8 + +// linkTarget returns the node whose entry holds this node's content: the node itself when it is not a +// link, and what it points at when it is, following a chain. +// +// nil means there is nothing to read: the chain leaves the archive, ends at a directory, or does not +// end. Such a link still exists as a path, and is answered under that path with no content - which is +// all the archive says about it. +func (r *ArchiveIndex) linkTarget(node *indexNode) *indexNode { + for hops := 0; ; hops++ { + if node.entry == nil || node.isDir { + return nil + } + header := node.entry.Header + if header.Linkname == "" { + return node + } + if hops == maxLinkHops { + return nil + } + next, ok := r.byPath[resolveLinkWithinArchive(node.path, header.Linkname, header.Typeflag == tar.TypeLink)] + if !ok { + return nil + } + node = next + } +} + +// metadataOf describes one entry as a file, including the MIME type sniffed from its content, which +// FilesByMIMEType and nested archive detection both read. func (r *ArchiveIndex) metadataOf(entry *archive.Entry, entryPath string) file.Metadata { var contents io.Reader if opened, err := r.store.Open(entry); err == nil { @@ -192,59 +333,86 @@ func (r *ArchiveIndex) metadataOf(entry *archive.Entry, entryPath string) file.M } metadata := stereoscopeFile.NewMetadata(entry.Header, contents) + // NewMetadata copies the header by value and its fs.FileInfo points into that copy, pinning a second + // tar.Header per entry. Point it at the header the store already holds and charged for. + metadata.FileInfo = entry.Header.FileInfo() metadata.Path = reportedPath(entryPath) if entry.Header.Linkname != "" { - relative := path.Clean(entry.Header.Linkname) - if !path.IsAbs(relative) { - relative = path.Join("/", path.Dir(entryPath), entry.Header.Linkname) - } - // resolved in archive-relative space, where path.Join clamps a climb at the root, so a target - // naming the host filesystem lands somewhere inside this archive or nowhere at all - metadata.LinkDestination = reportedPath(relative) + metadata.LinkDestination = reportedPath( + resolveLinkWithinArchive(entryPath, entry.Header.Linkname, entry.Header.Typeflag == tar.TypeLink)) } return metadata } -// Records reports how many of the archive's entries this filesystem holds. +// resolveLinkWithinArchive returns where a link entry points, resolved against the archive's own root +// rather than the host filesystem. linkPath is the archive-rooted path of the link itself. +// +// A hard link's target is named from the archive root (per the tar format), and an absolute symlink +// target is read as archive-absolute, so "/etc/passwd" names an entry of this archive, not the host's. +// Only a relative symlink target is joined onto the link's own directory. The leading "/" makes +// path.Join clamp climbs at the archive root, so "../../../etc/passwd" lands at "/etc/passwd" inside +// the archive. +// +// Nothing here follows the link: the result is a path in this archive's frame, resolving to an entry +// of this archive or to nothing. +func resolveLinkWithinArchive(linkPath, linkname string, hardLink bool) string { + if hardLink || path.IsAbs(linkname) { + return path.Join("/", linkname) + } + return path.Join("/", path.Dir(linkPath), linkname) +} + func (r *ArchiveIndex) Records() int { return r.records } -// IndexTruncated reports that the archive's content ended before the archive did. -func (r *ArchiveIndex) IndexTruncated() bool { return r.truncated } +// Truncated reports whether the budget stopped indexing before the archive's last entry. What is +// held is usable either way, just not all of the archive. +func (r *ArchiveIndex) Truncated() bool { return r.truncated } -// Close releases the entry store this index was built over, and with it the overflow blob's file -// handle. Readers handed out by this resolver borrow the store, so the index must not be closed until -// the extracted archive is torn down - which is exactly when ExtractedArchive.Cleanup calls it, after -// the sub-pipeline and after recursion. Safe to call more than once. +// Close releases the entry store, and with it the overflow blob's file handle. Readers handed out by +// this resolver borrow the store, so Close must wait until the extracted archive is torn down; +// ExtractedArchive.Cleanup calls it there. Safe to call more than once. func (r *ArchiveIndex) Close() error { if r == nil || r.store == nil { return nil } - // the store reference is kept, not cleared: EntryStore.Close is itself idempotent, and a read that - // arrives after Close then fails gracefully through the store rather than dereferencing a nil one. + // the reference is kept, not cleared: EntryStore.Close is idempotent, and a read arriving after + // Close fails through the store rather than dereferencing nil return r.store.Close() } func (r *ArchiveIndex) location(node *indexNode) file.Location { - return file.NewLocationFromCoordinates(file.Coordinates{ + return file.NewLocationFromCoordinates(r.coordinates(node)) +} + +// accessedLocation names the file at target, reached by the path of access. The two differ when a glob +// matched a link: the content is the target's, while the path the archive was searched by - and the +// one a reader will recognize - is the link's. +func (r *ArchiveIndex) accessedLocation(target, access *indexNode) file.Location { + if access == target { + return r.location(target) + } + return file.NewVirtualLocationFromCoordinates(r.coordinates(target), reportedPath(access.path)) +} + +func (r *ArchiveIndex) coordinates(node *indexNode) file.Coordinates { + return file.Coordinates{ RealPath: reportedPath(node.path), FileSystemID: r.fileSystemID, ArchivePath: r.archivePath, - }) + } } -// reportedPath is how a path leaves this resolver: archive-relative and with no leading slash, which -// is the form the tar-backed resolver reports and therefore the form already in published SBOMs and in -// every filesystem-id chain composed from one. Nodes are keyed with the slash, because that is what -// path.Dir and path.Join want. +// reportedPath is how a path leaves this resolver: archive-relative with no leading slash, the form +// SBOMs and archive-path chains use. Nodes are keyed with the slash, which path.Dir and path.Join +// want. func reportedPath(nodePath string) string { return strings.TrimPrefix(nodePath, "/") } -// FileContentsByLocation returns a reader over the entry's content, from memory or from the overflow -// blob as it currently stands. The reader is Read, Seek and ReadAt either way, so a nested archive is -// read where it lies. +// FileContentsByLocation returns a reader over the entry's content, from memory or the overflow blob. +// Either way it supports Read, Seek and ReadAt, so a nested archive is read where it lies. func (r *ArchiveIndex) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { node, ok := r.byPath[r.requestPath(location.RealPath)] if !ok || node.entry == nil { @@ -272,7 +440,7 @@ func (r *ArchiveIndex) HasPath(userPath string) bool { return ok } -// FilesByPath returns the files at the given paths. Directories are not files, matching every other +// FilesByPath returns the files at the given paths. Directories are not files, as in every other // resolver. func (r *ArchiveIndex) FilesByPath(paths ...string) ([]file.Location, error) { var out []file.Location @@ -291,7 +459,6 @@ func (r *ArchiveIndex) FilesByPath(paths ...string) ([]file.Location, error) { return out, nil } -// FilesByMIMEType returns the files whose sniffed content type is one of the given types. func (r *ArchiveIndex) FilesByMIMEType(types ...string) ([]file.Location, error) { wanted := map[string]struct{}{} for _, t := range types { @@ -300,7 +467,9 @@ func (r *ArchiveIndex) FilesByMIMEType(types ...string) ([]file.Location, error) var out []file.Location for _, node := range r.ordered() { - if node.entry == nil { + if node.target != node { + // only a file that holds bytes has a sniffed type; a link has none of its own. The target + // answers for itself under its own path. continue } if _, ok := wanted[node.metadata.MIMEType]; !ok { @@ -311,16 +480,10 @@ func (r *ArchiveIndex) FilesByMIMEType(types ...string) ([]file.Location, error) return out, nil } -// ordered returns the archive's files in a stable order. +// ordered returns the archive's files (not directories) in a stable path order. // -// Every answer this resolver gives has to be ordered, because a resolver that returns the same set in -// a different order each run is not the same resolver twice. The archive walk enters archives in the -// order they are handed to it, and what it finds inside them is merged into one shared SBOM - so the -// order decides which locations end up grouped into which package. Measured on trinodb/trino, whose -// plugin jars are hard links to one another: unordered, one libzstd with 24 locations became three -// libzstd packages holding 6, 9 and 9 of the same 24. -// -// Files are kept rather than every node, since every caller of this wants files. +// Order decides which locations group into which package when results merge into the SBOM, so an +// unstable order splits one package into several across runs. func (r *ArchiveIndex) ordered() []*indexNode { if r.sorted != nil { return r.sorted @@ -337,8 +500,7 @@ func (r *ArchiveIndex) ordered() []*indexNode { return out } -// RelativeFileByPath resolves a path relative to the archive's root, which is the only frame an -// archive has. +// RelativeFileByPath resolves against the archive's root, the only frame an archive has. func (r *ArchiveIndex) RelativeFileByPath(_ file.Location, userPath string) *file.Location { locations, err := r.FilesByPath(userPath) if err != nil || len(locations) == 0 { @@ -347,7 +509,6 @@ func (r *ArchiveIndex) RelativeFileByPath(_ file.Location, userPath string) *fil return &locations[0] } -// AllLocations returns every file in the archive. func (r *ArchiveIndex) AllLocations(ctx context.Context) <-chan file.Location { out := make(chan file.Location) go func() { @@ -366,8 +527,7 @@ func (r *ArchiveIndex) AllLocations(ctx context.Context) <-chan file.Location { return out } -// requestPath puts a caller's path into the archive's frame: a path may arrive absolute (the way a -// cataloger writes it) or already rooted, and both name the same entry. +// requestPath puts a caller's path into the archive's frame; a leading slash is optional. func (r *ArchiveIndex) requestPath(userPath string) string { if userPath == "" { return "/" @@ -377,8 +537,8 @@ func (r *ArchiveIndex) requestPath(userPath string) string { var _ file.Resolver = (*ArchiveIndex)(nil) -// nopCloserAt gives an entry reader the Close the resolver interface asks for. The store owns the -// bytes and the overflow file, so closing a reader over them must not close anything. +// nopCloserAt gives an entry reader the Close the resolver interface requires. The store owns the +// bytes and the overflow file, so closing a reader over them must close nothing. type nopCloserAt struct { archive.ReaderAtSeeker } diff --git a/syft/internal/fileresolver/archive_index_bench_test.go b/syft/internal/fileresolver/archive_index_bench_test.go new file mode 100644 index 000000000..0ac9c2b18 --- /dev/null +++ b/syft/internal/fileresolver/archive_index_bench_test.go @@ -0,0 +1,80 @@ +package fileresolver + +import ( + "fmt" + "testing" +) + +// BenchmarkArchiveIndex_FilesByGlob covers the shapes a cataloger asks for: the ones the name index +// answers outright, the ones reducing to a finite set of lookups, and the ones reaching doublestar +// over whatever the literal bounds narrow them to. +// +// Every pattern but the last two is one syft's own catalogers register, verbatim. That matters because +// a benchmark of invented patterns measures the wrong thing: no cataloger glob uses a character class, +// so `class-in-extension` is kept for coverage rather than as a cost anyone pays, and `many-matches` +// is a control - it matches thousands of files, so it measures building and sorting locations rather +// than finding them, and is expected to stay flat while the others improve. +func BenchmarkArchiveIndex_FilesByGlob(b *testing.B) { + r := benchArchive(b) + + patterns := map[string]string{ + "exact-name": "**/go", + "extension": "**/*.jar", + "alternation": "**/{go,go.exe}", + "question-marks": "**/libstd-????????????????.so", + "alternation-suffix": "**/*.{jar,war}", + "alternation-with-sep": "**/var/lib/dpkg/{status,status.d/**}", + "alternation-both-ends": "**/{var/lib,usr/share,usr/lib/sysimage}/rpm/" + + "{Packages,Packages.db,rpmdb.sqlite}", + "alternation-mid-segment": "**/{java,jvm}/*/release", + "alternation-mid-path": "/{usr,opt}/**/*.jar", + "class-in-extension": "**/*.[jw]ar", + "many-matches": "**/[lm]*", + } + + for name, pattern := range patterns { + b.Run(name, func(b *testing.B) { + for b.Loop() { + if _, err := r.FilesByGlob(pattern); err != nil { + b.Fatal(err) + } + } + }) + } +} + +// benchArchive is shaped like a fat application image layer: thousands of entries with thousands of +// distinct base names, spread over nested directories. +// +// The names must be distinct: an archive whose files are all called README.md puts two keys in the +// name index and makes every lookup look free. The decoys serve the same purpose - names ending "ar" +// that are not archives, names beginning "go" that are not the go binary - separating a lookup keyed +// on a whole extension from one keyed on the two characters a weak bound fixes. +func benchArchive(tb testing.TB) *ArchiveIndex { + tb.Helper() + entries := map[string]string{ + "usr/local/go/bin/go": "binary", + "usr/lib/rustlib/libstd-0123456789abcdef.so": "binary", + "usr/bin/mariadb": "binary", + "opt/app/lib/app.jar": "PK\x03\x04", + "opt/app/lib/dep.war": "PK\x03\x04", + "var/lib/dpkg/status": "dpkg", + "var/lib/dpkg/status.d/base": "dpkg", + "var/lib/rpm/Packages": "rpm", + "usr/lib/jvm/java-21/release": "jvm", + } + for i := range 1500 { + // decoys for the weak tail bound "ar" that `*.[jw]ar` and `*.{jar,war}` would share + entries[fmt.Sprintf("var/cache/archive%04d.tar", i)] = "data" + entries[fmt.Sprintf("var/cache/bundle%04d.rar", i)] = "data" + entries[fmt.Sprintf("usr/share/cal%04dendar", i)] = "data" + // decoys for the head bound "go" + entries[fmt.Sprintf("usr/share/gopher%04d.md", i)] = "docs" + // decoys for the head bounds "l" and "m", and for "libstd-" + entries[fmt.Sprintf("usr/lib/libstd-%04d.a", i)] = "archive" + entries[fmt.Sprintf("usr/lib/mod%04d.py", i)] = "code" + // ordinary bulk, with distinct names + entries[fmt.Sprintf("usr/share/doc/pkg%04d/NOTES%04d.md", i, i)] = "docs" + } + return indexOver(tb, 1<<26, entries) +} diff --git a/syft/internal/fileresolver/archive_index_charge_test.go b/syft/internal/fileresolver/archive_index_charge_test.go new file mode 100644 index 000000000..5ad90b8b0 --- /dev/null +++ b/syft/internal/fileresolver/archive_index_charge_test.go @@ -0,0 +1,228 @@ +package fileresolver + +import ( + "archive/tar" + "context" + "fmt" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/tmpdir" +) + +// Test_indexChargeCoversWhatTheIndexKeeps is what the cost constants in archive_index.go rest on. +// +// The charge exists to stop an archive before its index is built, so it must over-estimate what the +// index really retains. An under-count is worse than no limit at all: it reads as a configured bound +// while admitting some multiple of it. A previous accounting charged 216 bytes plus the name per +// entry where the index kept 627, and charged nothing for the directory nodes an entry's path +// implies - which a single entry can name 200,000 of. +func Test_indexChargeCoversWhatTheIndexKeeps(t *testing.T) { + tests := []struct { + name string + names []string + }{ + { + name: "flat, short names", + names: func() []string { + out := make([]string, 60000) + for i := range out { + out[i] = fmt.Sprintf("d/%06d", i) + } + return out + }(), + }, + { + name: "nested, jar-shaped names", + names: func() []string { + out := make([]string, 60000) + for i := range out { + out[i] = fmt.Sprintf("com/example/pkg%03d/sub%02d/Thing%06d.class", i%500, i%50, i) + } + return out + }(), + }, + { + name: "one entry per directory, so every entry synthesizes a chain", + names: func() []string { + out := make([]string, 20000) + for i := range out { + out[i] = fmt.Sprintf("a%05d/b%05d/c%05d/d%05d/file.txt", i, i, i, i) + } + return out + }(), + }, + { + name: "deep single entry, at the path cap", + names: []string{strings.Repeat("a/", (maxEntryPathBytes-2)/2) + "f"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := storeOf(t, tt.names) + limiter, charge := unboundedCharge() + + var index *ArchiveIndex + kept := heapGrowth(t, func() { + var err error + index, err = NewFromArchiveEntries("", "outer.jar", store, charge) + require.NoError(t, err) + }) + require.False(t, index.Truncated(), "an unbounded charge must never truncate") + + charged, _ := limiter.InUse() + t.Logf("entries=%d records=%d charged=%d kept=%d (charged/kept=%.2fx)", + len(tt.names), index.Records(), charged, kept, float64(charged)/float64(kept)) + + assert.GreaterOrEqual(t, charged, kept, + "the index charged less than it kept, so the configured bound admits more than it says") + + // an over-estimate is the point, but one wildly over wastes the budget on archives doing + // nothing wrong + assert.Less(t, charged, kept*4, + "the index charged several times what it kept, which spends a scan's budget on nothing") + + runtime.KeepAlive(index) + }) + } +} + +// Test_deepEntryPathsAreRefused covers the bound on an entry's path, which keeps one name from +// becoming an arbitrarily long chain of nodes. +func Test_deepEntryPathsAreRefused(t *testing.T) { + atCap := strings.Repeat("a/", (maxEntryPathBytes-2)/2) + "f" + overCap := strings.Repeat("a/", maxEntryPathBytes) + "f" + + require.LessOrEqual(t, len(atCap), maxEntryPathBytes) + require.Greater(t, len(overCap), maxEntryPathBytes) + + index, err := NewFromArchiveEntries("", "outer.jar", + storeOf(t, []string{"kept.txt", atCap, overCap}), nil) + require.NoError(t, err) + + assert.Equal(t, 2, index.Records(), "the entry over the cap is dropped and the rest are kept") + assert.True(t, index.HasPath("/kept.txt")) + assert.True(t, index.HasPath("/"+atCap)) + assert.False(t, index.HasPath("/"+overCap)) + assert.False(t, index.Truncated(), "dropping an unusable name is not a budget truncation") +} + +// Test_deepEntryPathsStayAffordable guards the two-pass placement in ArchiveIndex.node. +// +// Placing a node used to recurse from the leaf up, rebuilding the path with path.Dir at every level, +// making a chain cost work quadratic in its depth. Against the replacement: 400 entries sharing one +// deep prefix went from 9.8ms to 0.77ms, 400 entries each with their own chain from 2.87s to 0.33s, +// and a single 200,000-component name from 59.3s to 1.3s. That last shape no longer reaches here - +// sanitizeEntryName refuses it - which bounds the residual per-node cost neither pass can remove: +// placing a node means keying it by its full path. +// +// The budget below is enormous next to the ~0.1s this takes: it guards against an order-of-magnitude +// regression, not a few percent. +func Test_deepEntryPathsStayAffordable(t *testing.T) { + if testing.Short() { + t.Skip("timing guard") + } + + names := make([]string, 400) + for i := range names { + names[i] = fmt.Sprintf("%s%04d.txt", strings.Repeat("a/", (maxEntryPathBytes-16)/2), i) + } + + store := storeOf(t, names) + started := time.Now() + index, err := NewFromArchiveEntries("", "outer.jar", store, nil) + require.NoError(t, err) + took := time.Since(started) + + require.Equal(t, len(names), index.Records()) + assert.Less(t, took, 30*time.Second, "placing deep paths has regressed by an order of magnitude") + t.Logf("indexed %d entries of %d bytes of path in %s", len(names), len(names[0]), took) +} + +// Test_indexTruncatesWhenTheBudgetRefuses covers what an index does when the budget runs out part way +// through: it keeps what it has, says so, and stays usable. +func Test_indexTruncatesWhenTheBudgetRefuses(t *testing.T) { + names := make([]string, 500) + for i := range names { + names[i] = fmt.Sprintf("dir/file%04d.txt", i) + } + + // room for a handful of entries, nowhere near all 500 + limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: 10 * indexNodeCost, MaxDiskBytes: 0}) + index, err := NewFromArchiveEntries("", "outer.jar", storeOf(t, names), limiter.Charge()) + require.NoError(t, err) + + assert.True(t, index.Truncated(), "the budget refused a record, so the index is truncated") + assert.Positive(t, index.Records(), "what was indexed before the refusal is kept") + assert.Less(t, index.Records(), len(names)) + + // and what it did index is a working filesystem, not a half-built one + locations, err := index.FilesByGlob("**/*.txt") + require.NoError(t, err) + assert.Len(t, locations, index.Records()) + for _, loc := range locations { + assert.True(t, index.HasPath("/"+loc.RealPath)) + } +} + +// Test_zeroMemoryBudgetStillIndexes covers why index records fall back to the disk budget: +// MaxMemoryBytes of zero is a supported configuration meaning "spill everything", and an archive must +// still be reachable under it. +func Test_zeroMemoryBudgetStillIndexes(t *testing.T) { + limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) + index, err := NewFromArchiveEntries("", "outer.jar", + storeOf(t, []string{"a/b/c.txt", "a/d.txt"}), limiter.Charge()) + require.NoError(t, err) + + assert.False(t, index.Truncated()) + assert.Equal(t, 2, index.Records()) + + memory, disk := limiter.InUse() + assert.Zero(t, memory, "a zero memory budget admits nothing to memory") + assert.Positive(t, disk, "so the index records are charged against the disk budget instead") +} + +// storeOf puts the given entry names into a store with no budget of its own, so what a test measures +// afterwards is the index's charge and nothing else. +func storeOf(t testing.TB, names []string) *archive.EntryStore { + t.Helper() + workDir := archive.NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) + t.Cleanup(workDir.Remove) + store := archive.NewEntryStore(workDir, "test.jar", nil) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + for _, name := range names { + hdr := tar.Header{Name: name, Mode: 0o600, Typeflag: tar.TypeReg} + _, err := store.Add(hdr, strings.NewReader(""), nil) + require.NoError(t, err) + } + return store +} + +func unboundedCharge() (*archive.Limiter, *archive.Charge) { + limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) + return limiter, limiter.Charge() +} + +// heapGrowth reports how much live heap build left behind, which for an index kept alive is what it +// retains. +func heapGrowth(t testing.TB, build func()) int64 { + t.Helper() + var before, after runtime.MemStats + runtime.GC() + runtime.GC() + runtime.ReadMemStats(&before) + + build() + + runtime.GC() + runtime.GC() + runtime.ReadMemStats(&after) + return int64(after.HeapAlloc) - int64(before.HeapAlloc) +} diff --git a/syft/internal/fileresolver/archive_index_glob.go b/syft/internal/fileresolver/archive_index_glob.go index 58073f460..7fe1c60c0 100644 --- a/syft/internal/fileresolver/archive_index_glob.go +++ b/syft/internal/fileresolver/archive_index_glob.go @@ -6,171 +6,504 @@ import ( "github.com/bmatcuk/doublestar/v4" + syftindex "github.com/anchore/syft/internal/index" "github.com/anchore/syft/syft/file" ) +// The search here is ported from the dirscan prototype the key-split index came from. The shape is +// the prototype's: walk the pattern segment by segment, and wherever a segment fixes enough text, +// answer it from an index instead of from the children of the directory the walk has reached. +// +// Three deliberate departures from the prototype: +// +// - doublestar decides every segment the indexes cannot answer, rather than the prototype's +// hand-built regex. syft's globs are doublestar's dialect, and a segment carrying `?` or a `[...]` +// class has to mean there what it means everywhere else in syft. +// - a pattern the prototype panics on is answered instead: an embedded `**` (`a**b`) is matched as +// an ordinary segment, and a trailing `**` collects everything below. +// - alternations are expanded over the whole pattern rather than one segment at a time, and on `,` +// rather than the prototype's `|`. syft's catalogers write `{status,status.d/**}`, where the +// alternatives carry separators, and `|` is not an alternation to doublestar at all. const ( anyDir = "**" + sep = "/" wildcard = '*' ) // FilesByGlob returns the files matching any of the patterns. // -// The walk is segment by segment, with one deliberate shortcut: a pattern that begins `**/` and whose -// remainder is a single segment is answered from the global base-name index instead of by descending -// the tree. That covers `**/*.jar`, `**/pom.properties` and `**/*pom.xml` - which is most of what -// syft's catalogers ask for - and turns "match every path in this archive" into a lookup keyed on the -// name. -// -// Patterns are archive-relative. A leading slash is not a filesystem root here, it is the archive's -// own root, so `/META-INF/*` and `META-INF/*` name the same entries - and `/*` names the entries at -// the top of the archive, which is what the tar-backed resolver could not match at all. +// Patterns are archive-relative. A leading slash is the archive's own root rather than a filesystem +// root, so `/META-INF/*` and `META-INF/*` name the same entries. func (r *ArchiveIndex) FilesByGlob(patterns ...string) ([]file.Location, error) { - seen := map[string]struct{}{} - var out []file.Location - - keep := func(node *indexNode) { - if node.isDir || node.entry == nil { - return - } - if _, dup := seen[node.path]; dup { - return - } - seen[node.path] = struct{}{} - out = append(out, r.location(node)) - } - + found := map[*indexNode]struct{}{} for _, pattern := range patterns { - segments := splitPattern(pattern) - if len(segments) == 0 { - continue + // braces are expanded before splitting into segments, since an alternative may carry its own + // separators: `{status,status.d/**}` is one segment and two, and splitting first would cut the + // group in half. Past maxAlternatives the pattern is searched as written - see segmentProvider. + alternatives, ok := expandBraces(pattern) + if !ok { + alternatives = []string{pattern} } - if nodes, ok := r.byNameIndex(segments); ok { - for _, node := range nodes { - keep(node) + for _, alternative := range alternatives { + trimmed := strings.Trim(alternative, sep) + if trimmed == "" { + continue } - continue + r.search(found, r.root, strings.Split(trimmed, sep), nil, true) } - r.match(r.root, segments, keep) } - // ordered for the same reason every other answer here is: what a cataloger is handed in what order - // decides how its findings merge, and the index's own traversal order is a map's + out := make([]file.Location, 0, len(found)) + for target, access := range collapseToOnePathPerFile(found) { + out = append(out, r.accessedLocation(target, access)) + } + // location order decides how a cataloger merges its findings, and the search accumulates into a map. + // This sort is only for a stable answer; which path names each file is settled by preferredAccess. sort.Slice(out, func(i, j int) bool { return out[i].RealPath < out[j].RealPath }) return out, nil } -// splitPattern turns a glob into its segments, dropping the leading and trailing separators that -// distinguish "rooted at the archive" from "relative to it" - inside one archive there is no -// difference. -func splitPattern(pattern string) []string { - trimmed := strings.Trim(pattern, "/") - if trimmed == "" { - return nil +// collapseToOnePathPerFile reduces the matched nodes to one path per file, keyed by the node holding +// that file's content. +// +// Several paths can name one file - a link and its target, or two links to it - and a resolver answers +// with one of them (see file.Resolver: "if multiple paths to the same file are found, the best single +// match should be returned"). A dangling link names no content, so it stands alone under its own path. +func collapseToOnePathPerFile(found map[*indexNode]struct{}) map[*indexNode]*indexNode { + best := make(map[*indexNode]*indexNode, len(found)) + for node := range found { + target := node.target + if target == nil { + target = node + } + current, seen := best[target] + if !seen || preferredAccess(node, current, target) == node { + best[target] = node + } } - return strings.Split(trimmed, "/") + return best } -// byNameIndex answers the `**/` shapes from the global file-name index, and reports false -// for anything else so the caller falls back to the walk. +// preferredAccess picks which of two paths to one file the answer carries: the path that is the file +// itself, and otherwise the lower-sorting one. // -// Only the forms the index can answer exactly are taken: a suffix match (`*foo`), a prefix match -// (`foo*`), and an exact name. A segment with a wildcard in the middle would need every name anyway, -// so it goes to the walk rather than pretending. -func (r *ArchiveIndex) byNameIndex(segments []string) ([]*indexNode, bool) { - if len(segments) != 2 || segments[0] != anyDir { - return nil, false +// Both clauses read only the candidates, never their arrival order, so the winner is stable across +// runs regardless of map iteration order or how results are later sorted. +func preferredAccess(a, b, target *indexNode) *indexNode { + switch { + case a == target: + return a + case b == target: + return b + case a.path < b.path: + return a } - name := segments[1] - if strings.Contains(name, anyDir) { - return nil, false + return b +} + +// childProvider yields the nodes of one directory that a segment can reach. A provider may ignore the +// directory it is handed and answer from a global index instead, which is how `**` is skipped over. +type childProvider func(parent *indexNode) []*indexNode + +// search matches segments against the tree from parent, collecting every file the pattern reaches. +// +// visited is how `**` avoids reporting the same file twice when the directories it expands to overlap, +// and checkVisited is false on the recursions that are not a descent - an alternation being retried +// against the same directory, or `**` matching the directory it starts from. +func (r *ArchiveIndex) search(found map[*indexNode]struct{}, parent *indexNode, segments []string, visited []*indexNode, checkVisited bool) { + if len(segments) == 0 { + return } - first := strings.IndexRune(name, wildcard) - last := strings.LastIndexByte(name, byte(wildcard)) + if checkVisited { + for _, seen := range visited { + if seen == parent { + return + } + } + visited = append(visited, parent) + } + + segment := segments[0] + + if segment == anyDir { + r.searchAnyDir(found, parent, segments, visited) + return + } + + provider := r.segmentProvider(segment) + + if len(segments) == 1 { + for _, child := range provider(parent) { + keepFile(found, child) + } + return + } + + for _, child := range provider(parent) { + if !child.isDir { + continue + } + r.search(found, child, segments[1:], visited, true) + } +} + +// searchAnyDir handles a `**` segment. +// +// `**` matches the directory it sits at and every directory below it, so the general answer is to +// re-run what follows from each of those. The index is what makes that avoidable: when the pattern +// starts with `**`, the segment after it can often be looked up by name across the whole archive, +// which reaches the same directories - or, at the end of a pattern, the same files - without walking. +func (r *ArchiveIndex) searchAnyDir(found map[*indexNode]struct{}, parent *indexNode, segments []string, visited []*indexNode) { + rest := segments[1:] + + if len(rest) == 0 { + // a pattern ending in `**`: everything below here. The prototype rejects this shape; syft's + // catalogers do write it. + r.collect(found, parent) + return + } + + // The segment after a `**` is looked up across the whole archive, then confined to what sits below + // parent. The global lookup is what the name indexes key on; the filter is what lets a `**` appear + // anywhere in the pattern, not just as the leading segment. + // + // This matters for split patterns: `/{usr,opt}/**/*.jar` becomes `/usr/**/*.jar` and `/opt/**/*.jar`, + // each reaching its `**` with a directory already selected, where the prototype would walk every + // directory beneath it. + // + // Alternations are already expanded by FilesByGlob, so `{go,go.exe}` reaches here as two exact names. + if len(rest) == 1 { + // `**/`, the shape most cataloger globs take + if nodes, ok := r.byName(&r.fileNames, rest[0]); ok { + for _, node := range nodes { + if under(node, parent) { + keepFile(found, node) + } + } + return + } + } else if nodes, ok := r.byName(&r.dirNames, rest[0]); ok { + // `**//`: the directory segment is looked up across the archive, and the search + // continues from each match with both segments consumed + for _, node := range nodes { + if under(node, parent) { + r.search(found, node, rest[1:], visited, true) + } + } + return + } + + // `**` matches this directory and every directory beneath it + r.search(found, parent, rest, visited, false) + for _, dir := range r.dirsBeneath(parent) { + r.search(found, dir, rest, visited, true) + } +} + +func dedupeNodes(nodes []*indexNode) []*indexNode { + if len(nodes) < 2 { + return nodes + } + seen := make(map[*indexNode]struct{}, len(nodes)) + out := nodes[:0] + for _, node := range nodes { + if _, dup := seen[node]; dup { + continue + } + seen[node] = struct{}{} + out = append(out, node) + } + return out +} + +// byName answers one segment from a global name index, reporting false when the segment fixes too +// little for any lookup to reach every name it could match. +// +// Three shapes the index answers outright: an exact name, a single leading `*`, and a single trailing +// `*`. Everything else goes to byNameNarrowed, which either reduces the segment to a set of those or +// narrows by the literal text it fixes and lets doublestar decide. +func (r *ArchiveIndex) byName(idx *syftindex.PrefixSuffix[[]*indexNode], segment string) ([]*indexNode, bool) { + if strings.Contains(segment, anyDir) { + // `a**b` crosses directories, so no base-name lookup can answer it + return nil, false + } + if strings.ContainsAny(segment, nonWildcardMeta) { + // not a literal-with-one-`*`, so the plain lookups below would read the metacharacters as text + return r.byNameNarrowed(idx, segment) + } + + first := strings.IndexRune(segment, wildcard) + last := strings.LastIndexByte(segment, byte(wildcard)) switch { case first < 0: // an exact name, e.g. **/pom.properties - return flattenNodes(r.fileNames.Get(name)), true - case first == 0 && last == 0 && len(name) > 1: - // *something, e.g. **/*.jar - the reverse index is keyed for exactly this - return flattenNodes(r.fileNames.BySuffix(name[1:])...), true - case last == len(name)-1 && first == last && len(name) > 1: - // something*, e.g. **/pom* - return flattenNodes(r.fileNames.ByPrefix(name[:len(name)-1])...), true + return idx.Get(segment), true + case first == 0 && last == 0: + // `*something`, e.g. **/*.jar - what the reverse index is keyed for. A lone `*` gives the empty + // suffix, which reaches every name, and that is the right answer for `**/*`. + return flattenNodes(idx.BySuffix(segment[1:])), true + case last == len(segment)-1 && first == last: + // `something*`, e.g. **/pom* + return flattenNodes(idx.ByPrefix(segment[:len(segment)-1])), true } - return nil, false + + // anything else - a `?`, a class, a wildcard in the middle - is not a lookup on its own, but the + // literal text it fixes still narrows one + return r.byNameNarrowed(idx, segment) } -func flattenNodes(groups ...[]*indexNode) []*indexNode { +// byNameNarrowed answers a segment the three shapes above cannot, by narrowing the index to a +// candidate set and letting doublestar decide over it. It reports false when the segment fixes nothing +// to narrow by, which leaves the caller to walk. +// +// Two ways in. A segment whose only metacharacters are alternations and classes denotes a finite set +// of names, so it reduces to exact lookups and doublestar is not needed at all. Anything else is +// narrowed by the literal text it fixes at its head or tail - `libstd-????????????????.so` fixes +// "libstd-" and ".so" - and doublestar runs over what that leaves. +func (r *ArchiveIndex) byNameNarrowed(idx *syftindex.PrefixSuffix[[]*indexNode], segment string) ([]*indexNode, bool) { + if lookups := enumerateLookups(segment); len(lookups) > 0 { + // each lookup is exactly its alternative: `*` stands for any run of characters and a base name + // holds no separator for it to stop at + return byLookups(idx, lookups), true + } + + candidates, ok := r.narrowByBounds(idx, segment) + if !ok { + return nil, false + } + return matchingNodes(candidates, segment), true +} + +func byLookups(idx *syftindex.PrefixSuffix[[]*indexNode], lookups []nameLookup) []*indexNode { var out []*indexNode - for _, group := range groups { - out = append(out, group...) + for _, lookup := range lookups { + switch lookup.kind { + case segExact: + out = append(out, idx.Get(lookup.literal)...) + case segPrefix: + out = append(out, flattenNodes(idx.ByPrefix(lookup.literal))...) + case segSuffix: + out = append(out, flattenNodes(idx.BySuffix(lookup.literal))...) + } + } + if len(lookups) < 2 { + return out + } + // alternatives can overlap, e.g. `{lib,l}*` looks up "lib" and "l" + return dedupeNodes(out) +} + +// boundProbe is how many names a bound may reach before it is treated as the broad side, capping what +// the probe below can waste: a side that gives up costs this much, not the size of its full result. +// +// Sized for what a cataloger's glob seeks, not what an archive holds. A bound reaching more names than +// this is not narrowing much. +const boundProbe = 64 + +// narrowByBounds returns the nodes whose name carries one of the literal bounds the segment fixes, or +// false when it fixes nothing to look up by. +// +// A segment fixing text at both ends has two possible lookups, and the text alone does not say which +// is selective: among one `libstd-.so` beside 1,500 `libstd-NNNN.a`, "libstd-" reaches all 1,501 +// and ".so" reaches one, so the longer literal is the worse lookup. +// +// The selective side is found by running one lookup bounded: one that completes is both the answer and +// proof it was cheap; one that gives up costs only boundProbe. So choosing costs the probe, not the +// broad side's full result. +func (r *ArchiveIndex) narrowByBounds(idx *syftindex.PrefixSuffix[[]*indexNode], segment string) ([]*indexNode, bool) { + prefixes, suffixes := segmentBounds(segment) + + switch { + case len(prefixes) == 0 && len(suffixes) == 0: + return nil, false + case len(suffixes) == 0: + return gather(prefixes, idx.ByPrefix), true + case len(prefixes) == 0: + return gather(suffixes, idx.BySuffix), true + } + + // the tail first: a name's extension is usually the more selective end of it + if nodes, ok := gatherUpTo(suffixes, idx.BySuffixUpTo); ok { + return nodes, true + } + if nodes, ok := gatherUpTo(prefixes, idx.ByPrefixUpTo); ok { + return nodes, true + } + + // neither end narrows to less than the probe, so nothing distinguishes them and doublestar has the + // same work either way + return gather(prefixes, idx.ByPrefix), true +} + +// gatherUpTo runs a set of bounds under one shared probe budget, reporting false as soon as they +// reach past it. What it returns on false is partial and is discarded. +func gatherUpTo(values []string, lookup func(string, int) ([][]*indexNode, bool)) ([]*indexNode, bool) { + var out []*indexNode + for _, value := range values { + groups, complete := lookup(value, boundProbe) + if !complete { + return nil, false + } + out = append(out, flattenNodes(groups)...) + if len(out) > boundProbe { + // one name per key, but a name repeats across directories - the budget is on what comes back + return nil, false + } + } + if len(values) < 2 { + return out, true + } + return dedupeNodes(out), true +} + +func gather(values []string, lookup func(string) [][]*indexNode) []*indexNode { + var out []*indexNode + for _, value := range values { + out = append(out, flattenNodes(lookup(value))...) + } + if len(values) < 2 { + return out + } + return dedupeNodes(out) +} + +// matchingNodes keeps the candidates whose name the segment actually matches: the bounds narrow, +// doublestar decides. +func matchingNodes(candidates []*indexNode, segment string) []*indexNode { + var out []*indexNode + for _, node := range candidates { + if ok, err := doublestar.Match(segment, node.name); err == nil && ok { + out = append(out, node) + } } return out } -// match walks the pattern's segments from a node, calling keep for every file the pattern reaches. -func (r *ArchiveIndex) match(from *indexNode, segments []string, keep func(*indexNode)) { - if len(segments) == 0 { - keep(from) - return +// nonWildcardMeta are the glob metacharacters the index lookups cannot reason about. A lookup reads a +// segment as literal text around at most one `*`, so a segment carrying any of these goes to +// doublestar rather than being looked up as a literal name that matches nothing. +const nonWildcardMeta = `?[]{}\` + +// segmentProvider returns how to reach the children of a directory that one segment matches, choosing +// the narrowest answer the segment allows. +func (r *ArchiveIndex) segmentProvider(segment string) childProvider { + if strings.ContainsAny(segment, nonWildcardMeta) || strings.Contains(segment, anyDir) { + return narrowedChildren(segment) } - segment := segments[0] - rest := segments[1:] - - if segment == anyDir { - // ** matches this directory and every directory beneath it - if len(rest) == 0 { - r.collect(from, keep) - return - } - r.match(from, rest, keep) - for _, dir := range r.dirsBeneath(from) { - r.match(dir, rest, keep) - } - return - } - - for _, child := range r.childrenMatching(from, segment) { - if len(rest) == 0 { - keep(child) - continue - } - if child.isDir { - r.match(child, rest, keep) - } - } -} - -// childrenMatching returns the children of a node whose names match one glob segment, using the -// directory's own name index where the segment allows it. -func (r *ArchiveIndex) childrenMatching(node *indexNode, segment string) []*indexNode { first := strings.IndexRune(segment, wildcard) - switch { case first < 0: // an exact name: one lookup in this directory's index - if child := node.children.Get(segment); child != nil { - return []*indexNode{child} + return func(parent *indexNode) []*indexNode { + if child := parent.children.Get(segment); child != nil { + return []*indexNode{child} + } + return nil } - return nil case segment == string(wildcard): - return node.childList + return func(parent *indexNode) []*indexNode { return parent.childList } case first == len(segment)-1: - // name*, which the directory's index answers by prefix - return node.children.ByPrefix(segment[:len(segment)-1]) + // `name*`, which this directory's index answers by prefix + prefix := segment[:len(segment)-1] + return func(parent *indexNode) []*indexNode { return parent.children.ByPrefix(prefix) } } - var out []*indexNode - for _, child := range node.childList { - if ok, err := doublestar.Match(segment, child.name); err == nil && ok { - out = append(out, child) + return narrowedChildren(segment) +} + +// narrowedChildren is the fallback, narrowing a directory's children as far as the segment allows +// before doublestar sees them. +// +// A directory's children are indexed forwards only, so unlike the global indexes only a leading bound +// is usable here - which is why a segment that reduces to a set of exact and prefix lookups is worth +// detecting separately, and why `*tail` falls back to comparing each child. +func narrowedChildren(segment string) childProvider { + if lookups := enumerateLookups(segment); len(lookups) > 0 { + return func(parent *indexNode) []*indexNode { + return childrenByLookups(parent, lookups) } } - return out + + prefixes, _ := segmentBounds(segment) + if len(prefixes) == 0 { + return matchingChildren(segment, func(parent *indexNode) []*indexNode { return parent.childList }) + } + return matchingChildren(segment, func(parent *indexNode) []*indexNode { + var out []*indexNode + for _, prefix := range prefixes { + out = append(out, parent.children.ByPrefix(prefix)...) + } + if len(prefixes) < 2 { + return out + } + return dedupeNodes(out) + }) +} + +// childrenByLookups answers a segment that reduces to a finite set of lookups. Each lookup is exactly +// its alternative, so nothing here needs doublestar - see byLookups. +func childrenByLookups(parent *indexNode, lookups []nameLookup) []*indexNode { + var out []*indexNode + for _, lookup := range lookups { + switch lookup.kind { + case segExact: + if child := parent.children.Get(lookup.literal); child != nil { + out = append(out, child) + } + case segPrefix: + out = append(out, parent.children.ByPrefix(lookup.literal)...) + case segSuffix: + // children are indexed forwards only, so a fixed tail is compared per child + for _, child := range parent.childList { + if strings.HasSuffix(child.name, lookup.literal) { + out = append(out, child) + } + } + } + } + if len(lookups) < 2 { + return out + } + return dedupeNodes(out) +} + +// matchingChildren runs doublestar over whatever candidates narrows down to. +func matchingChildren(segment string, narrow childProvider) childProvider { + return func(parent *indexNode) []*indexNode { + var out []*indexNode + for _, child := range narrow(parent) { + if ok, err := doublestar.Match(segment, child.name); err == nil && ok { + out = append(out, child) + } + } + return out + } +} + +// under reports whether node sits strictly below dir, which is what confines a global name lookup to +// the subtree a `**` is standing in. +// +// Paths are archive-rooted and cleaned, so this is a prefix test with the separator checked: without +// it "/usrlocal/x" would read as below "/usr". The archive root is above everything. +func under(node, dir *indexNode) bool { + if dir.path == "/" { + return true + } + return len(node.path) > len(dir.path) && + node.path[len(dir.path)] == '/' && + strings.HasPrefix(node.path, dir.path) +} + +// keepFile records a node if it is a file this archive actually holds. Directories are never a glob's +// answer, and a synthesized directory has no entry to read. +func keepFile(found map[*indexNode]struct{}, node *indexNode) { + if node == nil || node.isDir || node.entry == nil { + return + } + found[node] = struct{}{} } // dirsBeneath returns every directory below a node, which is what `**` has to consider when more @@ -191,13 +524,21 @@ func (r *ArchiveIndex) dirsBeneath(node *indexNode) []*indexNode { return out } -// collect reports every file at or below a node. -func (r *ArchiveIndex) collect(node *indexNode, keep func(*indexNode)) { +// collect records every file at or below a node. +func (r *ArchiveIndex) collect(found map[*indexNode]struct{}, node *indexNode) { for _, child := range node.childList { if child.isDir { - r.collect(child, keep) + r.collect(found, child) continue } - keep(child) + keepFile(found, child) } } + +func flattenNodes(groups [][]*indexNode) []*indexNode { + var out []*indexNode + for _, group := range groups { + out = append(out, group...) + } + return out +} diff --git a/syft/internal/fileresolver/archive_index_glob_test.go b/syft/internal/fileresolver/archive_index_glob_test.go new file mode 100644 index 000000000..871356471 --- /dev/null +++ b/syft/internal/fileresolver/archive_index_glob_test.go @@ -0,0 +1,150 @@ +package fileresolver + +import ( + "sort" + "testing" + + "github.com/bmatcuk/doublestar/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Test_FilesByGlob_agreesWithBruteForce is the safety net under every index shortcut. +// +// Each shortcut - the exact-name lookup, the prefix and suffix lookups, and the literal bounds that +// narrow what doublestar is shown - answers a pattern from the index rather than the archive. A +// shortcut that is too eager fails silently, returning fewer files, so every shape is checked against +// matching every path the slow way. +func Test_FilesByGlob_agreesWithBruteForce(t *testing.T) { + paths := []string{ + "LICENSE", + "go", + "go.exe", + "gopher.txt", + "opt/a.jar", + "opt/b.war", + "opt/c.rar", + "opt/notes.txt", + "opt/app/lib/inner.jar", + "usr/local/go/bin/go", + "usr/lib/libstd-0123456789abcdef.so", + "usr/lib/libstd-short.so", + "usr/lib/libz.so", + "usr/bin/mariadb", + "usr/bin/mysql", + "usr/bin/postgres", + "var/lib/dpkg/status", + "var/lib/dpkg/status.d/base", + "var/lib/dpkg/status.d/nested/deep", + "var/lib/rpm/Packages", + "usr/lib/sysimage/rpm/rpmdb.sqlite", + "usr/lib/firefox/firefox", + "META-INF/MANIFEST.MF", + "META-INF/maven/com.example/lib/pom.xml", + "META-INF/maven/com.example/lib/pom.properties", + "WEB-INF/lib/dep.jar", + ".hidden.jar", + "xyz", + "xaz", + "lib.so", + "log.txt", + "mn.txt", + } + + entries := make(map[string]string, len(paths)) + for _, p := range paths { + entries[p] = "body of " + p + } + r := indexOver(t, 1<<20, entries) + + patterns := []string{ + // the shapes the index answers outright + "**/pom.xml", "**/*.jar", "**/pom*", "**/*", "/*", "/opt/*", + // the shapes that reach doublestar, with and without usable bounds + "**/{go,go.exe}", + "**/libstd-????????????????.so", + "**/*.[jw]ar", + "**/libstd-*.so", + "**/*.?ar", + "**/*o*", + "**/?o", + "**/{a,b}.{jar,war}", + // segments that reduce to a finite set of names, answered by exact lookups + "**/{mariadb,mysql}", + "**/[lm]*", + "**/*.{jar,war}", + "**/pom.{xml,properties}", + "**/{lib,libz}.so", + "**/[a-c]*", + "**/libstd-[s0]*.so", + "**/x[!y]z", + // alternations where only some branches reduce to a lookup: dropping the rest would silently + // lose every file those branches match + "**/{go,*}", + "**/{lib,l}*", + "**/{lo,m}*", + "**/{*.jar,pom.xml}", + // metacharacters in a middle segment, where only the leading bound is usable + "/{META-INF,WEB-INF}/**/*.xml", + "/usr/{bin,lib}/*", + "/opt/*/lib/*.jar", + "/us?/lib/libstd-*.so", + "**/lib/**/*.jar", + // a `**` below a selected directory, where the name it looks up also exists outside that + // directory - the lookup is archive-wide, so only the subtree filter keeps the stranger out + "/opt/**/*.jar", + "/opt/**/lib/*.jar", + "/usr/**/go", + "/{opt,WEB-INF}/**/*.jar", + "/META-INF/**/pom.*", + "/usr/**/lib*.so", + // the alternations syft's catalogers actually write, whose branches carry separators of their + // own - splitting the pattern into segments before expanding these cuts a group in half + "**/var/lib/dpkg/{status,status.d/**}", + "**/var/lib/dpkg/{status,status.d/*}", + "**/{var/lib,usr/share,usr/lib/sysimage}/rpm/{Packages,Packages.db,rpmdb.sqlite}", + "**/{firefox,firefox.exe}", + "**/{java,jvm}/*/release", + // and the degenerate ones + "**", "**/**", "/**/*.so", + } + + for _, pattern := range patterns { + t.Run(pattern, func(t *testing.T) { + locations, err := r.FilesByGlob(pattern) + require.NoError(t, err) + + var got []string + for _, loc := range locations { + got = append(got, loc.RealPath) + } + sort.Strings(got) + + assert.Equal(t, bruteForceMatches(t, pattern, paths), got) + }) + } +} + +// bruteForceMatches is what the index computes faster: every path in the archive, matched one at a +// time. Patterns and paths are both archive-relative, so the leading slash is dropped from both. +func bruteForceMatches(t *testing.T, pattern string, paths []string) []string { + t.Helper() + trimmed := trimArchiveRoot(pattern) + var out []string + for _, p := range paths { + matched, err := doublestar.Match(trimmed, p) + require.NoError(t, err) + if matched { + out = append(out, p) + } + } + sort.Strings(out) + return out +} + +func trimArchiveRoot(pattern string) string { + for len(pattern) > 0 && pattern[0] == '/' { + pattern = pattern[1:] + } + return pattern +} diff --git a/syft/internal/fileresolver/archive_index_links_test.go b/syft/internal/fileresolver/archive_index_links_test.go new file mode 100644 index 000000000..02d789808 --- /dev/null +++ b/syft/internal/fileresolver/archive_index_links_test.go @@ -0,0 +1,336 @@ +package fileresolver + +import ( + "archive/tar" + "bytes" + "context" + "io" + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/tmpdir" +) + +// linkEntry is one entry of a fixture archive: a regular file when linkname is empty, otherwise a +// symlink or, with hard set, a hard link. +type linkEntry struct { + name string + body string + linkname string + hard bool +} + +func indexWithLinks(t testing.TB, entries ...linkEntry) *ArchiveIndex { + t.Helper() + workDir := archive.NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) + t.Cleanup(workDir.Remove) + store := archive.NewEntryStore(workDir, "test.tar", nil) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + charge := archive.NewLimiter(archive.Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}).Charge() + + for _, e := range entries { + hdr := tar.Header{Name: e.name, Mode: 0o644, Typeflag: tar.TypeReg, Size: int64(len(e.body))} + switch { + case e.linkname != "" && e.hard: + hdr = tar.Header{Name: e.name, Mode: 0o644, Typeflag: tar.TypeLink, Linkname: e.linkname} + case e.linkname != "": + hdr = tar.Header{Name: e.name, Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: e.linkname} + } + _, err := store.Add(hdr, bytes.NewReader([]byte(e.body)), charge) + require.NoError(t, err) + } + + r, err := NewFromArchiveEntries("", "outer.tar", store, nil) + require.NoError(t, err) + return r +} + +func globPaths(t *testing.T, r *ArchiveIndex, pattern string) []string { + t.Helper() + locations, err := r.FilesByGlob(pattern) + require.NoError(t, err) + var out []string + for _, l := range locations { + out = append(out, l.RealPath) + } + return out +} + +// TestArchiveIndex_linksCollapseToOnePath covers the resolver contract that several paths to one file +// answer as one location, and which path that is. +// +// The rule is a property of the candidates, not of the order they were found in: the path that is the +// file itself wins, and otherwise the lowest-sorting one. That matters because the search accumulates +// into a map and the answer is sorted afterwards - neither of which the choice may depend on, or the +// same archive would name a different path from run to run. +func TestArchiveIndex_linksCollapseToOnePath(t *testing.T) { + tests := []struct { + name string + entries []linkEntry + pattern string + want []string + reason string + }{ + { + name: "the file itself beats every link to it", + entries: []linkEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/a-sym.jar", linkname: "real.jar"}, + {name: "opt/z-sym.jar", linkname: "real.jar"}, + {name: "opt/hard.jar", linkname: "opt/real.jar", hard: true}, + }, + pattern: "**/*.jar", + want: []string{"opt/real.jar"}, + reason: "one file, so one location - and it is named by the path that holds it", + }, + { + name: "with no candidate naming the file, the lowest-sorting link wins", + entries: []linkEntry{ + {name: "opt/real.bin", body: "PK\x03\x04"}, + {name: "opt/z-sym.jar", linkname: "real.bin"}, + {name: "opt/a-sym.jar", linkname: "real.bin"}, + {name: "opt/m-sym.jar", linkname: "real.bin"}, + }, + pattern: "**/*.jar", + want: []string{"opt/real.bin"}, + reason: "the content is the target's, so that is the real path the answer carries", + }, + { + name: "distinct files stay distinct", + entries: []linkEntry{ + {name: "opt/one.jar", body: "one"}, + {name: "opt/two.jar", body: "two"}, + {name: "opt/one-sym.jar", linkname: "one.jar"}, + }, + pattern: "**/*.jar", + want: []string{"opt/one.jar", "opt/two.jar"}, + reason: "collapsing is per file, not per pattern", + }, + { + name: "a dangling link keeps its own path", + entries: []linkEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/gone.jar", linkname: "missing.jar"}, + }, + pattern: "**/*.jar", + want: []string{"opt/gone.jar", "opt/real.jar"}, + reason: "it names no content, so it belongs to no other file's group", + }, + { + name: "two dangling links are two paths, not one", + entries: []linkEntry{ + {name: "opt/a.jar", linkname: "missing.jar"}, + {name: "opt/b.jar", linkname: "missing.jar"}, + }, + pattern: "**/*.jar", + want: []string{"opt/a.jar", "opt/b.jar"}, + reason: "nothing resolves them to a common file, so neither may absorb the other", + }, + { + name: "a chain of links collapses onto the file at its end", + entries: []linkEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/mid.jar", linkname: "real.jar"}, + {name: "opt/outer.jar", linkname: "mid.jar"}, + }, + pattern: "**/*.jar", + want: []string{"opt/real.jar"}, + reason: "following one hop at a time reaches the same file from every path", + }, + { + name: "a cycle resolves to nothing rather than looping", + entries: []linkEntry{ + {name: "opt/a.jar", linkname: "b.jar"}, + {name: "opt/b.jar", linkname: "a.jar"}, + }, + pattern: "**/*.jar", + want: []string{"opt/a.jar", "opt/b.jar"}, + reason: "neither reaches content, so each stands under its own path", + }, + { + name: "a hard link resolves from the archive root, not from its own directory", + entries: []linkEntry{ + {name: "opt/real.jar", body: "PK\x03\x04"}, + {name: "opt/nested/hard.jar", linkname: "opt/real.jar", hard: true}, + }, + pattern: "**/*.jar", + want: []string{"opt/real.jar"}, + reason: "read as relative it would resolve to opt/nested/opt/real.jar and dangle", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := indexWithLinks(t, tt.entries...) + assert.Equal(t, tt.want, globPaths(t, r, tt.pattern), tt.reason) + }) + } +} + +// TestArchiveIndex_collapseIgnoresTheOrderItFinds is what holds the rule to being a rule. +// +// Sorting the answer would hide a choice made by iteration order, so this exercises the choice +// directly: the same candidates in shuffled order must always yield the same path. +func TestArchiveIndex_collapseIgnoresTheOrderItFinds(t *testing.T) { + r := indexWithLinks(t, + linkEntry{name: "opt/real.jar", body: "PK\x03\x04"}, + linkEntry{name: "opt/a-sym.jar", linkname: "real.jar"}, + linkEntry{name: "opt/m-sym.jar", linkname: "real.jar"}, + linkEntry{name: "opt/z-sym.jar", linkname: "real.jar"}, + ) + + candidates := []*indexNode{ + r.byPath["/opt/real.jar"], r.byPath["/opt/a-sym.jar"], + r.byPath["/opt/m-sym.jar"], r.byPath["/opt/z-sym.jar"], + } + for _, node := range candidates { + require.NotNil(t, node) + } + + rng := rand.New(rand.NewSource(1)) + for range 64 { + rng.Shuffle(len(candidates), func(i, j int) { + candidates[i], candidates[j] = candidates[j], candidates[i] + }) + + found := map[*indexNode]struct{}{} + for _, node := range candidates { + found[node] = struct{}{} + } + + best := collapseToOnePathPerFile(found) + require.Len(t, best, 1, "four paths, one file") + for target, access := range best { + assert.Equal(t, "/opt/real.jar", target.path) + assert.Equal(t, "/opt/real.jar", access.path, "the file itself must win from any order") + } + } + + // and with the file itself not among the candidates, the lowest-sorting link wins from any order + links := candidates[:0:0] + for _, node := range candidates { + if node.path != "/opt/real.jar" { + links = append(links, node) + } + } + for range 64 { + rng.Shuffle(len(links), func(i, j int) { links[i], links[j] = links[j], links[i] }) + + found := map[*indexNode]struct{}{} + for _, node := range links { + found[node] = struct{}{} + } + + for _, access := range collapseToOnePathPerFile(found) { + assert.Equal(t, "/opt/a-sym.jar", access.path, "the lowest-sorting link must win from any order") + } + } +} + +// TestArchiveIndex_collapsedLocationReadsTheTarget covers what the collapsed answer is for: the +// location has to name the link a reader recognizes while reading the bytes the file actually holds. +func TestArchiveIndex_collapsedLocationReadsTheTarget(t *testing.T) { + r := indexWithLinks(t, + linkEntry{name: "opt/real.bin", body: "PK\x03\x04 real contents"}, + linkEntry{name: "opt/a-sym.jar", linkname: "real.bin"}, + linkEntry{name: "opt/z-sym.jar", linkname: "real.bin"}, + ) + + locations, err := r.FilesByGlob("**/*.jar") + require.NoError(t, err) + require.Len(t, locations, 1) + + assert.Equal(t, "opt/real.bin", locations[0].RealPath, "content comes from the file") + assert.Equal(t, "opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched") + + reader, err := r.FileContentsByLocation(locations[0]) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reader.Close()) }) + + body, err := io.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "PK\x03\x04 real contents", string(body), + "a location standing for a link must still read the file's bytes, not the link's none") +} + +// TestArchiveIndex_dangingLinkHasNoContent covers the other half of keeping a dangling link: it is +// answered as a path, and reading it fails rather than returning something invented. +func TestArchiveIndex_dangingLinkHasNoContent(t *testing.T) { + r := indexWithLinks(t, linkEntry{name: "opt/gone.jar", linkname: "missing.jar"}) + + locations, err := r.FilesByGlob("**/*.jar") + require.NoError(t, err) + require.Len(t, locations, 1) + assert.Equal(t, "opt/gone.jar", locations[0].RealPath) + + reader, err := r.FileContentsByLocation(locations[0]) + if err == nil { + t.Cleanup(func() { _ = reader.Close() }) + body, readErr := io.ReadAll(reader) + require.NoError(t, readErr) + assert.Empty(t, string(body), "a link to nothing holds nothing") + } + + metadata, err := r.FileMetadataByLocation(locations[0]) + require.NoError(t, err) + assert.Equal(t, "opt/missing.jar", metadata.LinkDestination, + "where it pointed is still reported, resolved against the link's own directory") +} + +// TestArchiveIndex_mimeTypeAnswersOnlyWithFiles covers why a link is never a MIME answer: the type is +// sniffed from content, a link has none, and the file it points at answers under its own path. +// +// Without the rule this passes by accident - a link's empty entry sniffs to no type, so no real query +// reaches one - and the accident shows through on a query for the empty type. +func TestArchiveIndex_mimeTypeAnswersOnlyWithFiles(t *testing.T) { + r := indexWithLinks(t, + linkEntry{name: "opt/real.jar", body: "PK\x03\x04 zip contents here"}, + linkEntry{name: "opt/sym.jar", linkname: "real.jar"}, + linkEntry{name: "opt/hard.jar", linkname: "opt/real.jar", hard: true}, + linkEntry{name: "opt/gone.jar", linkname: "missing.jar"}, + linkEntry{name: "opt/notes.txt", body: "hello world"}, + ) + + byMIME := func(types ...string) []string { + t.Helper() + locations, err := r.FilesByMIMEType(types...) + require.NoError(t, err) + var out []string + for _, l := range locations { + out = append(out, l.RealPath) + } + return out + } + + assert.Equal(t, []string{"opt/real.jar"}, byMIME("application/zip"), + "the file answers, not the two links to it") + assert.Equal(t, []string{"opt/notes.txt"}, byMIME("text/plain")) + assert.Empty(t, byMIME(""), + "a link sniffs to no type, and must not be reachable by asking for that") + assert.Equal(t, []string{"opt/notes.txt", "opt/real.jar"}, byMIME("application/zip", "text/plain")) +} + +// TestArchiveIndex_allLocationsKeepsEveryPath covers the opposite rule, and why it is opposite. +// +// AllLocations is an enumeration of what the archive holds, not a search for a file. The file metadata +// cataloger reads it to record a row per path - a symlink's row being its type and where it points - +// so collapsing links here would delete the very records it exists to produce. Callers wanting only +// real files filter by type, as file.cataloger/internal.AllRegularFiles does. +func TestArchiveIndex_allLocationsKeepsEveryPath(t *testing.T) { + r := indexWithLinks(t, + linkEntry{name: "opt/real.jar", body: "PK\x03\x04"}, + linkEntry{name: "opt/sym.jar", linkname: "real.jar"}, + linkEntry{name: "opt/gone.jar", linkname: "missing.jar"}, + ) + + var got []string + for l := range r.AllLocations(context.Background()) { + got = append(got, l.RealPath) + } + assert.Equal(t, []string{"opt/gone.jar", "opt/real.jar", "opt/sym.jar"}, got, + "every path the archive names, links included") +} diff --git a/syft/internal/fileresolver/archive_index_test.go b/syft/internal/fileresolver/archive_index_test.go index eaba8ec82..2da403825 100644 --- a/syft/internal/fileresolver/archive_index_test.go +++ b/syft/internal/fileresolver/archive_index_test.go @@ -8,74 +8,16 @@ import ( "os" "path/filepath" "sort" - "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/tmpdir" "github.com/anchore/syft/syft/file" ) -// storeWith builds an entry store holding the given entries, in name order so a test that depends on -// order is not depending on a map. -func storeWith(t *testing.T, maxInMemory int64, entries map[string]string) *archive.EntryStore { - t.Helper() - store := archive.NewEntryStore(t.TempDir(), "test.jar", nil) - // how much stays in memory is the limiter's answer, so a test that wants content on disk sets a - // memory bound and leaves disk unbounded - charge := archive.NewLimiter(archive.Limits{MaxMemoryBytes: maxInMemory, MaxDiskBytes: -1}).Charge() - t.Cleanup(func() { require.NoError(t, store.Close()) }) - - names := make([]string, 0, len(entries)) - for name := range entries { - names = append(names, name) - } - sort.Strings(names) - - for _, name := range names { - body := entries[name] - hdr := tar.Header{Name: name, Size: int64(len(body)), Mode: 0o600, Typeflag: tar.TypeReg} - if body == "" && name[len(name)-1] == '/' { - hdr = tar.Header{Name: name, Mode: 0o755, Typeflag: tar.TypeDir} - _, err := store.Add(hdr, nil, charge) - require.NoError(t, err) - continue - } - _, err := store.Add(hdr, bytes.NewReader([]byte(body)), charge) - require.NoError(t, err) - } - return store -} - -func indexOver(t *testing.T, maxInMemory int64, entries map[string]string, filters ...PathIndexVisitor) *ArchiveIndex { - t.Helper() - root := t.TempDir() - r, err := NewFromArchiveEntries(root, "", "outer.jar", storeWith(t, maxInMemory, entries), false, filters...) - require.NoError(t, err) - return r -} - -func relPaths(t *testing.T, r *ArchiveIndex, locations []file.Location) []string { - t.Helper() - var out []string - for _, loc := range locations { - out = append(out, "/"+loc.RealPath) - } - sort.Strings(out) - return out -} - -var jarEntriesFixture = map[string]string{ - "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", - "META-INF/maven/com.example/lib/pom.properties": "groupId=com.example\n", - "META-INF/maven/com.example/lib/pom.xml": "", - "LICENSE": "Apache License 2.0", - "BOOT-INF/lib/inner.jar": "PK\x03\x04 not really", - "com/example/Thing.class": "bytecode", -} - func TestArchiveIndex_globByExtensionUsesTheNameIndex(t *testing.T) { r := indexOver(t, 1<<20, jarEntriesFixture) @@ -93,8 +35,8 @@ func TestArchiveIndex_globByExactNameAtAnyDepth(t *testing.T) { } func TestArchiveIndex_globFindsRootLevelFiles(t *testing.T) { - // what the tar-backed resolver could not do: `/*` matched nothing there, so a jar carrying its - // license at the root - kafka-clients does - lost it + // `/*` names the entries at the top of the archive, where a jar such as kafka-clients carries its + // license r := indexOver(t, 1<<20, jarEntriesFixture) locations, err := r.FilesByGlob("/*") @@ -112,8 +54,8 @@ func TestArchiveIndex_globOneDirectoryDeep(t *testing.T) { } func TestArchiveIndex_globAcrossSyntheticDirectories(t *testing.T) { - // the archive never names META-INF/ or META-INF/maven/, so the directories a middle segment needs - // have to be synthesized from the paths of the entries that were named + // the archive never names META-INF/ or META-INF/maven/, so the directories a middle segment needs are + // synthesized from the paths of the entries that were named r := indexOver(t, 1<<20, jarEntriesFixture) locations, err := r.FilesByGlob("**/maven/*/*/pom.xml") @@ -121,6 +63,54 @@ func TestArchiveIndex_globAcrossSyntheticDirectories(t *testing.T) { assert.Equal(t, []string{"/META-INF/maven/com.example/lib/pom.xml"}, relPaths(t, r, locations)) } +func TestArchiveIndex_globHonorsNonWildcardMetacharacters(t *testing.T) { + // the name-index shortcuts read a segment as literal text around at most one `*`, so a segment + // carrying any other metacharacter must reach doublestar instead. Taking the shortcut anyway looks + // the pattern up as a literal name and returns nothing, losing every `**/{go,go.exe}` and + // `**/libstd-????????????????.so` glob the binary classifier cataloger has. + r := indexOver(t, 1<<20, map[string]string{ + "usr/local/go/bin/go": "binary", + "usr/lib/libstd-0123456789abcdef.so": "binary", + "opt/a.jar": "PK\x03\x04", + "opt/b.war": "PK\x03\x04", + "opt/notes.txt": "prose", + }) + + tests := []struct { + pattern string + want []string + }{ + {"**/{go,go.exe}", []string{"/usr/local/go/bin/go"}}, + {"**/libstd-????????????????.so", []string{"/usr/lib/libstd-0123456789abcdef.so"}}, + {"**/*.[jw]ar", []string{"/opt/a.jar", "/opt/b.war"}}, + {"**/{a,b}.{jar,war}", []string{"/opt/a.jar", "/opt/b.war"}}, + // the shapes the shortcuts do answer must keep working + {"**/*.jar", []string{"/opt/a.jar"}}, + {"**/go", []string{"/usr/local/go/bin/go"}}, + {"**/notes*", []string{"/opt/notes.txt"}}, + } + for _, tt := range tests { + t.Run(tt.pattern, func(t *testing.T) { + locations, err := r.FilesByGlob(tt.pattern) + require.NoError(t, err) + assert.Equal(t, tt.want, relPaths(t, r, locations)) + }) + } +} + +func TestArchiveIndex_globMetacharactersInAMiddleSegment(t *testing.T) { + // the same case one level down, where childrenMatching could take the directory index's exact-name + // and prefix shortcuts on any segment holding no `*` + r := indexOver(t, 1<<20, map[string]string{ + "META-INF/maven/com.example/lib/pom.xml": "", + "WEB-INF/lib/dep.jar": "PK\x03\x04", + }) + + locations, err := r.FilesByGlob("/{META-INF,WEB-INF}/**/*.xml") + require.NoError(t, err) + assert.Equal(t, []string{"/META-INF/maven/com.example/lib/pom.xml"}, relPaths(t, r, locations)) +} + func TestArchiveIndex_pathsAndContents(t *testing.T) { r := indexOver(t, 1<<20, jarEntriesFixture) @@ -169,14 +159,13 @@ func TestArchiveIndex_mimeTypesAreSniffedFromContent(t *testing.T) { } func TestArchiveIndex_exclusionsPruneWholeSubtrees(t *testing.T) { - // a filter answering SkipDir removes the directory and everything under it, the way it would in a - // directory walk. A tar is a sequence rather than a walk, so this is checked per entry against - // what was pruned. - skipMetaInf := func(_, entryPath string, info os.FileInfo, _ error) error { - if strings.HasSuffix(entryPath, "/META-INF") || strings.HasSuffix(entryPath, "/META-INF/") { + // a filter answering SkipDir removes the directory and everything under it, as in a directory walk. + // An archive is a sequence rather than a walk, so this is checked per entry against what was pruned. + // the filter sees each entry's archive-relative path, the only frame an archive has + skipMetaInf := func(_, entryPath string, _ os.FileInfo, _ error) error { + if entryPath == "META-INF" { return filepath.SkipDir } - _ = info return nil } @@ -219,17 +208,13 @@ func TestArchiveIndex_allLocationsCoversEveryFile(t *testing.T) { func TestArchiveIndex_recordsCountsWhatTheFilesystemHolds(t *testing.T) { r := indexOver(t, 1<<20, jarEntriesFixture) assert.Equal(t, len(jarEntriesFixture), r.Records()) - assert.False(t, r.IndexTruncated()) } func TestArchiveIndex_answersAreOrdered(t *testing.T) { - // every answer has to come back in the same order every time. The archive walk enters archives in - // the order it is handed them and merges what it finds into one shared SBOM, so the order decides - // which locations group into which package - and an index built over a Go map has no order at all. - // - // Measured on trinodb/trino, whose plugin jars are hard links to one another: unordered, one - // libzstd package holding 24 locations came back as three packages holding 6, 9 and 9 of the same - // 24, and the image reported 7508 packages instead of 7506. + // every answer must come back in the same order every time: results merge into one shared SBOM, so + // order decides which locations group into which package, and an index built over a Go map has none. + // Unordered, an image whose jars are hard links to one another split one package of 24 locations into + // three holding 6, 9 and 9 of the same 24. entries := map[string]string{} for _, name := range []string{ "b/z.jar", "a/y.jar", "c/x.jar", "a/b/c/w.jar", "z.jar", "a.jar", @@ -269,8 +254,8 @@ func TestArchiveIndex_answersAreOrdered(t *testing.T) { } func TestArchiveIndex_CloseReleasesStore(t *testing.T) { - // forcing content to disk (memory 0) so the store holds an open overflow blob, then Close must - // release it; reads after Close no longer resolve, and a second Close is a no-op. + // forcing content to disk (memory 0) so the store holds an open overflow blob: Close must release it, + // reads after Close no longer resolve, and a second Close is a no-op r := indexOver(t, 0, map[string]string{"a.txt": "hello"}) locs, err := r.FilesByPath("a.txt") @@ -291,3 +276,64 @@ func TestArchiveIndex_CloseReleasesStore(t *testing.T) { assert.NoError(t, r.Close(), "Close is idempotent") } + +// storeWith builds an entry store holding the given entries, in name order so a test that depends on +// order does not depend on a map. +func storeWith(t testing.TB, maxInMemory int64, entries map[string]string) *archive.EntryStore { + t.Helper() + workDir := archive.NewWorkDir(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) + t.Cleanup(workDir.Remove) + store := archive.NewEntryStore(workDir, "test.jar", nil) + // the limiter decides how much stays in memory, so a test wanting content on disk sets a memory bound + // and leaves disk unbounded + charge := archive.NewLimiter(archive.Limits{MaxMemoryBytes: maxInMemory, MaxDiskBytes: -1}).Charge() + t.Cleanup(func() { require.NoError(t, store.Close()) }) + + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + body := entries[name] + hdr := tar.Header{Name: name, Size: int64(len(body)), Mode: 0o600, Typeflag: tar.TypeReg} + if body == "" && name[len(name)-1] == '/' { + hdr = tar.Header{Name: name, Mode: 0o755, Typeflag: tar.TypeDir} + _, err := store.Add(hdr, nil, charge) + require.NoError(t, err) + continue + } + _, err := store.Add(hdr, bytes.NewReader([]byte(body)), charge) + require.NoError(t, err) + } + return store +} + +// indexOver builds an index with no budget of its own, so these tests see every entry they put in. +// What it charges, and what it does when a charge is refused, is in archive_index_charge_test.go. +func indexOver(t testing.TB, maxInMemory int64, entries map[string]string, filters ...PathIndexVisitor) *ArchiveIndex { + t.Helper() + r, err := NewFromArchiveEntries("", "outer.jar", storeWith(t, maxInMemory, entries), nil, filters...) + require.NoError(t, err) + return r +} + +func relPaths(t *testing.T, r *ArchiveIndex, locations []file.Location) []string { + t.Helper() + var out []string + for _, loc := range locations { + out = append(out, "/"+loc.RealPath) + } + sort.Strings(out) + return out +} + +var jarEntriesFixture = map[string]string{ + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n", + "META-INF/maven/com.example/lib/pom.properties": "groupId=com.example\n", + "META-INF/maven/com.example/lib/pom.xml": "", + "LICENSE": "Apache License 2.0", + "BOOT-INF/lib/inner.jar": "PK\x03\x04 not really", + "com/example/Thing.class": "bytecode", +} diff --git a/syft/internal/fileresolver/archive_tar.go b/syft/internal/fileresolver/archive_tar.go deleted file mode 100644 index e9c0bd2df..000000000 --- a/syft/internal/fileresolver/archive_tar.go +++ /dev/null @@ -1,323 +0,0 @@ -package fileresolver - -import ( - "archive/tar" - "fmt" - "io" - "os" - "path" - "path/filepath" - "strings" - - stereoscopeFile "github.com/anchore/stereoscope/pkg/file" - "github.com/anchore/stereoscope/pkg/filetree" - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/file" - "github.com/anchore/syft/syft/internal/windows" -) - -var _ file.Resolver = (*ArchiveTar)(nil) - -// ArchiveTar implements path and content access for one archive whose entries were overflowed into a -// single tar. -// -// It is the same FiletreeResolver a directory scan uses, differing only in where the tree comes from -// and how content is opened. That separation already existed: FiletreeResolver holds Tree, Index and -// an Opener, and nativeOSFileOpener - which opens a path on the real filesystem - is one -// implementation of that Opener among several. Reading an entry out of a tar by seek offset is -// another, so no resolver interface changes to accommodate it. -// -// Paths are reported relative to the archive's own root, exactly as they were when the archive was -// expanded into a directory: the tree is rooted at an empty directory standing in for the archive -// root, and the chroot context relativizes against it. Nothing about the storage - the tar, its -// offsets, the scratch directory holding it - is reachable from a Location. -type ArchiveTar struct { - FiletreeResolver - tarPath string - - // records is how many entries of the tar were indexed. - records int - - // truncated reports that indexing stopped before the end of the tar. - truncated bool -} - -// NewFromArchiveTar builds a resolver over the archive entries held in the tar at tarPath, whose -// Locations are reported relative to rootDir and stamped with fileSystemID and archivePath. -// -// rootDir must exist and is expected to be empty: it is the archive's logical root and nothing is -// ever written into it. Every entry the tar holds is indexed: what bounds a crafted archive is the -// disk limit charged as its entries were written, which is the resource the entries actually consume. -// -// pathFilters are applied to each entry the way they are applied to each path of a directory walk, so -// an exclusion pattern reaches inside an archive by exactly the mechanism it did before: the entry it -// matches is absent from the archive's filesystem rather than present and skipped. -func NewFromArchiveTar(rootDir, tarPath, fileSystemID, archivePath string, pathFilters ...PathIndexVisitor) (*ArchiveTar, error) { - chroot, err := NewChrootContextFromCWD(rootDir, "") - if err != nil { - return nil, fmt.Errorf("unable to interpret chroot context for archive root %q: %w", rootDir, err) - } - - r := &ArchiveTar{ - tarPath: tarPath, - FiletreeResolver: FiletreeResolver{ - Chroot: *chroot, - FileSystemID: fileSystemID, - ArchivePath: archivePath, - }, - } - - if err := r.buildIndex(chroot.Root(), pathFilters); err != nil { - return nil, err - } - - return r, nil -} - -// Records is how many entries this archive's filesystem holds. -func (r *ArchiveTar) Records() int { - return r.records -} - -// IndexTruncated reports that indexing stopped before the end of the tar, so the filesystem holds -// only part of what the archive contained. It is not an error: what was indexed is complete and -// usable, there is simply less of it. -func (r *ArchiveTar) IndexTruncated() bool { - return r.truncated -} - -func (r *ArchiveTar) String() string { - return fmt.Sprintf("archive:%s", r.tarPath) -} - -// buildIndex walks the overflow tar once, adding a node to the filetree and a record to the index for -// each entry, and keeping the entry itself so content can later be opened at its offset. -// -// A visitor error is never fatal to the archive. A crafted tar can hold two entries whose names -// disagree about what kind of node a path is, and letting that cost the whole archive would hand an -// attacker a way to hide everything else in it. -func (r *ArchiveTar) buildIndex(root string, pathFilters []PathIndexVisitor) error { - tree := filetree.New() - index := filetree.NewIndex() - builder := filetree.NewBuilder(tree, index) - entries := make(map[stereoscopeFile.ID]stereoscopeFile.TarIndexEntry) - - rootPath := treeRoot(root) - var pruned []string - - add := func(entry stereoscopeFile.TarIndexEntry) { - if ref, ok := r.addEntry(builder, entry, root, rootPath, pathFilters, &pruned); ok { - entries[ref] = entry - r.records++ - } - } - - // One entry behind: an entry is added only once the header after it has been read, so an entry - // whose data the tar stops part way through is never in the filesystem. It matters because a tar - // CAN end mid-entry - that is what the disk limit refusing a chunk of a copy leaves behind - and a - // half-present jar or pom that a cataloger parses anyway is worse than an absent one. The lag is - // what makes "the entries before the cut" mean the entries whose content is all there. - var pending *stereoscopeFile.TarIndexEntry - - visitor := func(entry stereoscopeFile.TarIndexEntry) error { - if pending != nil { - add(*pending) - pending = nil - } - pending = &entry - return nil - } - - if _, err := stereoscopeFile.NewTarIndex(r.tarPath, visitor); err != nil { - if r.records == 0 { - return fmt.Errorf("unable to index archive contents %q: %w", r.tarPath, err) - } - // the tar ends part way through an entry. Everything complete before that point is intact, so - // it is kept and reported as a truncation rather than discarding an archive that was cataloged - // perfectly well up to the cut. - log.WithFields("archive", r.tarPath, "error", err). - Debug("archive contents end part way through an entry; cataloging what was read") - r.truncated = true - } else if pending != nil { - // the tar ended cleanly, so the last entry read is whole - add(*pending) - } - - r.Tree = tree - r.Index = index - r.SearchContext = filetree.NewSearchContext(tree, index) - r.Opener = tarEntryOpener(entries) - - return nil -} - -// addEntry puts one tar entry into the filetree and the index, reporting the reference it was given, -// or false when the entry is not part of this archive's filesystem: a name that names nothing, a path -// under a directory a filter pruned, a path a filter excluded, or a path a crafted archive has -// already claimed as a different kind of node. -func (r *ArchiveTar) addEntry(builder *filetree.Builder, entry stereoscopeFile.TarIndexEntry, root, rootPath string, pathFilters []PathIndexVisitor, pruned *[]string) (stereoscopeFile.ID, bool) { - hdr := entry.ToTarFileEntry().Header - - name, ok := sanitizeEntryName(hdr.Name) - if !ok { - return 0, false - } - entryPath := path.Join(rootPath, name) - - if isPruned(entryPath, *pruned) { - return 0, false - } - - switch runPathFilters(pathFilters, root, entryPath, hdr.FileInfo()) { - case skipEntry: - return 0, false - case skipTree: - *pruned = append(*pruned, entryPath) - return 0, false - } - - ref, err := builder.Add(entryMetadata(entry, hdr, rootPath, entryPath)) - if err != nil { - log.WithFields("entry", hdr.Name, "error", err). - Trace("unable to add archive entry to its filesystem, skipping it") - return 0, false - } - return ref.ID(), true -} - -// entryMetadata describes one tar entry as a file in the archive's filesystem. -// -// The path and the link destination are both rewritten into the archive's own root. A link is -// resolved in archive-relative space first, where path.Join clamps a climb at the root, so a target -// naming the host filesystem - "/etc/passwd", or a chain of ".." - lands somewhere inside this -// archive or nowhere at all. That is stronger than the check it replaces: the previous code refused -// to create such a link, and here there is nothing to create in the first place. -func entryMetadata(entry stereoscopeFile.TarIndexEntry, hdr tar.Header, rootPath, entryPath string) file.Metadata { - contents := entry.Open() - metadata := stereoscopeFile.NewMetadata(hdr, contents) - if err := contents.Close(); err != nil { - log.WithFields("entry", hdr.Name, "error", err).Trace("unable to close archive entry after reading its type") - } - - metadata.Path = entryPath - - if hdr.Linkname != "" { - relative := path.Clean(hdr.Linkname) - if !path.IsAbs(relative) { - relative = path.Join("/", path.Dir(strings.TrimPrefix(entryPath, rootPath)), hdr.Linkname) - } - metadata.LinkDestination = path.Join(rootPath, relative) - } - - return metadata -} - -// tarEntryOpener opens an entry's content by seeking to its offset in the tar. -// -// The reader is bounded to the entry, and it is Read, Seek and ReadAt - so reading the last entry of -// a large archive costs a seek rather than a pass over everything before it, and a nested archive can -// be handed straight to an archive format that needs random access instead of being copied out first. -func tarEntryOpener(entries map[stereoscopeFile.ID]stereoscopeFile.TarIndexEntry) func(stereoscopeFile.Reference) (io.ReadCloser, error) { - return func(ref stereoscopeFile.Reference) (io.ReadCloser, error) { - entry, ok := entries[ref.ID()] - if !ok { - return nil, fmt.Errorf("no archive entry for path=%q", ref.RealPath) - } - contents := entry.Open() - seekable, ok := contents.(entryReadCloser) - if !ok { - // still readable, just not readable at an offset: whatever wants random access will copy - // it out first, which is what happened to every nested archive before this - return contents, nil - } - return overflowEntryReader{seekable}, nil - } -} - -// overflowEntryReader is a reader over one entry of an archive's overflow tar. -// -// The marker method is what lets the archive cataloger recognize a nested archive as content it can -// read where it lies - already random access, already charged to the archive holding it - without -// either package importing the other. -type overflowEntryReader struct { - entryReadCloser -} - -// OverflowArchiveEntry marks this reader as content that needs no copy to be read at random. -func (overflowEntryReader) OverflowArchiveEntry() {} - -// entryReadCloser is what a tar index entry hands back: a read closer that also seeks and reads at an -// offset, because it is a bounded view onto the tar file. -type entryReadCloser interface { - io.ReadCloser - io.ReaderAt - io.Seeker -} - -type filterOutcome int - -const ( - keepEntry filterOutcome = iota - skipEntry - skipTree -) - -// runPathFilters asks each filter about one entry, translating the answers a directory walk uses - -// skip this path, skip this whole directory - into what an unordered pass over tar entries can act on. -func runPathFilters(filters []PathIndexVisitor, root, entryPath string, info os.FileInfo) filterOutcome { - for _, filter := range filters { - if filter == nil { - continue - } - switch err := filter(root, entryPath, info, nil); err { - case nil: - continue - case filepath.SkipDir, filepath.SkipAll: - return skipTree - default: - return skipEntry - } - } - return keepEntry -} - -// isPruned reports whether an entry sits inside a directory a filter already pruned. A tar is a -// sequence, not a walk, so a pruned directory cannot simply end a subtree the way returning SkipDir -// from a walk does: each later entry is checked against what was pruned. -func isPruned(entryPath string, pruned []string) bool { - for _, dir := range pruned { - if entryPath == dir || strings.HasPrefix(entryPath, dir+"/") { - return true - } - } - return false -} - -// sanitizeEntryName turns the name an archive gave an entry into a path inside that archive, and -// reports false for a name that names nothing. -// -// Nothing is written through an entry's name any more - it is a field in a tar header - so this is -// not about arbitrary writes. It is about the path the SBOM reports: a name like "../../etc/passwd" -// left alone would put a node above the archive's own root, which renders as a path outside the -// archive and reads as a claim about the host filesystem. Cleaning it against the root clamps the -// climb, so the worst an attacker-controlled name produces is a wrong path inside the right archive. -func sanitizeEntryName(name string) (string, bool) { - if strings.ContainsRune(name, 0) { - return "", false - } - cleaned := strings.TrimPrefix(path.Clean("/"+name), "/") - if cleaned == "" || cleaned == "." { - // the archive's own root, which is not an entry in it - return "", false - } - return cleaned, true -} - -// treeRoot is the archive root as a filetree path: posix, absolute, and on Windows with the volume -// encoded the way every other resolver encodes it. -func treeRoot(root string) string { - if windows.HostRunningOnWindows() { - return windows.ToPosix(root) - } - return path.Clean(filepath.ToSlash(root)) -} diff --git a/syft/internal/fileresolver/archive_tar_test.go b/syft/internal/fileresolver/archive_tar_test.go deleted file mode 100644 index 69b07babc..000000000 --- a/syft/internal/fileresolver/archive_tar_test.go +++ /dev/null @@ -1,287 +0,0 @@ -package fileresolver - -import ( - "archive/tar" - "context" - "io" - "os" - "path/filepath" - "sort" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/anchore/syft/syft/file" -) - -// tarEntry describes one entry to write into a test tar: a symlink when link is set, a directory when -// the name ends in "/", a regular file otherwise. -type tarEntry struct { - name string - body string - link string -} - -// writeTestTar builds a plain tar on disk, which is what an archive's entries are overflowed into. -func writeTestTar(t *testing.T, dir string, entries ...tarEntry) string { - t.Helper() - path := filepath.Join(dir, "contents.tar") - f, err := os.Create(path) - require.NoError(t, err) - defer f.Close() - - tw := tar.NewWriter(f) - for _, e := range entries { - hdr := &tar.Header{Name: e.name, Mode: 0o644, Size: int64(len(e.body))} - switch { - case e.link != "": - hdr.Typeflag = tar.TypeSymlink - hdr.Linkname = e.link - hdr.Size = 0 - case strings.HasSuffix(e.name, "/"): - hdr.Typeflag = tar.TypeDir - default: - hdr.Typeflag = tar.TypeReg - } - require.NoError(t, tw.WriteHeader(hdr)) - if hdr.Typeflag == tar.TypeReg { - _, err := tw.Write([]byte(e.body)) - require.NoError(t, err) - } - } - require.NoError(t, tw.Close()) - return path -} - -// archiveRoot is the empty directory an archive's entries are reported relative to. Nothing is -// written into it; it exists so a path in the SBOM is relative to its own archive. -func archiveRoot(t *testing.T, dir string) string { - t.Helper() - root := filepath.Join(dir, "contents") - require.NoError(t, os.MkdirAll(root, 0o755)) - - // resolved, because that is the form the resolver reports paths in: it normalizes its own root - // with EvalSymlinks, and a test comparing against the unresolved form would be comparing against - // a path the resolver never produces - on macOS t.TempDir sits under /var, a symlink to - // /private/var, so every prefix check here would fail on that alone - resolved, err := filepath.EvalSymlinks(root) - require.NoError(t, err) - return resolved -} - -// locationOf finds the location the resolver reports for one path, going through AllLocations rather -// than FilesByPath: a link whose target is not in the archive resolves to nothing, and a dead link is -// exactly what some of these cases are about. -func locationOf(t *testing.T, r file.Resolver, path string) file.Location { - t.Helper() - // the channel is drained to the end rather than broken out of early: AllLocations runs a goroutine - // that fills it, and abandoning it leaks that goroutine into whatever else the package tests check - var found *file.Location - for loc := range r.AllLocations(context.Background()) { - if loc.RealPath == path { - match := loc - found = &match - } - } - if found == nil { - t.Fatalf("no location reported for %q", path) - } - return *found -} - -// resolvedPaths is every path the resolver reports, sorted. -func resolvedPaths(t *testing.T, r file.Resolver) []string { - t.Helper() - var out []string - for loc := range r.AllLocations(context.Background()) { - out = append(out, loc.RealPath) - } - sort.Strings(out) - return out -} - -func TestArchiveTar_pathsAreRelativeToTheArchiveRoot(t *testing.T) { - // the storage is an implementation detail of the scan: nothing about the tar, the offsets or the - // scratch directory holding them may be reachable from a Location, so an SBOM produced from a - // overflow archive says the same thing it said when the archive was expanded into a directory - dir := t.TempDir() - tarPath := writeTestTar(t, dir, - tarEntry{name: "nested/marker.txt", body: "hello"}, - tarEntry{name: "top.txt", body: "top"}, - ) - - r, err := NewFromArchiveTar(archiveRoot(t, dir), tarPath, "", "outer.zip") - require.NoError(t, err) - - assert.Equal(t, []string{"nested/marker.txt", "top.txt"}, resolvedPaths(t, r)) - assert.Equal(t, 2, r.Records()) - assert.False(t, r.IndexTruncated()) - - for _, path := range resolvedPaths(t, r) { - assert.NotContains(t, path, dir, "no part of the scratch directory may reach a path") - assert.NotContains(t, path, "contents.tar", "the storage file must not name itself in a path") - assert.False(t, filepath.IsAbs(path), "a path inside an archive is relative to that archive") - } - - // and the filesystem id is stamped on every location, which is what keeps identically-named files - // in different archives apart - locs, err := r.FilesByPath("nested/marker.txt") - require.NoError(t, err) - require.Len(t, locs, 1) - assert.Equal(t, "outer.zip", locs[0].ArchivePath) - - contents, err := r.FileContentsByLocation(locs[0]) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, contents.Close()) }) - body, err := io.ReadAll(contents) - require.NoError(t, err) - assert.Equal(t, "hello", string(body)) -} - -func TestArchiveTar_entryNamesAreSanitized(t *testing.T) { - // nothing is written through an entry's name any more - it is a field in a tar header - so this is - // not about arbitrary writes. It is about the path the SBOM reports: a name climbing out of the - // archive would put a node above the archive's own root, which renders as a path outside the - // archive and reads as a claim about the host filesystem. - dir := t.TempDir() - tarPath := writeTestTar(t, dir, - tarEntry{name: "../../etc/passwd", body: "climbing"}, - tarEntry{name: "/etc/hosts", body: "absolute"}, - tarEntry{name: "a/./b/../c.txt", body: "noisy"}, - tarEntry{name: "./", body: ""}, - tarEntry{name: "ok.txt", body: "plain"}, - ) - - r, err := NewFromArchiveTar(archiveRoot(t, dir), tarPath, "", "outer.zip") - require.NoError(t, err) - - // every name lands inside the archive, and the archive's own root is not an entry in it - assert.Equal(t, []string{"a/c.txt", "etc/hosts", "etc/passwd", "ok.txt"}, resolvedPaths(t, r)) - - // the two hostile names collapse onto the same reported path, which is the residual: a misleading - // path inside the right archive rather than an arbitrary write or a claim about the host - byPath, err := r.FilesByPath("etc/passwd") - require.NoError(t, err) - require.Len(t, byPath, 1) - - contents, err := r.FileContentsByLocation(byPath[0]) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, contents.Close()) }) - body, err := io.ReadAll(contents) - require.NoError(t, err) - assert.Equal(t, "climbing", string(body), "the entry is still readable under its sanitized name") -} - -func TestArchiveTar_linkTargetsAreResolvedInsideTheArchive(t *testing.T) { - // a link is resolved in archive-relative space, where a climb clamps at the root, so a target - // naming the host filesystem lands somewhere inside this archive or nowhere at all. That is - // stronger than the check it replaces: extraction refused to CREATE such a link, and here there is - // nothing to create. - dir := t.TempDir() - root := archiveRoot(t, dir) - tarPath := writeTestTar(t, dir, - tarEntry{name: "lib/real.txt", body: "real content"}, - tarEntry{name: "bin/link.txt", link: "../lib/real.txt"}, - tarEntry{name: "escape.txt", link: "../../../../etc/passwd"}, - tarEntry{name: "absolute.txt", link: "/etc/shadow"}, - ) - - r, err := NewFromArchiveTar(root, tarPath, "", "outer.zip") - require.NoError(t, err) - - t.Run("a legitimate relative link still resolves", func(t *testing.T) { - locs, err := r.FilesByPath("bin/link.txt") - require.NoError(t, err) - require.Len(t, locs, 1) - - contents, err := r.FileContentsByLocation(locs[0]) - require.NoError(t, err) - defer func() { require.NoError(t, contents.Close()) }() - body, err := io.ReadAll(contents) - require.NoError(t, err) - assert.Equal(t, "real content", string(body), "the link must resolve to its target's content") - }) - - for _, name := range []string{"escape.txt", "absolute.txt"} { - t.Run("a link out of the archive names nothing on the host: "+name, func(t *testing.T) { - // it is still a file of the archive, and it resolves to nothing rather than to the host: - // FilesByPath follows the link and finds no target - followed, err := r.FilesByPath(name) - require.NoError(t, err) - assert.Empty(t, followed, "the link must resolve to nothing, not to a path on the host") - - metadata, err := r.FileMetadataByLocation(locationOf(t, r, name)) - require.NoError(t, err) - assert.True(t, strings.HasPrefix(metadata.LinkDestination, root+"/"), - "the target must be re-rooted inside the archive, got %q", metadata.LinkDestination) - // the climb is clamped rather than followed: what is left of the target after the archive's - // own root is a path inside the archive, not a path on the host - inside := strings.TrimPrefix(metadata.LinkDestination, root+"/") - assert.NotContains(t, inside, "..", "a climb out of the archive must be clamped, got %q", inside) - }) - } -} - -func TestArchiveTar_aTarEndingPartWayThroughAnEntryIsUsable(t *testing.T) { - // what the disk limit stopping a copy leaves behind. Everything read before that point is intact, - // so it is kept and the tail is reported as a truncation - discarding an archive that was - // cataloged perfectly well up to the cut would lose more than it protects. - dir := t.TempDir() - tarPath := writeTestTar(t, dir, - tarEntry{name: "first.txt", body: "first"}, - tarEntry{name: "second.txt", body: strings.Repeat("s", 2048)}, - ) - - full, err := os.ReadFile(tarPath) - require.NoError(t, err) - cut := filepath.Join(dir, "cut.tar") - // through the first entry and into the second's data - require.NoError(t, os.WriteFile(cut, full[:3*512+16], 0o600)) - - r, err := NewFromArchiveTar(archiveRoot(t, dir), cut, "", "outer.tar") - require.NoError(t, err) - assert.Equal(t, []string{"first.txt"}, resolvedPaths(t, r)) - assert.True(t, r.IndexTruncated()) - - t.Run("a tar that holds nothing readable is an error, not an empty filesystem", func(t *testing.T) { - garbage := filepath.Join(dir, "garbage.tar") - require.NoError(t, os.WriteFile(garbage, []byte("this is not a tar at all"), 0o600)) - - _, err := NewFromArchiveTar(archiveRoot(t, t.TempDir()), garbage, "", "outer.tar") - require.Error(t, err) - }) -} - -func TestArchiveTar_pathFiltersDropEntriesAndPruneDirectories(t *testing.T) { - // an exclusion pattern reaches inside an archive by the same mechanism as before - a visitor per - // entry as the filesystem is indexed - so an excluded entry is ABSENT from the archive's filesystem - // rather than present and skipped. A tar is a sequence rather than a walk, so a pruned directory - // has to be remembered and later entries checked against it. - dir := t.TempDir() - root := archiveRoot(t, dir) - tarPath := writeTestTar(t, dir, - tarEntry{name: "keep.txt", body: "keep"}, - tarEntry{name: "drop.txt", body: "drop"}, - tarEntry{name: "vendor/", body: ""}, - tarEntry{name: "vendor/deep/inner.txt", body: "inner"}, - ) - - filter := func(_, path string, info os.FileInfo, _ error) error { - switch { - case strings.HasSuffix(path, "/drop.txt"): - return ErrSkipPath - case info != nil && info.IsDir() && strings.HasSuffix(path, "/vendor"): - return filepath.SkipDir - } - return nil - } - - r, err := NewFromArchiveTar(root, tarPath, "", "outer.zip", filter) - require.NoError(t, err) - - assert.Equal(t, []string{"keep.txt"}, resolvedPaths(t, r), - "the dropped file and everything under the pruned directory must be absent") - assert.Equal(t, 1, r.Records(), "an excluded entry is not a record either") -} diff --git a/syft/internal/fileresolver/directory.go b/syft/internal/fileresolver/directory.go index 2f079041b..750cfffd9 100644 --- a/syft/internal/fileresolver/directory.go +++ b/syft/internal/fileresolver/directory.go @@ -20,15 +20,7 @@ type Directory struct { } func NewFromDirectory(root, base string, pathFilters ...PathIndexVisitor) (*Directory, error) { - return NewFromDirectoryWithFS(root, base, "", pathFilters...) -} - -// NewFromDirectoryWithFS builds an indexed directory resolver whose Locations are stamped with the -// given fileSystemID. Pass a non-empty id when the directory is an archive that has been extracted -// to a temp dir and is being treated as its own standalone filesystem (see the archive cataloger); -// pass "" for an ordinary directory scan. -func NewFromDirectoryWithFS(root, base, fileSystemID string, pathFilters ...PathIndexVisitor) (*Directory, error) { - resolver, err := newFromDirectoryWithoutIndex(root, base, fileSystemID, pathFilters...) + resolver, err := newFromDirectoryWithoutIndex(root, base, pathFilters...) if err != nil { return nil, err } @@ -36,7 +28,7 @@ func NewFromDirectoryWithFS(root, base, fileSystemID string, pathFilters ...Path return resolver, resolver.buildIndex() } -func newFromDirectoryWithoutIndex(root, base, fileSystemID string, pathFilters ...PathIndexVisitor) (*Directory, error) { +func newFromDirectoryWithoutIndex(root, base string, pathFilters ...PathIndexVisitor) (*Directory, error) { chroot, err := NewChrootContextFromCWD(root, base) if err != nil { return nil, fmt.Errorf("unable to interpret chroot context: %w", err) @@ -48,11 +40,10 @@ func newFromDirectoryWithoutIndex(root, base, fileSystemID string, pathFilters . return &Directory{ path: cleanRoot, FiletreeResolver: FiletreeResolver{ - Chroot: *chroot, - Tree: filetree.New(), - Index: filetree.NewIndex(), - Opener: nativeOSFileOpener, - FileSystemID: fileSystemID, + Chroot: *chroot, + Tree: filetree.New(), + Index: filetree.NewIndex(), + Opener: nativeOSFileOpener, }, indexer: newDirectoryIndexer(cleanRoot, cleanBase, pathFilters...), }, nil diff --git a/syft/internal/fileresolver/directory_fsid_test.go b/syft/internal/fileresolver/directory_fsid_test.go deleted file mode 100644 index 8a5e24535..000000000 --- a/syft/internal/fileresolver/directory_fsid_test.go +++ /dev/null @@ -1,161 +0,0 @@ -//go:build !windows - -package fileresolver - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/anchore/syft/syft/file" -) - -// Test_DirectoryResolver_stampsFileSystemID verifies that a resolver built with a non-empty -// fileSystemID stamps it onto the Coordinates of every Location returned across all access paths. -// This is the identity mechanism that keeps identically-named files in different extracted -// archives from colliding in the coordinate-keyed SBOM tables. -func Test_DirectoryResolver_stampsFileSystemID(t *testing.T) { - const fsID = "some/path/to.zip/some/nested/path.tar.gz" - - resolver, err := NewFromDirectoryWithFS("./testdata/image-symlinks", "", fsID) - require.NoError(t, err) - - t.Run("FilesByPath", func(t *testing.T) { - locations, err := resolver.FilesByPath("/file-1.txt") - require.NoError(t, err) - require.NotEmpty(t, locations) - for _, l := range locations { - assert.Equal(t, fsID, l.FileSystemID, "FilesByPath did not stamp FileSystemID") - } - }) - - t.Run("FilesByGlob", func(t *testing.T) { - locations, err := resolver.FilesByGlob("**/*.txt") - require.NoError(t, err) - require.NotEmpty(t, locations) - for _, l := range locations { - assert.Equal(t, fsID, l.FileSystemID, "FilesByGlob did not stamp FileSystemID") - } - }) - - t.Run("FilesByMIMEType", func(t *testing.T) { - locations, err := resolver.FilesByMIMEType("text/plain") - require.NoError(t, err) - require.NotEmpty(t, locations) - for _, l := range locations { - assert.Equal(t, fsID, l.FileSystemID, "FilesByMIMEType did not stamp FileSystemID") - } - }) - - t.Run("AllLocations", func(t *testing.T) { - var count int - for l := range resolver.AllLocations(context.Background()) { - assert.Equal(t, fsID, l.FileSystemID, "AllLocations did not stamp FileSystemID") - count++ - } - require.NotZero(t, count) - }) -} - -// Test_DirectoryResolver_emptyFileSystemID confirms that an ordinary directory scan (via the -// unchanged NewFromDirectory) continues to produce empty FileSystemIDs. -func Test_DirectoryResolver_emptyFileSystemID(t *testing.T) { - resolver, err := NewFromDirectory("./testdata/image-symlinks", "") - require.NoError(t, err) - - locations, err := resolver.FilesByGlob("**/*.txt") - require.NoError(t, err) - require.NotEmpty(t, locations) - for _, l := range locations { - assert.Empty(t, l.FileSystemID, "ordinary directory scan should not stamp a FileSystemID") - } -} - -// Test_DirectoryResolver_archiveRootIsNotALocation covers -// syft/archive-content-identity#nested-paths-are-archive-relative, scenario "enumerating all -// locations yields nothing outside the archive tree". ToChrootPath trims the prefix root + "/", -// which cannot match the root path itself, so AllLocations used to emit the resolver's own root -// with an absolute path. For an extracted archive that path is a temp directory created fresh per -// run: it reached FileMetadata, was copied into the shared SBOM, and gained a CONTAINS edge, so -// identical input produced different SBOMs. -func Test_DirectoryResolver_archiveRootIsNotALocation(t *testing.T) { - // a directory standing in for an extracted archive, with a nested file so there is something - // legitimate to enumerate alongside the root - root := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(root, "lib", "dep.txt"), []byte("dep"), 0o644)) - - t.Run("an archive filesystem does not report its own root", func(t *testing.T) { - resolver, err := NewFromDirectoryWithFS(root, "", "app/bundle.zip") - require.NoError(t, err) - - var paths []string - for l := range resolver.AllLocations(context.Background()) { - assert.False(t, filepath.IsAbs(l.RealPath), - "no location may carry an absolute path: %q", l.RealPath) - assert.NotContains(t, l.RealPath, root, - "the extraction directory must not appear in any location") - paths = append(paths, l.RealPath) - } - assert.Contains(t, paths, "lib/dep.txt", "the archive's own contents must still be enumerated") - }) - - t.Run("a directory scan still reports its root", func(t *testing.T) { - // the suppression is gated on a non-empty FileSystemID precisely so that released behavior - // for directory and image scans is untouched: there the root is the path the user asked for - resolver, err := NewFromDirectory(root, "") - require.NoError(t, err) - - var sawAbsolute bool - for l := range resolver.AllLocations(context.Background()) { - if filepath.IsAbs(l.RealPath) { - sawAbsolute = true - } - } - assert.True(t, sawAbsolute, "a plain directory scan reports its root as it always has") - }) -} - -// Test_DirectoryResolver_archivePathsAreRelative covers the other half of -// #nested-paths-are-archive-relative: the path, glob and MIME code paths report archive-relative -// paths. Nothing asserted this - the FileSystemID test above checks the identifier and never the -// path. -func Test_DirectoryResolver_archivePathsAreRelative(t *testing.T) { - root := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(root, "lib", "dep.txt"), []byte("dep"), 0o644)) - - resolver, err := NewFromDirectoryWithFS(root, "", "app/bundle.zip") - require.NoError(t, err) - - assertRelative := func(t *testing.T, locations []file.Location) { - t.Helper() - require.NotEmpty(t, locations) - for _, l := range locations { - assert.Equal(t, "lib/dep.txt", l.RealPath) - assert.NotContains(t, l.AccessPath, root) - } - } - - t.Run("FilesByPath", func(t *testing.T) { - locations, err := resolver.FilesByPath("/lib/dep.txt") - require.NoError(t, err) - assertRelative(t, locations) - }) - - t.Run("FilesByGlob", func(t *testing.T) { - locations, err := resolver.FilesByGlob("**/*.txt") - require.NoError(t, err) - assertRelative(t, locations) - }) - - t.Run("FilesByMIMEType", func(t *testing.T) { - locations, err := resolver.FilesByMIMEType("text/plain") - require.NoError(t, err) - assertRelative(t, locations) - }) -} diff --git a/syft/internal/fileresolver/filetree_resolver.go b/syft/internal/fileresolver/filetree_resolver.go index 2a688314e..68b3b8395 100644 --- a/syft/internal/fileresolver/filetree_resolver.go +++ b/syft/internal/fileresolver/filetree_resolver.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "os" - "path/filepath" stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree" @@ -22,30 +21,6 @@ type FiletreeResolver struct { Index filetree.IndexReader SearchContext filetree.Searcher Opener func(stereoscopeFile.Reference) (io.ReadCloser, error) - - // FileSystemID, when non-empty, is stamped onto the Coordinates of every Location this - // resolver returns. For container images this is a layer digest; for an extracted archive - // treated as its own standalone filesystem it is the FileSystemID of the archive file's own - // filesystem (inherited unchanged down the nesting chain); for a plain root directory scan it - // is empty. - FileSystemID string - - // ArchivePath, when non-empty, is stamped onto the Coordinates of every Location this resolver - // returns. For an archive extracted and indexed as its own standalone filesystem it is the - // colon-delimited chain of archive paths from the scan root to that archive (e.g. - // "app.war:WEB-INF/lib/dep.jar"); for a plain root directory scan it is empty. This keeps - // identically-named files in different archives from colliding in the coordinate-keyed SBOM - // tables. - ArchivePath string -} - -// newVirtualLocation builds a directory location with a distinct access path, stamped with this -// resolver's FileSystemID and ArchivePath. -func (r FiletreeResolver) newVirtualLocation(responsePath, responseAccessPath string, ref stereoscopeFile.Reference) file.Location { - loc := file.NewVirtualLocationFromDirectory(responsePath, responseAccessPath, ref) - loc.FileSystemID = r.FileSystemID - loc.ArchivePath = r.ArchivePath - return loc } func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) { @@ -116,7 +91,7 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err if ref.HasReference() { references = append(references, - r.newVirtualLocation( + file.NewVirtualLocationFromDirectory( r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root *ref.Reference, @@ -160,7 +135,7 @@ func (r FiletreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro continue } - loc := r.newVirtualLocation( + loc := file.NewVirtualLocationFromDirectory( r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root *refVia.Reference, @@ -212,24 +187,10 @@ func (r *FiletreeResolver) AllLocations(ctx context.Context) <-chan file.Locatio go func() { defer close(results) for _, ref := range r.Tree.AllFiles(stereoscopeFile.AllTypes()...) { - responsePath := r.responsePath(string(ref.RealPath)) - if r.FileSystemID != "" && filepath.IsAbs(responsePath) { - // ToChrootPath relativizes by trimming the prefix root + "/", which cannot match - // the root path itself, so the resolver's own root arrives here still absolute. - // For a directory scan that is the path the user asked for and is left alone. For a - // filesystem this capability invented - an archive extracted into a directory - // created fresh per run - it is a path that must not reach a coordinate: it would - // make two scans of identical input produce different output, and it names a - // directory that no longer exists by the time the scan finishes. Gated on - // FileSystemID so directory and image scans behave exactly as before. - continue - } - loc := file.NewLocationFromDirectory(responsePath, r.FileSystemID, ref) - loc.ArchivePath = r.ArchivePath select { case <-ctx.Done(): return - case results <- loc: + case results <- file.NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), "", ref): continue } } @@ -261,7 +222,7 @@ func (r *FiletreeResolver) FilesByMIMEType(types ...string) ([]file.Location, er if uniqueFileIDs.Contains(*refVia.Reference) { continue } - location := r.newVirtualLocation( + location := file.NewVirtualLocationFromDirectory( r.responsePath(string(refVia.RealPath)), r.responsePath(string(refVia.RequestPath)), *refVia.Reference, diff --git a/syft/pkg/cataloger/java/archive_contents_cataloger.go b/syft/pkg/cataloger/java/archive_contents_cataloger.go deleted file mode 100644 index 71bff3533..000000000 --- a/syft/pkg/cataloger/java/archive_contents_cataloger.go +++ /dev/null @@ -1,129 +0,0 @@ -package java - -import ( - "context" - "fmt" - - "github.com/bmatcuk/doublestar/v4" - - "github.com/anchore/syft/internal/archive" - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/internal/unknown" - "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/file" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/java/internal/maven" -) - -const ArchiveContentsCatalogerName = "java-archive-contents-cataloger" - -// NewArchiveContentsCataloger returns a java archive cataloger for the world after extraction: it -// reads an archive's already-extracted filesystem rather than opening the archive itself. -// -// It is a second cataloger rather than a mode of the first because the two answer different -// questions. The archive-reading cataloger asks "is this file a jar?" of every file in a scan; this -// one asks "is the filesystem I have been handed a jar?", which is the only form the question can -// take once something else owns extraction. That inversion is also why it cannot be a parameter: -// when the archive cataloger task hands a jar's contents to the sub-pipeline, the jar file is one -// level up and already behind us. -// -// What it does not do is recurse. The task that extracted this archive will meet the archives inside -// it on its own, and one mechanism owning recursion is the whole point - see -// syft/java-archive-nesting#single-owner-of-archive-recursion. -func NewArchiveContentsCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger { - return archiveContentsCataloger{cfg: cfg} -} - -type archiveContentsCataloger struct { - cfg ArchiveCatalogerConfig -} - -func (c archiveContentsCataloger) Name() string { - return ArchiveContentsCatalogerName -} - -// Catalog reports the java package of the archive whose contents this resolver holds, if it is a java -// archive at all. -// -// Nothing outside an archive is cataloged. Without a traversal on the context there is no archive to -// describe - the identity of a java archive lives half in the file (its name, its digests) and half -// in its contents (the manifest, the poms), and the file half arrives only on the traversal. -func (c archiveContentsCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { - trav := archive.TraversalFromContext(ctx) - if trav == nil { - return nil, nil, nil - } - - if !isJavaArchiveName(trav.Location.Path()) { - // the same rule the archive-reading cataloger applies through its globs: a .zip is an archive - // but it is not a java archive, and its contents are not a jar's contents - return nil, nil, nil - } - - parser, err := newArchiveContentsParser(trav, resolver, c.cfg) - if err != nil { - return nil, nil, err - } - - pkgs, relationships, err := parser.parse(ctx, nil) - if err != nil { - // attributed to the archive rather than returned bare, the way the generic cataloger attributes - // a parser's failure to the file it was parsing. An archive that turns out to hold no java - // package at all reports exactly that, and the scan carries on. - return pkgs, relationships, unknown.New(trav.Location, err) - } - return pkgs, relationships, nil -} - -// newArchiveContentsParser builds the same parser the archive-reading cataloger uses, over the -// archive's extracted filesystem instead of over the archive file. Identity comes from the traversal: -// the location and virtual path it already carries, the archive's filename for the name and version -// fallbacks, and the digests taken when the archive was extracted. -func newArchiveContentsParser(trav *archive.Traversal, resolver file.Resolver, cfg ArchiveCatalogerConfig) (*archiveParser, error) { - if resolver == nil { - return nil, fmt.Errorf("no resolver for the contents of %q", trav.VirtualPath) - } - return &archiveParser{ - entries: newResolverEntries(resolver, trav.Digests), - location: trav.Location, - // the colon-delimited chain the task composed, which is the identity already-published SBOMs - // carry for a nested java package - virtualPath: trav.VirtualPath, - fileInfo: newJavaArchiveFilename(archiveFileNameOf(trav)), - // recursion belongs to the task that extracted this archive - detectNested: false, - cfg: cfg, - maven: maven.NewResolver(nil, cfg.mavenConfig()), - }, nil -} - -// archiveFileNameOf returns the archive's own file name, which is where a jar's name and version come -// from when its manifest does not say. The last element of the virtual path rather than of the real -// path, so a chain and a plain scan agree. -func archiveFileNameOf(trav *archive.Traversal) string { - name := trav.VirtualPath - if name == "" { - name = trav.Location.Path() - } - for i := len(name) - 1; i >= 0; i-- { - if name[i] == ':' || name[i] == '/' { - return name[i+1:] - } - } - return name -} - -// isJavaArchiveName reports whether the path is one of the java archive formats, by the same glob -// list the archive-reading cataloger registers against. -func isJavaArchiveName(path string) bool { - for _, glob := range archiveFormatGlobs { - if ok, err := doublestar.Match(glob, path); err == nil && ok { - return true - } - if ok, err := doublestar.Match(glob, "/"+path); err == nil && ok { - return true - } - } - log.WithFields("path", path).Trace("archive contents are not a java archive") - return false -} diff --git a/syft/pkg/cataloger/java/archive_entries.go b/syft/pkg/cataloger/java/archive_entries.go index 636c853ae..dccb801d7 100644 --- a/syft/pkg/cataloger/java/archive_entries.go +++ b/syft/pkg/cataloger/java/archive_entries.go @@ -16,30 +16,28 @@ import ( // archiveEntries is where an archive parser reads a java archive's entries from. // -// The parser needs three things of an archive and nothing else: which entries match a glob, the -// contents of some of them, and the digest of the archive as a whole. Everything else it does - -// which manifest wins, how pom.properties decides identity, license discovery - is logic over those -// answers. Naming the three makes them replaceable, which is what lets the same logic run over an -// archive file and over an archive that has already been extracted to a filesystem. +// A parser needs three things of an archive: which entries match a glob, the contents of some of +// them, and the digest of the archive as a whole. Everything else - which manifest wins, how +// pom.properties decides identity, license discovery - is logic over those answers, so naming the +// three lets the same logic run over an archive file and over one already extracted to a filesystem. // -// Entry names are archive-relative with no leading slash, matching zip entry names, because they -// become map keys and pom paths that identity is derived from. An implementation over a filesystem -// has to strip the root to match. +// Entry names are archive-relative with no leading slash, matching zip entry names, since they become +// the map keys and pom paths identity is derived from. An implementation over a filesystem must strip +// the root to match. type archiveEntries interface { - // glob returns the names of the entries matching any of the patterns. glob(caseInsensitive bool, patterns ...string) []string // contents reads the named entries, returning a map keyed by the same names. contents(ctx context.Context, names ...string) (map[string]string, error) - // digests returns the digests of the archive as a whole, which is a property of the archive file - // rather than of anything inside it. + // digests returns the digests of the archive as a whole, a property of the archive file rather + // than of anything inside it. digests(ctx context.Context) ([]file.Digest, error) } -// zipEntries reads entries out of a java archive on disk. This is the original source, kept for -// scans where nothing has extracted the archive: it copies the archive to a temp file, reads its -// central directory once, and opens it again for each set of entries it is asked for. +// zipEntries reads entries out of a java archive on disk, for scans where nothing has already +// extracted it. It copies the archive to a temp file, reads its central directory once, and reopens +// it for each set of entries it is asked for. type zipEntries struct { archivePath string fileManifest intFile.ZipFileManifest @@ -65,23 +63,18 @@ func (z *zipEntries) digests(ctx context.Context) ([]file.Digest, error) { return getDigestsFromArchive(ctx, z.archivePath) } -// resolverEntries reads entries out of an archive that has already been extracted and indexed - the -// filesystem the archive cataloger task builds for every archive it enters. -// -// Nothing is copied and nothing is re-opened: the entries are files in an indexed resolver, and the -// archive's own digests were taken by whoever extracted it, since the archive file itself is not -// present in its own contents. +// resolverEntries reads entries out of an archive already extracted and indexed by the archive +// cataloger task. Nothing is copied or reopened. The archive's own digests come from whoever +// extracted it, since the archive file is not present in its own contents. type resolverEntries struct { - resolver file.Resolver - archiveDigests []file.Digest - locationsByName map[string]file.Location + resolver file.Resolver + archiveDigests []file.Digest } func newResolverEntries(resolver file.Resolver, archiveDigests []file.Digest) *resolverEntries { return &resolverEntries{ - resolver: resolver, - archiveDigests: archiveDigests, - locationsByName: make(map[string]file.Location), + resolver: resolver, + archiveDigests: archiveDigests, } } @@ -90,10 +83,9 @@ func (r *resolverEntries) glob(caseInsensitive bool, patterns ...string) []strin return r.globCaseInsensitive(patterns...) } - // deduplicated within this call and not across calls: the parser globs the same pattern more than - // once (identity resolution asks for the poms, and so does aux package discovery), and a source - // that remembered what it had already returned would hand back nothing the second time. That - // silently cost every package derived from a bundled pom.properties. + // deduplicated within this call only: the parser globs the same pattern more than once (identity + // resolution and aux package discovery both ask for the poms), so remembering across calls would + // return nothing the second time and lose every package derived from a bundled pom.properties seen := strset.New() var names []string for _, pattern := range patterns { @@ -104,7 +96,6 @@ func (r *resolverEntries) glob(caseInsensitive bool, patterns ...string) []strin } for _, location := range locations { name := strings.TrimPrefix(location.Path(), "/") - r.locationsByName[name] = location if seen.Has(name) { continue } @@ -115,10 +106,9 @@ func (r *resolverEntries) glob(caseInsensitive bool, patterns ...string) []strin return names } -// globCaseInsensitive matches the way the zip-backed source does, by lowercasing both sides. The -// resolver's own glob is case sensitive, so this lists the archive's files once and matches them -// here. License discovery is the only caller that asks, and what it costs is one listing of a -// filesystem holding one archive's entries. +// globCaseInsensitive matches the way zipEntries does, by lowercasing both sides. The resolver's own +// glob is case sensitive, so this lists the archive's files once and matches them here. License +// discovery is the only caller. func (r *resolverEntries) globCaseInsensitive(patterns ...string) []string { all, err := r.resolver.FilesByGlob("**/*") if err != nil { @@ -130,16 +120,13 @@ func (r *resolverEntries) globCaseInsensitive(patterns ...string) []string { for _, pattern := range patterns { lowered := strings.ToLower(pattern) for _, location := range all { - // matched with a leading slash, the way the zip-backed source normalizes an entry before - // matching, because the patterns are written that way ("/META-INF/*"). An extracted - // archive's resolver reports paths without one - they are entry names, not absolute paths - - // so matching them raw silently matches nothing, which cost the license found in every - // jar that carries one as a file. + // matched with a leading slash, since the patterns are written that way ("/META-INF/*") and + // zipEntries normalizes the same way; an extracted archive's resolver reports names without + // one, so matching raw would match nothing name := strings.TrimPrefix(location.Path(), "/") if ok, matchErr := doublestar.Match(lowered, "/"+strings.ToLower(name)); matchErr != nil || !ok { continue } - r.locationsByName[name] = location if seen.Has(name) { continue } @@ -153,13 +140,14 @@ func (r *resolverEntries) globCaseInsensitive(patterns ...string) []string { func (r *resolverEntries) contents(_ context.Context, names ...string) (map[string]string, error) { out := make(map[string]string, len(names)) for _, name := range names { - location, ok := r.locationsByName[name] - if !ok { - // only names this source handed out can be read back; a name from anywhere else is a bug - // rather than a missing file - return nil, fmt.Errorf("entry %q was not found by any glob against this archive", name) + locations, err := r.resolver.FilesByPath(name) + if err != nil { + return nil, fmt.Errorf("unable to look up archive entry %q: %w", name, err) } - contents, err := r.read(location) + if len(locations) == 0 { + return nil, fmt.Errorf("entry %q is not in this archive", name) + } + contents, err := r.read(locations[0]) if err != nil { return nil, fmt.Errorf("unable to read archive entry %q: %w", name, err) } @@ -172,12 +160,11 @@ func (r *resolverEntries) digests(_ context.Context) ([]file.Digest, error) { return r.archiveDigests, nil } -// read reads one entry's content to the end and closes the stream. The parser wants entry contents as -// strings, since every consumer of them parses text. +// read reads one entry's content to the end and closes the stream, returning a string because every +// consumer parses text. // -// The copy is bounded the same way the zip-backed source bounds its own: an entry of an extracted -// archive is still attacker-supplied content, and reading it into a string is exactly the allocation -// a decompression bomb is aiming at. +// The copy is bounded the way zipEntries bounds its own: an entry of an extracted archive is still +// attacker-supplied, and reading it into a string is the allocation a decompression bomb targets. func (r *resolverEntries) read(location file.Location) (string, error) { reader, err := r.resolver.FileContentsByLocation(location) if err != nil { diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 6346babac..5b469826a 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -17,10 +17,8 @@ import ( "golang.org/x/exp/maps" "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/archive" intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/internal/tmpdir" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -58,18 +56,27 @@ var javaArchiveHashes = []crypto.Hash{ crypto.SHA1, } +// archiveParser reads one java archive, whichever source it came from: everything it does is logic +// over an archiveSource's entries and identity, so it does not know whether this cataloger opened an +// archive file or the archive cataloger task already extracted one. type archiveParser struct { - entries archiveEntries - location file.Location - virtualPath string - archivePath string - contentPath string - fileInfo archiveFilename + archiveSource detectNested bool cfg ArchiveCatalogerConfig maven *maven.Resolver } +// newArchiveParser builds the parser for one archive. detectNested is whether this parser recurses +// into the java archives inside this one, which only a file-backed source can do. +func newArchiveParser(src archiveSource, detectNested bool, cfg ArchiveCatalogerConfig) *archiveParser { + return &archiveParser{ + archiveSource: src, + detectNested: detectNested, + cfg: cfg, + maven: maven.NewResolver(nil, cfg.mavenConfig()), + } +} + type genericArchiveParserAdapter struct { cfg ArchiveCatalogerConfig } @@ -85,9 +92,9 @@ func (gap genericArchiveParserAdapter) parseJavaArchive(ctx context.Context, _ f // processJavaArchive processes an archive for java contents, returning all Java libraries and nested archives func (gap genericArchiveParserAdapter) processJavaArchive(ctx context.Context, reader file.LocationReadCloser, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) { - // when the generic archive cataloger task owns recursion into archives, this parser must not - // also unarchive nested archives, otherwise they would be cataloged twice - parser, cleanupFn, err := newJavaArchiveParser(ctx, reader, !gap.cfg.nestedArchivesHandledExternally(), gap.cfg) + // when the archive cataloger task owns recursion, unarchiving here would catalog nested archives + // twice + parser, cleanupFn, err := newJavaArchiveParser(ctx, reader, !gap.cfg.NestedArchivesHandledExternally, gap.cfg) // note: even on error, we should always run cleanup functions defer cleanupFn() if err != nil { @@ -104,43 +111,14 @@ func uniquePkgKey(groupID string, p *pkg.Package) string { return fmt.Sprintf("%s|%s|%s", groupID, p.Name, p.Version) } -// newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover -// and parse nested archives or ignore them. +// newJavaArchiveParser returns a new java archive parser for an archive file this cataloger opens +// itself. Can be configured to discover and parse nested archives or ignore them. func newJavaArchiveParser(ctx context.Context, reader file.LocationReadCloser, detectNested bool, cfg ArchiveCatalogerConfig) (*archiveParser, func(), error) { - // establish the full virtual path of this archive: when the generic archive cataloger task drove - // us here, the context traversal supplies the containing archive chain; otherwise the reader - // path already carries any colon-delimited nesting from this cataloger's own recursion - virtualPath := archive.TraversalFromContext(ctx).VirtualPathOf(reader.Path()) - - // fetch the last element of the virtual path - virtualElements := strings.Split(virtualPath, ":") - currentFilepath := virtualElements[len(virtualElements)-1] - - td := tmpdir.FromContext(ctx) - if td == nil { - return nil, func() {}, fmt.Errorf("no temp dir factory in context") - } - contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(td, currentFilepath, reader) - if err != nil { - return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err) - } - - entries, err := newZipEntries(ctx, archivePath) + src, cleanupFn, err := newFileArchiveSource(ctx, reader) if err != nil { return nil, cleanupFn, err } - - return &archiveParser{ - entries: entries, - location: reader.Location, - virtualPath: virtualPath, - archivePath: archivePath, - contentPath: contentPath, - fileInfo: newJavaArchiveFilename(currentFilepath), - detectNested: detectNested, - cfg: cfg, - maven: maven.NewResolver(nil, cfg.mavenConfig()), - }, cleanupFn, nil + return newArchiveParser(src, detectNested, cfg), cleanupFn, nil } // parse the loaded archive and return all packages found. @@ -199,9 +177,9 @@ func (j *archiveParser) parse(ctx context.Context, parentPkg *pkg.Package) ([]pk } pkgs = append(pkgs, nestedPkgs...) relationships = append(relationships, nestedRelationships...) - } else if !j.cfg.nestedArchivesHandledExternally() { - // nested archives will not be cataloged by anything: this parser is not recursing and the - // generic archive cataloger task is disabled + } else if !j.cfg.NestedArchivesHandledExternally { + // nothing will catalog these: this parser is not recursing and the archive cataloger task is + // disabled, so record them as unknowns // .jar and .war files are present in archives, are others? or generally just consider them top-level? nestedArchives := j.entries.glob(true, "**/*.jar", "**/*.war") if len(nestedArchives) > 0 { @@ -252,7 +230,7 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package, // parse the manifest file into a rich object manifestContents := contents[manifestMatches[0]] - manifest, err := parseJavaManifest(j.archivePath, strings.NewReader(manifestContents)) + manifest, err := parseJavaManifest(j.virtualPath, strings.NewReader(manifestContents)) if err != nil { log.Debugf("failed to parse java manifest (%s): %+v", j.location, err) return nil, nil @@ -641,7 +619,7 @@ func (j *archiveParser) versionFromPropertiesFile(ctx context.Context, manifest return "" } - contents, err := intFile.ContentsFromZip(ctx, j.archivePath, matches...) + contents, err := j.entries.contents(ctx, matches...) if err != nil { log.Debugf("unable to extract version.properties (%s): %v", j.location, err) return "" @@ -663,10 +641,9 @@ func (j *archiveParser) versionFromPropertiesFile(ctx context.Context, manifest return "" } -// discoverPkgsFromNestedArchives recurses into the java archives inside this one. It exists only for -// the zip-backed source: recursion needs an archive file to unzip, and the resolver-backed source has -// no file - by the time it runs, the archive cataloger task has already extracted everything and will -// meet those nested archives itself. +// discoverPkgsFromNestedArchives recurses into the java archives inside this one. Only zipEntries +// applies, since recursion needs an archive file to unzip; with resolverEntries the archive cataloger +// task has already extracted everything and meets those nested archives itself. func (j *archiveParser) discoverPkgsFromNestedArchives(ctx context.Context, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) { zipped, ok := j.entries.(*zipEntries) if !ok { diff --git a/syft/pkg/cataloger/java/archive_parser_traversal_test.go b/syft/pkg/cataloger/java/archive_parser_traversal_test.go index ea66c9bef..ac8c6a8d6 100644 --- a/syft/pkg/cataloger/java/archive_parser_traversal_test.go +++ b/syft/pkg/cataloger/java/archive_parser_traversal_test.go @@ -20,26 +20,6 @@ import ( "github.com/anchore/syft/syft/source/directorysource" ) -// makeTestZip builds a zip in memory so the parser can be exercised without the -// java-toolchain-built fixtures. -func makeTestZip(t *testing.T, entries map[string][]byte) []byte { - t.Helper() - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - for name, content := range entries { - w, err := zw.Create(name) - require.NoError(t, err) - _, err = w.Write(content) - require.NoError(t, err) - } - require.NoError(t, zw.Close()) - return buf.Bytes() -} - -func testManifest(title, version string) []byte { - return fmt.Appendf(nil, "Manifest-Version: 1.0\nImplementation-Title: %s\nImplementation-Version: %s\n", title, version) -} - func Test_virtualPathFromArchiveTraversal(t *testing.T) { jarBytes := makeTestZip(t, map[string][]byte{ "META-INF/MANIFEST.MF": testManifest("example-app", "1.0.0"), @@ -71,10 +51,8 @@ func Test_virtualPathFromArchiveTraversal(t *testing.T) { t.Run("traversal prepends containing archive chain", func(t *testing.T) { trav := &archive.Traversal{ - Location: file.NewLocation("/app.war"), - VirtualPath: "/app.war", - FileSystemID: "app.war", - Depth: 1, + Location: file.NewLocation("/app.war"), + VirtualPath: "/app.war", } p := parse(t, trav, "/WEB-INF/lib/example-app-1.0.0.jar") metadata, ok := p.Metadata.(pkg.JavaArchive) @@ -86,12 +64,9 @@ func Test_virtualPathFromArchiveTraversal(t *testing.T) { }) t.Run("multi-level traversal chains all parents", func(t *testing.T) { - outer := &archive.Traversal{VirtualPath: "/dist.zip", FileSystemID: "dist.zip", Depth: 1} + outer := &archive.Traversal{VirtualPath: "/dist.zip"} inner := &archive.Traversal{ - VirtualPath: outer.VirtualPathOf("/app.war"), - FileSystemID: "dist.zip/app.war", - Depth: 2, - Parent: outer, + VirtualPath: outer.VirtualPathOf("/app.war"), } p := parse(t, inner, "/WEB-INF/lib/example-app-1.0.0.jar") metadata, ok := p.Metadata.(pkg.JavaArchive) @@ -100,12 +75,12 @@ func Test_virtualPathFromArchiveTraversal(t *testing.T) { }) } -// Test_virtualPathFromArchiveTraversal_colonInEntryPath is the java-cataloger-level counterpart -// of internal/archive's escaping tests: a package reached through an entry whose own path -// contains a colon must carry that colon escaped as %3A end to end in its virtual path -// (decisions.md#colon-in-an-entry-path-is-escaped), and the maven-derived -// ":groupID:artifactID" suffix that distinguishes a pom.properties package from its containing -// archive must still be appended using real, unescaped delimiters. +// Test_virtualPathFromArchiveTraversal_colonInEntryPath is the java-cataloger-level counterpart of +// internal/archive's escaping tests: a package reached through an entry whose path contains a colon +// must carry it escaped as %3A end to end in its virtual path +// (decisions.md#colon-in-an-entry-path-is-escaped), while the maven-derived ":groupID:artifactID" +// suffix that distinguishes a pom.properties package from its containing archive is still appended +// with real, unescaped delimiters. func Test_virtualPathFromArchiveTraversal_colonInEntryPath(t *testing.T) { jarBytes := makeTestZip(t, map[string][]byte{ "META-INF/MANIFEST.MF": testManifest("outer-app", "1.0.0"), @@ -115,10 +90,8 @@ func Test_virtualPathFromArchiveTraversal_colonInEntryPath(t *testing.T) { }) trav := &archive.Traversal{ - Location: file.NewLocation("/outer.zip"), - VirtualPath: "outer.zip", - FileSystemID: "outer.zip", - Depth: 1, + Location: file.NewLocation("/outer.zip"), + VirtualPath: "outer.zip", } ctx := archive.WithTraversal(pkgtest.Context(t), trav) @@ -201,12 +174,12 @@ func Test_nestedArchiveOwnershipSwitch(t *testing.T) { }) t.Run("java's own archive-search depth does not hand off recursion", func(t *testing.T) { - // a library consumer setting java's squash-inlined MaxDepth must neither enable nor - // disable nested archive cataloging: that knob is Archive.MaxDepth on CreateSBOMConfig. - // Getting this wrong silently drops nested jars for every consumer that is not the CLI. + // a library consumer setting java's squash-inlined MaxDepth must neither enable nor disable + // nested archive cataloging: that knob is Archive.MaxDepth on CreateSBOMConfig. Getting it + // wrong silently drops nested jars for every consumer that is not the CLI. cfg := DefaultArchiveCatalogerConfig() cfg.ArchiveSearchConfig = cfg.ArchiveSearchConfig.WithMaxDepth(2) - require.False(t, cfg.nestedArchivesHandledExternally()) + require.False(t, cfg.NestedArchivesHandledExternally) pkgs := parse(t, cfg) require.Len(t, pkgs, 2, "this cataloger must still recurse itself") @@ -214,12 +187,11 @@ func Test_nestedArchiveOwnershipSwitch(t *testing.T) { } func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) { - // the wrapped-archive parsers exist only because the generic archive cataloger did not. When it - // is enabled it extracts zip and tar containers itself and this cataloger meets the same JARs - // one nesting level down, so registering them would double the work. + // the wrapped-archive parsers exist only because the archive cataloger did not. When it is enabled + // it extracts zip and tar containers itself and this cataloger meets the same jars one nesting level + // down, so registering them would double the work. // - // asserted through the globs the cataloger actually queries, since that is the observable - // consequence of a parser being registered + // Asserted through the globs the cataloger queries, the observable consequence of a registration. dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "placeholder.txt"), []byte("x"), 0o600)) @@ -243,11 +215,11 @@ func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) { cfg.NestedArchivesHandledExternally = true observed := queriedGlobs(t, cfg) + // outside a traversal the cataloger opens no files at all: the task brings every archive back + // as an extracted filesystem, and each is described there assert.False(t, observed.ObservedPathQuery("**/*.zip"), "no generic zip parser may be registered") assert.False(t, observed.ObservedPathQuery("**/*.tar"), "no generic tar parser may be registered") - - // the JAR-family globs are still registered: this cataloger is still the java parser - assert.True(t, observed.ObservedPathQuery("**/*.jar")) + assert.False(t, observed.ObservedPathQuery("**/*.jar"), "archive files are the task's to open") }) t.Run("wrapped parsers stay registered when the feature is off", func(t *testing.T) { @@ -272,11 +244,10 @@ func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) { } func Test_mavenVirtualPathSuffixUnderTraversal(t *testing.T) { - // a package derived from a pom.properties whose artifact does not match the containing archive - // gets a trailing "::" appended to the archive's virtual path. That suffix - // form is in published SBOMs, and once the archive cataloger drives the parser the base it is - // appended to comes from the traversal rather than from the reader path - so this asserts the - // two compose rather than one clobbering the other. + // a package derived from a pom.properties whose artifact does not match the containing archive gets + // a trailing "::" appended to the archive's virtual path. That suffix form is in + // published SBOMs, and once the archive cataloger drives the parser the base comes from the traversal + // rather than the reader path - so the two must compose rather than one clobbering the other. jarBytes := makeTestZip(t, map[string][]byte{ "META-INF/MANIFEST.MF": testManifest("example-app", "1.0.0"), "META-INF/maven/com.example.other/other-lib/pom.properties": []byte( @@ -320,8 +291,28 @@ func Test_mavenVirtualPathSuffixUnderTraversal(t *testing.T) { }) t.Run("under traversal: suffix hangs off the archive chain", func(t *testing.T) { - pkgs := parse(t, &archive.Traversal{VirtualPath: "app.war", Depth: 1}, "/WEB-INF/lib/example-app-1.0.0.jar") + pkgs := parse(t, &archive.Traversal{VirtualPath: "app.war"}, "/WEB-INF/lib/example-app-1.0.0.jar") assert.Equal(t, "app.war:WEB-INF/lib/example-app-1.0.0.jar:com.example.other:other-lib", virtualPathOf(t, pkgs, "other-lib")) }) } + +// makeTestZip builds a zip in memory so the parser can be exercised without the java-toolchain-built +// fixtures. +func makeTestZip(t *testing.T, entries map[string][]byte) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for name, content := range entries { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write(content) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +func testManifest(title, version string) []byte { + return fmt.Appendf(nil, "Manifest-Version: 1.0\nImplementation-Title: %s\nImplementation-Version: %s\n", title, version) +} diff --git a/syft/pkg/cataloger/java/archive_source.go b/syft/pkg/cataloger/java/archive_source.go new file mode 100644 index 000000000..29e0ee4c5 --- /dev/null +++ b/syft/pkg/cataloger/java/archive_source.go @@ -0,0 +1,119 @@ +package java + +import ( + "context" + "fmt" + "strings" + + "github.com/bmatcuk/doublestar/v4" + + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/syft/file" +) + +// archiveSource is one java archive to describe: where its entries are read from, and the identity +// that does not live inside them - the archive's own location, the colon-delimited virtual path +// SBOMs carry, and the file name that name and version fall back to. +// +// Two of these exist because two things can own extraction. When this cataloger owns it, the archive +// is a file it copies to a temp dir and reads as a zip. When the archive cataloger task owns it, the +// archive has already been extracted and indexed, and arrives as a resolver plus the traversal +// describing the file it came from. Everything past this point - which manifest wins, how +// pom.properties decides identity, license discovery - is logic over a source, so it runs unchanged +// either way. +type archiveSource struct { + entries archiveEntries + location file.Location + virtualPath string + fileInfo archiveFilename + + // contentPath is the directory nested archives are extracted into. Set only when this cataloger + // opened the archive itself: recursing needs somewhere to unzip into, and an already-extracted + // archive has none. The archive file itself is held by zipEntries. + contentPath string +} + +// newFileArchiveSource copies the archive to a temp dir and reads its central directory, for a scan +// where nothing has already extracted it. The returned cleanup must run even when an error is +// returned. +func newFileArchiveSource(ctx context.Context, reader file.LocationReadCloser) (archiveSource, func(), error) { + // the full virtual path of this archive: when the archive cataloger task drove us here, the context + // traversal supplies the containing chain; otherwise the reader path already carries any + // colon-delimited nesting from this cataloger's own recursion + virtualPath := archive.TraversalFromContext(ctx).VirtualPathOf(reader.Path()) + + // fetch the last element of the virtual path + virtualElements := strings.Split(virtualPath, ":") + currentFilepath := virtualElements[len(virtualElements)-1] + + td := tmpdir.FromContext(ctx) + if td == nil { + return archiveSource{}, func() {}, fmt.Errorf("no temp dir factory in context") + } + contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(td, currentFilepath, reader) + if err != nil { + return archiveSource{}, cleanupFn, fmt.Errorf("unable to process java archive: %w", err) + } + + entries, err := newZipEntries(ctx, archivePath) + if err != nil { + return archiveSource{}, cleanupFn, err + } + + return archiveSource{ + entries: entries, + location: reader.Location, + virtualPath: virtualPath, + fileInfo: newJavaArchiveFilename(currentFilepath), + contentPath: contentPath, + }, cleanupFn, nil +} + +// newExtractedArchiveSource describes the archive whose extracted contents the given resolver holds. +// Nothing is copied or reopened, and identity comes from the traversal: the location, the chain the +// archive cataloger task composed, the file name, and the digests taken at extraction - none of which +// are inside the archive. +func newExtractedArchiveSource(trav *archive.Traversal, resolver file.Resolver) (archiveSource, error) { + if resolver == nil { + return archiveSource{}, fmt.Errorf("no resolver for the contents of %q", trav.VirtualPath) + } + return archiveSource{ + entries: newResolverEntries(resolver, trav.Digests), + location: trav.Location, + virtualPath: trav.VirtualPath, + fileInfo: newJavaArchiveFilename(archiveFileNameOf(trav)), + }, nil +} + +// archiveFileNameOf returns the archive's own file name, where a jar's name and version come from +// when its manifest does not say. It takes the last element of the virtual path rather than the real +// path, so a nested chain and a plain scan agree. +func archiveFileNameOf(trav *archive.Traversal) string { + name := trav.VirtualPath + if name == "" { + name = trav.Location.Path() + } + for i := len(name) - 1; i >= 0; i-- { + if name[i] == ':' || name[i] == '/' { + return name[i+1:] + } + } + return name +} + +// isJavaArchiveName reports whether the path is one of the java archive formats, by the same globs +// this cataloger registers against when it is the one opening files. +func isJavaArchiveName(path string) bool { + for _, glob := range archiveFormatGlobs { + if ok, err := doublestar.Match(glob, path); err == nil && ok { + return true + } + if ok, err := doublestar.Match(glob, "/"+path); err == nil && ok { + return true + } + } + log.WithFields("path", path).Trace("archive contents are not a java archive") + return false +} diff --git a/syft/pkg/cataloger/java/archive_contents_cataloger_test.go b/syft/pkg/cataloger/java/archive_source_test.go similarity index 72% rename from syft/pkg/cataloger/java/archive_contents_cataloger_test.go rename to syft/pkg/cataloger/java/archive_source_test.go index 1f10db93b..e9ef020ad 100644 --- a/syft/pkg/cataloger/java/archive_contents_cataloger_test.go +++ b/syft/pkg/cataloger/java/archive_source_test.go @@ -20,9 +20,154 @@ import ( "github.com/anchore/syft/syft/pkg" ) +func Test_archiveContentsCataloger_agreesWithTheArchiveReadingCataloger(t *testing.T) { + // for the same archive, the contents-reading cataloger must report the same packages with the same + // identity as the one reading the archive file; any divergence shows up as an SBOM diff + entries := jarEntries() + cfg := DefaultArchiveCatalogerConfig() + + // the archive-reading cataloger, over a directory holding the jar + archiveDir := t.TempDir() + jarPath := writeJar(t, archiveDir, "example-lib-1.2.3.jar", entries) + archiveResolver, err := fileresolver.NewFromDirectory(archiveDir, "") + require.NoError(t, err) + // the archive-reading cataloger copies the archive to a temp dir first, which the contents-reading + // one does not + archiveCtx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + fromArchive, _, err := NewArchiveCataloger(cfg).Catalog(archiveCtx, archiveResolver) + require.NoError(t, err) + require.NotEmpty(t, fromArchive) + + // the contents-reading cataloger, over the same entries laid out as a filesystem, with the traversal + // the archive cataloger task would have placed on the context + contentsDir := t.TempDir() + explode(t, contentsDir, entries) + contentsResolver, err := fileresolver.NewFromDirectory(contentsDir, "") + require.NoError(t, err) + + // the traversal names the archive exactly as the resolver did for the archive-reading cataloger, so + // both sides derive identity from the same string and the comparison is about where the entries came + // from rather than how the test spelled a path + jarLocations, err := archiveResolver.FilesByGlob("**/*.jar") + require.NoError(t, err) + require.Len(t, jarLocations, 1) + archivePath := jarLocations[0].Path() + + digests := digestsOfFile(t, jarPath) + trav := &archive.Traversal{ + Location: file.NewLocation(archivePath), + VirtualPath: archivePath, + Digests: digests, + } + ctx := archive.WithTraversal(context.Background(), trav) + fromContents, _, err := NewArchiveCataloger(cfg).Catalog(ctx, contentsResolver) + require.NoError(t, err) + + assert.Equal(t, summarize(fromArchive), summarize(fromContents)) +} + +func Test_archiveContentsCataloger_reportsNothingOutsideAnArchive(t *testing.T) { + // with the archive cataloger task owning extraction, every archive comes back with a traversal + // naming it. Outside one there is nothing to describe: opening archive files here as well would + // catalog each of them a second time. + cfg := DefaultArchiveCatalogerConfig() + cfg.NestedArchivesHandledExternally = true + + dir := t.TempDir() + jarPath := writeJar(t, dir, "example-lib-1.2.3.jar", jarEntries()) + require.FileExists(t, jarPath) + resolver, err := fileresolver.NewFromDirectory(dir, "") + require.NoError(t, err) + + pkgs, _, err := NewArchiveCataloger(cfg).Catalog(context.Background(), resolver) + require.NoError(t, err) + assert.Empty(t, pkgs) +} + +func Test_archiveContentsCataloger_reportsNothingForAnArchiveThatIsNotJava(t *testing.T) { + // a .zip is an archive and its contents are not a jar's; the archive-reading cataloger draws that + // line with its globs and this one draws it with the same list + dir := t.TempDir() + explode(t, dir, jarEntries()) + resolver, err := fileresolver.NewFromDirectory(dir, "") + require.NoError(t, err) + + ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ + Location: file.NewLocation("/bundle.zip"), + VirtualPath: "/bundle.zip", + }) + pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) + require.NoError(t, err) + assert.Empty(t, pkgs) +} + +func Test_archiveContentsCataloger_noManifestIsNoPackage(t *testing.T) { + // unchanged from the archive-reading cataloger, and the whole coverage argument: a jar with no + // META-INF/MANIFEST.MF yields nothing today either, so requiring the contents to be traversed takes + // away less than it appears to. The maven metadata here is never consulted - without a main package + // there is nothing to attach it to. + entries := map[string]string{ + "META-INF/maven/com.other/bundled-dep/pom.properties": "groupId=com.other\nartifactId=bundled-dep\nversion=4.5.6\n", + "com/example/Thing.class": "not really bytecode", + } + + dir := t.TempDir() + explode(t, dir, entries) + resolver, err := fileresolver.NewFromDirectory(dir, "") + require.NoError(t, err) + + ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ + Location: file.NewLocation("no-manifest-1.0.0.jar"), + VirtualPath: "no-manifest-1.0.0.jar", + }) + pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) + assert.Empty(t, pkgs) + // reported against the archive rather than swallowed, and as an unknown rather than a scan failure + require.Error(t, err) + coordErrs, remaining := unknown.ExtractCoordinateErrors(err) + assert.NoError(t, remaining) + require.Len(t, coordErrs, 1) + assert.Equal(t, "no-manifest-1.0.0.jar", coordErrs[0].Coordinates.RealPath) +} + +func Test_archiveContentsCataloger_readsWhatWasExtractedAndNoMore(t *testing.T) { + // the sharp edge of reading contents rather than the archive: this cataloger sees exactly what the + // extraction produced. An archive truncated by a limit yields a package built from the entries that + // were written, so identity degrades rather than disappearing - the manifest is there, the maven + // metadata is not. + // + // Observed on a real artifact: jenkins-core-2.578.jar holds more than the default 10,000-entry bound, + // so its META-INF/maven pom is never extracted and its group id falls back from org.jenkins-ci.main + // to the artifact id. The archive-reading cataloger never felt this, reading the archive's own + // central directory with no bound on it. + entries := map[string]string{ + "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n" + + "Implementation-Title: example-lib\nImplementation-Version: 1.2.3\n", + // the pom that would have supplied the group id is missing, as truncation would leave it + "com/example/Thing.class": "not really bytecode", + } + + dir := t.TempDir() + explode(t, dir, entries) + resolver, err := fileresolver.NewFromDirectory(dir, "") + require.NoError(t, err) + + ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ + Location: file.NewLocation("example-lib-1.2.3.jar"), + VirtualPath: "example-lib-1.2.3.jar", + }) + pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) + require.NoError(t, err) + require.Len(t, pkgs, 1) + assert.Equal(t, "example-lib", pkgs[0].Name) + assert.Equal(t, "1.2.3", pkgs[0].Version) + assert.Equal(t, "pkg:maven/example-lib/example-lib@1.2.3", pkgs[0].PURL, + "with no pom to name the group, the group falls back to the artifact id") +} + // jarEntries is one java archive's worth of content, used both to build the archive the -// archive-reading cataloger parses and to lay out the filesystem the contents-reading one reads. -// The point of the pair of tests below is that these two must not disagree. +// archive-reading cataloger parses and to lay out the filesystem the contents-reading one reads. The +// tests below assert the two agree. func jarEntries() map[string]string { return map[string]string{ "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n" + @@ -63,8 +208,8 @@ func writeJar(t *testing.T, dir, name string, entries map[string]string) string return p } -// explode lays the same entries out as files, which is what the archive cataloger task hands the -// sub-pipeline once it has extracted an archive. +// explode lays the same entries out as files, what the archive cataloger task hands the sub-pipeline +// once it has extracted an archive. func explode(t *testing.T, dir string, entries map[string]string) { t.Helper() for name, contents := range entries { @@ -102,55 +247,6 @@ func joinSorted(in []string) string { return out } -func Test_archiveContentsCataloger_agreesWithTheArchiveReadingCataloger(t *testing.T) { - // the whole reason a second cataloger is tolerable: for the same archive it must report the same - // packages, with the same identity, as the one that reads the archive file. Anything else is a - // divergence that will be discovered as a diff in someone's SBOM. - entries := jarEntries() - cfg := DefaultArchiveCatalogerConfig() - - // the archive-reading cataloger, over a directory holding the jar - archiveDir := t.TempDir() - jarPath := writeJar(t, archiveDir, "example-lib-1.2.3.jar", entries) - archiveResolver, err := fileresolver.NewFromDirectory(archiveDir, "") - require.NoError(t, err) - // the archive-reading cataloger copies the archive to a temp dir before it can read it, which is - // the work the contents-reading one does not do - archiveCtx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) - fromArchive, _, err := NewArchiveCataloger(cfg).Catalog(archiveCtx, archiveResolver) - require.NoError(t, err) - require.NotEmpty(t, fromArchive) - - // the contents-reading cataloger, over the same entries laid out as a filesystem, with the - // traversal the archive cataloger task would have placed on the context - contentsDir := t.TempDir() - explode(t, contentsDir, entries) - contentsResolver, err := fileresolver.NewFromDirectory(contentsDir, "") - require.NoError(t, err) - - // the traversal names the archive exactly as the resolver did for the archive-reading cataloger, - // so both sides derive identity from the same string and the comparison is about the source of - // the entries rather than about how the test spelled a path - jarLocations, err := archiveResolver.FilesByGlob("**/*.jar") - require.NoError(t, err) - require.Len(t, jarLocations, 1) - archivePath := jarLocations[0].Path() - - digests := digestsOfFile(t, jarPath) - trav := &archive.Traversal{ - Location: file.NewLocation(archivePath), - VirtualPath: archivePath, - FileSystemID: "example-lib-1.2.3.jar", - Depth: 1, - Digests: digests, - } - ctx := archive.WithTraversal(context.Background(), trav) - fromContents, _, err := NewArchiveContentsCataloger(cfg).Catalog(ctx, contentsResolver) - require.NoError(t, err) - - assert.Equal(t, summarize(fromArchive), summarize(fromContents)) -} - func digestsOfFile(t *testing.T, path string) []file.Digest { t.Helper() entries, err := newZipEntries(context.Background(), path) @@ -160,101 +256,3 @@ func digestsOfFile(t *testing.T, path string) []file.Digest { require.NotEmpty(t, digests) return digests } - -func Test_archiveContentsCataloger_reportsNothingOutsideAnArchive(t *testing.T) { - // no traversal means the scan is not inside an archive, and the identity of a java archive is - // half in the file - so there is nothing this cataloger can say - dir := t.TempDir() - explode(t, dir, jarEntries()) - resolver, err := fileresolver.NewFromDirectory(dir, "") - require.NoError(t, err) - - pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()). - Catalog(context.Background(), resolver) - require.NoError(t, err) - assert.Empty(t, pkgs) -} - -func Test_archiveContentsCataloger_reportsNothingForAnArchiveThatIsNotJava(t *testing.T) { - // a .zip is an archive, and its contents are not a jar's contents. The archive-reading cataloger - // draws that line with its globs; this one draws it with the same list. - dir := t.TempDir() - explode(t, dir, jarEntries()) - resolver, err := fileresolver.NewFromDirectory(dir, "") - require.NoError(t, err) - - ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ - Location: file.NewLocation("/bundle.zip"), - VirtualPath: "/bundle.zip", - Depth: 1, - }) - pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) - require.NoError(t, err) - assert.Empty(t, pkgs) -} - -func Test_archiveContentsCataloger_noManifestIsNoPackage(t *testing.T) { - // unchanged from the archive-reading cataloger, and worth asserting because it is the whole - // coverage argument: a jar with no META-INF/MANIFEST.MF yields nothing today either, so requiring - // the contents to be traversed takes away less than it appears to. Note the maven metadata here is - // never consulted - without a main package there is nothing to attach it to. - entries := map[string]string{ - "META-INF/maven/com.other/bundled-dep/pom.properties": "groupId=com.other\nartifactId=bundled-dep\nversion=4.5.6\n", - "com/example/Thing.class": "not really bytecode", - } - - dir := t.TempDir() - explode(t, dir, entries) - resolver, err := fileresolver.NewFromDirectory(dir, "") - require.NoError(t, err) - - ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ - Location: file.NewLocation("no-manifest-1.0.0.jar"), - VirtualPath: "no-manifest-1.0.0.jar", - Depth: 1, - }) - pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) - assert.Empty(t, pkgs) - // reported against the archive rather than swallowed, and as an unknown rather than a scan failure - require.Error(t, err) - coordErrs, remaining := unknown.ExtractCoordinateErrors(err) - assert.NoError(t, remaining) - require.Len(t, coordErrs, 1) - assert.Equal(t, "no-manifest-1.0.0.jar", coordErrs[0].Coordinates.RealPath) -} - -func Test_archiveContentsCataloger_readsWhatWasExtractedAndNoMore(t *testing.T) { - // the sharp edge of reading contents rather than the archive: this cataloger sees exactly what the - // extraction produced. An archive truncated by a limit yields a package built from the entries - // that were written, so identity degrades rather than disappearing - the manifest is there and the - // maven metadata is not. - // - // Observed on a real artifact: jenkins-core-2.578.jar holds more than the default 10,000-entry - // bound, so its META-INF/maven pom is never extracted and its group id falls back from - // org.jenkins-ci.main to the artifact id. The archive-reading cataloger never felt this, because it - // read the archive's own central directory with no bound on it. - entries := map[string]string{ - "META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n" + - "Implementation-Title: example-lib\nImplementation-Version: 1.2.3\n", - // the pom that would have supplied the group id is missing, as truncation would leave it - "com/example/Thing.class": "not really bytecode", - } - - dir := t.TempDir() - explode(t, dir, entries) - resolver, err := fileresolver.NewFromDirectory(dir, "") - require.NoError(t, err) - - ctx := archive.WithTraversal(context.Background(), &archive.Traversal{ - Location: file.NewLocation("example-lib-1.2.3.jar"), - VirtualPath: "example-lib-1.2.3.jar", - Depth: 1, - }) - pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver) - require.NoError(t, err) - require.Len(t, pkgs, 1) - assert.Equal(t, "example-lib", pkgs[0].Name) - assert.Equal(t, "1.2.3", pkgs[0].Version) - assert.Equal(t, "pkg:maven/example-lib/example-lib@1.2.3", pkgs[0].PURL, - "with no pom to name the group, the group falls back to the artifact id") -} diff --git a/syft/pkg/cataloger/java/cataloger.go b/syft/pkg/cataloger/java/cataloger.go index 44fbbe9fd..8663fd55a 100644 --- a/syft/pkg/cataloger/java/cataloger.go +++ b/syft/pkg/cataloger/java/cataloger.go @@ -4,34 +4,97 @@ Package java provides a concrete Cataloger implementation for packages relating package java import ( + "context" + + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" ) +const ArchiveCatalogerName = "java-archive-cataloger" + // NewArchiveCataloger returns a new Java archive cataloger object for detecting packages with archives (jar, war, ear, par, sar, jpi, hpi, and native-image formats) func NewArchiveCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger { gap := newGenericArchiveParserAdapter(cfg) - c := generic.NewCataloger("java-archive-cataloger"). + files := generic.NewCataloger(ArchiveCatalogerName). WithParserByGlobs(gap.parseJavaArchive, archiveFormatGlobs...) - // when the generic archive cataloger task owns recursion into archives, it extracts zip/tar - // archives generically and this cataloger meets any java archives within them at the next - // nesting level, so the wrapped-archive parsers would only duplicate that work - if !cfg.nestedArchivesHandledExternally() { - if cfg.IncludeIndexedArchives { - // java archives wrapped within zip files - gzp := newGenericZipWrappedJavaArchiveParser(cfg) - c.WithParserByGlobs(gzp.parseZipWrappedJavaArchive, genericZipGlobs...) - } - - if cfg.IncludeUnindexedArchives { - // java archives wrapped within tar files - gtp := newGenericTarWrappedJavaArchiveParser(cfg) - c.WithParserByGlobs(gtp.parseTarWrappedJavaArchive, genericTarGlobs...) - } + if cfg.IncludeIndexedArchives { + // java archives wrapped within zip files + gzp := newGenericZipWrappedJavaArchiveParser(cfg) + files.WithParserByGlobs(gzp.parseZipWrappedJavaArchive, genericZipGlobs...) } - return c + + if cfg.IncludeUnindexedArchives { + // java archives wrapped within tar files + gtp := newGenericTarWrappedJavaArchiveParser(cfg) + files.WithParserByGlobs(gtp.parseTarWrappedJavaArchive, genericTarGlobs...) + } + + return &archiveCataloger{cfg: cfg, files: files} +} + +// archiveCataloger describes java archives from whichever source this scan makes them available in. +// +// Which source that is turns on who owns extraction, and there are only two answers. Inside an +// archive the archive cataloger task extracted, the resolver is that archive's contents and the +// traversal names the file it came from, so the archive to describe is the one this resolver *is*. +// Everywhere else the archives are files in the resolver, and the cataloger opens each itself. +// +// One cataloger rather than one per source, so a package carries the same FoundBy and answers to the +// same --select-catalogers name however the scan was configured. +type archiveCataloger struct { + cfg ArchiveCatalogerConfig + files *generic.Cataloger +} + +func (c *archiveCataloger) Name() string { + return ArchiveCatalogerName +} + +func (c *archiveCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + if trav := archive.TraversalFromContext(ctx); trav != nil { + return c.catalogExtracted(ctx, trav, resolver) + } + + if c.cfg.NestedArchivesHandledExternally { + // every archive in this scan comes back through the archive cataloger task with a traversal + // naming it, and is described above. Opening archive files here as well would catalog each of + // them a second time. + return nil, nil, nil + } + + return c.files.Catalog(ctx, resolver) +} + +// catalogExtracted reports the java package of the archive whose contents this resolver holds, if it +// is a java archive at all. +// +// It does not glob for the archives inside it: the task that extracted this one meets those on its +// own, at the next nesting level. See syft/java-archive-nesting#single-owner-of-archive-recursion. +func (c *archiveCataloger) catalogExtracted(ctx context.Context, trav *archive.Traversal, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + if !isJavaArchiveName(trav.Location.Path()) { + // the rule the globs apply when this cataloger opens files: a .zip is an archive but not a java + // archive + return nil, nil, nil + } + + src, err := newExtractedArchiveSource(trav, resolver) + if err != nil { + return nil, nil, err + } + + pkgs, relationships, err := newArchiveParser(src, false, c.cfg).parse(ctx, nil) + if err != nil { + // attributed to the archive rather than returned bare, as the generic cataloger attributes a + // parser's failure to the file it was parsing, so the scan carries on + return pkgs, relationships, unknown.New(trav.Location, err) + } + return pkgs, relationships, nil } // NewPomCataloger returns a cataloger capable of parsing dependencies from a pom.xml file. diff --git a/syft/pkg/cataloger/java/config.go b/syft/pkg/cataloger/java/config.go index 4ca66b466..460a31b76 100644 --- a/syft/pkg/cataloger/java/config.go +++ b/syft/pkg/cataloger/java/config.go @@ -34,13 +34,13 @@ type ArchiveCatalogerConfig struct { // app-config: java.resolve-transitive-dependencies ResolveTransitiveDependencies bool `yaml:"resolve-transitive-dependencies" json:"resolve-transitive-dependencies" mapstructure:"resolve-transitive-dependencies"` - // NestedArchivesHandledExternally indicates that some other mechanism owns recursion into - // nested archives, so this cataloger must not unarchive them itself. It is deliberately not a - // yaml/json/mapstructure key: no config file, flag, or environment variable reaches it. + // NestedArchivesHandledExternally indicates that another mechanism owns recursion into nested + // archives, so this cataloger must not unarchive them itself. It has no yaml/json/mapstructure + // key, so no config file, flag or environment variable reaches it. // - // It is written ONLY by syft.CreateSBOMConfig, which derives it from the nested-archive - // cataloging depth so the two sides cannot disagree. Callers building this config should leave - // it alone; the zero value is this cataloger's long-standing behavior of recursing itself. + // Only syft.CreateSBOMConfig writes it, deriving it from the nested-archive cataloging depth so + // the two cannot disagree. Callers building this config should leave it alone; the zero value + // recurses, this cataloger's default. NestedArchivesHandledExternally bool `yaml:"-" json:"-" mapstructure:"-"` } @@ -90,13 +90,6 @@ func (j ArchiveCatalogerConfig) WithArchiveTraversal(search cataloging.ArchiveSe return j } -// nestedArchivesHandledExternally indicates another mechanism (the generic archive cataloger task) -// owns recursion into all archives, in which case this cataloger must not unarchive nested archives -// itself. See the field of the same name for why this is not derived from MaxDepth. -func (j ArchiveCatalogerConfig) nestedArchivesHandledExternally() bool { - return j.NestedArchivesHandledExternally -} - func (j ArchiveCatalogerConfig) mavenConfig() maven.Config { return maven.Config{ UseNetwork: j.UseNetwork, diff --git a/syft/pkg/cataloger/java/parse_pom_xml.go b/syft/pkg/cataloger/java/parse_pom_xml.go index 357b426a6..4c56ac108 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml.go +++ b/syft/pkg/cataloger/java/parse_pom_xml.go @@ -303,17 +303,13 @@ func pomParent(ctx context.Context, r *maven.Resolver, pom *maven.Project) *pkg. } } -// isArchiveMetaPom returns true if the pom.xml location is inside a META-INF/maven directory, -// indicating it is archive metadata embedded by the Maven build process rather than a project -// pom.xml. These embedded pom.xml files are verbatim copies of the original build POM with -// potentially unresolved ${property} references and dependency versions inherited from parent -// POMs that are not available inside the archive. The java-archive-cataloger already uses -// pom.properties (which always contains resolved values) to identify these archives, so the -// pom cataloger should skip them to avoid creating phantom dependency packages. +// isArchiveMetaPom reports whether a pom.xml sits under META-INF/maven, making it build metadata +// Maven embedded rather than a project pom.xml. Such a pom is a verbatim copy of the build POM, with +// unresolved ${property} references and versions inherited from parent POMs absent from the archive. +// The java-archive-cataloger identifies these archives from pom.properties, whose values are always +// resolved, so skipping them here avoids phantom dependency packages. func isArchiveMetaPom(location file.Location) bool { - p := location.Path() - // normalize to forward slashes for consistent matching - p = filepath.ToSlash(p) + p := filepath.ToSlash(location.Path()) return strings.Contains(p, "META-INF/maven/") } diff --git a/syft/pkg/cataloger/java/parse_pom_xml_test.go b/syft/pkg/cataloger/java/parse_pom_xml_test.go index 94fde2542..dbb3669c6 100644 --- a/syft/pkg/cataloger/java/parse_pom_xml_test.go +++ b/syft/pkg/cataloger/java/parse_pom_xml_test.go @@ -732,8 +732,8 @@ func Test_isArchiveMetaPom(t *testing.T) { } func Test_pomCatalogerSkipsMetaInfPoms(t *testing.T) { - // A directory that only contains a pom.xml under META-INF/maven/ should produce - // no packages, since these are archive metadata not project files. + // a pom.xml under META-INF/maven/ alone yields no packages: that is archive metadata, not a + // project file cat := NewPomCataloger(ArchiveCatalogerConfig{ ArchiveSearchConfig: cataloging.ArchiveSearchConfig{ IncludeIndexedArchives: true, @@ -748,9 +748,8 @@ func Test_pomCatalogerSkipsMetaInfPoms(t *testing.T) { } func Test_pomCatalogerSkipsMetaInfButKeepsProjectPom(t *testing.T) { - // When a directory contains both a project pom.xml and a META-INF/maven pom.xml, - // only the project pom.xml should be cataloged. The META-INF pom.xml and its - // dependencies should be ignored. + // with both a project pom.xml and a META-INF/maven one, only the project pom.xml is + // cataloged; the META-INF pom and its dependencies are ignored pomLocation := file.NewLocationSet(file.NewLocation("pom.xml")) myApp := pkg.Package{ diff --git a/syft/pkg/cataloger/java/version_properties_test.go b/syft/pkg/cataloger/java/version_properties_test.go index f9d44d231..72e289c43 100644 --- a/syft/pkg/cataloger/java/version_properties_test.go +++ b/syft/pkg/cataloger/java/version_properties_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" @@ -21,10 +20,10 @@ func readManifest(t *testing.T, ctx context.Context, parser *archiveParser) *pkg matches := parser.entries.glob(false, manifestGlob) require.Len(t, matches, 1) - contents, err := intFile.ContentsFromZip(ctx, parser.archivePath, matches...) + contents, err := parser.entries.contents(ctx, matches...) require.NoError(t, err) - manifest, err := parseJavaManifest(parser.archivePath, strings.NewReader(contents[matches[0]])) + manifest, err := parseJavaManifest(parser.virtualPath, strings.NewReader(contents[matches[0]])) require.NoError(t, err) return manifest diff --git a/syft/source/directorysource/directory_source.go b/syft/source/directorysource/directory_source.go index e6bf6a516..7288215ea 100644 --- a/syft/source/directorysource/directory_source.go +++ b/syft/source/directorysource/directory_source.go @@ -138,14 +138,13 @@ func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileres return nil, nil } - // the indexer reports an absolute, symlink-resolved path to every visitor: the resolver normalizes - // its root with EvalSymlinks (fileresolver.NormalizeRootDirectory, via the chroot context) and the - // walk then takes filepath.Abs of it. The root these patterns are anchored to has to be derived - // the same way or they anchor to a path the walk never reports - on macOS a scan of a directory - // under /var, a symlink to /private/var, would exclude nothing at all. + // the indexer reports an absolute, symlink-resolved path to every visitor (the resolver normalizes + // its root with EvalSymlinks and the walk takes filepath.Abs of it), so patterns must anchor to a + // root derived the same way. Otherwise on macOS - where a directory under /var is really under + // /private/var - they exclude nothing. // - // A root that cannot be resolved is left as it was rather than failing the scan: EvalSymlinks - // wants the path to exist, and the resolver reports a missing root far better than this can. + // An unresolvable root is left alone rather than failing the scan: EvalSymlinks needs the path to + // exist, and the resolver reports a missing root better than this can. if resolved, err := filepath.EvalSymlinks(root); err == nil { root = resolved } @@ -162,10 +161,9 @@ func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileres root += "/" } - // the rooted patterns are built into a new slice rather than written back over the caller's: - // the patterns belong to the source's configuration, which is read elsewhere (see - // source.PathExcluder), and rewriting them in place would replace what the user configured with - // scan-root-absolute patterns as a side effect of building the resolver + // a new slice, not a rewrite in place: these patterns belong to the source's config and are read + // elsewhere (see source.PathExcluder), so building the resolver must not replace what the user + // configured with scan-root-absolute patterns rooted := make([]string, 0, len(exclusions)) var errors []string for _, exclusion := range exclusions { diff --git a/syft/source/exclude.go b/syft/source/exclude.go index dbc5a9428..64ddd06bd 100644 --- a/syft/source/exclude.go +++ b/syft/source/exclude.go @@ -4,21 +4,14 @@ type ExcludeConfig struct { Paths []string } -// PathExcluder is optionally implemented by a source that was configured with path exclusion -// patterns. It publishes those patterns so that a consumer which indexes content taken from the -// source as a filesystem of its own can honor the scan's exclusions rather than carry a second -// setting for the same purpose. The nested archive cataloger is that consumer: every archive it -// finds is extracted to a directory and indexed independently, and a pattern the scan excluded -// would otherwise be read inside an archive after being skipped outside one. +// PathExcluder is optionally implemented by a source configured with path exclusion patterns. It +// publishes them so a consumer that indexes source content as its own filesystem can honor the scan's +// exclusions without a second setting. The nested archive cataloger is that consumer: it indexes every +// archive independently. All patterns are published; which are depth-independent enough to be +// meaningful is the consumer's business. // -// Only patterns whose own shape is depth-independent are meaningful to such a consumer; a pattern -// anchored to the scan root describes a layout that exists only there. Deciding that is the -// consumer's business, so everything the source was given is published here. -// -// Implementations MUST return a copy. Sources that build directory resolvers hand their patterns to -// GetDirectoryExclusionFunctions, which rewrites what it is given to be absolute against the scan -// root, so a caller holding the same slice would find the patterns replaced the moment the source -// built its resolver. +// Implementations must return a copy: sources that build directory resolvers pass their patterns to +// GetDirectoryExclusionFunctions, which rewrites them absolute against the scan root. type PathExcluder interface { ExcludedPaths() []string } diff --git a/syft/source/stereoscopesource/image_source.go b/syft/source/stereoscopesource/image_source.go index fa257313e..8557e8c0a 100644 --- a/syft/source/stereoscopesource/image_source.go +++ b/syft/source/stereoscopesource/image_source.go @@ -129,11 +129,9 @@ func (s stereoscopeImageSource) FileResolver(scope source.Scope) (file.Resolver, // ExcludedPaths returns a copy of the exclusion patterns this source was configured with, so a // consumer indexing content taken from it can honor the same patterns. // -// An image filters its own tree after indexing, with NewExcludingDecorator above rather than an -// index visitor, because the tree is built by stereoscope and not walked here. That is a fact about -// the image's own filesystem: content taken out of it and indexed separately - an archive extracted -// to a directory - is a directory index like any other and is filtered like one. So the patterns -// travel, and the shape of each one still decides whether it reaches inside an archive. +// An image filters its own tree after indexing (NewExcludingDecorator above) rather than with an +// index visitor, since stereoscope builds that tree. Content taken out of the image is an ordinary +// index and is filtered like one, so the patterns travel unchanged. func (s stereoscopeImageSource) ExcludedPaths() []string { return slices.Clone(s.config.Exclude.Paths) }