feat: initial recursive archive cataloging impl

Signed-off-by: Keith Zantow <keith.zantow@anchore.com>

internal/archive/content.go

@@ -0,0 +1,175 @@
+package archive
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	intFile "github.com/anchore/syft/internal/file"
+	"github.com/anchore/syft/internal/log"
+)
+
+// copyChunkSize is how much is read, charged and written at a time. It bounds how far a charge can
+// be ahead of what has actually been written, and matches io.Copy's own buffer size.
+const copyChunkSize = 32 * 1024
+
+// ReaderAtSeeker is the random access an archive format needs. A zip reads its central directory
+// from the end of the stream, so a plain io.Reader will not do; both a *bytes.Reader over content
+// held in memory and an *os.File over content overflowed to disk satisfy it.
+type ReaderAtSeeker interface {
+	io.Reader
+	io.ReaderAt
+	io.Seeker
+}
+
+// Content is one archive's bytes, wherever the limiter routed them.
+//
+// Name is the archive's file name and is used for format identification only - never to open
+// anything - so content held in memory needs no file on disk at all.
+type Content struct {
+	Name   string
+	Reader ReaderAtSeeker
+
+	// closer releases the overflow file's handle; nil for content held in memory.
+	closer io.Closer
+}
+
+// Close releases the handle on overflow content. It is a no-op for content held in memory.
+func (c Content) Close() error {
+	if c.closer == nil {
+		return nil
+	}
+	return c.closer.Close()
+}
+
+// OpenFileContent opens an archive already on disk as Content. The caller closes it.
+func OpenFileContent(path string) (Content, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return Content{}, err
+	}
+	return Content{Name: filepath.Base(path), Reader: f, closer: f}, nil
+}
+
+// holdContent routes one archive's bytes: held in memory and charged to the memory limit while that
+// limit admits them, overflowed into workDir and charged to the disk limit once it does not. Where the
+// boundary falls is decided entirely by the memory limit - there is no separate configured size at
+// which overflowing begins.
+//
+// Content is read and charged incrementally, copyChunkSize at a time, rather than read speculatively
+// into a buffer ahead of any charge: nothing is ever held beyond what the memory limit would admit at
+// that moment, so a single archive far larger than the limit is never buffered past it. A zero
+// memory limit refuses every chunk, so its content overflows to disk in full; a negative memory limit
+// never refuses, so nothing overflows for want of memory.
+//
+// When the limit refuses a chunk part way through, what has already been read leads the overflow rather
+// than being re-read from the start or lost, and the memory it was charged against is refunded since
+// it is about to be charged to disk instead.
+//
+// Memory pressure degrades to disk - content that would take memory in use past the memory limit
+// overflows instead of being refused, because there is somewhere for it to go. The disk limit is
+// terminal: there is nowhere further, so content that would take it past the disk limit yields
+// ErrDiskLimitReached and the archive is skipped. Nothing waits for capacity; see ExtractToResolver.
+func holdContent(r io.Reader, workDir, name string, charge *Charge, notify Notify) (Content, error) {
+	var held bytes.Buffer
+	chunk := make([]byte, copyChunkSize)
+	for {
+		n, readErr := r.Read(chunk)
+		if n > 0 {
+			if !charge.Memory(int64(n)) {
+				// the memory limit refused this chunk: overflow what is already held plus this chunk
+				// plus whatever remains, without re-reading from the start
+				if notify != nil {
+					notify(ContentOverflowed{Archive: name, Bytes: int64(held.Len() + n), Reason: overflowReason(charge)})
+				}
+				charge.RefundMemory(int64(held.Len()))
+				return overflowContent(io.MultiReader(&held, bytes.NewReader(chunk[:n]), r), workDir, name, charge)
+			}
+			held.Write(chunk[:n])
+		}
+		if readErr != nil {
+			if errors.Is(readErr, io.EOF) {
+				return Content{Name: name, Reader: bytes.NewReader(held.Bytes())}, nil
+			}
+			return Content{}, fmt.Errorf("unable to read archive content: %w", readErr)
+		}
+	}
+}
+
+// overflowContent writes an archive's bytes into workDir, charging the disk limit as they land, and
+// returns a reader over the file positioned at its start.
+//
+// This is the only filesystem write left in the extraction path - an archive's entries are headers in
+// one tar and traverse no path - and the name it writes under is derived from where the archive was
+// found, which inside another archive is text the archive supplied. So the destination is checked the
+// same two ways an entry's path used to be: lexically, and then against where it really lands on the
+// filesystem.
+func overflowContent(r io.Reader, workDir, name string, charge *Charge) (Content, error) {
+	dest, err := intFile.SafeJoin(workDir, name)
+	if err != nil {
+		return Content{}, fmt.Errorf("refusing to overflow archive content to %q: %w", name, err)
+	}
+	if inside, err := resolvesInsideRoot(workDir, dest); err != nil || !inside {
+		return Content{}, fmt.Errorf("refusing to overflow archive content outside its work directory: %q", name)
+	}
+	f, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
+	if err != nil {
+		return Content{}, fmt.Errorf("unable to create temp archive file: %w", err)
+	}
+
+	written, limitReached, err := copyCharged(f, r, charge)
+	if err == nil && limitReached {
+		err = ErrDiskLimitReached
+	}
+	if err == nil {
+		_, err = f.Seek(0, io.SeekStart)
+	}
+	if err != nil {
+		charge.RefundDisk(written)
+		if closeErr := f.Close(); closeErr != nil {
+			log.WithFields("path", dest, "error", closeErr).Trace("unable to close overflow archive file")
+		}
+		if rmErr := os.Remove(dest); rmErr != nil {
+			log.WithFields("path", dest, "error", rmErr).Trace("unable to remove overflow archive file")
+		}
+		if errors.Is(err, ErrDiskLimitReached) {
+			return Content{}, err
+		}
+		return Content{}, fmt.Errorf("unable to write temp archive file: %w", err)
+	}
+
+	return Content{Name: name, Reader: f, closer: f}, nil
+}
+
+// copyCharged copies src into dst, charging every chunk to the disk limit before it is written, so
+// what bounds the copy is what actually lands rather than any size the archive declared. It stops at
+// the first refused charge and says so, leaving the caller to decide whether that truncates one
+// entry or skips a whole archive.
+func copyCharged(dst io.Writer, src io.Reader, charge *Charge) (written int64, limitReached bool, err error) {
+	buf := make([]byte, copyChunkSize)
+	for {
+		n, readErr := src.Read(buf)
+		if n > 0 {
+			if !charge.Disk(int64(n)) {
+				return written, true, nil
+			}
+			w, writeErr := dst.Write(buf[:n])
+			written += int64(w)
+			if w < n {
+				charge.RefundDisk(int64(n - w))
+			}
+			if writeErr != nil {
+				return written, false, writeErr
+			}
+		}
+		if readErr != nil {
+			if errors.Is(readErr, io.EOF) {
+				return written, false, nil
+			}
+			return written, false, readErr
+		}
+	}
+}

internal/archive/content_test.go

@@ -0,0 +1,255 @@
+package archive
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// filesIn lists the regular files directly in dir, which is how a test tells content held in memory
+// from content overflowed to disk.
+func filesIn(t *testing.T, dir string) []string {
+	t.Helper()
+	entries, err := os.ReadDir(dir)
+	require.NoError(t, err)
+	var names []string
+	for _, e := range entries {
+		if !e.IsDir() {
+			names = append(names, e.Name())
+		}
+	}
+	return names
+}
+
+func readAll(t *testing.T, c Content) []byte {
+	t.Helper()
+	_, err := c.Reader.Seek(0, io.SeekStart)
+	require.NoError(t, err)
+	body, err := io.ReadAll(c.Reader)
+	require.NoError(t, err)
+	return body
+}
+
+func TestHoldContent_heldWhileTheLimitAdmitsIt(t *testing.T) {
+	// the referent for the memory limit: an archive small enough to hold needs no file on disk at
+	// all, where every archive used to be copied to a temp file purely so an extractor could seek it
+	workDir := t.TempDir()
+	body := []byte(strings.Repeat("a", 100))
+	limiter := NewLimiter(Limits{MaxMemoryBytes: 1000, MaxDiskBytes: 1000})
+	charge := limiter.Charge()
+
+	held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil)
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, held.Close()) })
+
+	assert.IsType(t, &bytes.Reader{}, held.Reader)
+	assert.Equal(t, body, readAll(t, held))
+	assert.Empty(t, filesIn(t, workDir), "content held in memory must write nothing to disk")
+
+	mem, disk := charge.Held()
+	assert.Equal(t, int64(100), mem)
+	assert.Zero(t, disk)
+}
+
+func TestHoldContent_overflowsWhenTheLimitDoesNotAdmitIt(t *testing.T) {
+	// where the boundary falls is decided by the memory limit and nothing else: an archive that does
+	// not fit overflows in full, and the bytes already read to discover that lead the overflow rather than
+	// being lost
+	workDir := t.TempDir()
+	body := []byte(strings.Repeat("b", 250))
+	limiter := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 10_000})
+	charge := limiter.Charge()
+
+	held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil)
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, held.Close()) })
+
+	assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir))
+	assert.Equal(t, body, readAll(t, held),
+		"the bytes read to decide the routing must lead the overflow rather than being lost")
+
+	mem, disk := charge.Held()
+	assert.Zero(t, mem, "nothing is charged to a limit that refused")
+	assert.Equal(t, int64(250), disk)
+}
+
+// peakTrackingReader records the highest memory charge observed at any point while reading, which is
+// how a test tells that a large archive was never buffered beyond what the limit would admit -
+// nothing about the routing decision is otherwise observable from outside holdContent.
+type peakTrackingReader struct {
+	r      io.Reader
+	charge *Charge
+	peak   int64
+}
+
+func (p *peakTrackingReader) Read(b []byte) (int, error) {
+	n, err := p.r.Read(b)
+	if mem, _ := p.charge.Held(); mem > p.peak {
+		p.peak = mem
+	}
+	return n, err
+}
+
+func TestHoldContent_largeArchiveNeverBuffersBeyondTheLimit(t *testing.T) {
+	// an archive many times the memory limit must overflow, and at no point along the way may more than
+	// the limit would admit be held while the routing is being decided - it is never read
+	// speculatively into memory that has no room for it
+	workDir := t.TempDir()
+	const limit = int64(3 * copyChunkSize)
+	body := []byte(strings.Repeat("c", 10*copyChunkSize))
+	limiter := NewLimiter(Limits{MaxMemoryBytes: limit, MaxDiskBytes: int64(len(body)) + 1})
+	charge := limiter.Charge()
+	tracked := &peakTrackingReader{r: bytes.NewReader(body), charge: charge}
+
+	held, err := holdContent(tracked, workDir, "app.zip", charge, nil)
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, held.Close()) })
+
+	assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir), "an archive this much larger than the limit must overflow")
+	assert.Equal(t, body, readAll(t, held))
+	assert.LessOrEqual(t, tracked.peak, limit,
+		"memory charged must never exceed the limit, even transiently, while routing is decided")
+
+	mem, disk := charge.Held()
+	assert.Zero(t, mem)
+	assert.Equal(t, int64(len(body)), disk)
+}
+
+func TestHoldContent_diskLimitIsTerminal(t *testing.T) {
+	// there is nowhere further to overflow, so content that will not fit is refused and the caller skips
+	// the archive rather than waiting for capacity that nothing will release
+	workDir := t.TempDir()
+	body := []byte(strings.Repeat("d", 500))
+	limiter := NewLimiter(Limits{MaxDiskBytes: 100})
+	charge := limiter.Charge()
+
+	_, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil)
+	require.ErrorIs(t, err, ErrDiskLimitReached)
+
+	assert.Empty(t, filesIn(t, workDir), "the partial overflow must not be left behind")
+	mem, disk := charge.Held()
+	assert.Zero(t, mem)
+	assert.Zero(t, disk, "bytes no longer on disk must be refunded, or the limit only ever rises")
+}
+
+func TestHoldContent_zeroDiskLimitSkipsRatherThanFails(t *testing.T) {
+	// a zero disk limit has the same outcome as a positive disk limit exceeded: content that will
+	// not fit anywhere yields ErrDiskLimitReached so the caller skips the archive, rather than the
+	// scan failing
+	workDir := t.TempDir()
+	body := []byte(strings.Repeat("f", 500))
+	limiter := NewLimiter(Limits{MaxMemoryBytes: 10, MaxDiskBytes: 0})
+	charge := limiter.Charge()
+
+	_, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil)
+	require.ErrorIs(t, err, ErrDiskLimitReached)
+
+	assert.Empty(t, filesIn(t, workDir), "no partial overflow is left behind")
+	mem, disk := charge.Held()
+	assert.Zero(t, mem)
+	assert.Zero(t, disk)
+}
+
+func TestHoldContent_threeStateLimits(t *testing.T) {
+	// both limits read positive/zero/negative the same way: positive is the limit, zero means none
+	// of that resource at all, negative means unbounded. A caller can bound one without bounding the
+	// other.
+	body := []byte(strings.Repeat("e", 500))
+
+	t.Run("a zero memory limit sends everything to disk", func(t *testing.T) {
+		// zero does not mean unlimited memory: the memory limit decides a routing question rather
+		// than only capping an accumulator, so zero means the memory side of that routing is never
+		// taken
+		workDir := t.TempDir()
+		limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10_000})
+		charge := limiter.Charge()
+
+		held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil)
+		require.NoError(t, err)
+		t.Cleanup(func() { require.NoError(t, held.Close()) })
+
+		assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir), "a zero memory limit holds nothing in memory")
+		mem, disk := charge.Held()
+		assert.Zero(t, mem)
+		assert.Equal(t, int64(500), disk, "and the disk limit still applies to what it overflowed")
+	})
+
+	t.Run("a negative memory limit holds content regardless of how much is already held", func(t *testing.T) {
+		workDir := t.TempDir()
+		limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: 10_000})
+		charge := limiter.Charge()
+		require.True(t, charge.Memory(1_000_000), "a negative memory limit admits any amount already")
+
+		held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil)
+		require.NoError(t, err)
+		t.Cleanup(func() { require.NoError(t, held.Close()) })
+
+		assert.Empty(t, filesIn(t, workDir), "nothing overflows for want of memory when the limit is negative")
+		mem, disk := charge.Held()
+		assert.Equal(t, int64(1_000_000+len(body)), mem)
+		assert.Zero(t, disk)
+	})
+
+	t.Run("a zero disk limit writes nothing to disk while content the memory limit admits is still cataloged", func(t *testing.T) {
+		workDir := t.TempDir()
+		limiter := NewLimiter(Limits{MaxMemoryBytes: 10_000, MaxDiskBytes: 0})
+		charge := limiter.Charge()
+
+		held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil)
+		require.NoError(t, err)
+		t.Cleanup(func() { require.NoError(t, held.Close()) })
+
+		assert.Empty(t, filesIn(t, workDir), "content the memory limit admits never touches disk")
+		mem, disk := charge.Held()
+		assert.Equal(t, int64(len(body)), mem)
+		assert.Zero(t, disk)
+	})
+
+	t.Run("a negative disk limit overflows with no ceiling", func(t *testing.T) {
+		workDir := t.TempDir()
+		limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
+		charge := limiter.Charge()
+
+		held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", charge, nil)
+		require.NoError(t, err)
+		t.Cleanup(func() { require.NoError(t, held.Close()) })
+
+		assert.Equal(t, []string{"app.zip"}, filesIn(t, workDir))
+		assert.Equal(t, body, readAll(t, held))
+	})
+}
+
+func TestHoldContent_noLimitsAtAll(t *testing.T) {
+	// a nil charge is what an extraction with no configured bounds gets, and it must route content
+	// rather than panic - holding everything in memory, since nothing is there to refuse it
+	workDir := t.TempDir()
+	body := []byte("small enough")
+
+	held, err := holdContent(bytes.NewReader(body), workDir, "app.zip", nil, nil)
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, held.Close()) })
+
+	assert.Equal(t, body, readAll(t, held))
+	assert.Empty(t, filesIn(t, workDir))
+}
+
+func TestOpenFileContent(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "app.zip")
+	require.NoError(t, os.WriteFile(path, []byte("body"), 0o600))
+
+	content, err := OpenFileContent(path)
+	require.NoError(t, err)
+	assert.Equal(t, "app.zip", content.Name)
+	assert.Equal(t, []byte("body"), readAll(t, content))
+	require.NoError(t, content.Close())
+
+	_, err = OpenFileContent(filepath.Join(dir, "missing.zip"))
+	assert.Error(t, err)
+}

internal/archive/entrystore.go

@@ -0,0 +1,359 @@
+package archive
+
+import (
+	"archive/tar"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync"
+
+	"github.com/mholt/archives"
+
+	"github.com/anchore/syft/internal/log"
+)
+
+// Entry is one archive entry held by an EntryStore: its header, and a reference to its bytes wherever
+// they currently live.
+//
+// The bytes move; the entry does not. That is the point of this type - an index built over these
+// entries stays valid when content is pushed out to disk, because what changes is the entry's
+// reference and not its identity or its place in any index.
+type Entry struct {
+	Header tar.Header
+
+	// ref names where this entry's bytes are right now. Guarded by the store's mutex, because a swap
+	// to disk rewrites it while readers may be asking for it.
+	ref blobRef
+}
+
+// blobRef is either bytes held in memory or a region of the store's overflow file. Exactly one of the
+// two is live: mem is nil once the entry has been written out.
+type blobRef struct {
+	mem    []byte
+	offset int64
+	length int64
+}
+
+// EntryStore holds the entries of one extracted archive, in memory while the memory limit admits them
+// and in a single overflow file once it does not.
+//
+// This replaces writing every archive's entries into a tar and indexing that tar by seek offset. A
+// tar had to be on disk before there was anything to index, so a four-kilobyte jar paid for a file
+// write and a tar walk before a cataloger could read a byte of it. Here the index is over the entries
+// themselves, so a small archive never touches the disk at all, and a large one moves its bytes
+// without the index noticing.
+//
+// The store owns the overflow file and closes it on Close. Readers handed out by Open borrow it, so
+// they are only valid until then - which matches the lifetime the archive cataloger already gives an
+// extracted archive.
+type EntryStore struct {
+	mu sync.RWMutex
+
+	entries []*Entry
+
+	// heldInMemory is the bytes currently held by entries whose ref is memory-backed. It falls to zero
+	// when the store spills.
+	heldInMemory int64
+
+	workDir string
+	file    *os.File
+	written int64
+
+	// archive names this store's archive in the events it sends, so a spill can be attributed to the
+	// file that caused it rather than reported as an anonymous byte count.
+	archive string
+	notify  Notify
+}
+
+// overflowBlobName is the file an archive's entries are written into once they no longer fit in
+// memory. Unlike the tar it replaces it has no framing: the entries' offsets live in the store, so
+// nothing has to be parsed to find them again.
+const overflowBlobName = "contents.blob"
+
+// indexRecordOverhead approximates the fixed memory kept per entry no matter its content: the *Entry,
+// its tar.Header, and the resolver node built over it. It is an over-estimate of the fixed part, with
+// the variable part - the entry's name and link target - added on top, so the bound errs towards
+// charging more rather than letting the index outrun what was accounted for. See Charge.IndexRecord.
+var indexRecordOverhead = int64(reflect.TypeOf((*tar.Header)(nil)).Elem().Size())
+
+// indexRecordCost is what one entry's index record charges: the fixed overhead plus the bytes of the
+// two strings the header carries by value.
+func indexRecordCost(hdr tar.Header) int64 {
+	return indexRecordOverhead + int64(len(hdr.Name)) + int64(len(hdr.Linkname))
+}
+
+var _ EntrySink = (*EntryStore)(nil)
+
+// NewEntryStore returns a store that writes its overflow blob into workDir, naming archive in the
+// events it sends.
+//
+// How much it holds in memory is not a property of the store: it asks the scan's limiter, entry by
+// entry, and moves what it is holding out to disk the first time the limiter says no. That keeps one
+// answer to "how much memory is this scan using" rather than a per-archive budget that no one set.
+func NewEntryStore(workDir, archive string, notify Notify) *EntryStore {
+	return &EntryStore{workDir: workDir, archive: archive, notify: notify}
+}
+
+// Add stores one entry's header and content, and reports the entry.
+//
+// Content is read here rather than referenced: an archive's reader is sequential, so an entry that is
+// not read now cannot be read later. Where it lands is decided as it arrives, which is what keeps the
+// decision out of every read path downstream.
+//
+// It is read and charged copyChunkSize at a time rather than into a buffer sized by anything the
+// entry declared, for the reason holdContent reads an archive's own bytes that way: a header's size
+// is the archive's claim about itself, so an entry far larger than the memory limit would be held
+// past it in full before the limiter was ever asked. Here the limiter is asked per chunk, and the
+// first refusal moves this entry and everything the store is holding out to the overflow blob.
+func (s *EntryStore) Add(hdr tar.Header, contents io.Reader, charge *Charge) (*Entry, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// charge this entry's index record before it is held, so an archive of millions of near-empty
+	// entries is bounded by the same budgets its content is rather than growing the index without
+	// limit. The record overflows to disk when memory is full and is only refused when neither budget
+	// admits it; a refusal leaves the archive truncated at the entries already stored.
+	if !charge.IndexRecord(indexRecordCost(hdr)) {
+		return nil, ErrDiskLimitReached
+	}
+
+	entry := &Entry{Header: hdr}
+	s.entries = append(s.entries, entry)
+
+	if hdr.Typeflag != tar.TypeReg || contents == nil {
+		return entry, nil
+	}
+
+	var held bytes.Buffer
+	chunk := make([]byte, copyChunkSize)
+	for {
+		n, readErr := contents.Read(chunk)
+		if n > 0 {
+			if !charge.Memory(int64(n)) {
+				// memory is full: what is held of this entry joins the spill, and the rest of it
+				// streams straight into the blob behind it rather than being read into memory first
+				entry.ref = memRef(&held)
+				s.heldInMemory += entry.ref.length
+				rest := io.MultiReader(bytes.NewReader(chunk[:n]), contents)
+				return entry, s.spillThroughLocked(entry, rest, charge)
+			}
+			held.Write(chunk[:n])
+		}
+		if readErr != nil {
+			if errors.Is(readErr, io.EOF) {
+				break
+			}
+			return nil, fmt.Errorf("unable to read archive entry %q: %w", hdr.Name, readErr)
+		}
+	}
+
+	entry.ref = memRef(&held)
+	s.heldInMemory += entry.ref.length
+	return entry, nil
+}
+
+// memRef is a reference to bytes held in memory. The slice is taken non-nil even when it is empty,
+// because nil is how blobRef says "these bytes are on disk" and a zero-length entry is not.
+func memRef(held *bytes.Buffer) blobRef {
+	body := held.Bytes()
+	if body == nil {
+		body = []byte{}
+	}
+	return blobRef{mem: body, length: int64(len(body))}
+}
+
+// spillThroughLocked moves everything the store is holding out to the overflow blob and writes the
+// rest of entry's content in behind it, charging the disk limit as it lands.
+//
+// entry is the newest, so spillLocked writes it last and its region ends where the blob does - which
+// is exactly where the rest of its content is about to go, so the two are one contiguous region and
+// the entry keeps a single reference to it.
+func (s *EntryStore) spillThroughLocked(entry *Entry, rest io.Reader, charge *Charge) error {
+	spilled := entry.ref.length
+	moved, movedBytes, err := s.spillLocked(charge)
+	if err != nil {
+		return err
+	}
+
+	written, limitReached, err := copyCharged(io.NewOffsetWriter(s.file, s.written), rest, charge)
+	s.written += written
+	entry.ref = blobRef{offset: s.written - written - spilled, length: spilled + written}
+
+	// reported once the whole move is done, so the event counts the bytes that streamed through as
+	// well as the ones that were being held: they all went to disk for the same reason.
+	if s.notify != nil && moved > 0 {
+		s.notify(EntriesOverflowed{
+			Archive: s.archive,
+			Entries: moved,
+			Bytes:   movedBytes + written,
+			Reason:  overflowReason(charge),
+		})
+	}
+
+	if err != nil {
+		return fmt.Errorf("unable to write archive entry %q to the overflow blob: %w", entry.Header.Name, err)
+	}
+	if limitReached {
+		return ErrDiskLimitReached
+	}
+	return nil
+}
+
+// spillLocked writes every memory-backed entry into the overflow blob and repoints it there,
+// reporting what it moved. The index over these entries is untouched: an entry's identity is the
+// pointer, and only its reference to its bytes changes.
+//
+// It does not send the event for the move: the entry that triggered the spill is still streaming in
+// behind it, and an event sent from here would report a byte count short by whatever that entry has
+// left. spillThroughLocked sends it once both halves have landed.
+func (s *EntryStore) spillLocked(charge *Charge) (moved int, movedBytes int64, err error) {
+	if s.file == nil {
+		f, err := os.OpenFile(filepath.Join(s.workDir, overflowBlobName), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600)
+		if err != nil {
+			return 0, 0, fmt.Errorf("unable to create overflow blob: %w", err)
+		}
+		s.file = f
+	}
+
+	for _, entry := range s.entries {
+		if entry.ref.mem == nil {
+			continue
+		}
+		body := entry.ref.mem
+		if !charge.Disk(int64(len(body))) {
+			return moved, movedBytes, ErrDiskLimitReached
+		}
+		charged := charge != nil
+		if _, err := s.file.WriteAt(body, s.written); err != nil {
+			return moved, movedBytes, fmt.Errorf("unable to write archive entry %q to the overflow blob: %w", entry.Header.Name, err)
+		}
+		entry.ref = blobRef{offset: s.written, length: int64(len(body))}
+		s.written += int64(len(body))
+		moved++
+		movedBytes += int64(len(body))
+		if charged {
+			// the bytes are on disk now, so the memory they were charged for is no longer held
+			charge.RefundMemory(int64(len(body)))
+		}
+	}
+	s.heldInMemory = 0
+	return moved, movedBytes, nil
+}
+
+// overflowReason distinguishes an archive pushed to disk by pressure from one that was never going to
+// be held at all, which are different findings: the first says the scan is at its memory bound, the
+// second says the bound is zero.
+func overflowReason(charge *Charge) OverflowReason {
+	if charge != nil && charge.limiter != nil && charge.limiter.limits.MaxMemoryBytes == 0 {
+		return MemoryHoldsNothing
+	}
+	return MemoryLimitReached
+}
+
+// Entries returns the entries stored so far, in the order they were added.
+func (s *EntryStore) Entries() []*Entry {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	out := make([]*Entry, len(s.entries))
+	copy(out, s.entries)
+	return out
+}
+
+// Open returns a reader over one entry's content, from memory or from the overflow blob as the entry
+// currently stands. The reader is Read, Seek and ReadAt either way, so a nested archive is handed to
+// an archive format where it lies rather than being copied out first.
+func (s *EntryStore) Open(entry *Entry) (ReaderAtSeeker, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	if entry.ref.mem != nil {
+		return bytes.NewReader(entry.ref.mem), nil
+	}
+	if entry.ref.length == 0 {
+		return bytes.NewReader(nil), nil
+	}
+	if s.file == nil {
+		return nil, fmt.Errorf("archive entry %q has no content", entry.Header.Name)
+	}
+	return io.NewSectionReader(s.file, entry.ref.offset, entry.ref.length), nil
+}
+
+// HeldInMemory reports the entry content currently held in memory, for tests and for the scan's own
+// statistics.
+func (s *EntryStore) HeldInMemory() int64 {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.heldInMemory
+}
+
+// Close releases the overflow blob. Readers handed out by Open do not outlive it.
+func (s *EntryStore) Close() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.file == nil {
+		return nil
+	}
+	err := s.file.Close()
+	s.file = nil
+	return err
+}
+
+// AddEntry stores one archive entry, so an EntryStore can stand where an OverflowTar stands.
+//
+// Note what is NOT here: no header encoding, no block padding, and no end-of-archive marker, because
+// there is no framing to write. The disk limit is charged only when content actually reaches the disk,
+// which for a small archive is never - so an archive that fits in memory costs the limiter nothing on
+// disk and costs the machine no write at all.
+func (s *EntryStore) AddEntry(f archives.FileInfo, result *ExtractionResult, limits ExtractionLimits) error {
+	hdr, ok := entryHeader(f)
+	if !ok {
+		return nil
+	}
+
+	var contents io.ReadCloser
+	if f.Mode().IsRegular() {
+		opened, err := f.Open()
+		if err != nil {
+			log.WithFields("entry", f.NameInArchive, "error", err).
+				Trace("unable to open archive entry, skipping it")
+			return nil
+		}
+		contents = opened
+	}
+
+	entry, err := s.Add(*hdr, contents, limits.Charge)
+	if contents != nil {
+		if closeErr := contents.Close(); closeErr != nil {
+			log.WithFields("entry", f.NameInArchive, "error", closeErr).Trace("unable to close archive entry")
+		}
+	}
+	switch {
+	case errors.Is(err, ErrDiskLimitReached):
+		// nowhere left to put this archive's content: the entries stored so far stay, and the archive
+		// is reported as truncated rather than discarded
+		result.Truncation = TruncatedByDiskLimit
+		return errTruncated
+	case err != nil:
+		return err
+	}
+
+	result.FilesExtracted++
+	result.BytesWritten = s.OnDisk()
+	_ = entry
+	return nil
+}
+
+// Finish has nothing to write: a store's entries are complete as they are added, with no framing to
+// close off.
+func (s *EntryStore) Finish(*ExtractionResult, ExtractionLimits) {}
+
+// OnDisk reports the entry content written out so far, which is what the disk limit was charged for.
+func (s *EntryStore) OnDisk() int64 {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.written
+}

internal/archive/entrystore_test.go

@@ -0,0 +1,199 @@
+package archive
+
+import (
+	"archive/tar"
+	"bytes"
+	"io"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func regularHeader(name string, size int64) tar.Header {
+	return tar.Header{Name: name, Size: size, Mode: 0o600, Typeflag: tar.TypeReg}
+}
+
+func readEntry(t *testing.T, r ReaderAtSeeker) string {
+	t.Helper()
+	_, err := r.Seek(0, io.SeekStart)
+	require.NoError(t, err)
+	b, err := io.ReadAll(r)
+	require.NoError(t, err)
+	return string(b)
+}
+
+func TestEntryStore_holdsSmallContentInMemory(t *testing.T) {
+	dir := t.TempDir()
+	s := NewEntryStore(dir, "app.zip", nil)
+	charge := memCharge(1024)
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	entry, err := s.Add(regularHeader("a.txt", 5), bytes.NewReader([]byte("hello")), charge)
+	require.NoError(t, err)
+
+	assert.Equal(t, int64(5), s.HeldInMemory())
+	assert.NoFileExists(t, filepath.Join(dir, overflowBlobName),
+		"a small archive must not touch the disk at all - that is the whole point of the store")
+
+	r, err := s.Open(entry)
+	require.NoError(t, err)
+	assert.Equal(t, "hello", readEntry(t, r))
+}
+
+func TestEntryStore_spillsToDiskAndKeepsTheSameEntries(t *testing.T) {
+	// the index over these entries is not rebuilt when content moves, so the entry pointers and their
+	// contents must both survive the move
+	dir := t.TempDir()
+	s := NewEntryStore(dir, "app.zip", nil)
+	charge := memCharge(8)
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	first, err := s.Add(regularHeader("a.txt", 4), bytes.NewReader([]byte("aaaa")), charge)
+	require.NoError(t, err)
+	require.Equal(t, int64(4), s.HeldInMemory())
+
+	// the second entry takes the store past its memory bound, so everything held moves out
+	second, err := s.Add(regularHeader("b.txt", 6), bytes.NewReader([]byte("bbbbbb")), charge)
+	require.NoError(t, err)
+
+	assert.Zero(t, s.HeldInMemory(), "everything must have moved out, not just the entry that did not fit")
+	assert.FileExists(t, filepath.Join(dir, overflowBlobName))
+
+	// the same entry pointers still read their own content, from their new home
+	firstReader, err := s.Open(first)
+	require.NoError(t, err)
+	assert.Equal(t, "aaaa", readEntry(t, firstReader))
+
+	secondReader, err := s.Open(second)
+	require.NoError(t, err)
+	assert.Equal(t, "bbbbbb", readEntry(t, secondReader))
+
+	assert.Equal(t, []*Entry{first, second}, s.Entries(), "order and identity must be unchanged by the move")
+}
+
+func TestEntryStore_zeroMemoryHoldsNothing(t *testing.T) {
+	// the reading a zero memory limit has everywhere else: none of that resource may be used
+	dir := t.TempDir()
+	s := NewEntryStore(dir, "app.zip", nil)
+	charge := memCharge(0)
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	entry, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), charge)
+	require.NoError(t, err)
+
+	assert.Zero(t, s.HeldInMemory())
+	assert.FileExists(t, filepath.Join(dir, overflowBlobName))
+
+	r, err := s.Open(entry)
+	require.NoError(t, err)
+	assert.Equal(t, "abc", readEntry(t, r))
+}
+
+func TestEntryStore_negativeMemoryNeverSpills(t *testing.T) {
+	dir := t.TempDir()
+	s := NewEntryStore(dir, "app.zip", nil)
+	charge := memCharge(-1)
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	for _, body := range []string{"one", "two", "three"} {
+		_, err := s.Add(regularHeader(body+".txt", int64(len(body))), bytes.NewReader([]byte(body)), charge)
+		require.NoError(t, err)
+	}
+
+	assert.Equal(t, int64(11), s.HeldInMemory())
+	assert.NoFileExists(t, filepath.Join(dir, overflowBlobName))
+}
+
+func TestEntryStore_spillIsRefusedWhenDiskIsNotAvailable(t *testing.T) {
+	// content that does not fit in memory and cannot be written has nowhere to go, and the archive is
+	// skipped rather than cataloged as if it were empty
+	dir := t.TempDir()
+	s := NewEntryStore(dir, "app.zip", nil)
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	nowhereToPutIt := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).Charge()
+	_, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), nowhereToPutIt)
+	assert.ErrorIs(t, err, ErrDiskLimitReached)
+}
+
+func TestEntryStore_directoryEntriesHoldNoContent(t *testing.T) {
+	dir := t.TempDir()
+	s := NewEntryStore(dir, "app.zip", nil)
+	charge := memCharge(1024)
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	entry, err := s.Add(tar.Header{Name: "lib/", Typeflag: tar.TypeDir, Mode: 0o755}, nil, charge)
+	require.NoError(t, err)
+
+	assert.Zero(t, s.HeldInMemory())
+	r, err := s.Open(entry)
+	require.NoError(t, err)
+	assert.Empty(t, readEntry(t, r))
+}
+
+// memCharge is a charge against a memory bound with disk left unbounded, which is the shape the store
+// reads: hold while memory admits, move out when it does not.
+func memCharge(maxMemory int64) *Charge {
+	return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).Charge()
+}
+
+func TestEntryStore_chargesIndexRecordEvenForEmptyContent(t *testing.T) {
+	// an empty entry still charges its index record - the header and node held to reach it - which is
+	// what bounds entry count, the dimension the content-byte budgets never bounded.
+	dir := t.TempDir()
+	s := NewEntryStore(dir, "app.zip", nil)
+	limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1})
+	charge := limiter.Charge()
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	hdr := regularHeader("a.txt", 0)
+	_, err := s.Add(hdr, bytes.NewReader(nil), charge)
+	require.NoError(t, err)
+
+	mem, _ := limiter.InUse()
+	assert.Equal(t, indexRecordCost(hdr), mem, "the index record is charged to memory")
+	assert.Equal(t, int64(0), s.HeldInMemory(), "no content was held")
+}
+
+func TestEntryStore_indexOverflowsToDiskWhenMemoryIsZero(t *testing.T) {
+	// the index cannot be refused just because memory is zero: a resolver still needs the records to
+	// reach content that lives entirely on disk. With disk unbounded, every entry is admitted and its
+	// record charged to disk instead.
+	dir := t.TempDir()
+	s := NewEntryStore(dir, "app.zip", nil)
+	limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
+	charge := limiter.Charge()
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	hdr := regularHeader("a.txt", 0)
+	_, err := s.Add(hdr, bytes.NewReader(nil), charge)
+	require.NoError(t, err)
+
+	mem, disk := limiter.InUse()
+	assert.Equal(t, int64(0), mem, "memory budget is zero, so nothing is charged there")
+	assert.Equal(t, indexRecordCost(hdr), disk, "the index record overflows to the disk budget")
+}
+
+func TestEntryStore_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) {
+	// with memory zero and only a few index records' worth of disk, the store admits a bounded number
+	// of entries and then refuses - an archive of millions of empty entries can no longer grow the
+	// index without limit.
+	dir := t.TempDir()
+	s := NewEntryStore(dir, "app.zip", nil)
+	room := indexRecordCost(regularHeader("x", 0)) * 3
+	charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).Charge()
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	var err error
+	admitted := 0
+	for i := 0; i < 100; i++ {
+		if _, err = s.Add(regularHeader("x", 0), bytes.NewReader(nil), charge); err != nil {
+			break
+		}
+		admitted++
+	}
+	require.ErrorIs(t, err, ErrDiskLimitReached)
+	assert.Equal(t, 3, admitted, "exactly the index records that fit the disk budget were admitted")
+}

internal/archive/extractor.go

@@ -1,7 +1,9 @@
 package archive
 
 import (
+	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"io/fs"
@@ -18,31 +20,135 @@ import (
 
 // Extractor extracts archive contents to a destination directory.
 type Extractor interface {
-	// CanExtract returns true if this extractor can handle the given file.
-	CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool
+	// CanExtract returns true if this extractor can handle the given archive content.
+	CanExtract(ctx context.Context, content Content) bool
 
-	// Extract extracts the archive contents to destDir, respecting the given limits.
-	// Returns the number of files extracted and total bytes written.
-	Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error)
+	// Extract writes the archive's entries into sink - one store per archive, not one file per entry -
+	// respecting the given limits. Returns how many entries were written and how many bytes that took.
+	Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error)
 }
 
-// ExtractionLimits defines safety limits for archive extraction.
+// EntrySink is where an extractor puts the entries it reads out of an archive.
+//
+// Two implementations exist because they store an archive differently and the difference is
+// measurable: OverflowTar writes every entry into one tar on disk and is indexed by seek offset,
+// while EntryStore holds entries in memory until a bound says otherwise and is indexed by the entries
+// themselves. The extractors do not care which they were given, with one exception - see
+// VerbatimSink.
+type EntrySink interface {
+	// AddEntry stores one entry of the archive.
+	AddEntry(f archives.FileInfo, result *ExtractionResult, limits ExtractionLimits) error
+
+	// Finish is called once the last entry has been offered, for whatever a sink has to write after
+	// its content.
+	Finish(result *ExtractionResult, limits ExtractionLimits)
+}
+
+// VerbatimSink is an EntrySink that stores archives in tar form, so a tar-family archive can be
+// copied into it as it stands rather than decoded and re-encoded entry by entry.
+//
+// This is the one place the storage format shows through the extractor interface, and it is worth the
+// leak: a tar.gz copied verbatim is one streamed decompression, where walking it costs a read per
+// entry. A sink that does not store tars does not implement this and gets the entry-by-entry path.
+type VerbatimSink interface {
+	EntrySink
+	CopyVerbatim(r io.Reader, result *ExtractionResult, limits ExtractionLimits) error
+}
+
+// ExtractionLimits defines safety limits for extracting one archive.
 type ExtractionLimits struct {
-	MaxExtractionSizeBytes int64
-	MaxFileCount           int
+
+	// Charge is this archive's draw on the scan's resource limiter. Entry bytes are charged to it as
+	// they land, never from the size the entry declared, and a refused charge truncates the
+	// extraction. A nil Charge enforces no byte bound.
+	//
+	// It is a handle rather than a remaining-bytes value because what it charges has to be given back
+	// when the archive is released, and a value copied into a call cannot be.
+	Charge *Charge
 }
 
+// TruncationReason names the configured bound that stopped an extraction early. The empty value
+// means the archive was extracted in full.
+type TruncationReason string
+
+const (
+	// TruncatedByDiskLimit means placing more of this archive's content would have taken the scan's
+	// disk usage past the disk limit. Named rather than reported as a bare truncation because a breach
+	// has to be attributed to the archive that triggered it, and saying which bound was reached is
+	// half of that attribution.
+	TruncatedByDiskLimit TruncationReason = "disk limit"
+
+	// TruncatedByLimit means a bound stopped the extraction but the walker replaced the error that
+	// said which one. Defensive: both bounds record their own reason on the result before stopping.
+	TruncatedByLimit TruncationReason = "configured limit"
+)
+
+// errTruncated stops an in-progress archive walk when a configured limit is reached. It never
+// escapes this package: the extractors convert it into a successful, truncated ExtractionResult.
+var errTruncated = errors.New("archive extraction stopped at a configured limit")
+
 // ExtractionResult holds the result of an extraction operation.
+//
+// Reaching a configured limit is NOT an error: it is a truncation. Whatever was written before the
+// limit is left on disk and is usable, so callers can still catalog what was extracted rather than
+// discarding the whole archive. Check Truncated to tell a complete extraction from a partial one;
+// a non-nil error from Extract means the extraction failed and nothing usable was produced.
 type ExtractionResult struct {
+	// FilesExtracted counts the entries this archive's filesystem holds - the records its overflow tar
+	// was indexed into: regular files, symlinks, and directory entries. It is the count after indexing
+	// rather than the count written, so an entry dropped by an exclusion pattern is not counted as
+	// present. Reported, not bounded.
 	FilesExtracted int
-	BytesWritten   int64
+
+	// BytesWritten is what this archive put on disk: every byte of the overflow tar, headers and block
+	// padding included, because that is what the disk limit was charged for.
+	BytesWritten int64
+
+	// Truncation names the limit that stopped extraction early, or "" when the archive was
+	// extracted in full.
+	Truncation TruncationReason
 }
 
-// DefaultExtractionLimits returns limits from the given config.
-func DefaultExtractionLimits(cfg cataloging.ArchiveSearchConfig) ExtractionLimits {
-	return ExtractionLimits{
-		MaxExtractionSizeBytes: cfg.MaxExtractionSizeBytes,
-		MaxFileCount:           cfg.MaxFileCount,
+// Truncated reports whether a configured limit stopped this extraction before the end of the
+// archive.
+func (r ExtractionResult) Truncated() bool {
+	return r.Truncation != ""
+}
+
+// finishExtraction turns a walk that stopped at a configured limit into a successful, truncated
+// result.
+//
+// The limit signal is carried on the result rather than only in the returned error because the
+// underlying archive walkers are free to wrap or replace a handler's error, in which case
+// errors.Is(err, errTruncated) would not match and a truncation would be reported as a failure -
+// discarding an archive syft could have cataloged. The error is still checked, so a walker that
+// passes it through cleanly is handled too.
+func finishExtraction(result ExtractionResult, err error) (ExtractionResult, error) {
+	if result.Truncated() || errors.Is(err, errTruncated) {
+		if !result.Truncated() {
+			// the walker swallowed the reason but passed the sentinel through
+			result.Truncation = TruncatedByLimit
+		}
+		return result, nil
+	}
+	return result, err
+}
+
+// DefaultExtractionLimits returns the per-archive limits from the given config.
+//
+// Nothing is read from the config: every bound this scan enforces is an in-use limit held by the
+// scan's limiter, and the only per-archive field left is the Charge the caller draws against it.
+// The parameter stays so the two Default* constructors are called the same way, and so a per-archive
+// bound that is genuinely per-archive has somewhere to land.
+func DefaultExtractionLimits(_ cataloging.ArchiveSearchConfig) ExtractionLimits {
+	return ExtractionLimits{}
+}
+
+// DefaultLimits returns the scan-wide in-use limits from the given config.
+func DefaultLimits(cfg cataloging.ArchiveSearchConfig) Limits {
+	return Limits{
+		MaxMemoryBytes: cfg.MaxMemoryBytes,
+		MaxDiskBytes:   cfg.MaxDiskBytes,
 	}
 }
 
@@ -54,90 +160,82 @@ func DefaultExtractors() []Extractor {
 	}
 }
 
-// FindExtractor finds an extractor that can handle the given file, or nil if none can.
-func FindExtractor(ctx context.Context, extractors []Extractor, path string) Extractor {
-	f, err := os.Open(path)
-	if err != nil {
-		log.Tracef("unable to open file for archive detection: %v", err)
-		return nil
-	}
-	defer f.Close()
-
+// FindExtractor finds an extractor that can handle the given archive content, or nil if none can.
+func FindExtractor(ctx context.Context, extractors []Extractor, content Content) Extractor {
 	for _, ext := range extractors {
-		if ext.CanExtract(ctx, path, f) {
-			return ext
-		}
-		// reset for next extractor
-		if _, err := f.Seek(0, io.SeekStart); err != nil {
-			log.Tracef("unable to seek file for archive detection: %v", err)
+		// every extractor sniffs from the start, whether the content is a file or a buffer
+		if _, err := content.Reader.Seek(0, io.SeekStart); err != nil {
+			log.Tracef("unable to seek archive content for detection: %v", err)
 			return nil
 		}
+		if ext.CanExtract(ctx, content) {
+			return ext
+		}
 	}
 	return nil
 }
 
-// IsExcludedExtension checks if a file path has an excluded extension.
-func IsExcludedExtension(path string, excludeExtensions []string) bool {
-	for _, ext := range excludeExtensions {
-		if strings.HasSuffix(strings.ToLower(path), strings.ToLower(ext)) {
-			return true
-		}
-	}
-	return false
-}
-
 // ZipExtractor extracts zip-based archives (zip, jar, war, ear, etc.).
 type ZipExtractor struct{}
 
-func (z *ZipExtractor) CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool {
-	format, _, err := intFile.IdentifyArchive(ctx, path, reader)
-	if err != nil {
+// CanExtract reports whether this content is a zip, asking two questions in an order that matters.
+//
+// First, what does the content say it is, reading from the start? A format that identifies itself at
+// offset zero is what this content is, and if that format is not zip then it is not a zip - a tar
+// holding one jar ends in that jar's end-of-central-directory record, and reading it as a zip
+// instead of as a tar would lose everything else the tar holds.
+//
+// Only when nothing identifies from the head is the end-of-central-directory record consulted. That
+// record is what defines a zip - a reader locates the central directory through it - and it is the
+// only thing that can see an archive whose head is something else entirely. A self-extracting
+// archive is a launcher stub with a zip concatenated onto it, so no magic at offset zero can find
+// it; the canonical case is a Spring Boot executable jar, a shell script followed by a whole zip.
+//
+// Identification is intFile.IdentifyArchive, which matches on the filename as well as the content,
+// and it is left that way. Dropping the name makes the answer content's alone, which reads better -
+// but archives.Identify with no path has to try every format's content matcher against every
+// candidate, and measured on a stock ruby image that took the scan from 17 seconds to 45. The name
+// cannot admit a non-archive in practice: a file only reaches an extractor after its sniffed content
+// type put it in the candidate set, so the prose named notes.zip that this would wrongly identify is
+// never offered here at all. `#content-based-archive-detection` is upheld by the candidate set, and
+// Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError is where that is asserted.
+func (z *ZipExtractor) CanExtract(ctx context.Context, content Content) bool {
+	format, _, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader)
+	if err == nil {
+		_, isZip := format.(archives.Zip)
+		return isZip
+	}
+
+	if _, seekErr := content.Reader.Seek(0, io.SeekStart); seekErr != nil {
 		return false
 	}
-	_, ok := format.(archives.Zip)
-	return ok
+	return HasZipEndOfCentralDirectory(content.Reader)
 }
 
-func (z *ZipExtractor) Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) {
+func (z *ZipExtractor) Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error) {
 	var result ExtractionResult
 
-	f, err := os.Open(path)
-	if err != nil {
-		return result, fmt.Errorf("unable to open zip archive %q: %w", path, err)
+	if _, err := content.Reader.Seek(0, io.SeekStart); err != nil {
+		return result, fmt.Errorf("unable to seek zip archive %q: %w", content.Name, err)
 	}
-	defer f.Close()
 
-	return result, archives.Zip{}.Extract(ctx, f, func(_ context.Context, file archives.FileInfo) error {
-		if file.IsDir() {
-			joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
-			if err != nil {
-				return err
-			}
-			return os.MkdirAll(joinedPath, 0o755)
-		}
-
-		if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount {
-			return fmt.Errorf("archive file count limit reached (%d files)", limits.MaxFileCount)
-		}
-
-		joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
-		if err != nil {
-			return err
-		}
-
-		if err := os.MkdirAll(filepath.Dir(joinedPath), 0o755); err != nil {
-			return fmt.Errorf("unable to create parent dir: %w", err)
-		}
-
-		return extractFileWithLimits(file, joinedPath, destDir, &result, limits)
+	// mholt/archives imposes no extraction target: the FileHandler is the caller's, so an entry can
+	// be written into a tar header rather than onto the filesystem. Nothing about the library had to
+	// be customized - the old code wrote files because the handler wrote files.
+	err := archives.Zip{}.Extract(ctx, content.Reader, func(_ context.Context, file archives.FileInfo) error {
+		return sink.AddEntry(file, &result, limits)
 	})
+
+	sink.Finish(&result, limits)
+
+	return finishExtraction(result, err)
 }
 
 // TarExtractor extracts tar-based archives (tar, tar.gz, tar.bz2, tar.xz, tar.zst).
 type TarExtractor struct{}
 
-func (t *TarExtractor) CanExtract(ctx context.Context, path string, reader io.ReadSeeker) bool {
-	format, _, err := intFile.IdentifyArchive(ctx, path, reader)
+func (t *TarExtractor) CanExtract(ctx context.Context, content Content) bool {
+	format, _, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader)
 	if err != nil {
 		return false
 	}
@@ -149,125 +247,166 @@ func (t *TarExtractor) CanExtract(ctx context.Context, path string, reader io.Re
 	return ok
 }
 
-func (t *TarExtractor) Extract(ctx context.Context, path string, destDir string, limits ExtractionLimits) (ExtractionResult, error) {
+func (t *TarExtractor) Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error) {
 	var result ExtractionResult
 
-	f, err := os.Open(path)
-	if err != nil {
-		return result, fmt.Errorf("unable to open tar archive %q: %w", path, err)
+	if _, err := content.Reader.Seek(0, io.SeekStart); err != nil {
+		return result, fmt.Errorf("unable to seek tar archive %q: %w", content.Name, err)
 	}
-	defer f.Close()
 
-	format, readerAfterIdentify, err := intFile.IdentifyArchive(ctx, path, f)
+	format, readerAfterIdentify, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader)
 	if err != nil {
-		return result, fmt.Errorf("unable to identify archive format for %q: %w", path, err)
+		return result, fmt.Errorf("unable to identify archive format for %q: %w", content.Name, err)
 	}
 
 	extractor, ok := format.(archives.Extractor)
 	if !ok {
-		return result, fmt.Errorf("file format does not support extraction: %s", path)
+		return result, fmt.Errorf("file format does not support extraction: %s", content.Name)
 	}
 
 	// use the reader returned by IdentifyArchive since it may have consumed some bytes
-	extractReader := readerAfterIdentify
+	var extractReader = readerAfterIdentify
 	if extractReader == nil {
-		extractReader = f
+		extractReader = content.Reader
 	}
 
-	return result, extractor.Extract(ctx, extractReader, func(_ context.Context, file archives.FileInfo) error {
-		if file.IsDir() {
-			joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
-			if err != nil {
-				return err
+	// a tar handed to a sink that stores tars is not walked at all: a compressed tar is decompressed
+	// once into the sink's file and a plain one is copied, and the entries are first looked at when
+	// the file is indexed. Where the sink stores entries rather than a tar there is nothing to copy
+	// into, so the tar is walked like any other archive.
+	if verbatim, isVerbatim := sink.(VerbatimSink); isVerbatim {
+		if plain, ok := tarStream(format, extractReader); ok {
+			err = verbatim.CopyVerbatim(plain, &result, limits)
+			if closeErr := plain.Close(); closeErr != nil {
+				log.WithFields("archive", content.Name, "error", closeErr).Trace("unable to close decompressed tar stream")
 			}
-			return os.MkdirAll(joinedPath, 0o755)
+			return finishExtraction(result, err)
 		}
+	}
 
-		if limits.MaxFileCount > 0 && result.FilesExtracted >= limits.MaxFileCount {
-			return fmt.Errorf("archive file count limit reached (%d files)", limits.MaxFileCount)
-		}
-
-		joinedPath, err := intFile.SafeJoin(destDir, file.NameInArchive)
-		if err != nil {
-			return err
-		}
-
-		if err := os.MkdirAll(filepath.Dir(joinedPath), 0o755); err != nil {
-			return fmt.Errorf("unable to create parent dir: %w", err)
-		}
-
-		return extractFileWithLimits(file, joinedPath, destDir, &result, limits)
+	// anything else - 7z, rar, or a tar going into a sink that does not store tars - is walked entry
+	// by entry
+	err = extractor.Extract(ctx, extractReader, func(_ context.Context, file archives.FileInfo) error {
+		return sink.AddEntry(file, &result, limits)
 	})
+
+	sink.Finish(&result, limits)
+
+	return finishExtraction(result, err)
 }
 
-// extractFileWithLimits extracts a single file from an archive, respecting size limits.
-// Symlinks are written as real symlinks (only when the target stays inside destDir);
-// non-regular file types other than symlinks are skipped to avoid writing bogus content.
-func extractFileWithLimits(file archives.FileInfo, destPath, destDir string, result *ExtractionResult, limits ExtractionLimits) error {
-	mode := file.Mode()
-	if mode.Type()&fs.ModeSymlink != 0 {
-		if err := writeSafeSymlink(file.LinkTarget, destPath, destDir); err != nil {
-			log.WithFields("entry", file.NameInArchive, "target", file.LinkTarget, "error", err).Debug("skipping unsafe symlink in archive")
-			return nil
-		}
-		result.FilesExtracted++
-		return nil
-	}
-	if !mode.IsRegular() {
-		// skip device nodes, fifos, sockets, etc. — they can't safely round-trip
-		log.WithFields("entry", file.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry")
-		return nil
-	}
-
-	src, err := file.Open()
+// resolvesInsideRoot reports whether writing to path really lands inside root, following any symlink
+// that already exists along the way the same way the filesystem will.
+//
+// The one filesystem write left in this package is the archive's own bytes overflowing into its work
+// directory, under a name derived from where the archive was found - which inside another archive is
+// attacker-controlled text. That is what this now guards; entries are tar headers and traverse no
+// path at all. See overflowContent.
+//
+// It walks path one component at a time rather than calling filepath.EvalSymlinks on the whole
+// thing, because the leaf of an entry being extracted does not exist yet, which makes EvalSymlinks
+// fail rather than answer. The first component that does not exist ends the walk: nothing below it
+// exists either, so whatever is created there is created fresh under a path already known to be
+// inside the root.
+func resolvesInsideRoot(root, path string) (bool, error) {
+	// the root itself may sit under a symlinked directory (a temp dir on macOS, for one), so
+	// containment is judged against its real location
+	realRoot, err := filepath.EvalSymlinks(root)
 	if err != nil {
-		return fmt.Errorf("unable to open archive entry %q: %w", file.NameInArchive, err)
+		return false, fmt.Errorf("unable to resolve extraction root: %w", err)
 	}
-	defer src.Close()
 
-	dst, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
+	rel, err := filepath.Rel(root, path)
 	if err != nil {
-		return fmt.Errorf("unable to create dest file %q: %w", destPath, err)
+		return false, err
+	}
+	if !isWithin(filepath.Join(realRoot, rel), realRoot) {
+		return false, nil
 	}
-	defer dst.Close()
 
-	var reader io.Reader = src
-
-	// enforce per-archive total size limit
-	if limits.MaxExtractionSizeBytes > 0 {
-		remaining := limits.MaxExtractionSizeBytes - result.BytesWritten
-		if remaining <= 0 {
-			return fmt.Errorf("archive extraction size limit reached (%d bytes)", limits.MaxExtractionSizeBytes)
+	current := realRoot
+	for _, part := range strings.Split(filepath.ToSlash(rel), "/") {
+		if part == "" || part == "." {
+			continue
 		}
-		reader = io.LimitReader(src, remaining+1) // +1 to detect exceeding the limit
+		current = filepath.Join(current, part)
+
+		info, err := os.Lstat(current)
+		if err != nil {
+			if os.IsNotExist(err) {
+				break
+			}
+			return false, err
+		}
+		if info.Mode()&fs.ModeSymlink == 0 {
+			continue
+		}
+
+		resolved, err := resolveLink(current)
+		if err != nil {
+			return false, err
+		}
+		if !isWithin(resolved, realRoot) {
+			return false, nil
+		}
+		current = resolved
 	}
 
-	n, err := io.Copy(dst, reader)
-	result.BytesWritten += n
-	result.FilesExtracted++
-
-	if limits.MaxExtractionSizeBytes > 0 && result.BytesWritten > limits.MaxExtractionSizeBytes {
-		return fmt.Errorf("archive extraction size limit reached (%d bytes)", limits.MaxExtractionSizeBytes)
-	}
-
-	return err
+	return isWithin(current, realRoot), nil
 }
 
-// writeSafeSymlink creates a symlink at destPath only if target is relative and
-// resolves — relative to destPath's directory — to a location inside destDir.
+// resolveLink returns where a symlink really points, falling back to the lexically cleaned target
+// when the target does not exist yet - a dangling link still redirects a write.
+func resolveLink(path string) (string, error) {
+	target, err := os.Readlink(path)
+	if err != nil {
+		return "", err
+	}
+	if !filepath.IsAbs(target) {
+		target = filepath.Join(filepath.Dir(path), target)
+	}
+	resolved, err := filepath.EvalSymlinks(target)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return "", err
+		}
+		return filepath.Clean(target), nil
+	}
+	return resolved, nil
+}
+
+// isWithin reports whether path is root or sits beneath it, compared on a path boundary rather than
+// as a string prefix so a sibling sharing the prefix does not pass.
+func isWithin(path, root string) bool {
+	rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path))
+	if err != nil {
+		return false
+	}
+	return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)))
+}
+
+// writeSafeSymlink creates a symlink at destPath only if target is relative and really resolves,
+// relative to destPath's directory, to a location inside destDir.
 //
 // Absolute targets are rejected outright: os.Symlink writes the literal target
 // string, so an absolute target like "/etc/passwd" would resolve on the host
 // filesystem when read, regardless of any safety check we performed against
 // destDir at extraction time.
 //
-// NOTE: this guards against the single-symlink-escape pattern, but a malicious
-// archive can still attempt a multi-step chain where two cooperating link
-// entries together resolve outside destDir (e.g., one entry creates a benign
-// link, a later entry writes through it). Fully preventing that requires
-// either openat/O_NOFOLLOW per path component during extraction or
-// transitively resolving every newly-created symlink against all prior ones
-// — neither is implemented here.
+// A relative target is resolved against the REAL location of destPath's directory, not against its
+// lexical one, because two cooperating entries can otherwise each pass on their own and together
+// point outside: a link to "." followed by a link to ".." inside it resolves lexically to the root
+// while really landing above it, and repeating the pair climbs a level each time. The parent of
+// every write is checked the same way before the write happens, so a link that slipped through here
+// still could not be written through - this is the earlier of two refusals, not the only one.
+//
+// It has no caller since an archive's entries became headers in one tar rather than files on disk: a
+// symlink entry is now a header write, which creates nothing and traverses nothing, and where the
+// link points is decided inside the archive's own filetree. It is kept rather than retired because
+// that is a judgement about every path that could write a link, not about this one call site, and
+// because the reasoning above is the expensive part to reconstruct.
+//
+//nolint:unused // retained deliberately; see the note above
 func writeSafeSymlink(target, destPath, destDir string) error {
 	if target == "" {
 		return fmt.Errorf("empty link target")
@@ -275,11 +414,99 @@ func writeSafeSymlink(target, destPath, destDir string) error {
 	if filepath.IsAbs(target) {
 		return fmt.Errorf("absolute symlink target not allowed")
 	}
-	resolved := filepath.Join(filepath.Dir(destPath), target)
-	cleanRoot := filepath.Clean(destDir)
-	cleanResolved := filepath.Clean(resolved)
-	if cleanResolved != cleanRoot && !strings.HasPrefix(cleanResolved, cleanRoot+string(filepath.Separator)) {
+
+	realRoot, err := filepath.EvalSymlinks(destDir)
+	if err != nil {
+		return fmt.Errorf("unable to resolve extraction root: %w", err)
+	}
+	realParent, err := filepath.EvalSymlinks(filepath.Dir(destPath))
+	if err != nil {
+		return fmt.Errorf("unable to resolve link parent: %w", err)
+	}
+	if !isWithin(realParent, realRoot) {
+		return fmt.Errorf("symlink parent escapes extraction root")
+	}
+
+	resolved := filepath.Join(realParent, target)
+	if !isWithin(resolved, realRoot) {
 		return fmt.Errorf("symlink target escapes extraction root")
 	}
+	// the target itself may run through links planted by earlier entries
+	if existing, err := filepath.EvalSymlinks(resolved); err == nil && !isWithin(existing, realRoot) {
+		return fmt.Errorf("symlink target resolves outside extraction root")
+	}
+
 	return os.Symlink(target, destPath)
 }
+
+// zipEOCDSignature is the end-of-central-directory record's signature. A zip is defined by this
+// record: it is what a reader seeks to in order to find the central directory, which is why a zip
+// with something concatenated in front of it still opens.
+var zipEOCDSignature = []byte{'P', 'K', 0x05, 0x06}
+
+// maxZipEOCDSearch is how far back the record can be. The EOCD is 22 bytes and ends with a comment
+// of up to 65535 bytes, so it starts no earlier than this many bytes from the end.
+const maxZipEOCDSearch = 22 + 65535
+
+// zipLocalFileHeaderSignature starts every entry in a zip. In a self-extracting archive the first
+// one follows the stub, so it is what a bounded read of the head can find.
+var zipLocalFileHeaderSignature = []byte{'P', 'K', 0x03, 0x04}
+
+// MaxAppendedArchiveStubBytes is how far into a file a hidden archive is looked for. A Spring Boot
+// launcher stub is about 9 KB, so this is generous by a wide margin; a stub longer than this hides
+// its archive from the walk, which is the cost of not reading every candidate to the end.
+const MaxAppendedArchiveStubBytes = 64 * 1024
+
+// MayHideAnAppendedArchive reports whether the head of this stream contains a zip local file header,
+// and returns a reader that replays what it consumed.
+//
+// It exists to make the candidate set affordable. A file whose head sniffs as a script, an
+// executable, or as nothing recognizable may still be an archive wearing a stub, but almost none of
+// them are - and the authoritative test, the end-of-central-directory record, is at the far end of
+// the file, so applying it to every such candidate means reading every executable in a scan to its
+// end. Measured on a stock ruby image that doubled the scan.
+//
+// So the head is checked first, cheaply and with a bound, and only a file that shows an entry
+// signature there is read in full and put to the real test. This narrows what is examined; it never
+// admits anything, because a file that passes here is still refused by the extractor unless it ends
+// in a valid central directory.
+func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) {
+	head := make([]byte, MaxAppendedArchiveStubBytes)
+	n, err := io.ReadFull(r, head)
+	if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+		return io.MultiReader(bytes.NewReader(head[:n]), r), false
+	}
+	head = head[:n]
+	return io.MultiReader(bytes.NewReader(head), r), bytes.Contains(head, zipLocalFileHeaderSignature)
+}
+
+// HasZipEndOfCentralDirectory reports whether the content ends in a zip end-of-central-directory
+// record, which is the definition of a zip and the only part of one whose position is knowable
+// without parsing.
+//
+// This is what identifies an archive that content sniffing types by its head: a self-extracting
+// archive is a launcher stub with a zip concatenated onto it, so its first bytes are the stub's and
+// no magic-at-offset-zero test can see the archive. The record at the end is unambiguous, and
+// mholt/archives extracts such a file correctly once it is asked to - it reads through
+// archive/zip, which locates the central directory from the end.
+//
+// It is a content test, so it cannot make a non-archive extractable: a file named `notes.zip` that
+// is prose has no such record and is refused here exactly as it is refused by format identification.
+func HasZipEndOfCentralDirectory(r ReaderAtSeeker) bool {
+	size, err := r.Seek(0, io.SeekEnd)
+	if err != nil || size < int64(len(zipEOCDSignature)) {
+		return false
+	}
+
+	window := int64(maxZipEOCDSearch)
+	if size < window {
+		window = size
+	}
+
+	buf := make([]byte, window)
+	if _, err := r.ReadAt(buf, size-window); err != nil && !errors.Is(err, io.EOF) {
+		return false
+	}
+
+	return bytes.LastIndex(buf, zipEOCDSignature) >= 0
+}

internal/archive/limits.go

@@ -0,0 +1,255 @@
+package archive
+
+import (
+	"errors"
+	"sync"
+)
+
+// ErrDiskLimitReached reports that an archive's content could not be admitted because placing it
+// would have taken disk usage past the disk limit. It is not a failure of the scan: the archive is
+// skipped and the walk continues.
+var ErrDiskLimitReached = errors.New("archive content would exceed the disk limit")
+
+// Limits bounds how much archive content one scan may hold at once, and decides where an
+// archive's content is held.
+//
+// These are in-use limits rather than counters. An in-use limit measures what is held right now and
+// falls when an archive is released, so what it bounds is a scan's peak concurrent usage. A bound on
+// bytes ever written measures no state that ever exists, because every archive's extraction directory
+// is removed on the way out: it would restrict a long scan of small archives that never held much at
+// once, and permit a deep scan that held far more.
+//
+// Both fields read three states the same way: positive is the limit, zero means none of that
+// resource may be used at all, and negative means that resource is not bounded. Unbounded is a real
+// hazard on either - a crafted archive can exhaust memory or disk - so it is reachable only by asking
+// for it explicitly with a negative value, never as the reading of an unset or zeroed field.
+type Limits struct {
+	// MaxMemoryBytes bounds the archive content held in memory at once. Content is held in memory
+	// while this admits it and overflows to disk when it does not - there is no separate configured size
+	// at which overflowing begins. Zero holds nothing, sending every archive's content to disk;
+	// negative holds content regardless of how much is already held.
+	MaxMemoryBytes int64
+
+	// MaxDiskBytes bounds the archive content placed on disk at once: content overflowed there instead
+	// of being held in memory, plus the bytes of the entries extracted from it. Zero writes nothing
+	// to disk, so content that does not fit in memory has nowhere to go and its archive is skipped;
+	// negative overflows as needed with no ceiling.
+	MaxDiskBytes int64
+}
+
+// Limiter measures how much archive content a scan is holding right now. One instance per task run,
+// which is one scan - the only scoping under which "held at once" means anything.
+//
+// Guarded by a mutex even though the archive walk is sequential today: the archive task merges into
+// a shared builder while top-level catalogers run, so this code already lives in a concurrent
+// neighbourhood and an unguarded total here would be a data race the day someone parallelises the
+// walk.
+//
+// A nil *Limiter enforces nothing, which is what an extraction with no configured bounds gets.
+type Limiter struct {
+	limits Limits
+	mu     sync.Mutex
+	memory int64
+	disk   int64
+
+	// peaks are the high-water marks of the two gauges above: the most this scan held at any one
+	// moment, rather than the sum of what every archive held. They only ever rise, which is the one
+	// place in this type where that is correct - a peak that fell on release would be measuring the
+	// same thing the gauge already measures.
+	peakMemory int64
+	peakDisk   int64
+}
+
+// NewLimiter returns a limiter enforcing the given limits.
+func NewLimiter(limits Limits) *Limiter {
+	return &Limiter{limits: limits}
+}
+
+// MaxDiskBytes returns the configured disk bound, for attributing a skip in a log line.
+func (l *Limiter) MaxDiskBytes() int64 {
+	if l == nil {
+		return 0
+	}
+	return l.limits.MaxDiskBytes
+}
+
+// Peak reports the most archive content this scan held at any one moment, in memory and on disk.
+// These are the same quantities the limits bound, so they say how close a scan came to them.
+func (l *Limiter) Peak() (memory, disk int64) {
+	if l == nil {
+		return 0, 0
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.peakMemory, l.peakDisk
+}
+
+// InUse reports the archive content held right now, in memory and on disk.
+func (l *Limiter) InUse() (memory, disk int64) {
+	if l == nil {
+		return 0, 0
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.memory, l.disk
+}
+
+// Charge starts one archive's accounting against this limiter. Releasing the returned charge gives
+// back exactly what it took, which is what makes this a limiter rather than a counter.
+func (l *Limiter) Charge() *Charge {
+	if l == nil {
+		return nil
+	}
+	return &Charge{limiter: l}
+}
+
+// Charge is one archive's accumulated draw on the limiter.
+//
+// Bytes are charged to it as they land, never from a size an entry declared: both tar and zip
+// headers carry an entry size and a crafted archive is free to lie about it, so a reservation made
+// against a declared size bounds what an attacker claims rather than what the machine does. The
+// charge accumulates what was actually taken, so Release returns exactly that.
+//
+// A nil *Charge charges nothing and refuses nothing.
+type Charge struct {
+	limiter *Limiter
+	memory  int64
+	disk    int64
+}
+
+// admitsThreeState reports whether n more bytes may be charged against a limit that is already
+// holding held, reading the limit the same way on both bounds it governs: positive is the limit
+// itself, zero refuses unconditionally (none of that resource may be used at all), and negative
+// admits unconditionally (that resource is not bounded). One reading shared by Memory and Disk
+// rather than the same three-way condition written twice, since the two have already drifted once.
+func admitsThreeState(limit, held, n int64) bool {
+	switch {
+	case limit < 0:
+		return true
+	case limit == 0:
+		return false
+	default:
+		return held+n <= limit
+	}
+}
+
+// Memory charges n bytes of content held in memory, reporting false when the memory limit does not
+// admit it - which for a zero limit is every chunk, sending content to disk unconditionally, since
+// there is no separate threshold naming a size at which content overflows. Nothing is charged when it
+// reports false.
+func (c *Charge) Memory(n int64) bool {
+	if c == nil || c.limiter == nil || n <= 0 {
+		return true
+	}
+	l := c.limiter
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if !admitsThreeState(l.limits.MaxMemoryBytes, l.memory, n) {
+		return false
+	}
+	l.memory += n
+	c.memory += n
+	if l.memory > l.peakMemory {
+		l.peakMemory = l.memory
+	}
+	return true
+}
+
+// Disk charges n bytes of content placed on disk, reporting false when the disk limit does not admit
+// it - which for a zero limit is every chunk, so content with nowhere further to go leaves its
+// archive skipped exactly as content exceeding a positive disk limit does. Nothing is charged when it
+// reports false.
+func (c *Charge) Disk(n int64) bool {
+	if c == nil || c.limiter == nil || n <= 0 {
+		return true
+	}
+	l := c.limiter
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if !admitsThreeState(l.limits.MaxDiskBytes, l.disk, n) {
+		return false
+	}
+	l.disk += n
+	c.disk += n
+	if l.disk > l.peakDisk {
+		l.peakDisk = l.disk
+	}
+	return true
+}
+
+// IndexRecord charges the in-memory cost of holding one entry's index record - the *Entry, its
+// header, and the resolver node built over it - preferring the memory budget and overflowing to the
+// disk budget when memory will not admit it.
+//
+// It overflows rather than refusing because the index is not content: a resolver needs an entry's
+// record to reach that entry's bytes at all, and the memory budget may legitimately be zero, so an
+// index that could be refused would leave an archive whose content is entirely on disk with no way to
+// read it back. What this bounds instead is entry count - the one dimension the byte budgets never
+// bounded, so an archive of millions of empty entries could grow the index without limit. It reports
+// false only when neither budget admits the record, which truncates the archive. What it charges is
+// released by Release along with everything else this charge took, so the index accounting falls when
+// the archive does.
+func (c *Charge) IndexRecord(n int64) bool {
+	if c == nil || c.limiter == nil || n <= 0 {
+		return true
+	}
+	if c.Memory(n) {
+		return true
+	}
+	return c.Disk(n)
+}
+
+// RefundDisk gives back n bytes charged for content that was removed again, so a partial file
+// dropped at a limit does not leave the limiter holding bytes that are no longer on disk.
+func (c *Charge) RefundDisk(n int64) {
+	if c == nil || c.limiter == nil || n <= 0 {
+		return
+	}
+	l := c.limiter
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if n > c.disk {
+		n = c.disk
+	}
+	l.disk -= n
+	c.disk -= n
+}
+
+// RefundMemory gives back n bytes charged for content no longer held in memory.
+func (c *Charge) RefundMemory(n int64) {
+	if c == nil || c.limiter == nil || n <= 0 {
+		return
+	}
+	l := c.limiter
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if n > c.memory {
+		n = c.memory
+	}
+	l.memory -= n
+	c.memory -= n
+}
+
+// Held reports what this charge is currently holding, in memory and on disk.
+func (c *Charge) Held() (memory, disk int64) {
+	if c == nil || c.limiter == nil {
+		return 0, 0
+	}
+	c.limiter.mu.Lock()
+	defer c.limiter.mu.Unlock()
+	return c.memory, c.disk
+}
+
+// Release returns everything this charge took, so both limits fall by what this archive held. Safe
+// to call more than once.
+func (c *Charge) Release() {
+	if c == nil || c.limiter == nil {
+		return
+	}
+	l := c.limiter
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.memory -= c.memory
+	l.disk -= c.disk
+	c.memory, c.disk = 0, 0
+}

internal/archive/limits_test.go

@@ -0,0 +1,210 @@
+package archive
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLimiter_chargeAndRelease(t *testing.T) {
+	limiter := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 200})
+
+	first := limiter.Charge()
+	require.True(t, first.Memory(60))
+	require.True(t, first.Disk(150))
+
+	mem, disk := limiter.InUse()
+	assert.Equal(t, int64(60), mem)
+	assert.Equal(t, int64(150), disk)
+
+	second := limiter.Charge()
+	assert.False(t, second.Memory(50), "60 + 50 is over the memory limit")
+	assert.False(t, second.Disk(60), "150 + 60 is over the disk limit")
+	assert.True(t, second.Memory(40), "and exactly at the limit is admitted")
+
+	mem, _ = limiter.InUse()
+	assert.Equal(t, int64(100), mem, "a refused charge takes nothing")
+
+	first.Release()
+	mem, disk = limiter.InUse()
+	assert.Equal(t, int64(40), mem, "release gives back exactly what that charge took")
+	assert.Zero(t, disk)
+
+	first.Release()
+	mem, _ = limiter.InUse()
+	assert.Equal(t, int64(40), mem, "releasing twice must not give back what was never taken")
+}
+
+func TestLimiter_refund(t *testing.T) {
+	// a partial entry dropped at a bound is no longer on disk, so the limiter must not go on holding it
+	limiter := NewLimiter(Limits{MaxDiskBytes: 100})
+	charge := limiter.Charge()
+
+	require.True(t, charge.Disk(80))
+	charge.RefundDisk(80)
+
+	_, disk := limiter.InUse()
+	assert.Zero(t, disk)
+	assert.True(t, charge.Disk(100), "the refunded room is available again")
+
+	charge.RefundDisk(500)
+	_, disk = limiter.InUse()
+	assert.Zero(t, disk, "a refund larger than what was charged cannot drive the limiter negative")
+}
+
+func TestLimiter_boundsAreIndependent(t *testing.T) {
+	// a caller may bound one limit without bounding the other: a limit on one side of the pair does
+	// not change how the other side reads its own value.
+	memoryBounded := NewLimiter(Limits{MaxMemoryBytes: 10, MaxDiskBytes: -1}).Charge()
+	assert.False(t, memoryBounded.Memory(50))
+	assert.True(t, memoryBounded.Disk(1_000_000), "a negative disk limit admits anything")
+
+	diskBounded := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10}).Charge()
+	assert.False(t, diskBounded.Memory(1_000_000), "a zero memory limit refuses everything rather than admitting anything")
+	assert.False(t, diskBounded.Disk(50))
+}
+
+func TestLimiter_threeStateReadingIsUniform(t *testing.T) {
+	// Memory and Disk read their limit the same way: positive is the limit, zero refuses
+	// unconditionally, negative admits unconditionally. The two used to disagree in opposite
+	// directions - Memory got zero right and negative wrong, Disk got negative right and zero wrong.
+	t.Run("positive is the limit", func(t *testing.T) {
+		c := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 100}).Charge()
+		assert.True(t, c.Memory(100))
+		assert.False(t, c.Memory(1))
+		assert.True(t, c.Disk(100))
+		assert.False(t, c.Disk(1))
+	})
+
+	t.Run("zero refuses unconditionally on both", func(t *testing.T) {
+		c := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).Charge()
+		assert.False(t, c.Memory(1))
+		assert.False(t, c.Disk(1))
+	})
+
+	t.Run("negative admits unconditionally on both", func(t *testing.T) {
+		c := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}).Charge()
+		assert.True(t, c.Memory(1_000_000_000))
+		assert.True(t, c.Disk(1_000_000_000))
+	})
+}
+
+func TestLimiter_nilIsUnbounded(t *testing.T) {
+	// what an extraction with no configured bounds gets: charges succeed and nothing is measured.
+	// This is distinct from a configured but non-positive MaxMemoryBytes, which refuses instead -
+	// there is no limiter here at all to refuse anything.
+	var limiter *Limiter
+	charge := limiter.Charge()
+	assert.Nil(t, charge)
+
+	assert.True(t, charge.Memory(1_000_000))
+	assert.True(t, charge.Disk(1_000_000))
+	charge.RefundDisk(10)
+	charge.Release()
+
+	mem, disk := charge.Held()
+	assert.Zero(t, mem)
+	assert.Zero(t, disk)
+
+	mem, disk = limiter.InUse()
+	assert.Zero(t, mem)
+	assert.Zero(t, disk)
+	assert.Zero(t, limiter.MaxDiskBytes())
+}
+
+func TestLimiter_concurrentChargesAgree(t *testing.T) {
+	// the walk is sequential today, but the archive task merges into a shared builder while top-level
+	// catalogers run, so the accounting is guarded rather than left for whoever parallelises it.
+	// Both limits need an explicit value here: zero now refuses every charge on either bound, so
+	// MaxMemoryBytes is set comfortably above what this test charges and MaxDiskBytes is negative
+	// for unbounded, rather than leaving either at its zero value.
+	limiter := NewLimiter(Limits{MaxMemoryBytes: 1_000_000, MaxDiskBytes: -1})
+
+	var wg sync.WaitGroup
+	charges := make([]*Charge, 50)
+	for i := range charges {
+		charges[i] = limiter.Charge()
+		wg.Add(1)
+		go func(c *Charge) {
+			defer wg.Done()
+			for range 100 {
+				c.Memory(1)
+				c.Disk(2)
+			}
+		}(charges[i])
+	}
+	wg.Wait()
+
+	mem, disk := limiter.InUse()
+	assert.Equal(t, int64(50*100), mem)
+	assert.Equal(t, int64(50*100*2), disk)
+
+	for _, c := range charges {
+		c.Release()
+	}
+	mem, disk = limiter.InUse()
+	assert.Zero(t, mem)
+	assert.Zero(t, disk)
+}
+
+func Test_Limiter_peakIsAHighWaterMark(t *testing.T) {
+	// the peak is what the scan held at its worst moment, so it must survive the release that takes
+	// the in-use gauge back down - a peak that fell on release would measure what InUse already does
+	l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
+
+	first := l.Charge()
+	require.True(t, first.Memory(100))
+	require.True(t, first.Disk(300))
+	first.Release()
+
+	second := l.Charge()
+	require.True(t, second.Memory(40))
+	require.True(t, second.Disk(50))
+
+	memory, disk := l.InUse()
+	assert.Equal(t, int64(40), memory)
+	assert.Equal(t, int64(50), disk)
+
+	peakMemory, peakDisk := l.Peak()
+	assert.Equal(t, int64(100), peakMemory, "peak memory is the most held at once, not the most recently held")
+	assert.Equal(t, int64(300), peakDisk, "and the same for disk")
+}
+
+func Test_Limiter_peakCountsConcurrentHoldersTogether(t *testing.T) {
+	// two archives held at the same time peak at their sum, which is the number the limits are
+	// enforced against and therefore the number worth reporting
+	l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
+
+	outer := l.Charge()
+	require.True(t, outer.Memory(100))
+	inner := l.Charge()
+	require.True(t, inner.Memory(60))
+	inner.Release()
+	outer.Release()
+
+	peakMemory, _ := l.Peak()
+	assert.Equal(t, int64(160), peakMemory)
+}
+
+func Test_Limiter_peakOfANilLimiterIsZero(t *testing.T) {
+	var l *Limiter
+	memory, disk := l.Peak()
+	assert.Zero(t, memory)
+	assert.Zero(t, disk)
+}
+
+func Test_Limiter_refusedChargeDoesNotMoveThePeak(t *testing.T) {
+	// nothing is charged when a limit refuses, so nothing may be recorded either: a peak that rose
+	// on refusal would report bytes the scan never held
+	l := NewLimiter(Limits{MaxMemoryBytes: 50, MaxDiskBytes: 50})
+
+	c := l.Charge()
+	assert.False(t, c.Memory(500))
+	assert.False(t, c.Disk(500))
+
+	peakMemory, peakDisk := l.Peak()
+	assert.Zero(t, peakMemory)
+	assert.Zero(t, peakDisk)
+}

internal/archive/notify.go

@@ -0,0 +1,117 @@
+package archive
+
+import (
+	"github.com/anchore/syft/internal/log"
+)
+
+// Notify receives structured events from archive extraction. A nil Notify is the no-op.
+//
+// The signature is deliberately `func(any)` rather than a set of typed callbacks or an interface with
+// a method per event. Extraction is a hot path - a scan of a big image enters thousands of archives -
+// and what instrumentation costs there has to be nothing when it is off. A single func field is one
+// nil check, and a handler that cares about one event type asserts for it and ignores the rest, so
+// adding an event never changes a signature or breaks a handler that did not ask for it.
+//
+// Guard the call rather than relying on a nil check inside the callee:
+//
+//	if notify != nil {
+//		notify(EntriesOverflowed{Archive: name, Bytes: n})
+//	}
+//
+// Building the event value is what costs - putting a struct in an `any` allocates - so the guard is
+// what makes "off" free rather than merely cheap. Send exists for the paths where that does not
+// matter.
+type Notify func(msg any)
+
+// Send delivers one event, doing nothing when there is no handler. Convenient, but it builds the
+// event whether or not anyone is listening: on a hot path, guard the call instead.
+func (n Notify) Send(msg any) {
+	if n == nil {
+		return
+	}
+	n(msg)
+}
+
+// Tee returns a Notify that delivers to each of the given handlers, skipping the nil ones, and nil
+// when none of them are live - so a tee of nothing stays free.
+func Tee(handlers ...Notify) Notify {
+	var live []Notify
+	for _, h := range handlers {
+		if h != nil {
+			live = append(live, h)
+		}
+	}
+	switch len(live) {
+	case 0:
+		return nil
+	case 1:
+		return live[0]
+	}
+	return func(msg any) {
+		for _, h := range live {
+			h(msg)
+		}
+	}
+}
+
+// OverflowReason says why content was written to disk rather than held.
+type OverflowReason string
+
+const (
+	// MemoryLimitReached means the scan was already holding as much as the memory limit admits.
+	MemoryLimitReached OverflowReason = "memory limit reached"
+
+	// MemoryHoldsNothing means the memory limit is zero, so nothing is ever held and every archive's
+	// content goes to disk by policy rather than by pressure.
+	MemoryHoldsNothing OverflowReason = "memory limit is zero"
+)
+
+// ContentOverflowed reports that one archive's own bytes were written to disk instead of being held in
+// memory. Bytes is the size of the archive.
+type ContentOverflowed struct {
+	Archive string
+	Bytes   int64
+	Reason  OverflowReason
+}
+
+// EntriesOverflowed reports that the entries an archive's store was holding have been written to its
+// overflow blob. Entries and Bytes are what moved, not what the archive holds in total: a store spills
+// everything it is holding at once, and may spill more than once as it fills again.
+type EntriesOverflowed struct {
+	Archive string
+	Entries int
+	Bytes   int64
+	Reason  OverflowReason
+}
+
+// Skipped reports that an archive contributed nothing, and why. Not a failure - the scan
+// continues - but the packages inside it are absent from the SBOM, which is worth being able to see.
+type Skipped struct {
+	Archive string
+	Reason  string
+}
+
+// Truncated reports that an archive was cataloged from part of its contents, naming the bound
+// that stopped it.
+type Truncated struct {
+	Archive string
+	Reason  TruncationReason
+}
+
+// LogNotify is the default handler: it writes each event to the debug log. Anything it does not
+// recognize is ignored rather than logged blindly, so an event added for one consumer does not turn
+// into noise for everyone.
+func LogNotify(msg any) {
+	switch e := msg.(type) {
+	case ContentOverflowed:
+		log.WithFields("archive", e.Archive, "bytes", e.Bytes, "reason", string(e.Reason)).
+			Debug("archive content written to disk rather than held in memory")
+	case EntriesOverflowed:
+		log.WithFields("archive", e.Archive, "entries", e.Entries, "bytes", e.Bytes, "reason", string(e.Reason)).
+			Debug("archive entries written to disk rather than held in memory")
+	case Skipped:
+		log.WithFields("archive", e.Archive, "reason", e.Reason).Debug("archive skipped")
+	case Truncated:
+		log.WithFields("archive", e.Archive, "reason", string(e.Reason)).Debug("archive cataloged from part of its contents")
+	}
+}

internal/archive/notify_test.go

@@ -0,0 +1,98 @@
+package archive
+
+import (
+	"archive/tar"
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// recorder is what a consumer of these events looks like: one func, a type assertion for what it
+// cares about, and silence for everything else.
+type recorder struct {
+	overflows []EntriesOverflowed
+	other     int
+}
+
+func (r *recorder) notify(msg any) {
+	if e, ok := msg.(EntriesOverflowed); ok {
+		r.overflows = append(r.overflows, e)
+		return
+	}
+	r.other++
+}
+
+func TestNotify_reportsWhichArchiveSpilledAndWhy(t *testing.T) {
+	rec := &recorder{}
+	s := NewEntryStore(t.TempDir(), "lib/app.jar", rec.notify)
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	charge := memCharge(8)
+	for _, body := range []string{"aaaa", "bbbbbb"} {
+		_, err := s.Add(regularHeader(body+".txt", int64(len(body))), bytes.NewReader([]byte(body)), charge)
+		require.NoError(t, err)
+	}
+
+	require.Len(t, rec.overflows, 1)
+	assert.Equal(t, "lib/app.jar", rec.overflows[0].Archive, "the event has to name the archive, or it says nothing actionable")
+	assert.Equal(t, MemoryLimitReached, rec.overflows[0].Reason)
+	assert.Equal(t, 2, rec.overflows[0].Entries, "a store moves everything it is holding, not just the entry that did not fit")
+	assert.Equal(t, int64(10), rec.overflows[0].Bytes)
+}
+
+func TestNotify_distinguishesPressureFromPolicy(t *testing.T) {
+	// a scan at its memory bound and a scan whose bound is zero are different findings: the first says
+	// the limit is biting, the second says nothing was ever going to be held
+	rec := &recorder{}
+	s := NewEntryStore(t.TempDir(), "app.jar", rec.notify)
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	_, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), memCharge(0))
+	require.NoError(t, err)
+
+	require.Len(t, rec.overflows, 1)
+	assert.Equal(t, MemoryHoldsNothing, rec.overflows[0].Reason)
+}
+
+func TestNotify_nilHandlerIsTheNoOp(t *testing.T) {
+	s := NewEntryStore(t.TempDir(), "app.jar", nil)
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	_, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), memCharge(0))
+	assert.NoError(t, err, "no handler must not mean no extraction")
+}
+
+func TestNotify_teeDeliversToEveryLiveHandler(t *testing.T) {
+	first, second := &recorder{}, &recorder{}
+	tee := Tee(nil, first.notify, nil, second.notify)
+	require.NotNil(t, tee)
+
+	tee(EntriesOverflowed{Archive: "app.jar", Entries: 1})
+
+	assert.Len(t, first.overflows, 1)
+	assert.Len(t, second.overflows, 1)
+}
+
+func TestNotify_teeOfNothingIsNil(t *testing.T) {
+	// a tee that would deliver to no one stays nil, so the guard at every call site keeps it free
+	assert.Nil(t, Tee())
+	assert.Nil(t, Tee(nil, nil))
+}
+
+func TestNotify_sendIsNilSafe(t *testing.T) {
+	var n Notify
+	n.Send(EntriesOverflowed{})
+}
+
+func TestNotify_directoryEntriesDoNotSpill(t *testing.T) {
+	// a directory carries no content, so it can never be the thing that pushes a store to disk
+	rec := &recorder{}
+	s := NewEntryStore(t.TempDir(), "app.jar", rec.notify)
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	_, err := s.Add(tar.Header{Name: "lib/", Typeflag: tar.TypeDir, Mode: 0o755}, nil, memCharge(0))
+	require.NoError(t, err)
+	assert.Empty(t, rec.overflows)
+}

internal/archive/overflow.go

@@ -0,0 +1,352 @@
+package archive
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/mholt/archives"
+
+	"github.com/anchore/syft/internal/log"
+)
+
+// tarBlockSize is the tar format's fixed block size. Every header occupies a whole number of blocks
+// and every entry's data is padded up to one, which is why the bytes an archive puts on disk are not
+// the sum of its entry sizes - see OverflowTar.
+const tarBlockSize = 512
+
+// overflowTarName is the basename of the one file an archive's entries are written into. It sits beside
+// the archive's logical root rather than inside it, so nothing in the archive's own filesystem can
+// name it.
+const overflowTarName = "contents.tar"
+
+// OverflowTar is the single file one archive's entries are overflowed into.
+//
+// The point of the whole shape is that a scan's filesystem cost tracks the number of archives it
+// finds rather than the number of entries those archives hold. Expanding every entry as its own file
+// makes an archive of a hundred thousand entries a hundred thousand inodes - exhausting inodes on a
+// filesystem with bytes to spare, and making cleanup proportional to content as well. One tar per
+// archive is one inode per archive, and stereoscope's TarIndex then gives direct access to any entry
+// by seek offset, so nothing has to be re-read or re-decompressed to reach the end of it.
+//
+// It is written by hand rather than through a tar.Writer because tar wants an entry's size in the
+// header, before its data, and a size an archive declares is exactly what must not be trusted: both
+// tar and zip headers carry a size and a crafted archive is free to lie about it. So each entry's
+// header is reserved, its data is copied and charged as it lands, and the header is then written over
+// the reservation with the size that actually arrived. The reservation is exact because the header is
+// encoded as GNU, whose numeric fields are fixed width - the encoded length of a header therefore
+// depends on its name and link target and not at all on its size. That is asserted per entry rather
+// than assumed, and an entry whose two encodings disagree is dropped rather than corrupting the tar.
+type OverflowTar struct {
+	path string
+	file *os.File
+
+	// offset is where the next byte written will land, which is also the number of bytes this tar
+	// holds. Tracked rather than queried so a header can be written back over its reservation.
+	offset int64
+
+	// verbatim records that this tar was copied wholesale from an already-tar-shaped stream rather
+	// than assembled entry by entry, in which case it carries the source's own end-of-archive marker
+	// and must not be given a second one.
+	verbatim bool
+}
+
+func newOverflowTar(path string) (*OverflowTar, error) {
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create overflow tar %q: %w", path, err)
+	}
+	return &OverflowTar{path: path, file: f}, nil
+}
+
+// Path is where this tar lives, which is what stereoscope's TarIndex needs: it indexes by seek
+// offset and therefore wants a seekable, uncompressed file rather than a stream.
+func (s *OverflowTar) Path() string {
+	return s.path
+}
+
+func (s *OverflowTar) Close() error {
+	if s == nil || s.file == nil {
+		return nil
+	}
+	err := s.file.Close()
+	s.file = nil
+	return err
+}
+
+// put charges b to the disk limit and writes it, reporting whether the limit refused it. Nothing is
+// written when the charge is refused, so the tar never holds bytes the limiter is not accounting for.
+func (s *OverflowTar) put(b []byte, result *ExtractionResult, limits ExtractionLimits) (refused bool, err error) {
+	if len(b) == 0 {
+		return false, nil
+	}
+	if !limits.Charge.Disk(int64(len(b))) {
+		return true, nil
+	}
+	n, err := s.file.Write(b)
+	s.offset += int64(n)
+	result.BytesWritten += int64(n)
+	if n < len(b) {
+		limits.Charge.RefundDisk(int64(len(b) - n))
+	}
+	return false, err
+}
+
+// AddEntry writes one archive entry into the tar: a header, its data, and the padding that takes the
+// entry up to a block boundary.
+//
+// Directory entries carry no data, so what bounds an archive of a million of them is the disk limit
+// charging their headers and padding, which is the only resource they actually consume.
+func (s *OverflowTar) AddEntry(f archives.FileInfo, result *ExtractionResult, limits ExtractionLimits) error {
+	hdr, ok := entryHeader(f)
+	if !ok {
+		return nil
+	}
+
+	reserved, err := encodeHeader(hdr, 0)
+	if err != nil {
+		// GNU could not represent this entry; let the tar package pick a format it can, and rely on
+		// the length check below to drop the entry if that choice makes the length size-dependent
+		hdr.Format = tar.FormatUnknown
+		reserved, err = encodeHeader(hdr, 0)
+		if err != nil {
+			log.WithFields("entry", f.NameInArchive, "error", err).
+				Debug("skipping archive entry whose header cannot be encoded")
+			return nil
+		}
+	}
+
+	start := s.offset
+	if refused, err := s.put(reserved, result, limits); err != nil {
+		return err
+	} else if refused {
+		return s.dropEntry(start, result, limits)
+	}
+
+	written, refused, err := s.copyEntryData(f, hdr, result, limits)
+	if err != nil {
+		return err
+	}
+	if refused {
+		// this entry would have taken the scan's disk usage past the disk limit. Drop it rather than
+		// leaving a silently short one behind: a half-written jar or pom is worse than an absent one,
+		// because a cataloger will try to parse it and may believe what it finds.
+		return s.dropEntry(start, result, limits)
+	}
+
+	final, err := encodeHeader(hdr, written)
+	if err != nil || len(final) != len(reserved) {
+		// the reservation and the real header disagree on length, so writing the real header would
+		// either overwrite the entry's first bytes or leave a gap - either way an unreadable tar.
+		// GNU's fixed-width numeric fields make this unreachable in practice; it is checked because
+		// the alternative to checking is silent corruption.
+		log.WithFields("entry", f.NameInArchive, "error", err).
+			Debug("skipping archive entry whose header length depends on its size")
+		return s.dropEntry(start, result, limits)
+	}
+	if _, err := s.file.WriteAt(final, start); err != nil {
+		return fmt.Errorf("unable to write header for archive entry %q: %w", f.NameInArchive, err)
+	}
+
+	if pad := (tarBlockSize - written%tarBlockSize) % tarBlockSize; pad > 0 {
+		if refused, err := s.put(make([]byte, pad), result, limits); err != nil {
+			return err
+		} else if refused {
+			return s.dropEntry(start, result, limits)
+		}
+	}
+
+	result.FilesExtracted++
+	return nil
+}
+
+// copyEntryData copies one entry's content into the tar, charging every chunk as it lands, and reports
+// how much actually arrived. A size the archive declared is never used: both tar and zip headers carry
+// one and a crafted archive is free to lie about it, so the header is written afterwards from what was
+// counted here. An entry carrying no data - a directory, a link - copies nothing.
+func (s *OverflowTar) copyEntryData(f archives.FileInfo, hdr *tar.Header, result *ExtractionResult, limits ExtractionLimits) (written int64, refused bool, err error) {
+	if hdr.Typeflag != tar.TypeReg {
+		return 0, false, nil
+	}
+
+	src, err := f.Open()
+	if err != nil {
+		return 0, false, fmt.Errorf("unable to open archive entry %q: %w", f.NameInArchive, err)
+	}
+	defer func() {
+		if closeErr := src.Close(); closeErr != nil {
+			log.WithFields("entry", f.NameInArchive, "error", closeErr).Trace("unable to close archive entry")
+		}
+	}()
+
+	written, refused, err = copyCharged(s.file, src, limits.Charge)
+	s.offset += written
+	result.BytesWritten += written
+	return written, refused, err
+}
+
+// dropEntry rewinds the tar to where the entry being written began, refunding what that entry
+// charged since those bytes are no longer on disk, and stops the extraction. Whatever was written
+// before this entry is a complete, readable tar and is still cataloged; see ExtractionResult.
+func (s *OverflowTar) dropEntry(start int64, result *ExtractionResult, limits ExtractionLimits) error {
+	spent := s.offset - start
+	if err := s.file.Truncate(start); err != nil {
+		log.WithFields("path", s.path, "error", err).Trace("unable to rewind overflow tar past an over-limit entry")
+	}
+	if _, err := s.file.Seek(start, io.SeekStart); err != nil {
+		log.WithFields("path", s.path, "error", err).Trace("unable to seek overflow tar past an over-limit entry")
+	}
+	s.offset = start
+	limits.Charge.RefundDisk(spent)
+	result.BytesWritten -= spent
+	if result.Truncation == "" {
+		result.Truncation = TruncatedByDiskLimit
+	}
+	return errTruncated
+}
+
+// CopyVerbatim writes an already-tar-shaped stream into the overflow file as it stands, which is what a
+// tar-family archive gets: a compressed tar is decompressed once into a plain tar rather than
+// expanded entry by entry, and a plain tar is copied. Charging is per chunk as the bytes land, so a
+// decompression bomb is bounded by the disk limit rather than by what the archive claimed.
+//
+// A refused chunk leaves the tar ending part way through an entry. That is deliberately not repaired
+// here: indexing keeps every entry it could read and reports the tail as a truncation, which is the
+// same outcome as stopping on a whole-entry boundary and needs no second mechanism.
+func (s *OverflowTar) CopyVerbatim(r io.Reader, result *ExtractionResult, limits ExtractionLimits) error {
+	s.verbatim = true
+
+	written, refused, err := copyCharged(s.file, r, limits.Charge)
+	s.offset += written
+	result.BytesWritten += written
+	if err != nil {
+		return err
+	}
+	if refused {
+		result.Truncation = TruncatedByDiskLimit
+		return errTruncated
+	}
+	return nil
+}
+
+// Finish writes the tar's end-of-archive marker.
+//
+// A marker that does not fit within the disk limit is not an error: Go's tar reader stops at the
+// first zero block or at the end of the file, so a tar that simply ends reads exactly the same.
+func (s *OverflowTar) Finish(result *ExtractionResult, limits ExtractionLimits) {
+	if s.verbatim {
+		// the source stream carried its own marker
+		return
+	}
+	if refused, err := s.put(make([]byte, 2*tarBlockSize), result, limits); err != nil || refused {
+		log.WithFields("path", s.path, "error", err).Trace("overflow tar written without an end-of-archive marker")
+	}
+}
+
+// entryHeader builds the tar header for one archive entry, or reports false when the entry is not
+// something worth carrying.
+//
+// Symlinks are recorded as symlink headers rather than being created and checked as real links,
+// because writing a header traverses no path: nothing is created, so there is nothing for a crafted
+// target to redirect. Where a link points is decided when the archive's filetree is built, inside
+// that tree, so a target naming the host filesystem resolves to nothing rather than to the host.
+//
+// Device nodes, fifos and sockets are skipped, as they were when entries were files: they cannot
+// round-trip and no cataloger can read one.
+func entryHeader(f archives.FileInfo) (*tar.Header, bool) {
+	mode := f.Mode()
+	if !mode.IsRegular() && !mode.IsDir() && mode.Type()&fs.ModeSymlink == 0 {
+		log.WithFields("entry", f.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry")
+		return nil, false
+	}
+
+	hdr, err := tar.FileInfoHeader(f.FileInfo, f.LinkTarget)
+	if err != nil {
+		log.WithFields("entry", f.NameInArchive, "error", err).Debug("skipping archive entry with an unusable header")
+		return nil, false
+	}
+
+	// the name is carried through as the archive gave it, unsanitized. Nothing is written through it
+	// - it is a field in a header - and the one place it can still mislead is the logical filetree,
+	// so it is sanitized there, once, rather than in each of the two paths that write a tar.
+	hdr.Name = f.NameInArchive
+	if f.IsDir() && !strings.HasSuffix(hdr.Name, "/") {
+		hdr.Name += "/"
+	}
+	hdr.Linkname = f.LinkTarget
+
+	// GNU rather than the format the tar package would pick: its numeric fields are fixed width, so
+	// the encoded length of a header does not depend on the size recorded in it. That is what lets
+	// the header be reserved before the entry's real size is known and written afterwards. Fields GNU
+	// cannot carry are cleared instead of forcing the encoder into PAX, where a large size becomes an
+	// extra record and the length stops being predictable.
+	hdr.Format = tar.FormatGNU
+	hdr.PAXRecords = nil
+	hdr.Uname, hdr.Gname = "", ""
+	hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{}
+	hdr.ModTime = hdr.ModTime.Truncate(time.Second)
+
+	return hdr, true
+}
+
+// encodeHeader returns the bytes one header occupies in a tar, for the given size. It includes
+// whatever the format needs around it - a GNU long-name record for a name a header field cannot
+// hold - because all of it is written before the entry's data and all of it has to be reserved.
+func encodeHeader(hdr *tar.Header, size int64) ([]byte, error) {
+	h := *hdr
+	h.Size = size
+
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	// WriteHeader emits the header, and any record the format needs ahead of it, before it returns.
+	// Close is deliberately not called: it would append the end-of-archive marker, which belongs to
+	// the tar being assembled and not to one header in it.
+	if err := tw.WriteHeader(&h); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+// tarStream returns the plain-tar byte stream for a tar-family archive, decompressing once where the
+// archive is a compressed tar, and reports false for anything that is not a tar.
+//
+// This is what makes a compressed tar strictly cheaper than it was: it used to be expanded entry by
+// entry into a tree of files, and here it is decompressed once into a file that is already the
+// storage format, then indexed by seek offset. A plain tar is copied for the same reason - TarIndex
+// needs a seekable uncompressed file, and the archive's own bytes may be held in memory.
+func tarStream(format archives.Format, r io.Reader) (io.ReadCloser, bool) {
+	switch f := format.(type) {
+	case archives.Tar:
+		return io.NopCloser(r), true
+	case archives.CompressedArchive:
+		if !isTarFormat(f) {
+			return nil, false
+		}
+		if f.Compression == nil {
+			return io.NopCloser(r), true
+		}
+		rc, err := f.OpenReader(r)
+		if err != nil {
+			log.WithFields("error", err).Debug("unable to decompress tar-family archive")
+			return nil, false
+		}
+		return rc, true
+	}
+	return nil, false
+}
+
+// isTarFormat reports whether the archival half of a compressed archive is a tar. Identify sets both
+// halves for a compressed tar, so either answers; both are checked because only one is guaranteed to
+// be non-nil in general.
+func isTarFormat(f archives.CompressedArchive) bool {
+	if _, ok := f.Archival.(archives.Tar); ok {
+		return true
+	}
+	_, ok := f.Extraction.(archives.Tar)
+	return ok
+}

internal/archive/overflow_test.go

@@ -0,0 +1,221 @@
+package archive
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/syft/syft/file"
+)
+
+// manyEntryZip builds a zip of count small entries, in sorted name order.
+func manyEntryZip(t *testing.T, count int) *bytes.Buffer {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for i := range count {
+		w, err := zw.Create(fmt.Sprintf("e%06d.txt", i))
+		require.NoError(t, err)
+		_, err = w.Write([]byte("x"))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return &buf
+}
+
+// extractedOverflow extracts one archive and hands back where its content went, along with the work
+// directory holding it - which is what a filesystem cost has to be measured against.
+func extractedOverflow(t *testing.T, content io.Reader, limits ExtractionLimits) (*ExtractedArchive, Overflow, string) {
+	t.Helper()
+
+	var got Overflow
+	factory := func(overflow Overflow) (file.Resolver, IndexResult, error) {
+		got = overflow
+		return nil, IndexResult{}, nil
+	}
+
+	extracted, err := ExtractToResolver(
+		context.Background(), content, "app.zip", "", "app.zip",
+		DefaultExtractors(), NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), limits, factory, nil, nil,
+	)
+	require.NoError(t, err)
+	require.NotNil(t, extracted)
+	t.Cleanup(extracted.Cleanup)
+
+	return extracted, got, filepath.Dir(got.TarPath)
+}
+
+// filesystemEntries is every directory entry under dir, path by path, which is the cost the
+// requirement is about: an archive must not put one of these on the filesystem per entry it holds.
+func filesystemEntries(t *testing.T, dir string) []string {
+	t.Helper()
+	var out []string
+	require.NoError(t, filepath.WalkDir(dir, func(p string, _ fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, relErr := filepath.Rel(dir, p)
+		if relErr != nil {
+			return relErr
+		}
+		if rel == "." {
+			return nil
+		}
+		out = append(out, filepath.ToSlash(rel))
+		return nil
+	}))
+	return out
+}
+
+func TestOverflow_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
+	// the requirement this whole shape exists for: an archive of a hundred thousand entries must not
+	// put a hundred thousand files on the filesystem. Asserted as an exact set of paths for two
+	// archives three orders of magnitude apart in entry count, rather than as a bound - a count that
+	// merely grows slowly would pass a bound and still be proportional to content.
+	small, smallOverflow, smallWork := extractedOverflow(t, manyEntryZip(t, 2), ExtractionLimits{})
+	large, largeOverflow, largeWork := extractedOverflow(t, manyEntryZip(t, 5000), ExtractionLimits{})
+
+	want := []string{"contents", "contents.tar"}
+	assert.Equal(t, want, filesystemEntries(t, smallWork))
+	assert.Equal(t, want, filesystemEntries(t, largeWork),
+		"5000 entries must cost the same filesystem entries as 2: one tar, plus the empty root their "+
+			"paths are reported relative to")
+
+	// and the entries really are all there, so this is not passing because nothing was extracted
+	assert.Equal(t, 2, countTarEntries(t, smallOverflow.TarPath))
+	assert.Equal(t, 5000, countTarEntries(t, largeOverflow.TarPath))
+	// what one archive costs on disk, exactly: a header block and a padded data block per entry, plus
+	// the two-block end-of-archive marker. This is the figure the disk limit is charged, which is why
+	// a limit expressed in entry bytes now under-counts.
+	assert.Equal(t, int64((5000*2+2)*tarBlockSize), large.Result.BytesWritten)
+	assert.Equal(t, int64((2*2+2)*tarBlockSize), small.Result.BytesWritten)
+}
+
+func TestOverflow_cleanupRemovesOneFileAndDoesNotWalkPerEntry(t *testing.T) {
+	// cleanup was proportional to content too: releasing an archive meant removing one filesystem
+	// entry per archive entry. It is now bounded by the same constant as the extraction, so the count
+	// of things removed is asserted rather than just the fact that the directory is gone.
+	extracted, overflow, workDir := extractedOverflow(t, manyEntryZip(t, 5000), ExtractionLimits{})
+
+	before := filesystemEntries(t, workDir)
+	require.Len(t, before, 2, "the release has two things to remove whatever the archive held")
+
+	extracted.Cleanup()
+
+	_, err := os.Stat(overflow.TarPath)
+	assert.True(t, os.IsNotExist(err), "the one file holding every entry must be gone")
+	_, err = os.Stat(workDir)
+	assert.True(t, os.IsNotExist(err), "and so must the work directory")
+
+	// safe to call more than once, since the cataloger defers it past the sub-pipeline and past
+	// recursion
+	extracted.Cleanup()
+}
+
+func TestOverflow_anEntryAtTheEndIsReadBySeeking(t *testing.T) {
+	// "overflow content MUST remain directly readable without re-reading or re-decompressing what
+	// precedes it". A design that met the entry-count requirement by re-streaming the whole archive
+	// per read would not satisfy it, so what is asserted here is the mechanism: the index records
+	// where the last entry's content begins, near the end of a tar whose earlier entries are large,
+	// and the reader it hands back is a bounded window onto the file at that offset.
+	const filler = 64 * 1024
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for i := range 8 {
+		w, err := zw.Create(fmt.Sprintf("filler%d.bin", i))
+		require.NoError(t, err)
+		_, err = w.Write(bytes.Repeat([]byte{byte(i)}, filler))
+		require.NoError(t, err)
+	}
+	last, err := zw.Create("zz-last.txt")
+	require.NoError(t, err)
+	wanted := strings.Repeat("last", 64)
+	_, err = last.Write([]byte(wanted))
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+
+	_, overflow, _ := extractedOverflow(t, &buf, ExtractionLimits{})
+
+	info, err := os.Stat(overflow.TarPath)
+	require.NoError(t, err)
+	total := info.Size()
+	require.Greater(t, total, int64(8*filler), "the fixture must put real bulk ahead of the last entry")
+
+	// the last entry's content sits near the end of the tar, which is the only reason a seek to it is
+	// worth anything: everything before it is what a re-streaming design would have had to read
+	offset := int64(bytes.Index(readFile(t, overflow.TarPath), []byte(wanted)))
+	require.Positive(t, offset)
+	assert.Greater(t, offset, total*9/10, "the last entry must really be at the end of the tar")
+
+	entry := tarEntryNamed(t, overflow.TarPath, "zz-last.txt")
+
+	contents := entry.Open()
+	t.Cleanup(func() { require.NoError(t, contents.Close()) })
+
+	random, ok := contents.(interface {
+		io.ReaderAt
+		io.Seeker
+	})
+	require.True(t, ok, "the entry reader must be random access, or a read cannot be a seek")
+
+	// the reader is bounded to this entry rather than being a view of the whole tar: its end is the
+	// entry's own size, so reading it cannot have read what precedes it
+	end, err := random.Seek(0, io.SeekEnd)
+	require.NoError(t, err)
+	assert.Equal(t, int64(len(wanted)), end)
+
+	// and offset zero of the reader is the last entry's first byte, not the tar's. A reader over the
+	// whole file would answer this with the first filler entry's content.
+	head := make([]byte, 8)
+	_, err = random.ReadAt(head, 0)
+	require.NoError(t, err)
+	assert.Equal(t, wanted[:8], string(head))
+}
+
+func readFile(t *testing.T, path string) []byte {
+	t.Helper()
+	body, err := os.ReadFile(path)
+	require.NoError(t, err)
+	return body
+}
+
+// countTarEntries is how many entries a tar on disk holds, read through the same index the resolver
+// builds its filesystem from.
+func countTarEntries(t *testing.T, tarPath string) int {
+	t.Helper()
+	count := 0
+	_, err := stereoscopeFile.NewTarIndex(tarPath, func(stereoscopeFile.TarIndexEntry) error {
+		count++
+		return nil
+	})
+	require.NoError(t, err)
+	return count
+}
+
+// tarEntryNamed is the index record for one entry of a tar, which is what carries its seek offset.
+func tarEntryNamed(t *testing.T, tarPath, name string) stereoscopeFile.TarIndexEntry {
+	t.Helper()
+	var found *stereoscopeFile.TarIndexEntry
+	_, err := stereoscopeFile.NewTarIndex(tarPath, func(entry stereoscopeFile.TarIndexEntry) error {
+		if entry.ToTarFileEntry().Header.Name == name {
+			e := entry
+			found = &e
+		}
+		return nil
+	})
+	require.NoError(t, err)
+	require.NotNil(t, found, "no entry %q in %q", name, tarPath)
+	return *found
+}

internal/archive/path_guards_test.go

@@ -0,0 +1,176 @@
+package archive
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The path-safety guards used to be exercised through extraction, because extraction created a file
+// or a link per archive entry and every one of them went through them. An entry is a tar header now,
+// so those tests assert something else and these guards need their own.
+//
+// They are covered here rather than retired because their situations differ:
+//
+//   - resolvesInsideRoot, resolveLink and isWithin guard the one filesystem write left in this
+//     package - an archive's own bytes overflowing into its work directory, under a name derived from
+//     where the archive was found, which inside another archive is text the archive supplied.
+//   - writeSafeSymlink has no caller at all. A symlink entry is a header write, which creates
+//     nothing, and where the link points is decided inside the archive's own filetree. It is kept
+//     with its reasoning intact, and whether it retires is a judgement about every path that could
+//     write a link rather than about one call site.
+
+func TestResolvesInsideRoot_multiStepSymlinkChainIsRefused(t *testing.T) {
+	// the escape the guard exists for: two cooperating links each pass a lexical check and together
+	// resolve above the root. "d" -> "." lands on the root, so "d/up" -> ".." is created as
+	// "<root>/up" -> ".." and points above it, and a write through "d/up/x" lands outside. A lexical
+	// check sees <root>/d/../x and allows it; only resolving against the real filesystem catches it.
+	base := t.TempDir()
+	root := filepath.Join(base, "contents")
+	require.NoError(t, os.MkdirAll(root, 0o755))
+
+	require.NoError(t, os.Symlink(".", filepath.Join(root, "d")))
+	require.NoError(t, os.Symlink("..", filepath.Join(root, "up")))
+
+	inside, err := resolvesInsideRoot(root, filepath.Join(root, "d", "up", "PWNED"))
+	require.NoError(t, err)
+	assert.False(t, inside, "a write through a link chain that leaves the root must be refused")
+}
+
+func TestResolvesInsideRoot_legitimatePathsAreAllowed(t *testing.T) {
+	base := t.TempDir()
+	root := filepath.Join(base, "contents")
+	require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755))
+
+	for _, path := range []string{
+		filepath.Join(root, "top.txt"),
+		filepath.Join(root, "lib", "deep", "file.txt"),
+	} {
+		inside, err := resolvesInsideRoot(root, path)
+		require.NoError(t, err)
+		assert.True(t, inside, "%s is inside the root and must be allowed", path)
+	}
+}
+
+func TestResolvesInsideRoot_siblingSharingThePrefixIsRefused(t *testing.T) {
+	// the string-prefix hole: "contents-evil" shares "contents" as a prefix and is not inside it
+	base := t.TempDir()
+	root := filepath.Join(base, "contents")
+	require.NoError(t, os.MkdirAll(root, 0o755))
+	require.NoError(t, os.MkdirAll(filepath.Join(base, "contents-evil"), 0o755))
+
+	inside, err := resolvesInsideRoot(root, filepath.Join(base, "contents-evil", "x.txt"))
+	require.NoError(t, err)
+	assert.False(t, inside)
+}
+
+func TestWriteSafeSymlink(t *testing.T) {
+	// asserted with no production caller, deliberately: see the note at the top of this file. The
+	// reasoning in these cases is what makes the function worth keeping, so it is kept executable
+	// rather than left as prose about code nothing runs.
+	newRoot := func(t *testing.T) string {
+		t.Helper()
+		base := t.TempDir()
+		root := filepath.Join(base, "contents")
+		require.NoError(t, os.MkdirAll(filepath.Join(root, "bin"), 0o755))
+		require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(root, "lib", "real.txt"), []byte("real"), 0o600))
+		return root
+	}
+
+	t.Run("a relative target inside the root is written", func(t *testing.T) {
+		root := newRoot(t)
+		dest := filepath.Join(root, "bin", "link.txt")
+		require.NoError(t, writeSafeSymlink("../lib/real.txt", dest, root))
+
+		body, err := os.ReadFile(dest)
+		require.NoError(t, err)
+		assert.Equal(t, "real", string(body))
+	})
+
+	for _, tc := range []struct {
+		name   string
+		target string
+	}{
+		{name: "an empty target", target: ""},
+		{
+			// os.Symlink writes the literal string, so reading the link resolves on the host whatever
+			// was checked when it was created
+			name:   "an absolute target",
+			target: "/etc/passwd",
+		},
+		{name: "a target climbing out of the root", target: "../../../../etc/passwd"},
+	} {
+		t.Run(tc.name+" is refused", func(t *testing.T) {
+			root := newRoot(t)
+			dest := filepath.Join(root, "bin", "link.txt")
+			require.Error(t, writeSafeSymlink(tc.target, dest, root))
+
+			_, err := os.Lstat(dest)
+			assert.True(t, os.IsNotExist(err), "nothing may be created for a refused link")
+		})
+	}
+
+	t.Run("a target that resolves out of the root through an earlier link is refused", func(t *testing.T) {
+		// the target itself can run through a link an earlier entry left behind
+		root := newRoot(t)
+		base := filepath.Dir(root)
+		require.NoError(t, os.Symlink("..", filepath.Join(root, "up")))
+		require.NoError(t, os.WriteFile(filepath.Join(base, "PWNED"), []byte("x"), 0o600))
+
+		dest := filepath.Join(root, "bin", "link.txt")
+		require.Error(t, writeSafeSymlink("../up/PWNED", dest, root))
+
+		_, err := os.Lstat(dest)
+		assert.True(t, os.IsNotExist(err))
+	})
+
+	t.Run("KNOWN LIMIT: a target whose own path runs out of the root is not caught by either guard", func(t *testing.T) {
+		// Recorded as a finding rather than a passing assertion of safety, because it is one.
+		//
+		// Three entries: "up" -> "..", then "bin/link.txt" -> "../up/PWNED", then a write to
+		// "bin/link.txt/through". writeSafeSymlink computes <root>/up/PWNED, which is lexically inside
+		// the root, and its last check - does the target already resolve outside? - does not fire,
+		// because <root>/up/PWNED does not exist yet. So the link is created. resolvesInsideRoot then
+		// walks the write path, meets bin/link.txt, and resolves its target the same lexical way: the
+		// target's own path is not walked, so the fact that <root>/up is itself a link out of the root
+		// is never seen, and the write is allowed. On the real filesystem it lands above the root.
+		//
+		// It is not a live exposure. The only caller of resolvesInsideRoot writes one flat name into a
+		// work directory it just created, where no link exists and none can be planted, and
+		// writeSafeSymlink has no caller at all - a symlink entry is a header write now. It matters
+		// because it is what the retirement question actually hangs on: these two are not a working
+		// defence being kept in reserve, they are a defence with a hole in it and no exposure, and
+		// anything that gives them a caller again has to close this first.
+		root := newRoot(t)
+		require.NoError(t, os.Symlink("..", filepath.Join(root, "up")))
+
+		dest := filepath.Join(root, "bin", "link.txt")
+		require.NoError(t, writeSafeSymlink("../up/PWNED", dest, root),
+			"the link is created: its target does not resolve outside the root YET")
+
+		inside, err := resolvesInsideRoot(root, filepath.Join(root, "bin", "link.txt", "through"))
+		require.NoError(t, err)
+		assert.True(t, inside,
+			"the guard allows this write, and the real filesystem puts it above the root - the hole this "+
+				"case exists to record")
+	})
+}
+
+func TestArchiveFileName_refusesNamesThatWouldLeaveTheWorkDirectory(t *testing.T) {
+	// the name an archive's own bytes are overflowed under is the basename of where the archive was
+	// found, and Base returns ".." for an access path ending in it - which joined to the work
+	// directory would name the directory above. overflowContent checks its destination as well, so this
+	// is the earlier of two refusals rather than the only one.
+	for _, accessPath := range []string{"", ".", "..", "a/..", string(filepath.Separator)} {
+		assert.Equal(t, "archive", archiveFileName(accessPath),
+			"%q must not become a name joined to the work directory", accessPath)
+	}
+
+	// and a real name is preserved, compound extension included, so format detection has the best
+	// chance of identifying the archive
+	assert.Equal(t, "bundle.tar.gz", archiveFileName("some/path/bundle.tar.gz"))
+}

internal/archive/resolver.go

@@ -0,0 +1,412 @@
+package archive
+
+import (
+	"context"
+	"crypto"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	intFile "github.com/anchore/syft/internal/file"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/internal/tmpdir"
+	"github.com/anchore/syft/syft/file"
+)
+
+// ResolverFactory builds an indexed resolver over one archive's overflow tar. It is injected (rather
+// than calling syft/internal/fileresolver directly) so this module-root package does not depend on
+// the syft-subtree internal fileresolver package.
+//
+// It reports what indexing found as well as the resolver, because a tar-family archive is copied
+// wholesale and its entries are first counted where the tar is indexed.
+type ResolverFactory func(Overflow) (file.Resolver, IndexResult, error)
+
+// StoreResolverFactory builds a resolver over an archive's entries rather than over a tar of them.
+//
+// Where a ResolverFactory can only index content already written to a file - its index IS that file's
+// offsets - this one indexes the entries themselves, so an archive small enough to stay in memory
+// never reaches the disk and one that outgrows memory moves its bytes without the index changing.
+// Injected the same way and for the same reason: this package must not depend on the resolver
+// packages it hands work to.
+type StoreResolverFactory func(*EntryStore, Overflow) (file.Resolver, IndexResult, error)
+
+// Overflow names what one archive's entries were written into, and what the resolver built over them
+// must know.
+type Overflow struct {
+	// RootDir is the archive's logical root: an empty directory that every entry's path is reported
+	// relative to. It exists on disk, and stays empty - nothing is written into it - because the
+	// resolver relativizes an entry's path against a real root the way a directory scan does, and
+	// that is what keeps a path in the SBOM relative to its own archive rather than naming the
+	// scan's scratch space.
+	RootDir string
+
+	// TarPath is the one file holding every entry of this archive, indexed by seek offset.
+	TarPath string
+
+	// FileSystemID is the identifier of the filesystem the archive file itself lives on (a layer
+	// digest, or blank), inherited unchanged down the nesting chain, stamped onto every Location.
+	FileSystemID string
+
+	// ArchivePath is the colon-delimited chain of archive paths from the scan root to this archive,
+	// stamped onto every Location. It is what keeps identically-named files in different archives
+	// from colliding in the coordinate-keyed SBOM tables.
+	ArchivePath string
+}
+
+// IndexResult reports what indexing one archive's overflow tar produced.
+type IndexResult struct {
+	// Records is how many entries the archive's filesystem holds.
+	Records int
+
+	// Truncated reports that indexing stopped before the end of the tar, which now means only one
+	// thing: the tar itself ends part way through an entry because the disk limit stopped the write.
+	// What was indexed is complete and usable; there is simply less of it.
+	Truncated bool
+}
+
+// archiveDigestHashes are the digests taken of an archive file as a whole. SHA-1 alone, matching
+// what the java archive cataloger has always reported for a jar, because these are the digests it
+// reads when it can no longer open the archive itself.
+var archiveDigestHashes = []crypto.Hash{crypto.SHA1}
+
+// digestsOf takes the archive's own digests from the content already acquired. One pass over a
+// seekable source - memory or an overflow file - and it replaces a read the java cataloger used to
+// do for itself, so it is not new work so much as work moved to where the bytes already are.
+//
+// A failure is logged and returns nothing: a digest is metadata about a package, and an archive
+// whose contents cataloged fine should not be discarded because its hash could not be taken.
+func digestsOf(ctx context.Context, content Content, accessPath string) []file.Digest {
+	if _, err := content.Reader.Seek(0, io.SeekStart); err != nil {
+		log.WithFields("archive", accessPath, "error", err).Trace("unable to rewind archive content for digests")
+		return nil
+	}
+	digests, err := intFile.NewDigestsFromFile(ctx, io.NopCloser(content.Reader), archiveDigestHashes)
+	if err != nil {
+		log.WithFields("archive", accessPath, "error", err).Trace("unable to take archive digests")
+		return nil
+	}
+	return digests
+}
+
+// ExtractedArchive is a standalone, fully-indexed filesystem built from an extracted archive.
+type ExtractedArchive struct {
+	// Resolver is an indexed resolver over the extracted contents; its Locations are stamped with FileSystemID.
+	Resolver file.Resolver
+
+	// Digests are the digests of the archive file itself, taken while its bytes were in hand.
+	Digests []file.Digest
+	// FileSystemID is the identifier of the filesystem the archive file itself lives on (inherited unchanged down the nesting chain).
+	FileSystemID string
+	// ArchivePath is the colon-delimited chain of archive paths from the scan root to this archive.
+	ArchivePath string
+	// Result reports how much was extracted and whether a configured bound truncated the
+	// extraction. A truncated archive still yields a usable Resolver over whatever was written
+	// before the bound; see ExtractionResult.
+	Result ExtractionResult
+
+	// content is this archive's own bytes, held in memory or overflowed to disk. It is kept here rather
+	// than dropped after extraction for two reasons: content held in memory must stay reachable for
+	// as long as the memory limit accounts for it as held, otherwise the limiter measures something
+	// the garbage collector has already taken back; and openers over the extracted tree are heading
+	// towards reading lazily, which cannot work if the backing bytes are gone.
+	content Content
+
+	cleanup func()
+}
+
+// Cleanup removes the temp directory backing this archive's filesystem and releases what this
+// archive was holding against the scan's limiter, so both limits fall by exactly what it took. Safe
+// to call more than once.
+func (e *ExtractedArchive) Cleanup() {
+	if e == nil || e.cleanup == nil {
+		return
+	}
+	// close the resolver before the work directory is removed, so the entry store's overflow blob
+	// handle is released while the file it points at still exists. A resolver that holds nothing open
+	// - the tar path opens its file per read - does not implement io.Closer and is skipped.
+	if closer, ok := e.Resolver.(io.Closer); ok {
+		if err := closer.Close(); err != nil {
+			log.WithFields("fsid", e.FileSystemID, "error", err).Trace("unable to close archive resolver")
+		}
+	}
+	e.cleanup()
+	e.cleanup = nil
+	// drop the last reference to content held in memory, so the limit falling and the bytes becoming
+	// collectable happen together
+	e.content = Content{}
+}
+
+// ExtractToResolver holds the given archive content - in memory while the memory limit admits it, on
+// disk once it does not - extracts it, and returns a standalone indexed resolver over its contents
+// whose Locations are stamped with the given fileSystemID (the filesystem the archive file lives on)
+// and archivePath (the colon-delimited archive nesting chain from the scan root to this archive). It
+// returns (nil, nil) when the content is not a supported/extractable archive. The caller owns the
+// returned value and must call Cleanup on it.
+//
+// Everything this archive holds is charged to one handle against the scan's limiter and released by
+// Cleanup, which processArchive already defers past the sub-pipeline and past recursion - so the
+// limits fall exactly where the resources are freed, and nothing new has to remember to do it.
+//
+// Reaching a limit never waits for capacity. The walk descends before it unwinds, so a containing
+// archive is still holding its content while its children are cataloged and nothing will be released
+// while this archive waits: waiting would be a deadlock dressed as backpressure. Content that cannot
+// be admitted yields ErrDiskLimitReached and the caller skips the archive.
+//
+// Reaching a bound part way through is not an error either: the returned ExtractedArchive is indexed
+// over the partial contents and Result.Truncated() reports it, so the caller can still catalog what
+// was extracted. Any other non-nil error means nothing usable was produced.
+func ExtractToResolver(ctx context.Context, content io.Reader, archiveAccessPath, fileSystemID, archivePath string, extractors []Extractor, limiter *Limiter, limits ExtractionLimits, newResolver ResolverFactory, newStoreResolver StoreResolverFactory, notify Notify) (*ExtractedArchive, error) {
+	workDir, removeWorkDir, err := newWorkDir(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	charge := limiter.Charge()
+	limits.Charge = charge
+
+	var held Content
+	cleanup := func() {
+		if closeErr := held.Close(); closeErr != nil {
+			log.WithFields("archive", archiveAccessPath, "error", closeErr).Trace("unable to close archive content")
+		}
+		held = Content{}
+		removeWorkDir()
+		charge.Release()
+	}
+
+	held, err = acquireContent(content, workDir, archiveFileName(archiveAccessPath), charge, notify)
+	if err != nil {
+		cleanup()
+		if notify != nil && errors.Is(err, ErrDiskLimitReached) {
+			notify(Skipped{Archive: archiveAccessPath, Reason: "content would exceed the disk limit"})
+		}
+		// ErrDiskLimitReached travels unwrapped so the caller can attribute the skip to this archive
+		return nil, err
+	}
+
+	extractor := FindExtractor(ctx, extractors, held)
+	if extractor == nil {
+		// not an archive we can extract (e.g. an executable that happened to match a broad mime filter)
+		cleanup()
+		return nil, nil
+	}
+
+	resolver, result, err := overflowAndIndex(ctx, extractor, held, workDir, fileSystemID, archivePath, archiveAccessPath, limits, newResolver, newStoreResolver, notify)
+	if err != nil {
+		cleanup()
+		return nil, err
+	}
+
+	return &ExtractedArchive{
+		Resolver:     resolver,
+		Digests:      digestsOf(ctx, held, archiveAccessPath),
+		FileSystemID: fileSystemID,
+		ArchivePath:  archivePath,
+		Result:       result,
+		content:      held,
+		cleanup:      cleanup,
+	}, nil
+}
+
+// workDirName is the prefix given to the directory holding one archive's content and entries. Named
+// so a scan's leftovers are recognizable, and matched by the tests that sample what is on disk.
+const workDirName = "syft-archive"
+
+// newWorkDir creates the directory one archive's content and entries are written into, under the
+// scan's own temp root when the context carries one.
+//
+// The root is where every other cataloger that spills to disk already writes (see internal/tmpdir),
+// which is what makes a scan's temp files land where the caller configured them and get removed
+// together if an archive's own cleanup is ever missed. A context with no root - a caller reaching
+// this package directly - still gets a directory, from the same place it would have come from
+// before.
+func newWorkDir(ctx context.Context) (string, func(), error) {
+	if td := tmpdir.FromContext(ctx); td != nil {
+		dir, cleanup, err := td.NewChild(workDirName) //nolint:gocritic // cleanup is returned to the caller, not deferred here
+		if err != nil {
+			return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err)
+		}
+		return dir, cleanup, nil
+	}
+
+	dir, err := os.MkdirTemp("", workDirName+"-")
+	if err != nil {
+		return "", nil, fmt.Errorf("unable to create temp dir for archive extraction: %w", err)
+	}
+	return dir, func() {
+		if rmErr := os.RemoveAll(dir); rmErr != nil {
+			log.WithFields("dir", dir, "error", rmErr).Trace("unable to remove archive temp dir")
+		}
+	}, nil
+}
+
+// overflowAndIndex writes one archive's entries into the tar in workDir and builds the resolver over it.
+//
+// Errors travel unwrapped where the caller has to tell them apart: ErrDiskLimitReached attributes a
+// skip to this archive, and everything else is a failure of this archive alone.
+func overflowAndIndex(ctx context.Context, extractor Extractor, held Content, workDir, fsID, archivePath, accessPath string, limits ExtractionLimits, newResolver ResolverFactory, newStoreResolver StoreResolverFactory, notify Notify) (file.Resolver, ExtractionResult, error) {
+	if newStoreResolver != nil {
+		return storeAndIndex(ctx, extractor, held, workDir, fsID, archivePath, accessPath, limits, newStoreResolver, notify)
+	}
+	var result ExtractionResult
+
+	// the archive's logical root, which stays empty: entries live in the tar beside it, and this is
+	// only what their paths are reported relative to
+	contentsDir := filepath.Join(workDir, "contents")
+	if err := os.MkdirAll(contentsDir, 0o755); err != nil {
+		return nil, result, fmt.Errorf("unable to create contents dir for archive %q: %w", fsID, err)
+	}
+
+	overflow, err := newOverflowTar(filepath.Join(workDir, overflowTarName))
+	if err != nil {
+		return nil, result, err
+	}
+
+	result, err = extractor.Extract(ctx, held, overflow, limits)
+	if closeErr := overflow.Close(); closeErr != nil && err == nil {
+		err = closeErr
+	}
+	if err != nil {
+		return nil, result, fmt.Errorf("unable to extract archive %q: %w", fsID, err)
+	}
+
+	// nothing of this archive could be placed: the disk limit refused its first byte, so there is
+	// nowhere for its content to go and the archive is skipped rather than cataloged as empty. This is
+	// where the terminal reading of the disk limit lands now that a nested archive is read where it
+	// lies: its own bytes are never copied, so they are never the thing refused, and what is refused
+	// instead is the file its entries are written into. An archive that placed SOME of its entries is
+	// a truncation, not a skip - see ExtractionResult.
+	if result.Truncation == TruncatedByDiskLimit && result.BytesWritten == 0 {
+		return nil, result, ErrDiskLimitReached
+	}
+
+	resolver, indexed, err := newResolver(Overflow{
+		RootDir:      contentsDir,
+		TarPath:      overflow.Path(),
+		FileSystemID: fsID,
+		ArchivePath:  archivePath,
+	})
+	if err != nil {
+		return nil, result, fmt.Errorf("unable to index extracted archive %q: %w", fsID, err)
+	}
+
+	// the count of entries the archive's filesystem holds is the index's to report, not the writer's:
+	// a tar-family archive is copied wholesale and never walked, and an exclusion pattern drops
+	// entries after they were written
+	result.FilesExtracted = indexed.Records
+	if indexed.Truncated && !result.Truncated() {
+		// the tar ends part way through an entry and nothing recorded why. The disk limit is the only
+		// bound that can end a write early and it names itself, so reaching here means a ragged tar
+		// with no attributed cause - reported as a truncation without claiming which bound did it.
+		result.Truncation = TruncatedByLimit
+	}
+
+	return resolver, result, nil
+}
+
+// inPlaceContent is content that is already random-access on a file this scan wrote and is already
+// accounted for by the archive that wrote it.
+//
+// A nested archive arrives as a reader over one entry of its parent's overflow tar, and that reader is
+// Read, Seek AND ReadAt - which is everything an archive format needs, a zip's central directory
+// included. So it is read in place rather than copied into memory or overflowed again: the payoff for
+// choosing tar as the storage format, and the reason an inner zip needs no special handling.
+//
+// Declared as a marker method rather than by asserting on the reader's type, because the type belongs
+// to the resolver package that this one deliberately does not import. Nothing else in the process
+// answers to it, so nothing else is read in place by accident.
+type inPlaceContent interface {
+	io.Reader
+	io.ReaderAt
+	io.Seeker
+
+	// OverflowArchiveEntry marks a reader over one entry of an archive's overflow tar.
+	OverflowArchiveEntry()
+}
+
+// acquireContent gets one archive's bytes ready to be read at random, which is what every archive
+// format needs: it reads content already sitting in a file this scan wrote in place, and otherwise
+// routes the bytes through the limiter - held in memory while the memory limit admits them, overflowed
+// to disk once it does not.
+//
+// Content read in place is charged nothing, because nothing new is held: the bytes are part of the
+// parent archive's tar, which the parent already charged for and releases on its own Cleanup. Its
+// handle is the caller's to close, as it was before this ever reached here.
+func acquireContent(r io.Reader, workDir, name string, charge *Charge, notify Notify) (Content, error) {
+	if entry, ok := r.(inPlaceContent); ok {
+		return Content{Name: name, Reader: entry}, nil
+	}
+	return holdContent(r, workDir, name, charge, notify)
+}
+
+// archiveFileName derives a safe basename for the saved archive, preserving compound extensions
+// (e.g. ".tar.gz") so format detection has the best chance to identify the archive.
+//
+// ".." is refused along with the empty and root cases: Base returns it for an access path ending in
+// it, and joining it to the work directory would name the directory above. overflowContent checks its
+// destination as well, so this is the earlier of two refusals rather than the only one.
+func archiveFileName(accessPath string) string {
+	name := filepath.Base(accessPath)
+	if name == "" || name == "." || name == ".." || name == string(filepath.Separator) {
+		return "archive"
+	}
+	return name
+}
+
+// storeAndIndex is the entry-store path: the archive's entries go into an EntryStore, which holds
+// them in memory while the memory limit admits them, and the resolver is built over the entries.
+//
+// Compared to the tar path this writes nothing for an archive that fits in memory - no tar to encode,
+// no file to index - and where the archive does not fit, the content moves without the index being
+// rebuilt.
+func storeAndIndex(ctx context.Context, extractor Extractor, held Content, workDir, fsID, archivePath, accessPath string, limits ExtractionLimits, newStoreResolver StoreResolverFactory, notify Notify) (file.Resolver, ExtractionResult, error) {
+	var result ExtractionResult
+
+	// the archive's logical root, which stays empty: entries live in the store, and this is only what
+	// their paths are reported relative to
+	contentsDir := filepath.Join(workDir, "contents")
+	if err := os.MkdirAll(contentsDir, 0o755); err != nil {
+		return nil, result, fmt.Errorf("unable to create contents dir for archive %q: %w", fsID, err)
+	}
+
+	store := NewEntryStore(workDir, accessPath, notify)
+
+	result, err := extractor.Extract(ctx, held, store, limits)
+	if err != nil {
+		if closeErr := store.Close(); closeErr != nil {
+			log.WithFields("archive", fsID, "error", closeErr).Trace("unable to close archive entry store")
+		}
+		return nil, result, fmt.Errorf("unable to extract archive %q: %w", fsID, err)
+	}
+
+	// nothing of this archive could be placed: the disk limit refused its first byte, so the entries
+	// it holds are content-less husks and the archive is skipped rather than cataloged as empty. The
+	// same terminal reading of the disk limit the tar path takes, measured the way this path stores
+	// its content - an archive that placed SOME of its entries, in memory or on disk, is a truncation.
+	if result.Truncation == TruncatedByDiskLimit && store.OnDisk() == 0 {
+		if closeErr := store.Close(); closeErr != nil {
+			log.WithFields("archive", fsID, "error", closeErr).Trace("unable to close archive entry store")
+		}
+		return nil, result, ErrDiskLimitReached
+	}
+
+	resolver, indexed, err := newStoreResolver(store, Overflow{
+		RootDir:      contentsDir,
+		FileSystemID: fsID,
+		ArchivePath:  archivePath,
+	})
+	if err != nil {
+		if closeErr := store.Close(); closeErr != nil {
+			log.WithFields("archive", fsID, "error", closeErr).Trace("unable to close archive entry store")
+		}
+		return nil, result, fmt.Errorf("unable to index extracted archive %q: %w", fsID, err)
+	}
+
+	result.FilesExtracted = indexed.Records
+	if notify != nil && result.Truncated() {
+		notify(Truncated{Archive: accessPath, Reason: result.Truncation})
+	}
+	return resolver, result, nil
+}

internal/archive/resolver_test.go

@@ -0,0 +1,145 @@
+package archive
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/internal/tmpdir"
+	"github.com/anchore/syft/syft/file"
+)
+
+func TestExtractToResolver_writesUnderTheScansTempRoot(t *testing.T) {
+	// where an archive's scratch space lands is the scan's decision, not this package's: the root on
+	// the context is what a caller configured and what gets cleaned up if an archive's own cleanup is
+	// ever missed. Reaching for os.MkdirTemp instead put archive work outside all of that.
+	root := t.TempDir()
+	ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))
+
+	var got Overflow
+	factory := func(overflow Overflow) (file.Resolver, IndexResult, error) {
+		got = overflow
+		return nil, IndexResult{Records: 1}, nil
+	}
+
+	extracted, err := ExtractToResolver(
+		ctx, newTestZip(t, map[string]string{"hello.txt": "hi"}), "app.zip", "", "app.zip",
+		DefaultExtractors(), nil, ExtractionLimits{}, factory, nil, nil,
+	)
+	require.NoError(t, err)
+	require.NotNil(t, extracted)
+	t.Cleanup(extracted.Cleanup)
+
+	assert.True(t, strings.HasPrefix(got.TarPath, root+string(os.PathSeparator)),
+		"the archive's entries must be written under the scan's temp root, got %q", got.TarPath)
+
+	// and the archive still cleans up after itself, so the root is a safety net rather than the only
+	// thing reclaiming the space
+	extracted.Cleanup()
+	_, statErr := os.Stat(got.RootDir)
+	assert.True(t, os.IsNotExist(statErr), "expected the archive work directory to be removed by Cleanup")
+}
+
+func TestExtractToResolver(t *testing.T) {
+	content := newTestZip(t, map[string]string{"dir/hello.txt": "hello world"})
+
+	var got Overflow
+	factory := func(overflow Overflow) (file.Resolver, IndexResult, error) {
+		got = overflow
+		return nil, IndexResult{Records: 1}, nil
+	}
+
+	extracted, err := ExtractToResolver(
+		context.Background(), content, "some/path/app.zip", "parentFS", "app.war:some/path/app.zip",
+		DefaultExtractors(), nil, ExtractionLimits{}, factory, nil, nil,
+	)
+	require.NoError(t, err)
+	require.NotNil(t, extracted)
+	t.Cleanup(extracted.Cleanup)
+
+	// the filesystem the archive lives on is passed through unchanged; the nesting chain is carried
+	// separately as the archive path, and both are handed to the resolver factory
+	assert.Equal(t, "parentFS", extracted.FileSystemID)
+	assert.Equal(t, "parentFS", got.FileSystemID)
+	assert.Equal(t, "app.war:some/path/app.zip", extracted.ArchivePath)
+	assert.Equal(t, "app.war:some/path/app.zip", got.ArchivePath)
+
+	// the archive's entries are in the one tar handed to the factory, and its logical root is an
+	// empty directory beside it: an entry's path is reported relative to that root, and nothing is
+	// written into it
+	entries, err := os.ReadDir(got.RootDir)
+	require.NoError(t, err)
+	assert.Empty(t, entries, "the archive root holds no files: the entries are in the tar")
+
+	body := readTarEntry(t, got.TarPath, "dir/hello.txt")
+	assert.Equal(t, "hello world", body)
+
+	// the count of entries this archive contributes is the index's answer, not the extractor's
+	assert.Equal(t, 1, extracted.Result.FilesExtracted)
+
+	// Cleanup removes the temp tree
+	extracted.Cleanup()
+	_, statErr := os.Stat(got.TarPath)
+	assert.True(t, os.IsNotExist(statErr), "expected the overflow tar to be removed by Cleanup")
+	_, statErr = os.Stat(got.RootDir)
+	assert.True(t, os.IsNotExist(statErr), "expected temp dir to be removed by Cleanup")
+}
+
+// readTarEntry returns the content of one entry of a tar on disk.
+func readTarEntry(t *testing.T, tarPath, name string) string {
+	t.Helper()
+	f, err := os.Open(tarPath)
+	require.NoError(t, err)
+	defer f.Close()
+
+	tr := tar.NewReader(f)
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		require.NoError(t, err)
+		if hdr.Name != name {
+			continue
+		}
+		body, err := io.ReadAll(tr)
+		require.NoError(t, err)
+		return string(body)
+	}
+	t.Fatalf("no entry %q in %q", name, tarPath)
+	return ""
+}
+
+func TestExtractToResolver_notAnArchive(t *testing.T) {
+	factory := func(Overflow) (file.Resolver, IndexResult, error) { return nil, IndexResult{}, nil }
+
+	extracted, err := ExtractToResolver(
+		context.Background(), strings.NewReader("this is not an archive"), "notes.txt", "", "notes.txt",
+		DefaultExtractors(), nil, ExtractionLimits{}, factory, nil, nil,
+	)
+	require.NoError(t, err)
+	assert.Nil(t, extracted, "non-archive content should yield a nil ExtractedArchive")
+}
+
+func newTestZip(t *testing.T, files map[string]string) *bytes.Buffer {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for name, body := range files {
+		w, err := zw.Create(name)
+		require.NoError(t, err)
+		_, err = w.Write([]byte(body))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return &buf
+}

internal/archive/traversal.go

@@ -0,0 +1,85 @@
+package archive
+
+import (
+	"context"
+	"strings"
+
+	"github.com/anchore/syft/syft/file"
+)
+
+// Traversal describes one level of archive nesting during recursive archive cataloging. The archive
+// cataloger task places one on the context before running the cataloger sub-pipeline against an
+// archive's contents, so ecosystem catalogers (e.g. java) can reconstruct nesting-aware identity
+// such as colon-delimited virtual paths (e.g. "app.war:WEB-INF/lib/dep.jar") without owning the
+// recursion themselves.
+type Traversal struct {
+	// Location is the archive file's location within its parent filesystem.
+	Location file.Location
+
+	// VirtualPath is the colon-delimited chain of archive paths from the scan root to this archive,
+	// e.g. "app.war:WEB-INF/lib/dep.jar".
+	VirtualPath string
+
+	// FileSystemID is the identifier of the filesystem the archive file itself lives on (a container
+	// image layer digest, or empty for a directory source), inherited unchanged down the nesting
+	// chain. The nesting chain is carried by VirtualPath, not by this field.
+	FileSystemID string
+
+	// Depth is 1 for an archive found directly in the scan source, incrementing per nesting level.
+	Depth int
+
+	// Digests are the digests of the archive file itself. They are carried here because they are a
+	// property of the file and not of its contents: a cataloger running against an archive's extracted
+	// filesystem cannot compute them, since the archive is not inside itself. Taken once by the
+	// extraction, which already has the bytes in hand.
+	Digests []file.Digest
+
+	// Parent is the traversal for the containing archive; nil at depth 1.
+	Parent *Traversal
+}
+
+// VirtualPathOf returns the colon-delimited virtual path for an entry within this archive. It is
+// nil-safe: with no traversal the entry path is returned unchanged, matching non-nested behavior.
+// The entry path's leading slash is trimmed so chains match the historical java cataloger format,
+// which joins slash-less zip entry names onto the containing archive's path.
+func (t *Traversal) VirtualPathOf(entryPath string) string {
+	if t == nil {
+		return entryPath
+	}
+	entryPath = strings.ReplaceAll(entryPath, ":", "%3A")
+	if t.VirtualPath == "" {
+		return entryPath
+	}
+	return t.VirtualPath + ":" + strings.TrimPrefix(entryPath, "/")
+}
+
+type limiterCtxKey struct{}
+
+// WithLimiter returns a context carrying the scan's archive limiter, so a cataloger running within
+// the recursive walk can read the scan's live draw on the memory and disk budgets (Limiter.InUse) -
+// the same totals the limits are enforced against. Carried on the context rather than copied onto
+// each Traversal so there is one source of truth for the charge, not a snapshot per level.
+func WithLimiter(ctx context.Context, l *Limiter) context.Context {
+	return context.WithValue(ctx, limiterCtxKey{}, l)
+}
+
+// LimiterFromContext returns the scan's archive limiter, or nil when the current pass is not running
+// within the recursive archive walk.
+func LimiterFromContext(ctx context.Context) *Limiter {
+	l, _ := ctx.Value(limiterCtxKey{}).(*Limiter)
+	return l
+}
+
+type traversalCtxKey struct{}
+
+// WithTraversal returns a context carrying the given archive traversal.
+func WithTraversal(ctx context.Context, t *Traversal) context.Context {
+	return context.WithValue(ctx, traversalCtxKey{}, t)
+}
+
+// TraversalFromContext returns the archive traversal on the context, or nil when the current
+// cataloging pass is not running within an archive.
+func TraversalFromContext(ctx context.Context) *Traversal {
+	t, _ := ctx.Value(traversalCtxKey{}).(*Traversal)
+	return t
+}

internal/archive/traversal_test.go

@@ -0,0 +1,197 @@
+package archive
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/syft/file"
+)
+
+func TestTraversalContextRoundTrip(t *testing.T) {
+	assert.Nil(t, TraversalFromContext(context.Background()))
+
+	trav := &Traversal{
+		Location:     file.NewLocation("app.war"),
+		VirtualPath:  "app.war",
+		FileSystemID: "",
+		Depth:        1,
+	}
+	ctx := WithTraversal(context.Background(), trav)
+	require.Same(t, trav, TraversalFromContext(ctx))
+
+	child := &Traversal{
+		Location:     file.NewLocation("/WEB-INF/lib/dep.jar"),
+		VirtualPath:  trav.VirtualPathOf("/WEB-INF/lib/dep.jar"),
+		FileSystemID: "",
+		Depth:        2,
+		Parent:       trav,
+	}
+	ctx = WithTraversal(ctx, child)
+	require.Same(t, child, TraversalFromContext(ctx))
+	assert.Same(t, trav, TraversalFromContext(ctx).Parent)
+}
+
+func TestTraversal_VirtualPathOf(t *testing.T) {
+	tests := []struct {
+		name      string
+		traversal *Traversal
+		entryPath string
+		expected  string
+	}{
+		{
+			name:      "nil traversal returns entry path unchanged",
+			traversal: nil,
+			entryPath: "/WEB-INF/lib/dep.jar",
+			expected:  "/WEB-INF/lib/dep.jar",
+		},
+		{
+			name:      "nil traversal preserves relative path",
+			traversal: nil,
+			entryPath: "some/dir/example.jar",
+			expected:  "some/dir/example.jar",
+		},
+		{
+			name:      "single level trims leading slash",
+			traversal: &Traversal{VirtualPath: "app.war"},
+			entryPath: "/WEB-INF/lib/dep.jar",
+			expected:  "app.war:WEB-INF/lib/dep.jar",
+		},
+		{
+			name:      "single level with slash-less entry",
+			traversal: &Traversal{VirtualPath: "app.war"},
+			entryPath: "WEB-INF/lib/dep.jar",
+			expected:  "app.war:WEB-INF/lib/dep.jar",
+		},
+		{
+			name:      "multi-level chain",
+			traversal: &Traversal{VirtualPath: "app.war:WEB-INF/lib/dep.jar"},
+			entryPath: "/inner/most.jar",
+			expected:  "app.war:WEB-INF/lib/dep.jar:inner/most.jar",
+		},
+		{
+			// a colon in the entry name itself must not be mistaken for a chain delimiter -
+			// see decisions.md#colon-in-an-entry-path-is-escaped
+			name:      "colon in entry path is escaped",
+			traversal: &Traversal{VirtualPath: "app.war"},
+			entryPath: "WEB-INF/lib/dep:special.jar",
+			expected:  "app.war:WEB-INF/lib/dep%3Aspecial.jar",
+		},
+		{
+			name:      "every colon in the entry path is escaped",
+			traversal: &Traversal{VirtualPath: "app.war"},
+			entryPath: "lib/a:b:c.jar",
+			expected:  "app.war:lib/a%3Ab%3Ac.jar",
+		},
+		{
+			// a traversal carrying no virtual path (as distinct from a nil traversal) still
+			// yields the entry path alone, never a bare leading colon
+			name:      "empty virtual path yields the entry path alone, no leading colon",
+			traversal: &Traversal{VirtualPath: ""},
+			entryPath: "WEB-INF/lib/dep.jar",
+			expected:  "WEB-INF/lib/dep.jar",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.expected, tt.traversal.VirtualPathOf(tt.entryPath))
+		})
+	}
+}
+
+// TestTraversal_VirtualPathOf_noColonIsByteIdenticalToPreEscaping is the assertion that carries
+// the compatibility guarantee behind java-virtual-path-identity-preserved
+// (decisions.md#colon-in-an-entry-path-is-escaped): escaping a colon in an entry path must not
+// alter the result for any entry path that does not contain one. Every case below has no colon,
+// and every expected value is exactly what VirtualPathOf produced before escaping existed - do
+// not "fix" a failure here by updating the expected string, since that is precisely the
+// regression this test exists to catch.
+func TestTraversal_VirtualPathOf_noColonIsByteIdenticalToPreEscaping(t *testing.T) {
+	tests := []struct {
+		name      string
+		traversal *Traversal
+		entryPath string
+		expected  string
+	}{
+		{
+			name:      "nil traversal, absolute entry path",
+			traversal: nil,
+			entryPath: "/WEB-INF/lib/dep.jar",
+			expected:  "/WEB-INF/lib/dep.jar",
+		},
+		{
+			name:      "nil traversal, relative entry path",
+			traversal: nil,
+			entryPath: "some/dir/example.jar",
+			expected:  "some/dir/example.jar",
+		},
+		{
+			name:      "single level, leading slash trimmed",
+			traversal: &Traversal{VirtualPath: "app.war"},
+			entryPath: "/WEB-INF/lib/dep.jar",
+			expected:  "app.war:WEB-INF/lib/dep.jar",
+		},
+		{
+			name:      "single level, no leading slash",
+			traversal: &Traversal{VirtualPath: "app.war"},
+			entryPath: "WEB-INF/lib/dep.jar",
+			expected:  "app.war:WEB-INF/lib/dep.jar",
+		},
+		{
+			name:      "multi-level chain",
+			traversal: &Traversal{VirtualPath: "app.war:WEB-INF/lib/dep.jar"},
+			entryPath: "/inner/most.jar",
+			expected:  "app.war:WEB-INF/lib/dep.jar:inner/most.jar",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.expected, tt.traversal.VirtualPathOf(tt.entryPath))
+		})
+	}
+}
+
+// TestTraversal_VirtualPathOf_escapingComposesAcrossNestingLevels asserts that escaping is
+// applied at every level of a chain, not only the first: a colon in an entry name at a deeper
+// nesting level is just as ambiguous as one at the outermost level, so it gets the same
+// treatment. After composing, splitting the result on ':' must yield exactly one segment per
+// real archive boundary - none of the escaped colons survive as a literal ':' byte, so they
+// cannot be mistaken for delimiters by a naive splitter.
+func TestTraversal_VirtualPathOf_theEscapeCharacterIsNotItselfEscaped(t *testing.T) {
+	// a known and accepted limit, not a defect: an entry path holding the literal text %3A is
+	// indistinguishable from one holding a colon. Escaping % as %25 would close it and is
+	// deliberately not done, because % is far more common in real paths than : and closing this
+	// would move a large number of well-defined identities to resolve a collision between two
+	// inputs both already assumed absent. Asserted so the limit is a decision on the record
+	// rather than something a later reader discovers and "fixes".
+	t.Parallel()
+
+	outer := &Traversal{VirtualPath: "outer.zip"}
+
+	fromColon := outer.VirtualPathOf("a:b.jar")
+	fromLiteral := outer.VirtualPathOf("a%3Ab.jar")
+
+	assert.Equal(t, "outer.zip:a%3Ab.jar", fromColon)
+	assert.Equal(t, "outer.zip:a%3Ab.jar", fromLiteral)
+	assert.Equal(t, fromColon, fromLiteral,
+		"the collision is accepted; if this ever differs, the escape character started being escaped")
+}
+
+func TestTraversal_VirtualPathOf_escapingComposesAcrossNestingLevels(t *testing.T) {
+	level1 := &Traversal{VirtualPath: "outer.zip"}
+	level2VirtualPath := level1.VirtualPathOf("lib/mid:dle.war")
+	require.Equal(t, "outer.zip:lib/mid%3Adle.war", level2VirtualPath)
+
+	level2 := &Traversal{VirtualPath: level2VirtualPath}
+	final := level2.VirtualPathOf("WEB-INF/lib/inn:er.jar")
+	require.Equal(t, "outer.zip:lib/mid%3Adle.war:WEB-INF/lib/inn%3Aer.jar", final)
+
+	assert.Equal(t,
+		[]string{"outer.zip", "lib/mid%3Adle.war", "WEB-INF/lib/inn%3Aer.jar"},
+		strings.Split(final, ":"),
+		"splitting on ':' must recover exactly the three real archive boundaries",
+	)
+}