Add nested archive cataloging (#5260)

Signed-off-by: Kudryavcev Nikolay <kydry.nikolau@gmail.com> Signed-off-by: Thomas Bechtold <thomasbechtold@jpberlin.de> Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> Signed-off-by: Keith Zantow <kzantow@gmail.com> Co-authored-by: Kudryavcev Nikolay <kydry.nikolau@gmail.com> Co-authored-by: Thomas Bechtold <thomasbechtold@jpberlin.de> Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>
2026-11-03 07:56:36 +01:00 · 2026-10-29 17:33:07 -04:00
parent 14a0beb93c
commit 4b8a3db5f3
224 changed files with 15918 additions and 1281 deletions
@@ -0,0 +1,263 @@
+package archive
+
+import (
+	"bytes"
+	"context"
+	"crypto"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+
+	"github.com/mholt/archives"
+
+	"github.com/anchore/syft/internal"
+	intFile "github.com/anchore/syft/internal/file"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/file"
+)
+
+// ErrUnsupportedFormat is returned for an archive in a format that is not extracted (see supportedExtractor).
+var ErrUnsupportedFormat = errors.New("archive format is not supported")
+
+// Extract opens the archive read from r as its own filesystem, holding its bytes and entries within
+// the limiter's bounds. It returns nil when the content is not an archive, ErrUnsupportedFormat when it
+// is one that is not extracted, and ErrDiskLimitReached when the archive's own bytes cannot be placed
+// within the disk limit. The caller must Cleanup the result.
+//
+// archiveCoordinates are those of the archive file itself; every location the resolver returns
+// extends them with its entry path (see file.Coordinates).
+func Extract(ctx context.Context, r io.Reader, archiveCoordinates file.Coordinates, limiter *Limiter, exclusions Exclusions) (*Resolver, error) {
+	charge := limiter.charge()
+	resolver := newResolver(ctx, archiveCoordinates, charge)
+	archivePath := archiveCoordinates.FullNestedPath()
+
+	// deferred so that a panic in a decoder still gives back the charge and removes any spill file
+	succeeded := false
+	defer func() {
+		if !succeeded {
+			resolver.Cleanup()
+		}
+	}()
+
+	content, release, err := resolver.acquire(r)
+	if err != nil {
+		return nil, err
+	}
+	// the archive's own bytes are needed only until they are extracted and digested
+	defer release()
+
+	format, err := identifyFormat(ctx, path.Base(archiveCoordinates.LocalPath()), content)
+	if format == nil || err != nil {
+		return nil, err
+	}
+
+	if err := resolver.extract(ctx, format, content, exclusions); err != nil {
+		return nil, fmt.Errorf("unable to extract archive %q: %w", archivePath, err)
+	}
+	resolver.Digests = digestsOf(ctx, content, archivePath)
+	succeeded = true
+	return resolver, nil
+}
+
+// acquire makes the archive's own bytes random-access, which identifying, extracting and digesting
+// them needs. Bytes that already are, such as a file on disk or an entry of a parent archive, are
+// read in place and cost nothing. Anything else is held within the limits like an entry, and the
+// returned release gives back what memory it takes; bytes spilled to disk stay until Cleanup.
+// Reaching the disk limit yields ErrDiskLimitReached.
+func (r *Resolver) acquire(content io.Reader) (ReaderAtSeeker, func(), error) {
+	if ras, ok := content.(ReaderAtSeeker); ok {
+		return ras, func() {}, nil
+	}
+	var b blob
+	if err := r.put(&b, content); err != nil {
+		if errors.Is(err, ErrDiskLimitReached) {
+			return nil, nil, err
+		}
+		return nil, nil, fmt.Errorf("unable to read archive content: %w", err)
+	}
+	return r.open(&b), func() { r.discard(&b) }, nil
+}
+
+// identifyFormat returns the format able to extract this content, or nil when it is not an archive.
+//
+// Identification from the head of the stream wins: a tar holding one jar ends in that jar's central
+// directory, and reading the tar as a zip would lose the rest. The end of the stream is consulted
+// only when the head identifies nothing supported, which is how a zip appended to a launcher script (a
+// Spring Boot executable jar) is found, and how a java resource adapter (a zip named .rar) is read.
+func identifyFormat(ctx context.Context, name string, content ReaderAtSeeker) (archives.Extractor, error) {
+	if _, err := content.Seek(0, io.SeekStart); err != nil {
+		return nil, nil
+	}
+	format, _, err := intFile.IdentifyArchive(ctx, name, content)
+	if err == nil {
+		if extractor := supportedExtractor(format); extractor != nil {
+			return extractor, nil
+		}
+	}
+	if HasZipEndOfCentralDirectory(content) {
+		return archives.Zip{}, nil
+	}
+	if err == nil && holdsFiles(format) {
+		return nil, fmt.Errorf("%w: %s", ErrUnsupportedFormat, format.MediaType())
+	}
+	return nil, nil
+}
+
+// holdsFiles reports whether the format is an archive of files rather than a bare compression format,
+// such as a gzipped man page, which holds one stream and is not cataloged as an archive.
+func holdsFiles(format archives.Format) bool {
+	switch f := format.(type) {
+	case archives.CompressedArchive:
+		return f.Extraction != nil
+	case archives.Extractor:
+		return true
+	}
+	return false
+}
+
+// supportedExtractor admits only the zip and tar families, the tar family optionally compressed. Other
+// formats mholt identifies, such as 7z and rar, pull in decoders that have not been exercised against
+// untrusted input here, so they are not opened until they are deliberately supported. A bare
+// compression format, such as a gzipped file that is not a tar, is not extractable either.
+func supportedExtractor(format archives.Format) archives.Extractor {
+	switch f := format.(type) {
+	case archives.Zip:
+		return f
+	case archives.Tar:
+		return f
+	case archives.CompressedArchive:
+		if _, ok := f.Extraction.(archives.Tar); ok {
+			return f
+		}
+	}
+	return nil
+}
+
+// extract adds every entry of the archive. Reaching the disk limit stops the walk and marks the
+// resolver truncated rather than failing.
+func (r *Resolver) extract(ctx context.Context, format archives.Extractor, content ReaderAtSeeker, exclusions Exclusions) error {
+	if _, err := content.Seek(0, io.SeekStart); err != nil {
+		return err
+	}
+	var source io.Reader = content
+	if ca, ok := format.(archives.CompressedArchive); ok && ca.Compression != nil {
+		// a compressed tar decompresses everything to reach each header, including entries skipped whole
+		// and whatever an entry leaves unread, so the budget is charged for every byte the decoder
+		// produces rather than for what is kept. A zip or a plain tar seeks past what it skips
+		rc, err := ca.OpenReader(content)
+		if err != nil {
+			return err
+		}
+		defer internal.CloseAndLogError(rc, "decompressor")
+		source, format, r.metered = &meteredReader{ctx: ctx, reader: rc, budget: r.budget}, ca.Extraction, true
+	}
+	err := format.Extract(ctx, source, func(_ context.Context, f archives.FileInfo) error {
+		hdr, ok := r.entryHeader(f)
+		if !ok || exclusions.Excludes(hdr.Name) {
+			return nil
+		}
+		if !f.Mode().IsRegular() {
+			return r.add(hdr, nil)
+		}
+		entry, err := f.Open()
+		if err != nil {
+			log.WithFields("entry", hdr.Name, "error", err).Trace("unable to open archive entry, skipping it")
+			return nil
+		}
+		defer func() {
+			if err := entry.Close(); err != nil {
+				log.WithFields("entry", hdr.Name, "error", err).Trace("unable to close archive entry")
+			}
+		}()
+		return r.add(hdr, entry)
+	})
+	r.finish()
+	if errors.Is(err, ErrDiskLimitReached) {
+		r.truncate(fmt.Errorf("extraction stopped: %w", err))
+		return nil
+	}
+	if errors.Is(err, errBudgetSpent) {
+		r.truncate(fmt.Errorf("extraction stopped, possible decompression bomb: %w", err))
+		return nil
+	}
+	if err != nil && ctx.Err() == nil && r.entries > 0 {
+		// the stream broke partway (a truncated .tar.gz); what was read before that is still good
+		r.truncate(fmt.Errorf("extraction stopped early: %w", err))
+		return nil
+	}
+	return err
+}
+
+// archiveDigestHashes are the digests taken of an archive as a whole: SHA-1, matching what the java
+// cataloger reports for a jar.
+var archiveDigestHashes = []crypto.Hash{crypto.SHA1}
+
+// digestsOf digests the archive's own bytes while they are in hand. A failure is logged and returns
+// nothing rather than discarding an archive that extracted fine.
+func digestsOf(ctx context.Context, content ReaderAtSeeker, archivePath string) []file.Digest {
+	if _, err := content.Seek(0, io.SeekStart); err != nil {
+		log.WithFields("archive", archivePath, "error", err).Trace("unable to rewind archive content for digests")
+		return nil
+	}
+	digests, err := intFile.NewDigestsFromFile(ctx, io.NopCloser(content), archiveDigestHashes)
+	if err != nil {
+		log.WithFields("archive", archivePath, "error", err).Trace("unable to digest archive")
+		return nil
+	}
+	return digests
+}
+
+var (
+	// zipLocalFileHeaderSignature starts every entry in a zip.
+	zipLocalFileHeaderSignature = []byte{'P', 'K', 0x03, 0x04}
+
+	// zipEOCDSignature starts the end-of-central-directory record that closes every zip. Readers
+	// locate the central directory from it, which is why a zip with data prepended still opens.
+	zipEOCDSignature = []byte{'P', 'K', 0x05, 0x06}
+)
+
+const (
+	// maxAppendedArchiveStubBytes is how far into a file an appended zip's first entry is looked for. A
+	// Spring Boot launcher stub is about 9 KB.
+	maxAppendedArchiveStubBytes = 64 * 1024
+
+	// maxZipEOCDSearch is how far from the end the end-of-central-directory record can start: the 22
+	// byte record plus a comment of up to 65535 bytes.
+	maxZipEOCDSearch = 22 + 65535
+)
+
+// MayHideAnAppendedArchive reports whether a zip entry signature appears in the head of the stream,
+// which a zip appended to a launcher stub does and a file that merely has an archive's extension
+// almost never does. The returned reader replays what was consumed.
+func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) {
+	head := make([]byte, maxAppendedArchiveStubBytes)
+	n, err := io.ReadFull(r, head)
+	if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+		return r, false
+	}
+	head = head[:n]
+	found := bytes.Contains(head, zipLocalFileHeaderSignature)
+
+	if seeker, ok := r.(io.Seeker); ok {
+		if _, err := seeker.Seek(0, io.SeekStart); err == nil {
+			return r, found
+		}
+	}
+	return io.MultiReader(bytes.NewReader(head), r), found
+}
+
+// HasZipEndOfCentralDirectory reports whether the content ends in a zip end-of-central-directory
+// record, the one part of a zip whose position is known without parsing it.
+func HasZipEndOfCentralDirectory(r ReaderAtSeeker) bool {
+	size, err := r.Seek(0, io.SeekEnd)
+	if err != nil || size < int64(len(zipEOCDSignature)) {
+		return false
+	}
+	window := min(size, int64(maxZipEOCDSearch))
+	buf := make([]byte, window)
+	if _, err := r.ReadAt(buf, size-window); err != nil && !errors.Is(err, io.EOF) {
+		return false
+	}
+	return bytes.Contains(buf, zipEOCDSignature)
+}