oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-02 23:46:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

chore: revert custom index in favor of key_Split

Browse Source 
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
Keith Zantow
2026-09-20 11:22:10 -04:00
parent 99615b7dfa
commit 53cd4254e4
81 changed files with 7461 additions and 6624 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/archive/extractor.go
+65 -237
View File
@@ -18,111 +18,56 @@ import (
"github.com/anchore/syft/syft/cataloging"
)
// Extractor extracts archive contents to a destination directory.
// Extractor reads the entries of one archive into an EntrySink.
type Extractor interface {
// CanExtract returns true if this extractor can handle the given archive content.
CanExtract(ctx context.Context, content Content) bool
// Extract writes the archive's entries into sink - one store per archive, not one file per entry -
// respecting the given limits. Returns how many entries were written and how many bytes that took.
Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error)
Extract(ctx context.Context, content Content, sink EntrySink, charge *Charge) (ExtractionResult, error)
}
// EntrySink is where an extractor puts the entries it reads out of an archive.
//
// Two implementations exist because they store an archive differently and the difference is
// measurable: OverflowTar writes every entry into one tar on disk and is indexed by seek offset,
// while EntryStore holds entries in memory until a bound says otherwise and is indexed by the entries
// themselves. The extractors do not care which they were given, with one exception - see
// VerbatimSink.
type EntrySink interface {
// AddEntry stores one entry of the archive.
AddEntry(f archives.FileInfo, result *ExtractionResult, limits ExtractionLimits) error
AddEntry(f archives.FileInfo, result *ExtractionResult, charge *Charge) error
// Finish is called once the last entry has been offered, for whatever a sink has to write after
// its content.
Finish(result *ExtractionResult, limits ExtractionLimits)
// Finish is called once the last entry has been offered.
Finish(result *ExtractionResult)
}
// VerbatimSink is an EntrySink that stores archives in tar form, so a tar-family archive can be
// copied into it as it stands rather than decoded and re-encoded entry by entry.
//
// This is the one place the storage format shows through the extractor interface, and it is worth the
// leak: a tar.gz copied verbatim is one streamed decompression, where walking it costs a read per
// entry. A sink that does not store tars does not implement this and gets the entry-by-entry path.
type VerbatimSink interface {
EntrySink
CopyVerbatim(r io.Reader, result *ExtractionResult, limits ExtractionLimits) error
}
// ExtractionLimits defines safety limits for extracting one archive.
type ExtractionLimits struct {
// Charge is this archive's draw on the scan's resource limiter. Entry bytes are charged to it as
// they land, never from the size the entry declared, and a refused charge truncates the
// extraction. A nil Charge enforces no byte bound.
//
// It is a handle rather than a remaining-bytes value because what it charges has to be given back
// when the archive is released, and a value copied into a call cannot be.
Charge *Charge
}
// TruncationReason names the configured bound that stopped an extraction early. The empty value
// means the archive was extracted in full.
// TruncationReason names the bound that stopped an extraction early; empty means extracted in full.
type TruncationReason string
const (
// TruncatedByDiskLimit means placing more of this archive's content would have taken the scan's
// disk usage past the disk limit. Named rather than reported as a bare truncation because a breach
// has to be attributed to the archive that triggered it, and saying which bound was reached is
// half of that attribution.
// TruncatedByDiskLimit: storing more content would exceed the scan's disk limit.
TruncatedByDiskLimit TruncationReason = "disk limit"
// TruncatedByLimit means a bound stopped the extraction but the walker replaced the error that
// said which one. Defensive: both bounds record their own reason on the result before stopping.
// TruncatedByIndexLimit: indexing more entries would exceed the record budget. Distinct from the
// disk limit because the index is never written to disk, though the disk budget backstops it - see
// Charge.IndexRecord.
TruncatedByIndexLimit TruncationReason = "index limit"
// TruncatedByLimit: a bound stopped extraction but the walker replaced the error naming which one.
TruncatedByLimit TruncationReason = "configured limit"
)
// errTruncated stops an in-progress archive walk when a configured limit is reached. It never
// escapes this package: the extractors convert it into a successful, truncated ExtractionResult.
// errTruncated stops an in-progress walk at a configured limit. It never escapes this package:
// extractors convert it into a successful, truncated ExtractionResult.
var errTruncated = errors.New("archive extraction stopped at a configured limit")
// ExtractionResult holds the result of an extraction operation.
//
// Reaching a configured limit is NOT an error: it is a truncation. Whatever was written before the
// limit is left on disk and is usable, so callers can still catalog what was extracted rather than
// discarding the whole archive. Check Truncated to tell a complete extraction from a partial one;
// a non-nil error from Extract means the extraction failed and nothing usable was produced.
// ExtractionResult holds the result of an extraction. Reaching a limit is a truncation, not an error:
// content stored before the limit stays usable. A non-nil error from Extract means nothing usable was
// produced.
type ExtractionResult struct {
// FilesExtracted counts the entries this archive's filesystem holds - the records its overflow tar
// was indexed into: regular files, symlinks, and directory entries. It is the count after indexing
// rather than the count written, so an entry dropped by an exclusion pattern is not counted as
// present. Reported, not bounded.
FilesExtracted int
// BytesWritten is what this archive put on disk: every byte of the overflow tar, headers and block
// padding included, because that is what the disk limit was charged for.
BytesWritten int64
// Truncation names the limit that stopped extraction early, or "" when the archive was
// extracted in full.
// Truncation names the limit that stopped extraction early, or "" if none did.
Truncation TruncationReason
}
// Truncated reports whether a configured limit stopped this extraction before the end of the
// archive.
func (r ExtractionResult) Truncated() bool {
return r.Truncation != ""
}
// finishExtraction turns a walk that stopped at a configured limit into a successful, truncated
// result.
//
// The limit signal is carried on the result rather than only in the returned error because the
// underlying archive walkers are free to wrap or replace a handler's error, in which case
// errors.Is(err, errTruncated) would not match and a truncation would be reported as a failure -
// discarding an archive syft could have cataloged. The error is still checked, so a walker that
// passes it through cleanly is handled too.
// finishExtraction turns a walk stopped at a configured limit into a successful, truncated result.
// Both result and error are checked: walkers may wrap or replace the handler's error, so
// errors.Is(err, errTruncated) alone would report some truncations as failures.
func finishExtraction(result ExtractionResult, err error) (ExtractionResult, error) {
if result.Truncated() || errors.Is(err, errTruncated) {
if !result.Truncated() {
@@ -134,16 +79,6 @@ func finishExtraction(result ExtractionResult, err error) (ExtractionResult, err
return result, err
}
// DefaultExtractionLimits returns the per-archive limits from the given config.
//
// Nothing is read from the config: every bound this scan enforces is an in-use limit held by the
// scan's limiter, and the only per-archive field left is the Charge the caller draws against it.
// The parameter stays so the two Default* constructors are called the same way, and so a per-archive
// bound that is genuinely per-archive has somewhere to land.
func DefaultExtractionLimits(_ cataloging.ArchiveSearchConfig) ExtractionLimits {
return ExtractionLimits{}
}
// DefaultLimits returns the scan-wide in-use limits from the given config.
func DefaultLimits(cfg cataloging.ArchiveSearchConfig) Limits {
return Limits{
@@ -178,27 +113,16 @@ func FindExtractor(ctx context.Context, extractors []Extractor, content Content)
// ZipExtractor extracts zip-based archives (zip, jar, war, ear, etc.).
type ZipExtractor struct{}
// CanExtract reports whether this content is a zip, asking two questions in an order that matters.
// CanExtract reports whether this content is a zip. Test order matters: head identification wins,
// because a tar holding one jar ends in that jar's EOCD record, so reading it as a zip would lose the
// rest of the tar. The EOCD record is consulted only when nothing identifies from the head, where it
// is the only test that sees a zip concatenated onto a launcher stub (e.g. a Spring Boot executable
// jar).
//
// First, what does the content say it is, reading from the start? A format that identifies itself at
// offset zero is what this content is, and if that format is not zip then it is not a zip - a tar
// holding one jar ends in that jar's end-of-central-directory record, and reading it as a zip
// instead of as a tar would lose everything else the tar holds.
//
// Only when nothing identifies from the head is the end-of-central-directory record consulted. That
// record is what defines a zip - a reader locates the central directory through it - and it is the
// only thing that can see an archive whose head is something else entirely. A self-extracting
// archive is a launcher stub with a zip concatenated onto it, so no magic at offset zero can find
// it; the canonical case is a Spring Boot executable jar, a shell script followed by a whole zip.
//
// Identification is intFile.IdentifyArchive, which matches on the filename as well as the content,
// and it is left that way. Dropping the name makes the answer content's alone, which reads better -
// but archives.Identify with no path has to try every format's content matcher against every
// candidate, and measured on a stock ruby image that took the scan from 17 seconds to 45. The name
// cannot admit a non-archive in practice: a file only reaches an extractor after its sniffed content
// type put it in the candidate set, so the prose named notes.zip that this would wrongly identify is
// never offered here at all. `#content-based-archive-detection` is upheld by the candidate set, and
// Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError is where that is asserted.
// IdentifyArchive matches on filename as well as content; dropping the name would run every format's
// content matcher against every candidate, far slower. The name cannot admit a non-archive, since
// files reach an extractor only after content sniffing (see
// Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError).
func (z *ZipExtractor) CanExtract(ctx context.Context, content Content) bool {
format, _, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader)
if err == nil {
@@ -212,21 +136,18 @@ func (z *ZipExtractor) CanExtract(ctx context.Context, content Content) bool {
return HasZipEndOfCentralDirectory(content.Reader)
}
func (z *ZipExtractor) Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error) {
func (z *ZipExtractor) Extract(ctx context.Context, content Content, sink EntrySink, charge *Charge) (ExtractionResult, error) {
var result ExtractionResult
if _, err := content.Reader.Seek(0, io.SeekStart); err != nil {
return result, fmt.Errorf("unable to seek zip archive %q: %w", content.Name, err)
}
// mholt/archives imposes no extraction target: the FileHandler is the caller's, so an entry can
// be written into a tar header rather than onto the filesystem. Nothing about the library had to
// be customized - the old code wrote files because the handler wrote files.
err := archives.Zip{}.Extract(ctx, content.Reader, func(_ context.Context, file archives.FileInfo) error {
return sink.AddEntry(file, &result, limits)
return sink.AddEntry(file, &result, charge)
})
sink.Finish(&result, limits)
sink.Finish(&result)
return finishExtraction(result, err)
}
@@ -234,20 +155,21 @@ func (z *ZipExtractor) Extract(ctx context.Context, content Content, sink EntryS
// TarExtractor extracts tar-based archives (tar, tar.gz, tar.bz2, tar.xz, tar.zst).
type TarExtractor struct{}
// CanExtract reports whether the content identifies as an extractable non-zip format.
func (t *TarExtractor) CanExtract(ctx context.Context, content Content) bool {
format, _, err := intFile.IdentifyArchive(ctx, content.Name, content.Reader)
if err != nil {
return false
}
// mholt/archives returns a compound type for tar+compression, check if it's an extractor but not a zip
if _, isZip := format.(archives.Zip); isZip {
return false
}
// tar+compression comes back as a compound type, so test for the Extractor interface
_, ok := format.(archives.Extractor)
return ok
}
func (t *TarExtractor) Extract(ctx context.Context, content Content, sink EntrySink, limits ExtractionLimits) (ExtractionResult, error) {
func (t *TarExtractor) Extract(ctx context.Context, content Content, sink EntrySink, charge *Charge) (ExtractionResult, error) {
var result ExtractionResult
if _, err := content.Reader.Seek(0, io.SeekStart); err != nil {
@@ -270,47 +192,24 @@ func (t *TarExtractor) Extract(ctx context.Context, content Content, sink EntryS
extractReader = content.Reader
}
// a tar handed to a sink that stores tars is not walked at all: a compressed tar is decompressed
// once into the sink's file and a plain one is copied, and the entries are first looked at when
// the file is indexed. Where the sink stores entries rather than a tar there is nothing to copy
// into, so the tar is walked like any other archive.
if verbatim, isVerbatim := sink.(VerbatimSink); isVerbatim {
if plain, ok := tarStream(format, extractReader); ok {
err = verbatim.CopyVerbatim(plain, &result, limits)
if closeErr := plain.Close(); closeErr != nil {
log.WithFields("archive", content.Name, "error", closeErr).Trace("unable to close decompressed tar stream")
}
return finishExtraction(result, err)
}
}
// anything else - 7z, rar, or a tar going into a sink that does not store tars - is walked entry
// by entry
err = extractor.Extract(ctx, extractReader, func(_ context.Context, file archives.FileInfo) error {
return sink.AddEntry(file, &result, limits)
return sink.AddEntry(file, &result, charge)
})
sink.Finish(&result, limits)
sink.Finish(&result)
return finishExtraction(result, err)
}
// resolvesInsideRoot reports whether writing to path really lands inside root, following any symlink
// that already exists along the way the same way the filesystem will.
// resolvesInsideRoot reports whether writing to path lands inside root, following existing symlinks
// as the filesystem would. It guards the one filesystem write left in this package: an archive
// spilling into its work directory under an attacker-controlled name.
//
// The one filesystem write left in this package is the archive's own bytes overflowing into its work
// directory, under a name derived from where the archive was found - which inside another archive is
// attacker-controlled text. That is what this now guards; entries are tar headers and traverse no
// path at all. See overflowContent.
//
// It walks path one component at a time rather than calling filepath.EvalSymlinks on the whole
// thing, because the leaf of an entry being extracted does not exist yet, which makes EvalSymlinks
// fail rather than answer. The first component that does not exist ends the walk: nothing below it
// exists either, so whatever is created there is created fresh under a path already known to be
// inside the root.
// The walk is component by component rather than one EvalSymlinks call because the leaf does not yet
// exist, which makes EvalSymlinks fail rather than answer. The first missing component ends the walk:
// anything created below it lands under an already-checked path.
func resolvesInsideRoot(root, path string) (bool, error) {
// the root itself may sit under a symlinked directory (a temp dir on macOS, for one), so
// containment is judged against its real location
// the root itself may sit under a symlinked directory (e.g. a macOS temp dir)
realRoot, err := filepath.EvalSymlinks(root)
if err != nil {
return false, fmt.Errorf("unable to resolve extraction root: %w", err)
@@ -355,8 +254,8 @@ func resolvesInsideRoot(root, path string) (bool, error) {
return isWithin(current, realRoot), nil
}
// resolveLink returns where a symlink really points, falling back to the lexically cleaned target
// when the target does not exist yet - a dangling link still redirects a write.
// resolveLink returns where a symlink points, falling back to the lexically cleaned target when it
// does not exist yet - a dangling link still redirects a write.
func resolveLink(path string) (string, error) {
target, err := os.Readlink(path)
if err != nil {
@@ -375,8 +274,8 @@ func resolveLink(path string) (string, error) {
return resolved, nil
}
// isWithin reports whether path is root or sits beneath it, compared on a path boundary rather than
// as a string prefix so a sibling sharing the prefix does not pass.
// isWithin reports whether path is root or beneath it, compared on path boundaries so a sibling
// sharing the prefix does not pass.
func isWithin(path, root string) bool {
rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path))
if err != nil {
@@ -385,93 +284,30 @@ func isWithin(path, root string) bool {
return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)))
}
// writeSafeSymlink creates a symlink at destPath only if target is relative and really resolves,
// relative to destPath's directory, to a location inside destDir.
//
// Absolute targets are rejected outright: os.Symlink writes the literal target
// string, so an absolute target like "/etc/passwd" would resolve on the host
// filesystem when read, regardless of any safety check we performed against
// destDir at extraction time.
//
// A relative target is resolved against the REAL location of destPath's directory, not against its
// lexical one, because two cooperating entries can otherwise each pass on their own and together
// point outside: a link to "." followed by a link to ".." inside it resolves lexically to the root
// while really landing above it, and repeating the pair climbs a level each time. The parent of
// every write is checked the same way before the write happens, so a link that slipped through here
// still could not be written through - this is the earlier of two refusals, not the only one.
//
// It has no caller since an archive's entries became headers in one tar rather than files on disk: a
// symlink entry is now a header write, which creates nothing and traverses nothing, and where the
// link points is decided inside the archive's own filetree. It is kept rather than retired because
// that is a judgement about every path that could write a link, not about this one call site, and
// because the reasoning above is the expensive part to reconstruct.
//
//nolint:unused // retained deliberately; see the note above
func writeSafeSymlink(target, destPath, destDir string) error {
if target == "" {
return fmt.Errorf("empty link target")
}
if filepath.IsAbs(target) {
return fmt.Errorf("absolute symlink target not allowed")
}
realRoot, err := filepath.EvalSymlinks(destDir)
if err != nil {
return fmt.Errorf("unable to resolve extraction root: %w", err)
}
realParent, err := filepath.EvalSymlinks(filepath.Dir(destPath))
if err != nil {
return fmt.Errorf("unable to resolve link parent: %w", err)
}
if !isWithin(realParent, realRoot) {
return fmt.Errorf("symlink parent escapes extraction root")
}
resolved := filepath.Join(realParent, target)
if !isWithin(resolved, realRoot) {
return fmt.Errorf("symlink target escapes extraction root")
}
// the target itself may run through links planted by earlier entries
if existing, err := filepath.EvalSymlinks(resolved); err == nil && !isWithin(existing, realRoot) {
return fmt.Errorf("symlink target resolves outside extraction root")
}
return os.Symlink(target, destPath)
}
// zipEOCDSignature is the end-of-central-directory record's signature. A zip is defined by this
// record: it is what a reader seeks to in order to find the central directory, which is why a zip
// with something concatenated in front of it still opens.
// zipEOCDSignature is the end-of-central-directory record's signature. Readers seek to this record
// to find the central directory, which is why a zip with something prepended still opens.
var zipEOCDSignature = []byte{'P', 'K', 0x05, 0x06}
// maxZipEOCDSearch is how far back the record can be. The EOCD is 22 bytes and ends with a comment
// of up to 65535 bytes, so it starts no earlier than this many bytes from the end.
// maxZipEOCDSearch is how far back from the end the EOCD record can start: a 22 byte record plus a
// comment of up to 65535 bytes.
const maxZipEOCDSearch = 22 + 65535
// zipLocalFileHeaderSignature starts every entry in a zip. In a self-extracting archive the first
// one follows the stub, so it is what a bounded read of the head can find.
// one follows the stub, within reach of a bounded read of the head.
var zipLocalFileHeaderSignature = []byte{'P', 'K', 0x03, 0x04}
// MaxAppendedArchiveStubBytes is how far into a file a hidden archive is looked for. A Spring Boot
// launcher stub is about 9 KB, so this is generous by a wide margin; a stub longer than this hides
// its archive from the walk, which is the cost of not reading every candidate to the end.
const MaxAppendedArchiveStubBytes = 64 * 1024
// maxAppendedArchiveStubBytes is how far into a file an appended archive is looked for. A Spring
// Boot launcher stub is about 9 KB; a longer stub hides its archive from the walk.
const maxAppendedArchiveStubBytes = 64 * 1024
// MayHideAnAppendedArchive reports whether the head of this stream contains a zip local file header,
// and returns a reader that replays what it consumed.
//
// It exists to make the candidate set affordable. A file whose head sniffs as a script, an
// executable, or as nothing recognizable may still be an archive wearing a stub, but almost none of
// them are - and the authoritative test, the end-of-central-directory record, is at the far end of
// the file, so applying it to every such candidate means reading every executable in a scan to its
// end. Measured on a stock ruby image that doubled the scan.
//
// So the head is checked first, cheaply and with a bound, and only a file that shows an entry
// signature there is read in full and put to the real test. This narrows what is examined; it never
// admits anything, because a file that passes here is still refused by the extractor unless it ends
// in a valid central directory.
// A cheap, bounded pre-filter for HasZipEndOfCentralDirectory: reading every script, executable and
// unrecognized file to its end roughly doubles a scan, so only files showing an entry signature in
// their head are read in full. It narrows what is examined; it admits nothing on its own.
func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) {
head := make([]byte, MaxAppendedArchiveStubBytes)
head := make([]byte, maxAppendedArchiveStubBytes)
n, err := io.ReadFull(r, head)
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
return io.MultiReader(bytes.NewReader(head[:n]), r), false
@@ -481,17 +317,9 @@ func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) {
}
// HasZipEndOfCentralDirectory reports whether the content ends in a zip end-of-central-directory
// record, which is the definition of a zip and the only part of one whose position is knowable
// without parsing.
//
// This is what identifies an archive that content sniffing types by its head: a self-extracting
// archive is a launcher stub with a zip concatenated onto it, so its first bytes are the stub's and
// no magic-at-offset-zero test can see the archive. The record at the end is unambiguous, and
// mholt/archives extracts such a file correctly once it is asked to - it reads through
// archive/zip, which locates the central directory from the end.
//
// It is a content test, so it cannot make a non-archive extractable: a file named `notes.zip` that
// is prose has no such record and is refused here exactly as it is refused by format identification.
// record - the only part of a zip whose position is knowable without parsing. It identifies archives
// that head-based sniffing types as something else, such as a launcher stub with a zip appended,
// which mholt/archives extracts correctly via archive/zip.
func HasZipEndOfCentralDirectory(r ReaderAtSeeker) bool {
size, err := r.Seek(0, io.SeekEnd)
if err != nil || size < int64(len(zipEOCDSignature)) {