mirror of
https://github.com/anchore/syft.git
synced 2026-11-03 07:56:36 +01:00
chore: revert custom index in favor of key_Split
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
+30
-53
@@ -6,19 +6,17 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
|
||||
// copyChunkSize is how much is read, charged and written at a time. It bounds how far a charge can
|
||||
// be ahead of what has actually been written, and matches io.Copy's own buffer size.
|
||||
// copyChunkSize is the read/charge/write chunk size, bounding how far a charge runs ahead of what is
|
||||
// written. Matches io.Copy's buffer size.
|
||||
const copyChunkSize = 32 * 1024
|
||||
|
||||
// ReaderAtSeeker is the random access an archive format needs. A zip reads its central directory
|
||||
// from the end of the stream, so a plain io.Reader will not do; both a *bytes.Reader over content
|
||||
// held in memory and an *os.File over content overflowed to disk satisfy it.
|
||||
// ReaderAtSeeker is the random access an archive format needs: a zip reads its central directory from
|
||||
// the end of the stream, so a plain io.Reader will not do.
|
||||
type ReaderAtSeeker interface {
|
||||
io.Reader
|
||||
io.ReaderAt
|
||||
@@ -26,9 +24,6 @@ type ReaderAtSeeker interface {
|
||||
}
|
||||
|
||||
// Content is one archive's bytes, wherever the limiter routed them.
|
||||
//
|
||||
// Name is the archive's file name and is used for format identification only - never to open
|
||||
// anything - so content held in memory needs no file on disk at all.
|
||||
type Content struct {
|
||||
Name string
|
||||
Reader ReaderAtSeeker
|
||||
@@ -45,43 +40,22 @@ func (c Content) Close() error {
|
||||
return c.closer.Close()
|
||||
}
|
||||
|
||||
// OpenFileContent opens an archive already on disk as Content. The caller closes it.
|
||||
func OpenFileContent(path string) (Content, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return Content{}, err
|
||||
}
|
||||
return Content{Name: filepath.Base(path), Reader: f, closer: f}, nil
|
||||
}
|
||||
|
||||
// holdContent routes one archive's bytes: held in memory and charged to the memory limit while that
|
||||
// limit admits them, overflowed into workDir and charged to the disk limit once it does not. Where the
|
||||
// boundary falls is decided entirely by the memory limit - there is no separate configured size at
|
||||
// which overflowing begins.
|
||||
// holdContent holds one archive's bytes in memory while the memory limit admits them, and spills into
|
||||
// workDir against the disk limit once it does not. Reads and charges copyChunkSize at a time so an
|
||||
// oversized archive is never held past the limit. A zero memory limit spills in full; a negative one
|
||||
// never refuses.
|
||||
//
|
||||
// Content is read and charged incrementally, copyChunkSize at a time, rather than read speculatively
|
||||
// into a buffer ahead of any charge: nothing is ever held beyond what the memory limit would admit at
|
||||
// that moment, so a single archive far larger than the limit is never buffered past it. A zero
|
||||
// memory limit refuses every chunk, so its content overflows to disk in full; a negative memory limit
|
||||
// never refuses, so nothing overflows for want of memory.
|
||||
//
|
||||
// When the limit refuses a chunk part way through, what has already been read leads the overflow rather
|
||||
// than being re-read from the start or lost, and the memory it was charged against is refunded since
|
||||
// it is about to be charged to disk instead.
|
||||
//
|
||||
// Memory pressure degrades to disk - content that would take memory in use past the memory limit
|
||||
// overflows instead of being refused, because there is somewhere for it to go. The disk limit is
|
||||
// terminal: there is nowhere further, so content that would take it past the disk limit yields
|
||||
// ErrDiskLimitReached and the archive is skipped. Nothing waits for capacity; see ExtractToResolver.
|
||||
func holdContent(r io.Reader, workDir, name string, charge *Charge, notify Notify) (Content, error) {
|
||||
// On a refused chunk, held bytes and the rest of the stream spill without re-reading, and the memory
|
||||
// charge is refunded as it is charged to disk. The disk limit is terminal: exceeding it yields
|
||||
// ErrDiskLimitReached and the archive is skipped (see ExtractToResolver).
|
||||
func holdContent(r io.Reader, workDir *WorkDir, name string, charge *Charge, notify Notify) (Content, error) {
|
||||
var held bytes.Buffer
|
||||
chunk := make([]byte, copyChunkSize)
|
||||
for {
|
||||
n, readErr := r.Read(chunk)
|
||||
if n > 0 {
|
||||
if !charge.Memory(int64(n)) {
|
||||
// the memory limit refused this chunk: overflow what is already held plus this chunk
|
||||
// plus whatever remains, without re-reading from the start
|
||||
// spill held bytes, this chunk, and the rest without re-reading
|
||||
if notify != nil {
|
||||
notify(ContentOverflowed{Archive: name, Bytes: int64(held.Len() + n), Reason: overflowReason(charge)})
|
||||
}
|
||||
@@ -99,20 +73,24 @@ func holdContent(r io.Reader, workDir, name string, charge *Charge, notify Notif
|
||||
}
|
||||
}
|
||||
|
||||
// overflowContent writes an archive's bytes into workDir, charging the disk limit as they land, and
|
||||
// returns a reader over the file positioned at its start.
|
||||
// overflowContent writes an archive's bytes into its work directory, charging the disk limit as they
|
||||
// land, and returns a reader over the file positioned at its start. The work directory is created
|
||||
// here on first write, so an in-memory archive never creates one.
|
||||
//
|
||||
// This is the only filesystem write left in the extraction path - an archive's entries are headers in
|
||||
// one tar and traverse no path - and the name it writes under is derived from where the archive was
|
||||
// found, which inside another archive is text the archive supplied. So the destination is checked the
|
||||
// same two ways an entry's path used to be: lexically, and then against where it really lands on the
|
||||
// filesystem.
|
||||
func overflowContent(r io.Reader, workDir, name string, charge *Charge) (Content, error) {
|
||||
dest, err := intFile.SafeJoin(workDir, name)
|
||||
// This is the only filesystem write in the extraction path, and the name derives from where the
|
||||
// archive was found - attacker-controlled inside another archive - so the destination is checked
|
||||
// both lexically and against where it really lands.
|
||||
func overflowContent(r io.Reader, workDir *WorkDir, name string, charge *Charge) (Content, error) {
|
||||
dir, err := workDir.Path()
|
||||
if err != nil {
|
||||
return Content{}, err
|
||||
}
|
||||
|
||||
dest, err := intFile.SafeJoin(dir, name)
|
||||
if err != nil {
|
||||
return Content{}, fmt.Errorf("refusing to overflow archive content to %q: %w", name, err)
|
||||
}
|
||||
if inside, err := resolvesInsideRoot(workDir, dest); err != nil || !inside {
|
||||
if inside, err := resolvesInsideRoot(dir, dest); err != nil || !inside {
|
||||
return Content{}, fmt.Errorf("refusing to overflow archive content outside its work directory: %q", name)
|
||||
}
|
||||
f, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
|
||||
@@ -144,10 +122,9 @@ func overflowContent(r io.Reader, workDir, name string, charge *Charge) (Content
|
||||
return Content{Name: name, Reader: f, closer: f}, nil
|
||||
}
|
||||
|
||||
// copyCharged copies src into dst, charging every chunk to the disk limit before it is written, so
|
||||
// what bounds the copy is what actually lands rather than any size the archive declared. It stops at
|
||||
// the first refused charge and says so, leaving the caller to decide whether that truncates one
|
||||
// entry or skips a whole archive.
|
||||
// copyCharged copies src into dst, charging each chunk to the disk limit before writing, so the copy
|
||||
// is bounded by what lands rather than any declared size. It stops at the first refused charge and
|
||||
// reports it; the caller decides whether that truncates one entry or skips the archive.
|
||||
func copyCharged(dst io.Writer, src io.Reader, charge *Charge) (written int64, limitReached bool, err error) {
|
||||
buf := make([]byte, copyChunkSize)
|
||||
for {
|
||||
|
||||
Reference in New Issue
Block a user