mirror of
https://github.com/anchore/syft.git
synced 2026-10-30 22:12:28 +01:00
Merge remote-tracking branch 'upstream/main' into add-nested-archive-cataloging
This commit is contained in:
Keith Zantow
@@ -0,0 +1,66 @@
|
||||
package file
|
||||
|
||||
import "io"
|
||||
|
||||
// ReaderSize reports the number of bytes behind r. The bool is false when that cannot be determined, and
|
||||
// callers must branch on it rather than treating the zero as a size.
|
||||
//
|
||||
// Binary parsers read sizes out of the files they are parsing and allocate against them. Those sizes are
|
||||
// 32- and 64-bit fields under the control of whoever produced the file, so a truncated or hostile input
|
||||
// can declare gigabytes that are not there. The read would fail afterwards either way; what matters is
|
||||
// refusing before the allocation, and that needs a real byte count to weigh the claim against.
|
||||
//
|
||||
// The bool is the whole point of the signature. An earlier version returned 0 for every failure mode, and
|
||||
// a caller that forgot to special-case it got a bound of "nothing is allowed" or, more often, silently
|
||||
// fell back to an unbounded path. Making the caller name the failure keeps that from happening by
|
||||
// omission.
|
||||
//
|
||||
// Both *bytes.Reader and *io.SectionReader answer directly. A file handle only has Seek, so fall back to
|
||||
// a save-and-restore seek, which leaves the caller's cursor where it found it. That fallback is not
|
||||
// atomic: it moves the cursor and puts it back, so a reader being read concurrently can observe the
|
||||
// intermediate position.
|
||||
//
|
||||
// Neither answer is taken on faith, because a reader can report a length it cannot deliver: an
|
||||
// *io.SectionReader answers with the length it was constructed with, and debug/elf and friends construct
|
||||
// theirs with a nominal one, so io.NewSectionReader(r, 0, 1<<63-1) would otherwise report a bound that
|
||||
// permits everything as if it had been measured. The last byte is read back to confirm the count, which is
|
||||
// one ReadAt whatever the size, and the read is what the caller was going to weigh anyway.
|
||||
//
|
||||
// A wrapper that embeds io.ReaderAt as an interface promotes only ReadAt, so it answers no size here
|
||||
// however sizable the reader underneath it is. A type meant to be bounded needs to forward Size() itself.
|
||||
func ReaderSize(r io.ReaderAt) (int64, bool) {
|
||||
size, ok := reportedSize(r)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
// ReadAt may return io.EOF alongside a full read, so the count is what says the byte is there
|
||||
var last [1]byte
|
||||
if n, _ := r.ReadAt(last[:], size-1); n != len(last) {
|
||||
return 0, false
|
||||
}
|
||||
return size, true
|
||||
}
|
||||
|
||||
// reportedSize is what the reader says about itself, before ReaderSize checks whether it can back it up.
|
||||
func reportedSize(r io.ReaderAt) (int64, bool) {
|
||||
if sr, ok := r.(interface{ Size() int64 }); ok {
|
||||
size := sr.Size()
|
||||
return size, size > 0
|
||||
}
|
||||
s, ok := r.(io.Seeker)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
cur, err := s.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
size, err := s.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
if _, err := s.Seek(cur, io.SeekStart); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return size, size > 0
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestReaderSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
r io.ReaderAt
|
||||
want int64
|
||||
wantOK bool
|
||||
}{
|
||||
{
|
||||
name: "bytes.Reader answers directly",
|
||||
r: bytes.NewReader(make([]byte, 1234)),
|
||||
want: 1234,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
// an empty reader is not a size to bound against, so it reports not-ok along with everything
|
||||
// else that cannot be measured
|
||||
name: "empty bytes.Reader",
|
||||
r: bytes.NewReader(nil),
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "SectionReader over a real length",
|
||||
r: io.NewSectionReader(bytes.NewReader(make([]byte, 500)), 100, 300),
|
||||
want: 300,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "seek-only reader falls back to seeking",
|
||||
r: seekOnly{bytes.NewReader(make([]byte, 4096))},
|
||||
want: 4096,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "reader with neither Size nor Seek cannot be measured",
|
||||
r: readAtOnly{bytes.NewReader(make([]byte, 4096))},
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// the shape debug/elf, debug/pe and debug/macho build internally. Taking this at its word hands
|
||||
// back a bound that permits everything, reported as if it had been measured.
|
||||
name: "SectionReader over a nominal length reports what it can deliver",
|
||||
r: io.NewSectionReader(bytes.NewReader(make([]byte, 500)), 0, 1<<63-1),
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// a Size() a type simply got wrong is the same failure: the bytes are what decides
|
||||
name: "a size the reader cannot back up is refused",
|
||||
r: overstatedSize{ReaderAt: bytes.NewReader(make([]byte, 10)), size: 1 << 30},
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
size, ok := ReaderSize(tt.r)
|
||||
assert.Equal(t, tt.want, size)
|
||||
assert.Equal(t, tt.wantOK, ok)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReaderSize_FileHandle(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "f.bin")
|
||||
require.NoError(t, os.WriteFile(path, make([]byte, 7777), 0o600))
|
||||
f, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
// the cursor has to come back where it was found: the caller may be mid-read, and an *os.File is the
|
||||
// one input here that carries a shared position
|
||||
_, err = f.Seek(101, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
size, ok := ReaderSize(f)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, int64(7777), size)
|
||||
|
||||
at, err := f.Seek(0, io.SeekCurrent)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(101), at, "measuring the file must not move the caller's cursor")
|
||||
}
|
||||
|
||||
// TestReaderSize_WrapperMustForwardSize pins the shape that made every bound expressed against this
|
||||
// helper go silently inert. A type that embeds io.ReaderAt as an interface promotes only ReadAt, so it
|
||||
// answers 0 here however sizable the reader underneath is, and a caller that treats 0 as "fall back to
|
||||
// unbounded" then stops bounding anything. Forwarding Size() is what fixes it, and this is the assertion
|
||||
// that a new wrapper has to satisfy.
|
||||
func TestReaderSize_WrapperMustForwardSize(t *testing.T) {
|
||||
inner := bytes.NewReader(make([]byte, 2048))
|
||||
|
||||
size, ok := ReaderSize(embedsReaderAt{inner})
|
||||
assert.False(t, ok, "a wrapper that only embeds the interface cannot be measured")
|
||||
assert.Equal(t, int64(0), size)
|
||||
|
||||
size, ok = ReaderSize(forwardsSize{inner})
|
||||
assert.True(t, ok, "a wrapper meant to be bounded has to forward Size itself")
|
||||
assert.Equal(t, int64(2048), size)
|
||||
}
|
||||
|
||||
type seekOnly struct{ inner *bytes.Reader }
|
||||
|
||||
func (s seekOnly) ReadAt(p []byte, off int64) (int, error) { return s.inner.ReadAt(p, off) }
|
||||
func (s seekOnly) Seek(off int64, whence int) (int64, error) {
|
||||
return s.inner.Seek(off, whence)
|
||||
}
|
||||
|
||||
type readAtOnly struct{ inner *bytes.Reader }
|
||||
|
||||
func (r readAtOnly) ReadAt(p []byte, off int64) (int, error) { return r.inner.ReadAt(p, off) }
|
||||
|
||||
type embedsReaderAt struct{ io.ReaderAt }
|
||||
|
||||
type forwardsSize struct{ io.ReaderAt }
|
||||
|
||||
func (f forwardsSize) Size() int64 {
|
||||
size, _ := ReaderSize(f.ReaderAt)
|
||||
return size
|
||||
}
|
||||
|
||||
// overstatedSize answers with a size larger than the bytes behind it, the way a wrapper that returns a
|
||||
// declared length rather than a measured one would.
|
||||
type overstatedSize struct {
|
||||
io.ReaderAt
|
||||
size int64
|
||||
}
|
||||
|
||||
func (o overstatedSize) Size() int64 { return o.size }
|
||||
@@ -0,0 +1,330 @@
|
||||
// Package spillbuf provides a sparse, offset-addressed byte buffer that keeps a bounded amount in memory
|
||||
// and spills the rest to a temp file.
|
||||
//
|
||||
// It exists for the shape where output arrives out of order, at offsets the input itself declares: a
|
||||
// decompressor placing extents, an archive rebuilding a file from chunks. A plain []byte cannot do that
|
||||
// without either pre-sizing to a length the input claims or growing to the furthest offset it names, and
|
||||
// both hand a hostile input an allocation knob.
|
||||
//
|
||||
// The buffer reports only what it actually stored. Size is the contiguous run written from offset zero,
|
||||
// and a read past it is io.EOF rather than the zeros an unwritten region would otherwise hand back. That
|
||||
// distinction is the whole point: sparse storage serves holes as zeros for free, so a reader that reported
|
||||
// a length covering them would let a write offset stand in for real output. Anything sizing an allocation
|
||||
// against this reader (debug/elf's saferio does exactly that) would then allocate bytes nothing produced.
|
||||
//
|
||||
// Allocation is bounded by the memory limit rather than by how much is written: filling 1MB and filling
|
||||
// 64MB cost about the same, because everything past the limit goes to the file. Only memory is bounded
|
||||
// here: how much ends up on disk is whatever the caller writes, so bounding that stays the caller's job.
|
||||
package spillbuf
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"slices"
|
||||
|
||||
"github.com/anchore/syft/internal/tmpdir"
|
||||
)
|
||||
|
||||
// DefaultMemLimit is how much a buffer keeps in memory before it spills, when the caller does not say.
|
||||
// Small on purpose: the memory tier saves a temp file for the common small case, it is not a place to
|
||||
// hold output. A limit large enough to matter is a limit large enough to be an amplification path.
|
||||
const DefaultMemLimit int64 = 1 << 20 // 1MB
|
||||
|
||||
// ErrNoTempDir is returned when a buffer needs to spill and has nowhere to spill to.
|
||||
var ErrNoTempDir = errors.New("no temp dir available to spill to")
|
||||
|
||||
// Buffer is a sparse, offset-addressed buffer. The zero value is not usable; call New.
|
||||
//
|
||||
// Storage is two tiers with a fixed boundary: bytes below memLimit live in mem, bytes at or past it live
|
||||
// in file. Nothing ever moves between them, so an offset belongs to exactly one tier for the life of the
|
||||
// buffer and the only case needing care is a range that straddles the boundary.
|
||||
//
|
||||
// Not safe for concurrent use. Every consumer writes from a single goroutine, and guarding it would cost
|
||||
// each write an uncontended lock for no reader.
|
||||
type Buffer struct {
|
||||
td *tmpdir.TempDir
|
||||
memLimit int64
|
||||
|
||||
// mem holds [0, memLimit). Grown geometrically to fit what has been written, never past the limit.
|
||||
mem []byte
|
||||
|
||||
// file holds [memLimit, ...), created on the first write that reaches there, so a buffer that stays
|
||||
// small never touches disk. remove deletes it on Close.
|
||||
file *os.File
|
||||
remove func()
|
||||
|
||||
// written is the set of ranges stored, sorted by start and merged, so runs of adjacent writes collapse
|
||||
// to one entry rather than growing per call. Everything the buffer reports comes from it.
|
||||
written []extent
|
||||
|
||||
closed bool
|
||||
}
|
||||
|
||||
// extent is a half-open byte range that has been written.
|
||||
//
|
||||
// Unexported on purpose. What a caller needs to know about the contents is answered by Size and FirstGap;
|
||||
// handing out the bookkeeping invites re-deriving those at the call site and getting them subtly wrong,
|
||||
// which is the bug this package exists to prevent.
|
||||
type extent struct {
|
||||
start, end int64
|
||||
}
|
||||
|
||||
// Option configures a Buffer.
|
||||
type Option func(*Buffer)
|
||||
|
||||
// WithMemLimit sets how many bytes are held in memory before the buffer spills to disk. Zero sends every
|
||||
// write to the temp file. Negative is treated as zero.
|
||||
//
|
||||
// Peak allocation is a small multiple of this, since growing the tier holds the old copy alongside the
|
||||
// new one and the new one may be rounded up past what was asked for.
|
||||
func WithMemLimit(n int64) Option {
|
||||
return func(b *Buffer) {
|
||||
b.memLimit = max(n, 0)
|
||||
}
|
||||
}
|
||||
|
||||
// New returns a buffer that spills into td.
|
||||
//
|
||||
// td may be nil only when every write is known to stay inside the memory limit; a write past it then
|
||||
// fails with ErrNoTempDir rather than allocating. Nothing is created on disk until a write needs it.
|
||||
func New(td *tmpdir.TempDir, opts ...Option) *Buffer {
|
||||
b := &Buffer{td: td, memLimit: DefaultMemLimit}
|
||||
for _, opt := range opts {
|
||||
opt(b)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// split divides a range of n bytes starting at off across the two tiers, reporting how many bytes fall in
|
||||
// each. The file portion starts at off+inMem.
|
||||
func (b *Buffer) split(off, n int64) (inMem, inFile int64) {
|
||||
if off >= b.memLimit {
|
||||
return 0, n
|
||||
}
|
||||
inMem = min(n, b.memLimit-off)
|
||||
return inMem, n - inMem
|
||||
}
|
||||
|
||||
// fileOffset translates a buffer offset at or past the limit into an offset within the spill file.
|
||||
func (b *Buffer) fileOffset(off int64) int64 { return off - b.memLimit }
|
||||
|
||||
// WriteAt stores p at off. Writes may land anywhere, in any order, and may overlap. A write that fails
|
||||
// leaves the buffer as it was.
|
||||
func (b *Buffer) WriteAt(p []byte, off int64) (int, error) {
|
||||
if b.closed {
|
||||
return 0, fmt.Errorf("spillbuf: write to a closed buffer: %w", os.ErrClosed)
|
||||
}
|
||||
if off < 0 {
|
||||
return 0, fmt.Errorf("spillbuf: negative offset %d", off)
|
||||
}
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
// subtraction form so off+len(p) cannot wrap into a small in-range end
|
||||
if off > math.MaxInt64-int64(len(p)) {
|
||||
return 0, fmt.Errorf("spillbuf: write of %d bytes at %d overflows", len(p), off)
|
||||
}
|
||||
|
||||
inMem, inFile := b.split(off, int64(len(p)))
|
||||
// the fallible half goes first: once the memory tier is stamped the old bytes are gone, so a file
|
||||
// write that fails afterward would report failure over a buffer it had already changed
|
||||
if inFile > 0 {
|
||||
if err := b.writeFile(p[inMem:], b.fileOffset(off+inMem)); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if inMem > 0 {
|
||||
b.writeMem(p[:inMem], off)
|
||||
}
|
||||
|
||||
b.record(extent{start: off, end: off + int64(len(p))})
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (b *Buffer) writeMem(p []byte, off int64) {
|
||||
if need := off + int64(len(p)); int64(len(b.mem)) < need {
|
||||
// geometric, capped at the limit. Sizing to exactly what each write needs looks tidier and is
|
||||
// quadratic: a caller streaming a block in 32KB chunks reallocates and copies the whole tier every
|
||||
// chunk, so filling 1MB costs ~16MB of allocation.
|
||||
grown := min(b.memLimit, max(need, 2*int64(len(b.mem))))
|
||||
// sized to exactly the geometric target rather than grown through append: append rounds the new
|
||||
// capacity up past what was asked for, and the tier is already capped, so the slack is pure waste
|
||||
mem := make([]byte, grown)
|
||||
copy(mem, b.mem)
|
||||
b.mem = mem
|
||||
}
|
||||
copy(b.mem[off:], p)
|
||||
}
|
||||
|
||||
func (b *Buffer) writeFile(p []byte, off int64) error {
|
||||
if b.file == nil {
|
||||
if b.td == nil {
|
||||
return ErrNoTempDir
|
||||
}
|
||||
f, remove, err := b.td.NewFile("syft-spill-*.bin") //nolint:gocritic // removal happens in Close
|
||||
if err != nil {
|
||||
return fmt.Errorf("spillbuf: unable to create spill file: %w", err)
|
||||
}
|
||||
b.file, b.remove = f, remove
|
||||
}
|
||||
if _, err := b.file.WriteAt(p, off); err != nil {
|
||||
return fmt.Errorf("spillbuf: unable to write to spill file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadAt returns bytes this buffer actually stored. A read starting at or past Size is io.EOF, and one
|
||||
// running past it is short, because everything beyond is a hole the buffer never held.
|
||||
func (b *Buffer) ReadAt(p []byte, off int64) (int, error) {
|
||||
if b.closed {
|
||||
return 0, fmt.Errorf("spillbuf: read from a closed buffer: %w", os.ErrClosed)
|
||||
}
|
||||
if off < 0 {
|
||||
return 0, fmt.Errorf("spillbuf: negative offset %d", off)
|
||||
}
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
size := b.Size()
|
||||
if off >= size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// bounded by Size, so every byte in [off, off+want) was written and both tiers really hold theirs
|
||||
want := min(int64(len(p)), size-off)
|
||||
inMem, inFile := b.split(off, want)
|
||||
|
||||
// guarded rather than relying on a zero-length copy: when off is past the limit the slice expression
|
||||
// itself is out of range, since mem is only as long as what was written into it
|
||||
if inMem > 0 {
|
||||
copy(p[:inMem], b.mem[off:off+inMem])
|
||||
}
|
||||
if inFile > 0 {
|
||||
if err := readFullAt(b.file, p[inMem:want], b.fileOffset(off+inMem)); err != nil {
|
||||
return int(inMem), fmt.Errorf("spillbuf: unable to read spill file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if want < int64(len(p)) {
|
||||
return int(want), io.EOF
|
||||
}
|
||||
return int(want), nil
|
||||
}
|
||||
|
||||
// readFullAt fills p from r. A short read is an error rather than a short return: the caller has already
|
||||
// bounded the range by what was written, so the file failing to deliver means it is not what we left.
|
||||
func readFullAt(r io.ReaderAt, p []byte, off int64) error {
|
||||
for read := 0; read < len(p); {
|
||||
n, err := r.ReadAt(p[read:], off+int64(read))
|
||||
read += n
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Size reports the contiguous run written from offset zero, which is the length this buffer is willing to
|
||||
// stand behind. Deliberately not the furthest offset written: see the package doc for why the two are not
|
||||
// interchangeable.
|
||||
//
|
||||
// record keeps the set merged and sorted, so the run from zero is at most the first two entries and the
|
||||
// loop stops there: the cost this adds to every ReadAt is effectively constant, not a scan of the set.
|
||||
func (b *Buffer) Size() int64 {
|
||||
var covered int64
|
||||
for _, e := range b.written {
|
||||
if e.start > covered {
|
||||
break
|
||||
}
|
||||
covered = max(covered, e.end)
|
||||
}
|
||||
return covered
|
||||
}
|
||||
|
||||
// FirstGap returns the offset of the earliest unwritten run of at least size bytes lying entirely within
|
||||
// [0, within), and whether there is one. Callers that place output at offsets of their own choosing use it
|
||||
// to fill what they left behind, without needing the buffer's bookkeeping.
|
||||
//
|
||||
// The arithmetic is in subtraction form throughout: within comes from caller input and the result is used
|
||||
// as a write offset, so a wrap here would be a write nobody bounded.
|
||||
func (b *Buffer) FirstGap(size, within int64) (int64, bool) {
|
||||
if b.closed || size < 0 || within < 0 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
var at int64
|
||||
for _, e := range b.written {
|
||||
// once the cursor is at the bound there is nothing left to offer, and it only moves forward
|
||||
if at >= within {
|
||||
return 0, false
|
||||
}
|
||||
// the gap ahead of this extent ends at whichever comes first, the extent or the bound
|
||||
end := min(e.start, within)
|
||||
if end > at && end-at >= size {
|
||||
return at, true
|
||||
}
|
||||
at = max(at, e.end)
|
||||
}
|
||||
if at <= within && within-at >= size {
|
||||
return at, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// record merges e into the written set, keeping it sorted by start with no overlapping or adjacent pairs.
|
||||
func (b *Buffer) record(e extent) {
|
||||
if e.end <= e.start {
|
||||
return
|
||||
}
|
||||
|
||||
// inserted in place rather than appended and re-sorted: the set is already ordered, so a run of
|
||||
// ascending writes (the common case) stays linear
|
||||
i, _ := slices.BinarySearchFunc(b.written, e, func(x, y extent) int {
|
||||
return cmpInt64(x.start, y.start)
|
||||
})
|
||||
b.written = slices.Insert(b.written, i, e)
|
||||
|
||||
merged := b.written[:1]
|
||||
for _, cur := range b.written[1:] {
|
||||
last := &merged[len(merged)-1]
|
||||
if cur.start <= last.end {
|
||||
last.end = max(last.end, cur.end)
|
||||
continue
|
||||
}
|
||||
merged = append(merged, cur)
|
||||
}
|
||||
b.written = merged
|
||||
}
|
||||
|
||||
func cmpInt64(x, y int64) int {
|
||||
switch {
|
||||
case x < y:
|
||||
return -1
|
||||
case x > y:
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Close releases the spill file, if one was ever created, and drops the memory tier. Safe on a nil
|
||||
// receiver and safe to call more than once. Reads and writes after it fail with os.ErrClosed.
|
||||
func (b *Buffer) Close() error {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
b.mem = nil
|
||||
b.written = nil
|
||||
b.closed = true
|
||||
if b.file == nil {
|
||||
return nil
|
||||
}
|
||||
err := b.file.Close()
|
||||
if b.remove != nil {
|
||||
b.remove()
|
||||
}
|
||||
b.file, b.remove = nil, nil
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,864 @@
|
||||
package spillbuf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/testutils"
|
||||
"github.com/anchore/syft/internal/tmpdir"
|
||||
)
|
||||
|
||||
func newTestBuffer(t *testing.T, opts ...Option) *Buffer {
|
||||
t.Helper()
|
||||
b := New(tmpdir.FromPath(t.TempDir()), opts...)
|
||||
t.Cleanup(func() { _ = b.Close() })
|
||||
return b
|
||||
}
|
||||
|
||||
func mustWrite(t *testing.T, b *Buffer, off int64, p []byte) {
|
||||
t.Helper()
|
||||
n, err := b.WriteAt(p, off)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, len(p), n, "WriteAt must report the full write")
|
||||
}
|
||||
|
||||
// readAll returns everything the buffer is willing to hand back.
|
||||
func readAll(t *testing.T, b *Buffer) []byte {
|
||||
t.Helper()
|
||||
out := make([]byte, b.Size())
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
n, err := b.ReadAt(out, 0)
|
||||
require.NoError(t, err, "a read of exactly Size bytes is not short")
|
||||
require.Equal(t, len(out), n)
|
||||
return out
|
||||
}
|
||||
|
||||
func repeat(b byte, n int) []byte { return bytes.Repeat([]byte{b}, n) }
|
||||
|
||||
// --- the core property: report only what was stored -------------------------------------------------
|
||||
|
||||
// TestSizeIsTheContiguousPrefix is the property the package exists for. A sparse buffer hands its holes
|
||||
// back as zeros it never stored, so reporting a length that covers them lets a write offset stand in for
|
||||
// real output: whatever sizes an allocation against this reader then allocates bytes nothing produced.
|
||||
func TestSizeIsTheContiguousPrefix(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
writes []extent // start is the offset, end-start the length
|
||||
wantSize int64
|
||||
}{
|
||||
{
|
||||
name: "nothing written",
|
||||
wantSize: 0,
|
||||
},
|
||||
{
|
||||
name: "one run from zero",
|
||||
writes: []extent{{0, 100}},
|
||||
wantSize: 100,
|
||||
},
|
||||
{
|
||||
name: "a run that does not start at zero covers nothing",
|
||||
writes: []extent{{10, 100}},
|
||||
wantSize: 0,
|
||||
},
|
||||
{
|
||||
name: "adjacent runs join",
|
||||
writes: []extent{{0, 50}, {50, 100}},
|
||||
wantSize: 100,
|
||||
},
|
||||
{
|
||||
name: "a hole stops the prefix",
|
||||
writes: []extent{{0, 50}, {60, 100}},
|
||||
wantSize: 50,
|
||||
},
|
||||
{
|
||||
name: "the hole is filled later",
|
||||
writes: []extent{{0, 50}, {60, 100}, {50, 60}},
|
||||
wantSize: 100,
|
||||
},
|
||||
{
|
||||
name: "out of order still resolves",
|
||||
writes: []extent{{60, 100}, {0, 50}, {50, 60}},
|
||||
wantSize: 100,
|
||||
},
|
||||
{
|
||||
name: "overlapping runs",
|
||||
writes: []extent{{0, 60}, {40, 100}},
|
||||
wantSize: 100,
|
||||
},
|
||||
{
|
||||
name: "a run fully inside another",
|
||||
writes: []extent{{0, 100}, {20, 40}},
|
||||
wantSize: 100,
|
||||
},
|
||||
{
|
||||
name: "far placement is not coverage",
|
||||
writes: []extent{{0, 64}, {1 << 20, 1<<20 + 64}},
|
||||
wantSize: 64,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
b := newTestBuffer(t)
|
||||
for _, w := range tt.writes {
|
||||
mustWrite(t, b, w.start, repeat('x', int(w.end-w.start)))
|
||||
}
|
||||
assert.Equal(t, tt.wantSize, b.Size(), "Size is the contiguous prefix")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadPastSizeIsEOFNotZeros is the other half of the same property. Sparse storage returns zeros for
|
||||
// a hole for free; handing those out would make the hole indistinguishable from real content.
|
||||
func TestReadPastSizeIsEOFNotZeros(t *testing.T) {
|
||||
b := newTestBuffer(t)
|
||||
mustWrite(t, b, 0, repeat('A', 64))
|
||||
mustWrite(t, b, 4096, repeat('B', 64)) // past a hole
|
||||
|
||||
require.Equal(t, int64(64), b.Size())
|
||||
|
||||
t.Run("a read starting in the hole is EOF", func(t *testing.T) {
|
||||
p := make([]byte, 16)
|
||||
n, err := b.ReadAt(p, 100)
|
||||
assert.Zero(t, n)
|
||||
assert.ErrorIs(t, err, io.EOF)
|
||||
})
|
||||
|
||||
t.Run("a read starting at the far extent is EOF, not the bytes there", func(t *testing.T) {
|
||||
p := make([]byte, 64)
|
||||
n, err := b.ReadAt(p, 4096)
|
||||
assert.Zero(t, n)
|
||||
assert.ErrorIs(t, err, io.EOF)
|
||||
assert.Equal(t, make([]byte, 64), p, "nothing may be copied out")
|
||||
})
|
||||
|
||||
t.Run("a read running off the end is short and says so", func(t *testing.T) {
|
||||
p := make([]byte, 128)
|
||||
n, err := b.ReadAt(p, 0)
|
||||
assert.Equal(t, 64, n, "only the stored prefix comes back")
|
||||
assert.ErrorIs(t, err, io.EOF)
|
||||
assert.Equal(t, repeat('A', 64), p[:64])
|
||||
assert.Equal(t, make([]byte, 64), p[64:], "the rest is untouched")
|
||||
})
|
||||
}
|
||||
|
||||
// TestSparsePlacementIsNotAnAllocationKnob pins the amplification directly: a tiny payload parked at a
|
||||
// huge offset must not cost memory proportional to the offset.
|
||||
func TestSparsePlacementIsNotAnAllocationKnob(t *testing.T) {
|
||||
const far = 512 << 20 // 512MB out
|
||||
|
||||
b := newTestBuffer(t, WithMemLimit(1<<20))
|
||||
mustWrite(t, b, 0, repeat('A', 64))
|
||||
|
||||
allocated := testutils.MeasureAlloc(t, func() {
|
||||
mustWrite(t, b, far, repeat('B', 64))
|
||||
})
|
||||
t.Logf("writing 64 bytes at offset %d allocated %d bytes", far, allocated)
|
||||
assert.Less(t, allocated, uint64(1<<20),
|
||||
"a far placement must cost its own bytes, not its offset")
|
||||
assert.Equal(t, int64(64), b.Size(), "and it must not count toward what the buffer will deliver")
|
||||
}
|
||||
|
||||
// --- tiers ------------------------------------------------------------------------------------------
|
||||
|
||||
func TestMemoryTierNeverTouchesDisk(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := New(tmpdir.FromPath(dir), WithMemLimit(4096))
|
||||
t.Cleanup(func() { _ = b.Close() })
|
||||
|
||||
mustWrite(t, b, 0, repeat('A', 4096)) // exactly the limit
|
||||
|
||||
assert.Equal(t, repeat('A', 4096), readAll(t, b))
|
||||
assert.Nil(t, b.file, "a buffer that stays inside the limit must not create a file")
|
||||
assert.Empty(t, dirEntries(t, dir), "and must leave nothing on disk")
|
||||
}
|
||||
|
||||
func TestSpillCreatesTheFileOnlyWhenNeeded(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := New(tmpdir.FromPath(dir), WithMemLimit(4096))
|
||||
t.Cleanup(func() { _ = b.Close() })
|
||||
|
||||
mustWrite(t, b, 0, repeat('A', 4096))
|
||||
require.Empty(t, dirEntries(t, dir))
|
||||
|
||||
mustWrite(t, b, 4096, repeat('B', 1)) // one byte past the limit
|
||||
assert.NotNil(t, b.file)
|
||||
assert.Len(t, dirEntries(t, dir), 1, "exactly one spill file")
|
||||
}
|
||||
|
||||
func TestWriteStraddlingTheLimit(t *testing.T) {
|
||||
const limit = 1024
|
||||
b := newTestBuffer(t, WithMemLimit(limit))
|
||||
|
||||
payload := make([]byte, 2048)
|
||||
for i := range payload {
|
||||
payload[i] = byte(i % 251)
|
||||
}
|
||||
// starts below the limit and ends well past it, so the write is split across both tiers
|
||||
mustWrite(t, b, limit-512, payload)
|
||||
|
||||
require.Equal(t, int64(0), b.Size(), "nothing was written at zero yet")
|
||||
mustWrite(t, b, 0, repeat('Z', limit-512))
|
||||
|
||||
got := readAll(t, b)
|
||||
require.Len(t, got, limit-512+2048)
|
||||
assert.Equal(t, repeat('Z', limit-512), got[:limit-512], "the memory-only part")
|
||||
assert.Equal(t, payload, got[limit-512:], "the straddling part reads back whole")
|
||||
}
|
||||
|
||||
func TestMemLimitZeroSendsEverythingToDisk(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := New(tmpdir.FromPath(dir), WithMemLimit(0))
|
||||
t.Cleanup(func() { _ = b.Close() })
|
||||
|
||||
mustWrite(t, b, 0, repeat('A', 16))
|
||||
assert.Equal(t, repeat('A', 16), readAll(t, b))
|
||||
assert.Len(t, dirEntries(t, dir), 1, "with no memory tier the first byte spills")
|
||||
assert.Nil(t, b.mem)
|
||||
}
|
||||
|
||||
func TestNegativeMemLimitIsTreatedAsZero(t *testing.T) {
|
||||
b := newTestBuffer(t, WithMemLimit(-1))
|
||||
mustWrite(t, b, 0, repeat('A', 8))
|
||||
assert.Equal(t, repeat('A', 8), readAll(t, b))
|
||||
}
|
||||
|
||||
func TestDefaultMemLimitApplies(t *testing.T) {
|
||||
b := New(nil)
|
||||
t.Cleanup(func() { _ = b.Close() })
|
||||
assert.Equal(t, DefaultMemLimit, b.memLimit)
|
||||
}
|
||||
|
||||
// --- no temp dir ------------------------------------------------------------------------------------
|
||||
|
||||
func TestNoTempDir(t *testing.T) {
|
||||
t.Run("a write inside the memory limit needs no temp dir", func(t *testing.T) {
|
||||
b := New(nil, WithMemLimit(4096))
|
||||
t.Cleanup(func() { _ = b.Close() })
|
||||
|
||||
mustWrite(t, b, 0, repeat('A', 4096))
|
||||
assert.Equal(t, repeat('A', 4096), readAll(t, b))
|
||||
})
|
||||
|
||||
t.Run("a write past it fails rather than allocating", func(t *testing.T) {
|
||||
b := New(nil, WithMemLimit(4096))
|
||||
t.Cleanup(func() { _ = b.Close() })
|
||||
|
||||
_, err := b.WriteAt(repeat('B', 1), 4096)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrNoTempDir)
|
||||
})
|
||||
}
|
||||
|
||||
// --- input validation -------------------------------------------------------------------------------
|
||||
|
||||
func TestWriteAtRejectsBadInput(t *testing.T) {
|
||||
b := newTestBuffer(t)
|
||||
|
||||
t.Run("negative offset", func(t *testing.T) {
|
||||
_, err := b.WriteAt([]byte("x"), -1)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("an offset plus length that would overflow", func(t *testing.T) {
|
||||
_, err := b.WriteAt(repeat('x', 16), math.MaxInt64-8)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "overflow",
|
||||
"the check must be a subtraction, not off+len wrapping into an in-range end")
|
||||
})
|
||||
|
||||
t.Run("an empty write is a no-op, not an extent", func(t *testing.T) {
|
||||
n, err := b.WriteAt(nil, 500)
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, n)
|
||||
assert.Empty(t, b.written, "a zero-length write records nothing")
|
||||
})
|
||||
}
|
||||
|
||||
func TestReadAtRejectsBadInput(t *testing.T) {
|
||||
b := newTestBuffer(t)
|
||||
mustWrite(t, b, 0, repeat('A', 16))
|
||||
|
||||
t.Run("negative offset", func(t *testing.T) {
|
||||
_, err := b.ReadAt(make([]byte, 4), -1)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("empty read", func(t *testing.T) {
|
||||
n, err := b.ReadAt(nil, 0)
|
||||
assert.NoError(t, err)
|
||||
assert.Zero(t, n)
|
||||
})
|
||||
|
||||
t.Run("read from an empty buffer", func(t *testing.T) {
|
||||
empty := newTestBuffer(t)
|
||||
n, err := empty.ReadAt(make([]byte, 4), 0)
|
||||
assert.Zero(t, n)
|
||||
assert.ErrorIs(t, err, io.EOF)
|
||||
})
|
||||
}
|
||||
|
||||
// TestReadAtHonorsTheReaderAtContract pins what io.ReaderAt requires: a short read must come with a
|
||||
// non-nil error, and a full read must not invent one. Parsers built on io.ReaderAt (debug/elf among them)
|
||||
// rely on this to tell "the file ends here" from "try again".
|
||||
func TestReadAtHonorsTheReaderAtContract(t *testing.T) {
|
||||
b := newTestBuffer(t, WithMemLimit(64))
|
||||
mustWrite(t, b, 0, repeat('A', 200)) // spans both tiers
|
||||
|
||||
for _, size := range []int{1, 63, 64, 65, 199, 200} {
|
||||
p := make([]byte, size)
|
||||
n, err := b.ReadAt(p, 0)
|
||||
assert.Equal(t, size, n, "size=%d", size)
|
||||
assert.NoError(t, err, "a read fully inside the buffer must not report EOF (size=%d)", size)
|
||||
}
|
||||
|
||||
p := make([]byte, 201)
|
||||
n, err := b.ReadAt(p, 0)
|
||||
assert.Equal(t, 200, n)
|
||||
assert.ErrorIs(t, err, io.EOF, "one byte past the end is short and must say so")
|
||||
}
|
||||
|
||||
func TestReadAtEveryOffsetAndLength(t *testing.T) {
|
||||
const limit = 32
|
||||
b := newTestBuffer(t, WithMemLimit(limit))
|
||||
|
||||
want := make([]byte, 100)
|
||||
for i := range want {
|
||||
want[i] = byte(i)
|
||||
}
|
||||
mustWrite(t, b, 0, want)
|
||||
|
||||
// every offset x length pair, so a tier-boundary off-by-one cannot hide
|
||||
for off := 0; off <= len(want); off++ {
|
||||
for l := 0; l <= len(want)-off+2; l++ {
|
||||
p := make([]byte, l)
|
||||
n, err := b.ReadAt(p, int64(off))
|
||||
|
||||
expected := len(want) - off
|
||||
if l < expected {
|
||||
expected = l
|
||||
}
|
||||
assert.Equal(t, expected, n, "off=%d len=%d", off, l)
|
||||
assert.Equal(t, want[off:off+expected], p[:expected], "off=%d len=%d", off, l)
|
||||
|
||||
switch {
|
||||
case l == 0:
|
||||
assert.NoError(t, err, "off=%d len=0", off)
|
||||
case off >= len(want):
|
||||
assert.ErrorIs(t, err, io.EOF, "off=%d len=%d", off, l)
|
||||
case l > expected:
|
||||
assert.ErrorIs(t, err, io.EOF, "off=%d len=%d", off, l)
|
||||
default:
|
||||
assert.NoError(t, err, "off=%d len=%d", off, l)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- overwrite semantics ----------------------------------------------------------------------------
|
||||
|
||||
func TestOverwrite(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
memLimit int64
|
||||
}{
|
||||
{"in memory", 1 << 20},
|
||||
{"on disk", 0},
|
||||
{"across the boundary", 8},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
b := newTestBuffer(t, WithMemLimit(tt.memLimit))
|
||||
mustWrite(t, b, 0, repeat('A', 16))
|
||||
mustWrite(t, b, 4, repeat('B', 8))
|
||||
|
||||
want := append(append(repeat('A', 4), repeat('B', 8)...), repeat('A', 4)...)
|
||||
assert.Equal(t, want, readAll(t, b), "the later write wins over the range it covers")
|
||||
assert.Equal(t, int64(16), b.Size(), "an overwrite does not extend the buffer")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- extents ----------------------------------------------------------------------------------------
|
||||
|
||||
func TestWrittenExtentsAreSortedAndMerged(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
writes []extent
|
||||
want []extent
|
||||
}{
|
||||
{
|
||||
name: "disjoint stay separate, in order",
|
||||
writes: []extent{{100, 150}, {0, 50}},
|
||||
want: []extent{{0, 50}, {100, 150}},
|
||||
},
|
||||
{
|
||||
name: "adjacent merge",
|
||||
writes: []extent{{0, 50}, {50, 100}},
|
||||
want: []extent{{0, 100}},
|
||||
},
|
||||
{
|
||||
name: "overlapping merge",
|
||||
writes: []extent{{0, 60}, {40, 100}},
|
||||
want: []extent{{0, 100}},
|
||||
},
|
||||
{
|
||||
name: "a write bridging two runs collapses all three",
|
||||
writes: []extent{{0, 20}, {80, 100}, {20, 80}},
|
||||
want: []extent{{0, 100}},
|
||||
},
|
||||
{
|
||||
name: "a contained write changes nothing",
|
||||
writes: []extent{{0, 100}, {20, 40}},
|
||||
want: []extent{{0, 100}},
|
||||
},
|
||||
{
|
||||
name: "identical writes collapse",
|
||||
writes: []extent{{0, 10}, {0, 10}, {0, 10}},
|
||||
want: []extent{{0, 10}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
b := newTestBuffer(t)
|
||||
for _, w := range tt.writes {
|
||||
mustWrite(t, b, w.start, repeat('x', int(w.end-w.start)))
|
||||
}
|
||||
assert.Equal(t, tt.want, b.written)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrittenExtentsStayCompactUnderManyAdjacentWrites(t *testing.T) {
|
||||
b := newTestBuffer(t)
|
||||
for i := int64(0); i < 1000; i++ {
|
||||
mustWrite(t, b, i*8, repeat('A', 8))
|
||||
}
|
||||
assert.Len(t, b.written, 1, "a run of adjacent writes must collapse to one extent, not 1000")
|
||||
assert.Equal(t, int64(8000), b.Size())
|
||||
}
|
||||
|
||||
// --- lifecycle --------------------------------------------------------------------------------------
|
||||
|
||||
func TestCloseRemovesTheSpillFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := New(tmpdir.FromPath(dir), WithMemLimit(0))
|
||||
mustWrite(t, b, 0, repeat('A', 32))
|
||||
require.Len(t, dirEntries(t, dir), 1)
|
||||
|
||||
require.NoError(t, b.Close())
|
||||
assert.Empty(t, dirEntries(t, dir), "Close must take the spill file with it")
|
||||
}
|
||||
|
||||
func TestCloseIsIdempotent(t *testing.T) {
|
||||
b := New(tmpdir.FromPath(t.TempDir()), WithMemLimit(0))
|
||||
mustWrite(t, b, 0, repeat('A', 32))
|
||||
|
||||
require.NoError(t, b.Close())
|
||||
assert.NoError(t, b.Close(), "a second Close is a no-op, not a double-close")
|
||||
assert.NoError(t, b.Close())
|
||||
}
|
||||
|
||||
func TestCloseOnNilReceiver(t *testing.T) {
|
||||
var b *Buffer
|
||||
assert.NotPanics(t, func() {
|
||||
assert.NoError(t, b.Close())
|
||||
})
|
||||
}
|
||||
|
||||
func TestCloseWithoutASpillFile(t *testing.T) {
|
||||
b := New(nil, WithMemLimit(1<<20))
|
||||
mustWrite(t, b, 0, repeat('A', 32))
|
||||
assert.NoError(t, b.Close(), "a memory-only buffer closes cleanly with no file to remove")
|
||||
}
|
||||
|
||||
func TestWriteAfterCloseFails(t *testing.T) {
|
||||
b := New(tmpdir.FromPath(t.TempDir()))
|
||||
mustWrite(t, b, 0, repeat('A', 8))
|
||||
require.NoError(t, b.Close())
|
||||
|
||||
_, err := b.WriteAt(repeat('B', 8), 0)
|
||||
require.Error(t, err, "a closed buffer must refuse writes rather than resurrect its file")
|
||||
assert.ErrorIs(t, err, os.ErrClosed)
|
||||
}
|
||||
|
||||
// TestReadAfterCloseFails pins that a closed buffer reads as closed rather than as empty. Close drops the
|
||||
// extents, so without the check every read would come back io.EOF and a caller could not tell a buffer it
|
||||
// still owns from one somebody already released.
|
||||
func TestReadAfterCloseFails(t *testing.T) {
|
||||
b := New(tmpdir.FromPath(t.TempDir()))
|
||||
mustWrite(t, b, 0, repeat('A', 8))
|
||||
require.NoError(t, b.Close())
|
||||
|
||||
n, err := b.ReadAt(make([]byte, 8), 0)
|
||||
assert.Zero(t, n)
|
||||
assert.ErrorIs(t, err, os.ErrClosed)
|
||||
|
||||
at, ok := b.FirstGap(8, 100)
|
||||
assert.False(t, ok, "and a closed buffer offers nowhere to write")
|
||||
assert.Zero(t, at)
|
||||
}
|
||||
|
||||
// TestWriteFailureLeavesTheBufferUnchanged pins the ordering inside WriteAt: the fallible half runs first,
|
||||
// so a spill file that cannot be created does not take the memory tier down with it.
|
||||
func TestWriteFailureLeavesTheBufferUnchanged(t *testing.T) {
|
||||
// FromPath takes the directory as given, so a path that does not exist makes file creation fail
|
||||
b := New(tmpdir.FromPath(filepath.Join(t.TempDir(), "no-such-dir")), WithMemLimit(16))
|
||||
t.Cleanup(func() { _ = b.Close() })
|
||||
|
||||
mustWrite(t, b, 0, repeat('A', 8))
|
||||
|
||||
_, err := b.WriteAt(repeat('B', 32), 0) // straddles the limit, so it needs the spill file
|
||||
require.Error(t, err)
|
||||
|
||||
assert.Equal(t, int64(8), b.Size(), "a failed write records nothing")
|
||||
assert.Nil(t, b.file, "and leaves no half-made spill file behind")
|
||||
|
||||
p := make([]byte, 8)
|
||||
n, err := b.ReadAt(p, 0)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 8, n)
|
||||
assert.Equal(t, repeat('A', 8), p, "the memory tier still holds what it held")
|
||||
|
||||
mustWrite(t, b, 8, repeat('C', 8)) // inside the limit, so it still needs no file
|
||||
assert.Equal(t, append(repeat('A', 8), repeat('C', 8)...), readAll(t, b))
|
||||
}
|
||||
|
||||
// --- interface compliance ---------------------------------------------------------------------------
|
||||
|
||||
func TestBufferSatisfiesTheStandardInterfaces(t *testing.T) {
|
||||
b := newTestBuffer(t)
|
||||
var (
|
||||
_ io.WriterAt = b
|
||||
_ io.ReaderAt = b
|
||||
_ io.Closer = b
|
||||
)
|
||||
assert.Implements(t, (*io.ReaderAt)(nil), b)
|
||||
assert.Implements(t, (*io.WriterAt)(nil), b)
|
||||
}
|
||||
|
||||
// TestWorksThroughIOOffsetWriter pins the composition every consumer uses: stream a decoded block into
|
||||
// the buffer at a chosen offset without the writer knowing where it lands.
|
||||
func TestWorksThroughIOOffsetWriter(t *testing.T) {
|
||||
b := newTestBuffer(t, WithMemLimit(16))
|
||||
|
||||
w := io.NewOffsetWriter(b, 8)
|
||||
n, err := io.Copy(w, bytes.NewReader(repeat('B', 32)))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(32), n)
|
||||
|
||||
mustWrite(t, b, 0, repeat('A', 8))
|
||||
|
||||
got := readAll(t, b)
|
||||
assert.Equal(t, append(repeat('A', 8), repeat('B', 32)...), got)
|
||||
}
|
||||
|
||||
// TestWorksWithIOSectionReader pins the other direction: parsers wrap an io.ReaderAt in a SectionReader,
|
||||
// and it must see a consistent length.
|
||||
func TestWorksWithIOSectionReader(t *testing.T) {
|
||||
b := newTestBuffer(t)
|
||||
mustWrite(t, b, 0, repeat('A', 100))
|
||||
|
||||
sr := io.NewSectionReader(b, 0, b.Size())
|
||||
got, err := io.ReadAll(sr)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, repeat('A', 100), got)
|
||||
}
|
||||
|
||||
// --- model-based randomized testing -----------------------------------------------------------------
|
||||
|
||||
// TestAgainstAReferenceModel drives random writes through the buffer and a dumb reference at the same
|
||||
// time, then compares every readable byte. This is what catches tier-boundary and merge bugs that
|
||||
// hand-written cases miss.
|
||||
func TestAgainstAReferenceModel(t *testing.T) {
|
||||
const universe = 8192
|
||||
|
||||
for _, limit := range []int64{0, 1, 64, 1000, 4096, universe * 2} {
|
||||
t.Run("memLimit="+itoa(limit), func(t *testing.T) {
|
||||
for seed := int64(0); seed < 40; seed++ {
|
||||
rng := rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic test input, not crypto
|
||||
b := newTestBuffer(t, WithMemLimit(limit))
|
||||
|
||||
model := make([]byte, universe)
|
||||
written := make([]bool, universe)
|
||||
|
||||
for op := 0; op < 40; op++ {
|
||||
off := rng.Int63n(universe)
|
||||
l := rng.Int63n(256) + 1
|
||||
if off+l > universe {
|
||||
l = universe - off
|
||||
}
|
||||
payload := make([]byte, l)
|
||||
for i := range payload {
|
||||
payload[i] = byte(rng.Intn(256))
|
||||
}
|
||||
|
||||
mustWrite(t, b, off, payload)
|
||||
copy(model[off:], payload)
|
||||
for i := off; i < off+l; i++ {
|
||||
written[i] = true
|
||||
}
|
||||
}
|
||||
|
||||
// the reference contiguous prefix
|
||||
var want int64
|
||||
for want < universe && written[want] {
|
||||
want++
|
||||
}
|
||||
|
||||
require.Equal(t, want, b.Size(), "seed=%d limit=%d", seed, limit)
|
||||
if want == 0 {
|
||||
continue
|
||||
}
|
||||
assert.Equal(t, model[:want], readAll(t, b), "seed=%d limit=%d", seed, limit)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRandomReadsMatchTheModel checks partial reads at arbitrary offsets, which the whole-buffer
|
||||
// comparison above cannot reach.
|
||||
func TestRandomReadsMatchTheModel(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(7)) //nolint:gosec // deterministic test input, not crypto
|
||||
b := newTestBuffer(t, WithMemLimit(97))
|
||||
|
||||
model := make([]byte, 1000)
|
||||
for i := range model {
|
||||
model[i] = byte(rng.Intn(256))
|
||||
}
|
||||
// written in random-sized chunks, in order, so the whole range is covered
|
||||
for off := 0; off < len(model); {
|
||||
l := rng.Intn(50) + 1
|
||||
if off+l > len(model) {
|
||||
l = len(model) - off
|
||||
}
|
||||
mustWrite(t, b, int64(off), model[off:off+l])
|
||||
off += l
|
||||
}
|
||||
require.Equal(t, int64(len(model)), b.Size())
|
||||
|
||||
for i := 0; i < 500; i++ {
|
||||
off := rng.Intn(len(model))
|
||||
l := rng.Intn(len(model)-off) + 1
|
||||
p := make([]byte, l)
|
||||
n, err := b.ReadAt(p, int64(off))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, l, n)
|
||||
require.Equal(t, model[off:off+l], p, "off=%d len=%d", off, l)
|
||||
}
|
||||
}
|
||||
|
||||
// --- helpers ----------------------------------------------------------------------------------------
|
||||
|
||||
func dirEntries(t *testing.T, dir string) []os.DirEntry {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(dir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
// tmpdir.NewFile creates the spill file inside the root it is given
|
||||
var files []os.DirEntry
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
sub, err := os.ReadDir(filepath.Join(dir, e.Name()))
|
||||
require.NoError(t, err)
|
||||
for _, s := range sub {
|
||||
files = append(files, s)
|
||||
}
|
||||
continue
|
||||
}
|
||||
files = append(files, e)
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func itoa(n int64) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
var digits []byte
|
||||
for n > 0 {
|
||||
digits = append([]byte{byte('0' + n%10)}, digits...)
|
||||
n /= 10
|
||||
}
|
||||
return string(digits)
|
||||
}
|
||||
|
||||
// TestMemoryTierGrowsAmortized is the regression test for quadratic growth. Sizing the tier to exactly
|
||||
// what each write needs reallocates and copies the whole thing per call, so a caller streaming a block in
|
||||
// small chunks (which is what io.Copy through an OffsetWriter does) pays O(n^2): filling a 1MB tier in
|
||||
// 32KB chunks cost ~16MB of allocation and showed up as a 19MB spike decompressing a 16MB payload.
|
||||
func TestMemoryTierGrowsAmortized(t *testing.T) {
|
||||
const (
|
||||
limit = 1 << 20
|
||||
chunk = 32 << 10
|
||||
)
|
||||
b := newTestBuffer(t, WithMemLimit(limit))
|
||||
|
||||
allocated := testutils.MeasureAlloc(t, func() {
|
||||
for off := int64(0); off < limit; off += chunk {
|
||||
mustWrite(t, b, off, repeat('A', chunk))
|
||||
}
|
||||
})
|
||||
t.Logf("filling a %d byte tier in %d byte chunks allocated %d bytes", limit, chunk, allocated)
|
||||
assert.Less(t, allocated, uint64(4*limit),
|
||||
"growth has to be amortized, not a fresh copy of the whole tier per write")
|
||||
assert.Equal(t, int64(limit), b.Size())
|
||||
}
|
||||
|
||||
// TestMemoryTierNeverExceedsTheLimit pins the cap that geometric growth could otherwise overshoot.
|
||||
func TestMemoryTierNeverExceedsTheLimit(t *testing.T) {
|
||||
const limit = 1000
|
||||
b := newTestBuffer(t, WithMemLimit(limit))
|
||||
|
||||
for off := int64(0); off < 4*limit; off += 100 {
|
||||
mustWrite(t, b, off, repeat('A', 100))
|
||||
assert.LessOrEqual(t, int64(len(b.mem)), int64(limit),
|
||||
"the memory tier may never grow past its own limit (off=%d)", off)
|
||||
}
|
||||
assert.Equal(t, int64(4*limit), b.Size())
|
||||
}
|
||||
|
||||
// TestFirstGap covers the placement question a caller writing at offsets of its own choosing has to ask.
|
||||
// It moved here from the UPX cataloger, which used to walk an exported extent list to answer it itself.
|
||||
func TestFirstGap(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
writes []extent
|
||||
size int64
|
||||
within int64
|
||||
wantAt int64
|
||||
wantOK bool
|
||||
}{
|
||||
{
|
||||
name: "an empty buffer starts at zero",
|
||||
size: 16,
|
||||
within: 100,
|
||||
wantAt: 0, wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "the gap between two runs comes first",
|
||||
writes: []extent{{0, 1000}, {1016, 2000}},
|
||||
size: 16,
|
||||
within: 3000,
|
||||
wantAt: 1000, wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "with the gap filled the tail is what is left",
|
||||
writes: []extent{{0, 2000}},
|
||||
size: 1000,
|
||||
within: 3000,
|
||||
wantAt: 2000, wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "a gap too small is skipped for a later one",
|
||||
writes: []extent{{0, 100}, {108, 200}},
|
||||
size: 50,
|
||||
within: 1000,
|
||||
wantAt: 200, wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "nowhere left to fit is reported rather than squeezed in",
|
||||
writes: []extent{{0, 1000}, {1016, 2000}},
|
||||
size: 1001,
|
||||
within: 3000,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "a run past the declared total must not wrap the remaining-space subtraction",
|
||||
writes: []extent{{0, 200}},
|
||||
size: 8,
|
||||
within: 100,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "exactly filling the tail is allowed",
|
||||
writes: []extent{{0, 900}},
|
||||
size: 100,
|
||||
within: 1000,
|
||||
wantAt: 900, wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "a gap between two runs is still cut off by the bound",
|
||||
writes: []extent{{0, 10}, {1000, 1010}},
|
||||
size: 45,
|
||||
within: 50,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "a gap between two runs that the bound still leaves room in",
|
||||
writes: []extent{{0, 10}, {1000, 1010}},
|
||||
size: 30,
|
||||
within: 50,
|
||||
wantAt: 10, wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "a negative size is refused rather than treated as zero",
|
||||
size: -1,
|
||||
within: 100,
|
||||
wantOK: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
b := newTestBuffer(t)
|
||||
for _, w := range tt.writes {
|
||||
mustWrite(t, b, w.start, repeat('x', int(w.end-w.start)))
|
||||
}
|
||||
|
||||
at, ok := b.FirstGap(tt.size, tt.within)
|
||||
assert.Equal(t, tt.wantOK, ok)
|
||||
if tt.wantOK {
|
||||
assert.Equal(t, tt.wantAt, at)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFirstGapNeverOverlapsWhatWasWritten is the property behind the table: whatever offset comes back has
|
||||
// to be usable, so writing size bytes there must not land on top of anything already stored.
|
||||
func TestFirstGapNeverOverlapsWhatWasWritten(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(11)) //nolint:gosec // deterministic test input, not crypto
|
||||
const universe = 4096
|
||||
|
||||
for seed := 0; seed < 200; seed++ {
|
||||
b := newTestBuffer(t)
|
||||
occupied := make([]bool, universe)
|
||||
|
||||
for op := 0; op < 6; op++ {
|
||||
// deliberately reaches past universe too, so the bound has to cut a between-extents gap short
|
||||
off := rng.Int63n(2 * universe)
|
||||
l := rng.Int63n(200) + 1
|
||||
mustWrite(t, b, off, repeat('x', int(l)))
|
||||
for i := off; i < off+l && i < universe; i++ {
|
||||
occupied[i] = true
|
||||
}
|
||||
}
|
||||
|
||||
size := rng.Int63n(100) + 1
|
||||
at, ok := b.FirstGap(size, universe)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
require.LessOrEqual(t, at+size, int64(universe), "the result must fit inside the declared total")
|
||||
for i := at; i < at+size; i++ {
|
||||
require.False(t, occupied[i], "FirstGap returned %d for %d bytes, but %d is taken", at, size, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package testutils
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// MeasureAlloc reports the bytes allocated while fn ran.
|
||||
//
|
||||
// This exists for guards whose whole point is "a small input cannot make us reserve a large buffer".
|
||||
// Only a byte count states that property: asserting that an error comes back would keep passing if the
|
||||
// allocation were hoisted above the check, which is the bug these guards were written for.
|
||||
//
|
||||
// Two things to know when using it:
|
||||
//
|
||||
// - TotalAlloc is process-wide, so a test calling this must not call t.Parallel. Another test's
|
||||
// allocations would land in the measurement.
|
||||
// - a budget assertion is only meaningful if the fixture would really blow past it unguarded. Prefer
|
||||
// measuring the unguarded path too and asserting it exceeds the declared size, so a fixture that
|
||||
// quietly stops being a bomb fails instead of passing.
|
||||
func MeasureAlloc(t testing.TB, fn func()) uint64 {
|
||||
t.Helper()
|
||||
|
||||
var before, after runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
fn()
|
||||
runtime.ReadMemStats(&after)
|
||||
|
||||
return after.TotalAlloc - before.TotalAlloc
|
||||
}
|
||||
Reference in New Issue
Block a user