oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-03 07:56:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

chore: revert custom index in favor of key_Split

Browse Source 
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
Keith Zantow
2026-09-20 11:22:10 -04:00
parent 99615b7dfa
commit 53cd4254e4
81 changed files with 7461 additions and 6624 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/archive/entrystore_test.go
+45 -45
View File
@@ -11,29 +11,16 @@ import (
"github.com/stretchr/testify/require"
)
func regularHeader(name string, size int64) tar.Header {
return tar.Header{Name: name, Size: size, Mode: 0o600, Typeflag: tar.TypeReg}
}
func readEntry(t *testing.T, r ReaderAtSeeker) string {
t.Helper()
_, err := r.Seek(0, io.SeekStart)
require.NoError(t, err)
b, err := io.ReadAll(r)
require.NoError(t, err)
return string(b)
}
func TestEntryStore_holdsSmallContentInMemory(t *testing.T) {
dir := t.TempDir()
s := NewEntryStore(dir, "app.zip", nil)
s := NewEntryStore(WorkDirAt(dir), "app.zip", nil)
charge := memCharge(1024)
t.Cleanup(func() { require.NoError(t, s.Close()) })
entry, err := s.Add(regularHeader("a.txt", 5), bytes.NewReader([]byte("hello")), charge)
require.NoError(t, err)
assert.Equal(t, int64(5), s.HeldInMemory())
assert.Equal(t, int64(5), s.heldInMemory())
assert.NoFileExists(t, filepath.Join(dir, overflowBlobName),
"a small archive must not touch the disk at all - that is the whole point of the store")
@@ -44,21 +31,21 @@ func TestEntryStore_holdsSmallContentInMemory(t *testing.T) {
func TestEntryStore_spillsToDiskAndKeepsTheSameEntries(t *testing.T) {
// the index over these entries is not rebuilt when content moves, so the entry pointers and their
// contents must both survive the move
// contents must survive it
dir := t.TempDir()
s := NewEntryStore(dir, "app.zip", nil)
s := NewEntryStore(WorkDirAt(dir), "app.zip", nil)
charge := memCharge(8)
t.Cleanup(func() { require.NoError(t, s.Close()) })
first, err := s.Add(regularHeader("a.txt", 4), bytes.NewReader([]byte("aaaa")), charge)
require.NoError(t, err)
require.Equal(t, int64(4), s.HeldInMemory())
require.Equal(t, int64(4), s.heldInMemory())
// the second entry takes the store past its memory bound, so everything held moves out
second, err := s.Add(regularHeader("b.txt", 6), bytes.NewReader([]byte("bbbbbb")), charge)
require.NoError(t, err)
assert.Zero(t, s.HeldInMemory(), "everything must have moved out, not just the entry that did not fit")
assert.Zero(t, s.heldInMemory(), "everything must have moved out, not just the entry that did not fit")
assert.FileExists(t, filepath.Join(dir, overflowBlobName))
// the same entry pointers still read their own content, from their new home
@@ -76,14 +63,14 @@ func TestEntryStore_spillsToDiskAndKeepsTheSameEntries(t *testing.T) {
func TestEntryStore_zeroMemoryHoldsNothing(t *testing.T) {
// the reading a zero memory limit has everywhere else: none of that resource may be used
dir := t.TempDir()
s := NewEntryStore(dir, "app.zip", nil)
s := NewEntryStore(WorkDirAt(dir), "app.zip", nil)
charge := memCharge(0)
t.Cleanup(func() { require.NoError(t, s.Close()) })
entry, err := s.Add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")), charge)
require.NoError(t, err)
assert.Zero(t, s.HeldInMemory())
assert.Zero(t, s.heldInMemory())
assert.FileExists(t, filepath.Join(dir, overflowBlobName))
r, err := s.Open(entry)
@@ -93,7 +80,7 @@ func TestEntryStore_zeroMemoryHoldsNothing(t *testing.T) {
func TestEntryStore_negativeMemoryNeverSpills(t *testing.T) {
dir := t.TempDir()
s := NewEntryStore(dir, "app.zip", nil)
s := NewEntryStore(WorkDirAt(dir), "app.zip", nil)
charge := memCharge(-1)
t.Cleanup(func() { require.NoError(t, s.Close()) })
@@ -102,15 +89,15 @@ func TestEntryStore_negativeMemoryNeverSpills(t *testing.T) {
require.NoError(t, err)
}
assert.Equal(t, int64(11), s.HeldInMemory())
assert.Equal(t, int64(11), s.heldInMemory())
assert.NoFileExists(t, filepath.Join(dir, overflowBlobName))
}
func TestEntryStore_spillIsRefusedWhenDiskIsNotAvailable(t *testing.T) {
// content that does not fit in memory and cannot be written has nowhere to go, and the archive is
// content that does not fit in memory and cannot be written has nowhere to go, so the archive is
// skipped rather than cataloged as if it were empty
dir := t.TempDir()
s := NewEntryStore(dir, "app.zip", nil)
s := NewEntryStore(WorkDirAt(dir), "app.zip", nil)
t.Cleanup(func() { require.NoError(t, s.Close()) })
nowhereToPutIt := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).Charge()
@@ -120,30 +107,24 @@ func TestEntryStore_spillIsRefusedWhenDiskIsNotAvailable(t *testing.T) {
func TestEntryStore_directoryEntriesHoldNoContent(t *testing.T) {
dir := t.TempDir()
s := NewEntryStore(dir, "app.zip", nil)
s := NewEntryStore(WorkDirAt(dir), "app.zip", nil)
charge := memCharge(1024)
t.Cleanup(func() { require.NoError(t, s.Close()) })
entry, err := s.Add(tar.Header{Name: "lib/", Typeflag: tar.TypeDir, Mode: 0o755}, nil, charge)
require.NoError(t, err)
assert.Zero(t, s.HeldInMemory())
assert.Zero(t, s.heldInMemory())
r, err := s.Open(entry)
require.NoError(t, err)
assert.Empty(t, readEntry(t, r))
}
// memCharge is a charge against a memory bound with disk left unbounded, which is the shape the store
// reads: hold while memory admits, move out when it does not.
func memCharge(maxMemory int64) *Charge {
return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).Charge()
}
func TestEntryStore_chargesIndexRecordEvenForEmptyContent(t *testing.T) {
// an empty entry still charges its index record - the header and node held to reach it - which is
// what bounds entry count, the dimension the content-byte budgets never bounded.
// an empty entry still charges its index record - the header and node held to reach it - which bounds
// entry count, the dimension the content-byte budgets do not cover
dir := t.TempDir()
s := NewEntryStore(dir, "app.zip", nil)
s := NewEntryStore(WorkDirAt(dir), "app.zip", nil)
limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1})
charge := limiter.Charge()
t.Cleanup(func() { require.NoError(t, s.Close()) })
@@ -154,15 +135,15 @@ func TestEntryStore_chargesIndexRecordEvenForEmptyContent(t *testing.T) {
mem, _ := limiter.InUse()
assert.Equal(t, indexRecordCost(hdr), mem, "the index record is charged to memory")
assert.Equal(t, int64(0), s.HeldInMemory(), "no content was held")
assert.Equal(t, int64(0), s.heldInMemory(), "no content was held")
}
func TestEntryStore_indexOverflowsToDiskWhenMemoryIsZero(t *testing.T) {
// the index cannot be refused just because memory is zero: a resolver still needs the records to
// reach content that lives entirely on disk. With disk unbounded, every entry is admitted and its
// record charged to disk instead.
// the index cannot be refused just because memory is zero: a resolver needs the records to reach
// content living entirely on disk. With disk unbounded, every entry is admitted and its record
// charged to disk instead.
dir := t.TempDir()
s := NewEntryStore(dir, "app.zip", nil)
s := NewEntryStore(WorkDirAt(dir), "app.zip", nil)
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
charge := limiter.Charge()
t.Cleanup(func() { require.NoError(t, s.Close()) })
@@ -177,11 +158,11 @@ func TestEntryStore_indexOverflowsToDiskWhenMemoryIsZero(t *testing.T) {
}
func TestEntryStore_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) {
// with memory zero and only a few index records' worth of disk, the store admits a bounded number
// of entries and then refuses - an archive of millions of empty entries can no longer grow the
// index without limit.
// with memory zero and only a few index records' worth of disk, the store admits a bounded number of
// entries and then refuses, so an archive of millions of empty entries cannot grow the index without
// limit
dir := t.TempDir()
s := NewEntryStore(dir, "app.zip", nil)
s := NewEntryStore(WorkDirAt(dir), "app.zip", nil)
room := indexRecordCost(regularHeader("x", 0)) * 3
charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).Charge()
t.Cleanup(func() { require.NoError(t, s.Close()) })
@@ -197,3 +178,22 @@ func TestEntryStore_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) {
require.ErrorIs(t, err, ErrDiskLimitReached)
assert.Equal(t, 3, admitted, "exactly the index records that fit the disk budget were admitted")
}
func regularHeader(name string, size int64) tar.Header {
return tar.Header{Name: name, Size: size, Mode: 0o600, Typeflag: tar.TypeReg}
}
func readEntry(t *testing.T, r ReaderAtSeeker) string {
t.Helper()
_, err := r.Seek(0, io.SeekStart)
require.NoError(t, err)
b, err := io.ReadAll(r)
require.NoError(t, err)
return string(b)
}
// memCharge is a charge against a memory bound with disk unbounded, the shape the store reads: hold
// while memory admits, move out when it does not.
func memCharge(maxMemory int64) *Charge {
return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).Charge()
}