mirror of
https://github.com/anchore/syft.git
synced 2026-11-03 16:06:36 +01:00
feat: initial recursive archive cataloging impl
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
Keith Zantow
@@ -0,0 +1,221 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// manyEntryZip builds a zip of count small entries, in sorted name order.
|
||||
func manyEntryZip(t *testing.T, count int) *bytes.Buffer {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for i := range count {
|
||||
w, err := zw.Create(fmt.Sprintf("e%06d.txt", i))
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("x"))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, zw.Close())
|
||||
return &buf
|
||||
}
|
||||
|
||||
// extractedOverflow extracts one archive and hands back where its content went, along with the work
|
||||
// directory holding it - which is what a filesystem cost has to be measured against.
|
||||
func extractedOverflow(t *testing.T, content io.Reader, limits ExtractionLimits) (*ExtractedArchive, Overflow, string) {
|
||||
t.Helper()
|
||||
|
||||
var got Overflow
|
||||
factory := func(overflow Overflow) (file.Resolver, IndexResult, error) {
|
||||
got = overflow
|
||||
return nil, IndexResult{}, nil
|
||||
}
|
||||
|
||||
extracted, err := ExtractToResolver(
|
||||
context.Background(), content, "app.zip", "", "app.zip",
|
||||
DefaultExtractors(), NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), limits, factory, nil, nil,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
return extracted, got, filepath.Dir(got.TarPath)
|
||||
}
|
||||
|
||||
// filesystemEntries is every directory entry under dir, path by path, which is the cost the
|
||||
// requirement is about: an archive must not put one of these on the filesystem per entry it holds.
|
||||
func filesystemEntries(t *testing.T, dir string) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
require.NoError(t, filepath.WalkDir(dir, func(p string, _ fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, relErr := filepath.Rel(dir, p)
|
||||
if relErr != nil {
|
||||
return relErr
|
||||
}
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
return nil
|
||||
}))
|
||||
return out
|
||||
}
|
||||
|
||||
func TestOverflow_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
|
||||
// the requirement this whole shape exists for: an archive of a hundred thousand entries must not
|
||||
// put a hundred thousand files on the filesystem. Asserted as an exact set of paths for two
|
||||
// archives three orders of magnitude apart in entry count, rather than as a bound - a count that
|
||||
// merely grows slowly would pass a bound and still be proportional to content.
|
||||
small, smallOverflow, smallWork := extractedOverflow(t, manyEntryZip(t, 2), ExtractionLimits{})
|
||||
large, largeOverflow, largeWork := extractedOverflow(t, manyEntryZip(t, 5000), ExtractionLimits{})
|
||||
|
||||
want := []string{"contents", "contents.tar"}
|
||||
assert.Equal(t, want, filesystemEntries(t, smallWork))
|
||||
assert.Equal(t, want, filesystemEntries(t, largeWork),
|
||||
"5000 entries must cost the same filesystem entries as 2: one tar, plus the empty root their "+
|
||||
"paths are reported relative to")
|
||||
|
||||
// and the entries really are all there, so this is not passing because nothing was extracted
|
||||
assert.Equal(t, 2, countTarEntries(t, smallOverflow.TarPath))
|
||||
assert.Equal(t, 5000, countTarEntries(t, largeOverflow.TarPath))
|
||||
// what one archive costs on disk, exactly: a header block and a padded data block per entry, plus
|
||||
// the two-block end-of-archive marker. This is the figure the disk limit is charged, which is why
|
||||
// a limit expressed in entry bytes now under-counts.
|
||||
assert.Equal(t, int64((5000*2+2)*tarBlockSize), large.Result.BytesWritten)
|
||||
assert.Equal(t, int64((2*2+2)*tarBlockSize), small.Result.BytesWritten)
|
||||
}
|
||||
|
||||
func TestOverflow_cleanupRemovesOneFileAndDoesNotWalkPerEntry(t *testing.T) {
|
||||
// cleanup was proportional to content too: releasing an archive meant removing one filesystem
|
||||
// entry per archive entry. It is now bounded by the same constant as the extraction, so the count
|
||||
// of things removed is asserted rather than just the fact that the directory is gone.
|
||||
extracted, overflow, workDir := extractedOverflow(t, manyEntryZip(t, 5000), ExtractionLimits{})
|
||||
|
||||
before := filesystemEntries(t, workDir)
|
||||
require.Len(t, before, 2, "the release has two things to remove whatever the archive held")
|
||||
|
||||
extracted.Cleanup()
|
||||
|
||||
_, err := os.Stat(overflow.TarPath)
|
||||
assert.True(t, os.IsNotExist(err), "the one file holding every entry must be gone")
|
||||
_, err = os.Stat(workDir)
|
||||
assert.True(t, os.IsNotExist(err), "and so must the work directory")
|
||||
|
||||
// safe to call more than once, since the cataloger defers it past the sub-pipeline and past
|
||||
// recursion
|
||||
extracted.Cleanup()
|
||||
}
|
||||
|
||||
func TestOverflow_anEntryAtTheEndIsReadBySeeking(t *testing.T) {
|
||||
// "overflow content MUST remain directly readable without re-reading or re-decompressing what
|
||||
// precedes it". A design that met the entry-count requirement by re-streaming the whole archive
|
||||
// per read would not satisfy it, so what is asserted here is the mechanism: the index records
|
||||
// where the last entry's content begins, near the end of a tar whose earlier entries are large,
|
||||
// and the reader it hands back is a bounded window onto the file at that offset.
|
||||
const filler = 64 * 1024
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for i := range 8 {
|
||||
w, err := zw.Create(fmt.Sprintf("filler%d.bin", i))
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write(bytes.Repeat([]byte{byte(i)}, filler))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
last, err := zw.Create("zz-last.txt")
|
||||
require.NoError(t, err)
|
||||
wanted := strings.Repeat("last", 64)
|
||||
_, err = last.Write([]byte(wanted))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
_, overflow, _ := extractedOverflow(t, &buf, ExtractionLimits{})
|
||||
|
||||
info, err := os.Stat(overflow.TarPath)
|
||||
require.NoError(t, err)
|
||||
total := info.Size()
|
||||
require.Greater(t, total, int64(8*filler), "the fixture must put real bulk ahead of the last entry")
|
||||
|
||||
// the last entry's content sits near the end of the tar, which is the only reason a seek to it is
|
||||
// worth anything: everything before it is what a re-streaming design would have had to read
|
||||
offset := int64(bytes.Index(readFile(t, overflow.TarPath), []byte(wanted)))
|
||||
require.Positive(t, offset)
|
||||
assert.Greater(t, offset, total*9/10, "the last entry must really be at the end of the tar")
|
||||
|
||||
entry := tarEntryNamed(t, overflow.TarPath, "zz-last.txt")
|
||||
|
||||
contents := entry.Open()
|
||||
t.Cleanup(func() { require.NoError(t, contents.Close()) })
|
||||
|
||||
random, ok := contents.(interface {
|
||||
io.ReaderAt
|
||||
io.Seeker
|
||||
})
|
||||
require.True(t, ok, "the entry reader must be random access, or a read cannot be a seek")
|
||||
|
||||
// the reader is bounded to this entry rather than being a view of the whole tar: its end is the
|
||||
// entry's own size, so reading it cannot have read what precedes it
|
||||
end, err := random.Seek(0, io.SeekEnd)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(wanted)), end)
|
||||
|
||||
// and offset zero of the reader is the last entry's first byte, not the tar's. A reader over the
|
||||
// whole file would answer this with the first filler entry's content.
|
||||
head := make([]byte, 8)
|
||||
_, err = random.ReadAt(head, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wanted[:8], string(head))
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
body, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
return body
|
||||
}
|
||||
|
||||
// countTarEntries is how many entries a tar on disk holds, read through the same index the resolver
|
||||
// builds its filesystem from.
|
||||
func countTarEntries(t *testing.T, tarPath string) int {
|
||||
t.Helper()
|
||||
count := 0
|
||||
_, err := stereoscopeFile.NewTarIndex(tarPath, func(stereoscopeFile.TarIndexEntry) error {
|
||||
count++
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return count
|
||||
}
|
||||
|
||||
// tarEntryNamed is the index record for one entry of a tar, which is what carries its seek offset.
|
||||
func tarEntryNamed(t *testing.T, tarPath, name string) stereoscopeFile.TarIndexEntry {
|
||||
t.Helper()
|
||||
var found *stereoscopeFile.TarIndexEntry
|
||||
_, err := stereoscopeFile.NewTarIndex(tarPath, func(entry stereoscopeFile.TarIndexEntry) error {
|
||||
if entry.ToTarFileEntry().Header.Name == name {
|
||||
e := entry
|
||||
found = &e
|
||||
}
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, found, "no entry %q in %q", name, tarPath)
|
||||
return *found
|
||||
}
|
||||
Reference in New Issue
Block a user