mirror of
https://github.com/anchore/syft.git
synced 2026-11-03 07:56:36 +01:00
fix: keep RealPath as top-level, use FullPath and utility functions FullNestedPath, LocalPath, Path
Signed-off-by: Keith Zantow <kzantow@gmail.com>
This commit is contained in:
@@ -8,7 +8,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
|
||||
@@ -22,11 +21,12 @@ import (
|
||||
// ErrDiskLimitReached when the archive's own bytes cannot be placed within the disk limit. The caller
|
||||
// must Cleanup the result.
|
||||
//
|
||||
// fileSystemID and archivePath are stamped onto every location the resolver returns: the filesystem
|
||||
// the archive was found on, and the colon-delimited chain of archives from the scan root to this one.
|
||||
func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string, limiter *Limiter, exclusions Exclusions) (*Resolver, error) {
|
||||
// archiveCoordinates are those of the archive file itself; every location the resolver returns
|
||||
// extends them with its entry path (see file.Coordinates).
|
||||
func Extract(ctx context.Context, r io.Reader, archiveCoordinates file.Coordinates, limiter *Limiter, exclusions Exclusions) (*Resolver, error) {
|
||||
charge := limiter.charge()
|
||||
resolver := newResolver(ctx, fileSystemID, archivePath, charge)
|
||||
resolver := newResolver(ctx, archiveCoordinates, charge)
|
||||
archivePath := archiveCoordinates.FullNestedPath()
|
||||
|
||||
// deferred so that a panic in a decoder still gives back the charge and removes any spill file
|
||||
succeeded := false
|
||||
@@ -43,7 +43,7 @@ func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string,
|
||||
// the archive's own bytes are needed only until they are extracted and digested
|
||||
defer release()
|
||||
|
||||
format := identifyFormat(ctx, archiveFileName(archivePath), content)
|
||||
format := identifyFormat(ctx, path.Base(archiveCoordinates.LocalPath()), content)
|
||||
if format == nil {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -75,12 +75,6 @@ func (r *Resolver) acquire(content io.Reader) (ReaderAtSeeker, func(), error) {
|
||||
return r.open(&b), func() { r.discard(&b) }, nil
|
||||
}
|
||||
|
||||
// archiveFileName returns the archive's own file name from the end of its archive path, which format
|
||||
// identification reads for its extension.
|
||||
func archiveFileName(archivePath string) string {
|
||||
return path.Base(archivePath[strings.LastIndexByte(archivePath, ':')+1:])
|
||||
}
|
||||
|
||||
// identifyFormat returns the format able to extract this content, or nil when it is not an archive.
|
||||
//
|
||||
// Identification from the head of the stream wins: a tar holding one jar ends in that jar's central
|
||||
|
||||
@@ -118,28 +118,6 @@ func TestResolver_extract_diskLimitBitesMidEntry(t *testing.T) {
|
||||
assert.Equal(t, r.written, info.Size())
|
||||
}
|
||||
|
||||
func TestResolver_extract_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{"a.txt": strings.Repeat("a", 200)})
|
||||
oneArchive := approxIndexBytes(tar.Header{Name: "a.txt"}) + 200
|
||||
limiter := NewLimiter(Limits{MaxDiskBytes: oneArchive * 3 / 2})
|
||||
|
||||
first, _ := extractBytes(t, data, "test.zip", limiter.charge())
|
||||
require.NoError(t, first.Truncated)
|
||||
require.Equal(t, int64(200), first.written)
|
||||
|
||||
second, _ := extractBytes(t, data, "test.zip", limiter.charge())
|
||||
assert.Error(t, second.Truncated, "a second archive is bounded while the first is still held")
|
||||
second.Cleanup()
|
||||
|
||||
first.Cleanup()
|
||||
_, disk := limiter.InUse()
|
||||
require.Zero(t, disk)
|
||||
|
||||
third, _ := extractBytes(t, data, "test.zip", limiter.charge())
|
||||
assert.NoError(t, third.Truncated, "and extracts in full once the first is released")
|
||||
assert.Equal(t, int64(200), third.written)
|
||||
}
|
||||
|
||||
func TestResolver_extract_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) {
|
||||
var entries []testEntry
|
||||
for i := range 50 {
|
||||
@@ -262,7 +240,7 @@ func TestResolver_extract_excludedEntriesAreNeverStored(t *testing.T) {
|
||||
func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
|
||||
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), "parentFS", "app.war:some/path/app.zip", nil, nil)
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip"}, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
@@ -274,7 +252,7 @@ func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
|
||||
locations, err := extracted.FilesByGlob("**/*.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, file.Coordinates{RealPath: "dir/hello.txt", FileSystemID: "parentFS", ArchivePath: "app.war:some/path/app.zip"},
|
||||
assert.Equal(t, file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip!/dir/hello.txt"},
|
||||
locations[0].Coordinates)
|
||||
|
||||
reader, err := extracted.FileContentsByLocation(locations[0])
|
||||
@@ -284,39 +262,13 @@ func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
|
||||
assert.Equal(t, "hello world", string(body))
|
||||
}
|
||||
|
||||
func TestExtract_notAnArchive(t *testing.T) {
|
||||
extracted, err := Extract(context.Background(), strings.NewReader("this is not an archive"), "", "notes.txt", nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, extracted)
|
||||
}
|
||||
|
||||
func TestExtract_spillsUnderTheScansTempRootAndCleansUp(t *testing.T) {
|
||||
ctx, root := scanContext(t)
|
||||
limiter := NewLimiter(spillingLimits)
|
||||
|
||||
extracted, err := Extract(ctx, bytes.NewReader(zipBytes(t, sampleFiles)), "", "app.zip", limiter, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
require.Len(t, filesIn(t, root), 1)
|
||||
_, disk := limiter.InUse()
|
||||
assert.Positive(t, disk)
|
||||
|
||||
extracted.Cleanup()
|
||||
assert.Empty(t, filesIn(t, root))
|
||||
_, disk = limiter.InUse()
|
||||
assert.Zero(t, disk, "cleanup releases everything the archive held")
|
||||
extracted.Cleanup()
|
||||
}
|
||||
|
||||
func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) {
|
||||
// the archive's own bytes are only needed to extract and digest it; afterwards the limiter holds
|
||||
// the entries and the index estimate alone
|
||||
data := zipBytes(t, map[string]string{"hello.txt": "hi"})
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
|
||||
|
||||
extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil)
|
||||
extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
@@ -332,7 +284,7 @@ func TestExtract_heldContentWrittenToDiskStaysUntilCleanup(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{"hello.txt": "hi"})
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
|
||||
|
||||
extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil)
|
||||
extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
@@ -354,7 +306,7 @@ func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) {
|
||||
data := tarGzBytes(t, map[string]string{"a.txt": "first", "b.bin": string(noise)})
|
||||
cut := data[:len(data)/2]
|
||||
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(cut), "", "app.tar.gz", NewLimiter(unboundedLimits), nil)
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(cut), file.NewCoordinates("app.tar.gz", ""), NewLimiter(unboundedLimits), nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
@@ -362,7 +314,7 @@ func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) {
|
||||
assert.Error(t, extracted.Truncated)
|
||||
locations, err := extracted.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"a.txt"}, realPaths(locations))
|
||||
assert.Equal(t, []string{"a.txt"}, localPaths(locations))
|
||||
}
|
||||
|
||||
func TestExtract_aPanicStillReleasesEverything(t *testing.T) {
|
||||
@@ -371,7 +323,7 @@ func TestExtract_aPanicStillReleasesEverything(t *testing.T) {
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
|
||||
|
||||
require.Panics(t, func() {
|
||||
_, _ = Extract(ctx, &panicsAfter{n: 4096}, "", "app.zip", limiter, nil)
|
||||
_, _ = Extract(ctx, &panicsAfter{n: 4096}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
})
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
@@ -397,11 +349,11 @@ func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) {
|
||||
data := zipBytes(t, sampleFiles)
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0})
|
||||
|
||||
_, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil)
|
||||
_, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
assert.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
|
||||
// read in place, the same bytes cost nothing; only the entries are refused
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "app.zip", limiter, nil)
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
@@ -411,24 +363,6 @@ func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) {
|
||||
assert.Empty(t, locations)
|
||||
}
|
||||
|
||||
func TestExtract_aZipBehindALauncherScript(t *testing.T) {
|
||||
// a Spring Boot executable jar sniffs as a shell script; archive/zip still finds the central directory
|
||||
data := append([]byte("#!/bin/bash\necho launcher\nexit 0\n"), zipBytes(t, map[string]string{
|
||||
"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n",
|
||||
"BOOT-INF/lib/dep.jar": "not really a jar, just bytes",
|
||||
})...)
|
||||
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "/opt/app.jar", nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
assert.NoError(t, extracted.Truncated)
|
||||
locations, err := extracted.FilesByGlob("**/*")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, locations, 2)
|
||||
}
|
||||
|
||||
func Test_HasZipEndOfCentralDirectory(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -521,7 +455,7 @@ func TestExtract_decompressionBombIsBoundedByTheLimits(t *testing.T) {
|
||||
ctx, root := scanContext(t)
|
||||
limiter := NewLimiter(limits)
|
||||
|
||||
r, err := Extract(ctx, bytes.NewReader(data), "", name, limiter, nil)
|
||||
r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates(name, ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, r)
|
||||
t.Cleanup(r.Cleanup)
|
||||
@@ -547,7 +481,7 @@ func TestExtract_entryCountBombIsBoundedByTheMemoryLimit(t *testing.T) {
|
||||
limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}
|
||||
limiter := NewLimiter(limits)
|
||||
|
||||
r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), "", "many.zip", limiter, nil)
|
||||
r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), file.NewCoordinates("many.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(r.Cleanup)
|
||||
|
||||
@@ -575,7 +509,7 @@ func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) {
|
||||
limits := Limits{MaxMemoryBytes: 8 << 20, MaxDiskBytes: -1}
|
||||
limiter := NewLimiter(limits)
|
||||
|
||||
r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), "", "deep.zip", limiter, nil)
|
||||
r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), file.NewCoordinates("deep.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(r.Cleanup)
|
||||
|
||||
@@ -584,3 +518,44 @@ func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) {
|
||||
peakMemory, _ := limiter.Peak()
|
||||
assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes)
|
||||
}
|
||||
|
||||
func TestExtract_locationsKeepTheTopLevelRealPath(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
|
||||
|
||||
// a top-level archive's coordinates, with FullPath unset or set to RealPath, name entries alike
|
||||
for _, archive := range []file.Coordinates{
|
||||
{RealPath: "/app.zip"},
|
||||
{RealPath: "/app.zip", FullPath: "/app.zip"},
|
||||
} {
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), archive, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
locations, err := extracted.FilesByPath("dir/hello.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}, locations[0].Coordinates)
|
||||
assert.Equal(t, "dir/hello.txt", locations[0].LocalPath())
|
||||
|
||||
for _, loc := range []file.Location{
|
||||
locations[0],
|
||||
file.NewLocation("dir/hello.txt"),
|
||||
file.NewLocation("/dir/hello.txt"),
|
||||
file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}),
|
||||
} {
|
||||
reader, err := extracted.FileContentsByLocation(loc)
|
||||
require.NoError(t, err, "location %s", loc)
|
||||
body, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hello world", string(body))
|
||||
|
||||
metadata, err := extracted.FileMetadataByLocation(loc)
|
||||
require.NoError(t, err, "location %s", loc)
|
||||
assert.Equal(t, "/app.zip!/dir/hello.txt", metadata.Path)
|
||||
}
|
||||
|
||||
_, err = extracted.FileContentsByLocation(file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/other.zip", FullPath: "/other.zip!/dir/hello.txt"}))
|
||||
assert.Error(t, err, "an entry of another archive is not read from this one")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
@@ -179,7 +178,7 @@ func resolverIn(t testing.TB, charge *charge) (*Resolver, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))
|
||||
r := newResolver(ctx, "", "test-archive", charge)
|
||||
r := newResolver(ctx, file.NewCoordinates("test-archive", ""), charge)
|
||||
t.Cleanup(r.Cleanup)
|
||||
return r, root
|
||||
}
|
||||
@@ -197,7 +196,7 @@ func addAll(t testing.TB, r *Resolver, entries []testEntry) {
|
||||
func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Resolver {
|
||||
t.Helper()
|
||||
r, _ := resolverIn(t, memCharge(maxInMemory))
|
||||
r.archivePath = "outer.jar"
|
||||
r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
|
||||
addAll(t, r, fileEntries(files))
|
||||
return r
|
||||
}
|
||||
@@ -207,35 +206,11 @@ func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Res
|
||||
func resolverFrom(t testing.TB, entries ...testEntry) *Resolver {
|
||||
t.Helper()
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
r.archivePath = "outer.jar"
|
||||
r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
|
||||
addAll(t, r, entries)
|
||||
return r
|
||||
}
|
||||
|
||||
// indexCostOf is what add charges for the given entries in name order: every listed node and every
|
||||
// directory their paths imply.
|
||||
func indexCostOf(entries map[string]string) int64 {
|
||||
names := make([]string, 0, len(entries))
|
||||
for name := range entries {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
seen := map[string]bool{}
|
||||
var total int64
|
||||
for _, name := range names {
|
||||
p := path.Clean("/" + name)
|
||||
for i := 1; i < len(p); i++ {
|
||||
if p[i] == '/' && !seen[p[:i]] {
|
||||
seen[p[:i]] = true
|
||||
total += approxIndexBytesPerEntry
|
||||
}
|
||||
}
|
||||
seen[p] = true
|
||||
total += approxIndexBytes(tar.Header{Name: name})
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// extractBytes extracts an archive into a resolver built over the given charge.
|
||||
func extractBytes(t testing.TB, data []byte, name string, charge *charge) (*Resolver, string) {
|
||||
t.Helper()
|
||||
@@ -316,11 +291,11 @@ func diskCharge(maxDisk int64) *charge {
|
||||
return NewLimiter(Limits{MaxDiskBytes: maxDisk}).charge()
|
||||
}
|
||||
|
||||
// realPaths lists the real path of every location, sorted.
|
||||
func realPaths(locations []file.Location) []string {
|
||||
// localPaths lists the local entry path of every location (rather than full nested archive path), sorted.
|
||||
func localPaths(locations []file.Location) []string {
|
||||
var out []string
|
||||
for _, loc := range locations {
|
||||
out = append(out, loc.RealPath)
|
||||
out = append(out, loc.LocalPath())
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
@@ -342,7 +317,7 @@ func extractedResolver(t testing.TB, data []byte, limits Limits) (*Resolver, str
|
||||
t.Helper()
|
||||
ctx, root := scanContext(t)
|
||||
|
||||
r, err := Extract(ctx, bytes.NewReader(data), "", "app.zip", NewLimiter(limits), nil)
|
||||
r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates("app.zip", ""), NewLimiter(limits), nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, r)
|
||||
t.Cleanup(r.Cleanup)
|
||||
|
||||
@@ -28,7 +28,8 @@ import (
|
||||
// into it, holding their content within the scan's limits. Entries are indexed in a stereoscope file
|
||||
// tree, the same index the directory and image resolvers search.
|
||||
//
|
||||
// Paths are archive-relative with no leading slash.
|
||||
// Locations it returns keep the outermost archive's RealPath and carry the entry's FullPath. Queries
|
||||
// may name an entry either by its full path or by its path within the archive (see rootedPath).
|
||||
type Resolver struct {
|
||||
// Digests are of the archive file itself.
|
||||
Digests []file.Digest
|
||||
@@ -37,10 +38,9 @@ type Resolver struct {
|
||||
// or the decompression budget stopped extraction early, or an entry was too large or unreadable.
|
||||
Truncated error
|
||||
|
||||
fileSystemID string
|
||||
archivePath string
|
||||
charge *charge
|
||||
budget *Budget
|
||||
archiveCoordinates file.Coordinates
|
||||
charge *charge
|
||||
budget *Budget
|
||||
|
||||
// content is held in memory while the memory limit admits it and in one spill file once it does
|
||||
// not. The file is created on first use, so an archive that stays in memory never touches the
|
||||
@@ -61,18 +61,17 @@ type Resolver struct {
|
||||
|
||||
var _ file.Resolver = (*Resolver)(nil)
|
||||
|
||||
// newResolver returns an empty resolver whose locations carry the given fileSystemID and archivePath,
|
||||
// spilling into the scan's temp root when ctx carries one (see internal/tmpdir).
|
||||
func newResolver(ctx context.Context, fileSystemID, archivePath string, charge *charge) *Resolver {
|
||||
// newResolver returns an empty resolver whose locations extend the archive's coordinates, spilling into
|
||||
// the scan's temp root when ctx carries one (see internal/tmpdir).
|
||||
func newResolver(ctx context.Context, archiveCoordinates file.Coordinates, charge *charge) *Resolver {
|
||||
return &Resolver{
|
||||
fileSystemID: fileSystemID,
|
||||
archivePath: archivePath,
|
||||
charge: charge,
|
||||
budget: budgetFromContext(ctx),
|
||||
tempDir: tmpdir.FromContext(ctx),
|
||||
tree: filetree.New(),
|
||||
index: filetree.NewIndex(),
|
||||
content: map[stereoscopeFile.ID]*blob{},
|
||||
archiveCoordinates: archiveCoordinates,
|
||||
charge: charge,
|
||||
budget: budgetFromContext(ctx),
|
||||
tempDir: tmpdir.FromContext(ctx),
|
||||
tree: filetree.New(),
|
||||
index: filetree.NewIndex(),
|
||||
content: map[stereoscopeFile.ID]*blob{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,13 +159,13 @@ func (r *Resolver) add(hdr tar.Header, content io.Reader) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Resolver) store(entryPath string, hdr tar.Header, b *blob) error {
|
||||
ref, err := r.addToTree(entryPath, hdr)
|
||||
func (r *Resolver) store(rooted string, hdr tar.Header, b *blob) error {
|
||||
ref, err := r.addToTree(rooted, hdr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
metadata := stereoscopeFile.NewMetadata(hdr, r.open(b))
|
||||
metadata.Path = reportedPath(entryPath)
|
||||
metadata.Path = entryPath(rooted)
|
||||
r.index.Add(*ref, metadata)
|
||||
r.content[ref.ID()] = b
|
||||
return nil
|
||||
@@ -217,34 +216,63 @@ func (r *Resolver) finish() {
|
||||
r.search = filetree.NewSearchContext(r.tree, r.index)
|
||||
}
|
||||
|
||||
// reportedPath is how paths leave the resolver: archive-relative with no leading slash.
|
||||
func reportedPath(rooted string) string {
|
||||
// linkDestination is the rooted path a link points at inside the archive. A hard link names its
|
||||
// target from the archive root, as does an absolute symlink, so "/etc/passwd" is an entry of this
|
||||
// archive and never the host's. A relative symlink is joined onto the link's own directory, and
|
||||
// cleaning against the root keeps "../../x" inside the archive.
|
||||
func linkDestination(linkPath, linkname string, hard bool) string {
|
||||
if hard || path.IsAbs(linkname) {
|
||||
return path.Join("/", linkname)
|
||||
}
|
||||
return path.Join(path.Dir(linkPath), linkname)
|
||||
}
|
||||
|
||||
// entryPath is a node's path within the archive: archive-relative with no leading slash.
|
||||
func entryPath(rooted string) string {
|
||||
return strings.TrimPrefix(rooted, "/")
|
||||
}
|
||||
|
||||
func rootedPath(userPath string) string {
|
||||
return path.Clean("/" + userPath)
|
||||
// fullPath is how paths leave the resolver: the archive's own full path, then the entry path.
|
||||
func (r *Resolver) fullPath(rooted string) string {
|
||||
return r.archiveCoordinates.FullNestedPath() + "!/" + entryPath(rooted)
|
||||
}
|
||||
|
||||
// location names a resolution: a link is reported at its target's path with the link's own path as
|
||||
// the access path.
|
||||
func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location {
|
||||
coordinates := file.Coordinates{
|
||||
RealPath: reportedPath(string(res.RealPath)),
|
||||
FileSystemID: r.fileSystemID,
|
||||
ArchivePath: r.archivePath,
|
||||
// rootedPath is the tree path a query names, which is either form a path takes: a full path is trimmed to
|
||||
// the entry path within this archive, and anything else (a LocalPath) is taken as archive-relative
|
||||
// already. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root)
|
||||
// names the root.
|
||||
func (r *Resolver) rootedPath(userPath string) string {
|
||||
archivePath := r.archiveCoordinates.FullNestedPath() + "!"
|
||||
if userPath == archivePath {
|
||||
return "/"
|
||||
}
|
||||
accessPath := reportedPath(string(res.RequestPath))
|
||||
if accessPath == "" || accessPath == coordinates.RealPath {
|
||||
return path.Clean("/" + strings.TrimPrefix(userPath, archivePath+"/"))
|
||||
}
|
||||
|
||||
// coordinates name an entry by the outermost archive's real path, which never includes a path inside an
|
||||
// archive, and its full path through every archive.
|
||||
func (r *Resolver) coordinates(rooted string) file.Coordinates {
|
||||
return file.Coordinates{
|
||||
RealPath: r.archiveCoordinates.RealPath,
|
||||
FileSystemID: r.archiveCoordinates.FileSystemID,
|
||||
FullPath: r.fullPath(rooted),
|
||||
}
|
||||
}
|
||||
|
||||
// location names a resolution: a link is reported at its target's path with the link's own full path
|
||||
// as the access path.
|
||||
func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location {
|
||||
coordinates := r.coordinates(string(res.RealPath))
|
||||
if res.RequestPath == "" || res.RequestPath == res.RealPath {
|
||||
return file.NewLocationFromCoordinates(coordinates)
|
||||
}
|
||||
return file.NewVirtualLocationFromCoordinates(coordinates, accessPath)
|
||||
return file.NewVirtualLocationFromCoordinates(coordinates, r.fullPath(string(res.RequestPath)))
|
||||
}
|
||||
|
||||
// resolve finds the file a path names, following links, and reports false for directories and
|
||||
// paths the archive never listed.
|
||||
func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool) {
|
||||
res, err := r.search.SearchByPath(rootedPath(userPath), filetree.FollowBasenameLinks)
|
||||
res, err := r.search.SearchByPath(r.rootedPath(userPath), filetree.FollowBasenameLinks)
|
||||
if err != nil || res == nil || !res.HasReference() {
|
||||
return nil, false
|
||||
}
|
||||
@@ -256,27 +284,34 @@ func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool)
|
||||
}
|
||||
|
||||
func (r *Resolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
|
||||
res, ok := r.resolve(location.RealPath)
|
||||
res, ok := r.resolve(location.FullNestedPath())
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no archive entry for path=%q", location.RealPath)
|
||||
return nil, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
|
||||
}
|
||||
return entryReader{r.open(r.content[res.ID()])}, nil
|
||||
}
|
||||
|
||||
func (r *Resolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
|
||||
exists, res, err := r.tree.File(stereoscopeFile.Path(rootedPath(location.RealPath)))
|
||||
rooted := r.rootedPath(location.FullNestedPath())
|
||||
exists, res, err := r.tree.File(stereoscopeFile.Path(rooted))
|
||||
if err != nil || !exists || !res.HasReference() {
|
||||
return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.RealPath)
|
||||
return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
|
||||
}
|
||||
entry, err := r.index.Get(*res.Reference)
|
||||
if err != nil {
|
||||
return file.Metadata{}, err
|
||||
}
|
||||
return entry.Metadata, nil
|
||||
// the index holds entry paths; callers get full paths, as with locations
|
||||
metadata := entry.Metadata
|
||||
metadata.Path = r.fullPath(rooted)
|
||||
if metadata.LinkDestination != "" {
|
||||
metadata.LinkDestination = r.fullPath(linkDestination(rooted, metadata.LinkDestination, metadata.Type == stereoscopeFile.TypeHardLink))
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func (r *Resolver) HasPath(userPath string) bool {
|
||||
return r.tree.HasPath(stereoscopeFile.Path(rootedPath(userPath)))
|
||||
return r.tree.HasPath(stereoscopeFile.Path(r.rootedPath(userPath)))
|
||||
}
|
||||
|
||||
func (r *Resolver) FilesByPath(paths ...string) ([]file.Location, error) {
|
||||
@@ -297,7 +332,7 @@ func (r *Resolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
|
||||
var out []file.Location
|
||||
seen := stereoscopeFile.NewFileReferenceSet()
|
||||
for _, pattern := range patterns {
|
||||
results, err := r.searchByGlob(rootedPath(pattern))
|
||||
results, err := r.searchByGlob(r.rootedPath(pattern))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid glob %q: %w", pattern, err)
|
||||
}
|
||||
@@ -330,8 +365,8 @@ func (r *Resolver) searchByGlob(pattern string) ([]stereoscopeFile.Resolution, e
|
||||
// merge into the SBOM.
|
||||
func sortLocations(locations []file.Location) {
|
||||
sort.Slice(locations, func(i, j int) bool {
|
||||
if locations[i].RealPath != locations[j].RealPath {
|
||||
return locations[i].RealPath < locations[j].RealPath
|
||||
if a, b := locations[i].FullNestedPath(), locations[j].FullNestedPath(); a != b {
|
||||
return a < b
|
||||
}
|
||||
return locations[i].AccessPath < locations[j].AccessPath
|
||||
})
|
||||
|
||||
@@ -4,8 +4,8 @@ import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -55,31 +55,6 @@ func TestResolver_add_directoryEntriesHoldNoContent(t *testing.T) {
|
||||
assert.Empty(t, readStore(t, r)["lib"].body)
|
||||
}
|
||||
|
||||
func TestResolver_add_indexCostIsChargedEvenForEmptyContent(t *testing.T) {
|
||||
// the index estimate is what bounds an archive of many empty entries
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1})
|
||||
r, _ := resolverIn(t, limiter.charge())
|
||||
|
||||
hdr := regularHeader("a.txt", 0)
|
||||
require.NoError(t, r.add(hdr, bytes.NewReader(nil)))
|
||||
|
||||
mem, _ := limiter.InUse()
|
||||
assert.Equal(t, approxIndexBytes(hdr), mem)
|
||||
assert.Zero(t, r.heldInMemory())
|
||||
}
|
||||
|
||||
func TestResolver_add_indexCostFallsBackToDiskWhenMemoryIsZero(t *testing.T) {
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
|
||||
r, _ := resolverIn(t, limiter.charge())
|
||||
|
||||
hdr := regularHeader("a.txt", 0)
|
||||
require.NoError(t, r.add(hdr, bytes.NewReader(nil)))
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
assert.Zero(t, mem)
|
||||
assert.Equal(t, approxIndexBytes(hdr), disk)
|
||||
}
|
||||
|
||||
func TestResolver_add_indexDoesNotFallBackToDiskWhenMemoryIsBounded(t *testing.T) {
|
||||
// the index lives in memory, so a bounded memory limit bounds it; only a zero limit defers to disk
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 1024, MaxDiskBytes: -1})
|
||||
@@ -108,21 +83,6 @@ func TestResolver_add_indexMakesRoomBySpillingHeldContent(t *testing.T) {
|
||||
assert.Equal(t, strings.Repeat("a", 100), readStore(t, r)["a.txt"].body)
|
||||
}
|
||||
|
||||
func TestResolver_add_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) {
|
||||
room := approxIndexBytes(regularHeader("x0", 0)) * 3
|
||||
r, _ := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).charge())
|
||||
|
||||
var err error
|
||||
admitted := 0
|
||||
for i := 0; i < 100 && err == nil; i++ {
|
||||
if err = r.add(regularHeader(fmt.Sprintf("x%d", i), 0), bytes.NewReader(nil)); err == nil {
|
||||
admitted++
|
||||
}
|
||||
}
|
||||
require.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
assert.Equal(t, 3, admitted)
|
||||
}
|
||||
|
||||
func TestResolver_add_firstEntryAtAPathWins(t *testing.T) {
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
|
||||
@@ -153,22 +113,6 @@ func TestResolver_pathsAndContents(t *testing.T) {
|
||||
assert.Equal(t, "Manifest-Version: 1.0\n", string(body))
|
||||
}
|
||||
|
||||
func TestResolver_contentsSurviveTheMoveToDisk(t *testing.T) {
|
||||
// room for the index but only a few bytes of content, so the rest moves to disk
|
||||
r := resolverOver(t, indexCostOf(jarEntriesFixture)+8, jarEntriesFixture)
|
||||
|
||||
locations, err := r.FilesByGlob("**/*.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
|
||||
reader, err := r.FileContentsByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
body, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, reader.Close())
|
||||
assert.Equal(t, "PK\x03\x04 not really", string(body))
|
||||
}
|
||||
|
||||
func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, map[string]string{
|
||||
"notes.txt": "just some text",
|
||||
@@ -177,26 +121,7 @@ func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) {
|
||||
|
||||
locations, err := r.FilesByMIMEType("text/x-shellscript")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"run.sh"}, realPaths(locations))
|
||||
}
|
||||
|
||||
func TestResolver_allLocationsCoversEveryFile(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, jarEntriesFixture)
|
||||
|
||||
var got []string
|
||||
for loc := range r.AllLocations(context.Background()) {
|
||||
got = append(got, loc.RealPath)
|
||||
}
|
||||
sort.Strings(got)
|
||||
|
||||
assert.Equal(t, []string{
|
||||
"BOOT-INF/lib/inner.jar",
|
||||
"LICENSE",
|
||||
"META-INF/MANIFEST.MF",
|
||||
"META-INF/maven/com.example/lib/pom.properties",
|
||||
"META-INF/maven/com.example/lib/pom.xml",
|
||||
"com/example/Thing.class",
|
||||
}, got)
|
||||
assert.Equal(t, []string{"run.sh"}, localPaths(locations))
|
||||
}
|
||||
|
||||
func TestResolver_answersAreOrdered(t *testing.T) {
|
||||
@@ -227,14 +152,14 @@ func TestResolver_answersAreOrdered(t *testing.T) {
|
||||
|
||||
var all []string
|
||||
for loc := range r.AllLocations(context.Background()) {
|
||||
all = append(all, loc.RealPath)
|
||||
all = append(all, loc.LocalPath())
|
||||
}
|
||||
assert.True(t, sort.StringsAreSorted(all))
|
||||
}
|
||||
|
||||
var paths []string
|
||||
for _, loc := range first {
|
||||
paths = append(paths, loc.RealPath)
|
||||
paths = append(paths, loc.LocalPath())
|
||||
}
|
||||
assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable")
|
||||
}
|
||||
@@ -279,7 +204,7 @@ func globPaths(t *testing.T, r *Resolver, pattern string) []string {
|
||||
t.Helper()
|
||||
locations, err := r.FilesByGlob(pattern)
|
||||
require.NoError(t, err)
|
||||
return realPaths(locations)
|
||||
return localPaths(locations)
|
||||
}
|
||||
|
||||
// several paths to one file answer as one location: the file's own path when it matched, otherwise
|
||||
@@ -399,8 +324,8 @@ func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
|
||||
assert.Equal(t, "opt/real.bin", locations[0].RealPath, "content comes from the file")
|
||||
assert.Equal(t, "opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched")
|
||||
assert.Equal(t, "opt/real.bin", locations[0].LocalPath(), "content comes from the file")
|
||||
assert.Equal(t, "outer.jar!/opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched")
|
||||
|
||||
reader, err := r.FileContentsByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
@@ -412,6 +337,28 @@ func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) {
|
||||
"a location standing for a link must still read the file's bytes, not the link's none")
|
||||
}
|
||||
|
||||
func TestResolver_fullPathsAreTrimmedToEntryPaths(t *testing.T) {
|
||||
// catalogers build queries from the full paths the resolver hands out, e.g. path.Dir plus a sibling
|
||||
r := resolverOver(t, 1<<20, map[string]string{"pom.xml": "a", "lib/dep.jar": "b"})
|
||||
|
||||
locations, err := r.FilesByPath("outer.jar!/lib/dep.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, "outer.jar!/lib/dep.jar", locations[0].AccessPath)
|
||||
|
||||
locations, err = r.FilesByPath(path.Join(path.Dir(locations[0].AccessPath), "../pom.xml"))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1, "a sibling at the archive root is reached through the full path")
|
||||
assert.Equal(t, "outer.jar!/pom.xml", locations[0].FullNestedPath())
|
||||
|
||||
assert.True(t, r.HasPath(path.Dir("outer.jar!/pom.xml")), "the directory of a root entry is the root")
|
||||
assert.True(t, r.HasPath("lib/dep.jar"), "an entry path is still accepted")
|
||||
|
||||
locations, err = r.FilesByGlob("outer.jar!/**/*.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
}
|
||||
|
||||
func TestResolver_danglingLinkHasNoContent(t *testing.T) {
|
||||
r := resolverFrom(t, testEntry{name: "opt/gone.jar", link: "missing.jar"})
|
||||
|
||||
@@ -420,14 +367,15 @@ func TestResolver_danglingLinkHasNoContent(t *testing.T) {
|
||||
locations = append(locations, l)
|
||||
}
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, "opt/gone.jar", locations[0].RealPath)
|
||||
assert.Equal(t, "opt/gone.jar", locations[0].LocalPath())
|
||||
|
||||
_, err := r.FileContentsByLocation(locations[0])
|
||||
assert.Error(t, err, "a link to nothing has no content to read")
|
||||
|
||||
metadata, err := r.FileMetadataByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "missing.jar", metadata.LinkDestination, "where it pointed is still reported, as the archive named it")
|
||||
assert.Equal(t, "outer.jar!/opt/missing.jar", metadata.LinkDestination,
|
||||
"where it pointed is still reported, resolved against the link's own directory")
|
||||
}
|
||||
|
||||
func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) {
|
||||
@@ -443,7 +391,7 @@ func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) {
|
||||
t.Helper()
|
||||
locations, err := r.FilesByMIMEType(types...)
|
||||
require.NoError(t, err)
|
||||
return realPaths(locations)
|
||||
return localPaths(locations)
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{"opt/real.jar"}, byMIME("application/zip"),
|
||||
@@ -465,7 +413,7 @@ func TestResolver_allLocationsKeepsEveryPath(t *testing.T) {
|
||||
|
||||
var got []string
|
||||
for l := range r.AllLocations(context.Background()) {
|
||||
got = append(got, l.RealPath)
|
||||
got = append(got, l.LocalPath())
|
||||
}
|
||||
assert.Equal(t, []string{"opt/gone.jar", "opt/real.jar", "opt/sym.jar"}, got,
|
||||
"every path the archive names, links included")
|
||||
@@ -489,7 +437,7 @@ func TestResolver_add_anEntryOverTheCapIsSkippedAndReadNoFurther(t *testing.T) {
|
||||
assert.ErrorContains(t, r.Truncated, "per-entry size cap of 1.0 KiB")
|
||||
locations, err := r.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"ok.txt"}, realPaths(locations))
|
||||
assert.Equal(t, []string{"ok.txt"}, localPaths(locations))
|
||||
}
|
||||
|
||||
type countingReader struct {
|
||||
|
||||
@@ -109,7 +109,7 @@ func (r *Resolver) spill() error {
|
||||
if err := r.openFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
log.WithFields("archive", r.archivePath, "held", len(r.held), "bytes", r.written).
|
||||
log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "held", len(r.held), "bytes", r.written).
|
||||
Debug("archive content does not fit in memory; writing it to disk")
|
||||
|
||||
for len(r.held) > 0 {
|
||||
@@ -188,7 +188,7 @@ func (r *Resolver) releaseStorage() {
|
||||
return
|
||||
}
|
||||
if err := r.file.Close(); err != nil {
|
||||
log.WithFields("archive", r.archivePath, "error", err).Trace("unable to close archive temp file")
|
||||
log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "error", err).Trace("unable to close archive temp file")
|
||||
}
|
||||
r.remove()
|
||||
r.charge.refundDisk(r.written)
|
||||
|
||||
@@ -223,7 +223,7 @@ func TestResolver_put_discardRefundsWhatIsHeldInMemory(t *testing.T) {
|
||||
assert.Equal(t, "keep", readEntry(t, s.open(&keep)))
|
||||
}
|
||||
|
||||
func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
|
||||
func TestResolver_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
|
||||
small, smallRoot := extractedResolver(t, manyEntryZip(t, 2), spillingLimits)
|
||||
large, largeRoot := extractedResolver(t, manyEntryZip(t, 5000), spillingLimits)
|
||||
|
||||
@@ -238,7 +238,7 @@ func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
|
||||
assert.Equal(t, int64(5000), large.written)
|
||||
}
|
||||
|
||||
func TestStorage_anEntryAtTheEndIsReadBySeeking(t *testing.T) {
|
||||
func TestResolver_anEntryAtTheEndIsReadBySeeking(t *testing.T) {
|
||||
// the reader over an entry is a bounded window at its offset, not a re-read of what precedes it
|
||||
const filler = 64 * 1024
|
||||
|
||||
|
||||
@@ -13,9 +13,8 @@ import (
|
||||
// such as java can describe the archive it is inside without owning the recursion.
|
||||
type Traversal struct {
|
||||
// Location is the archive file's location within its parent filesystem, including the access path
|
||||
// it was reached by. Its ArchivePath names the archives above it, so VirtualPath(Location) is the
|
||||
// full chain from the scan root as seen through that access path. The ArchivePath of the files
|
||||
// inside it is built from the real path instead.
|
||||
// it was reached by, so VirtualPath(Location) is the chain from the scan root as seen through that
|
||||
// access path. The files inside it extend its real coordinates instead (see ContainerPath).
|
||||
Location file.Location
|
||||
|
||||
// Digests are of the archive file itself, taken during extraction; a cataloger running inside the
|
||||
@@ -31,41 +30,64 @@ type Traversal struct {
|
||||
Packages []pkg.Package
|
||||
}
|
||||
|
||||
// ContentsArchivePath is the ArchivePath every file inside the archive carries: the chain to it, built
|
||||
// from real paths. Outside any archive (a nil Traversal) it is blank, as for files at the scan root.
|
||||
func (t *Traversal) ContentsArchivePath() string {
|
||||
// FullTraversalArchivePath is the archive's own full path, which is the ContainerPath of every file inside it. Outside
|
||||
// any archive (a nil Traversal) it is blank, as for files on the scanned filesystem.
|
||||
func (t *Traversal) FullTraversalArchivePath() string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return VirtualPath(file.NewLocationFromCoordinates(t.Location.Coordinates))
|
||||
return t.Location.FullNestedPath()
|
||||
}
|
||||
|
||||
// ContainerPath returns the full path of the archive directly holding the file at c, or blank for a
|
||||
// file on the scanned filesystem, whether or not its FullPath is set.
|
||||
func ContainerPath(c file.Coordinates) string {
|
||||
i := strings.LastIndex(c.FullPath, file.ArchivePathSeparator)
|
||||
if i < 0 {
|
||||
return ""
|
||||
}
|
||||
return c.FullPath[:i]
|
||||
}
|
||||
|
||||
// archiveEntries is the "!/"-delimited chain of entry paths after RealPath, reporting false for a file
|
||||
// not inside an archive. RealPath is split off first, so a "!/" in it is not taken for a boundary.
|
||||
func archiveEntries(c file.Coordinates) (string, bool) {
|
||||
remain, ok := strings.CutPrefix(c.FullNestedPath(), c.RealPath)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return strings.CutPrefix(remain, file.ArchivePathSeparator)
|
||||
}
|
||||
|
||||
// VirtualPath returns the colon-delimited chain of archives from the scan root to loc, ending in loc
|
||||
// itself: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path unchanged. In an
|
||||
// entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':' recovers the archive
|
||||
// boundaries and no two entry names share an escaped form. The leading slash is dropped to match the
|
||||
// java cataloger, which joins slash-less zip entry names onto the containing archive's path.
|
||||
// itself as accessed: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path
|
||||
// unchanged. In an entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':'
|
||||
// recovers the archive boundaries and no two entry names share an escaped form. Leading slashes are
|
||||
// dropped to match the java cataloger, which joins slash-less zip entry names onto the containing
|
||||
// archive's path.
|
||||
func VirtualPath(loc file.Location) string {
|
||||
if loc.ArchivePath == "" {
|
||||
return loc.Path()
|
||||
accessPath := loc.AccessPath
|
||||
if accessPath == "" {
|
||||
accessPath = loc.FullNestedPath()
|
||||
}
|
||||
return loc.ArchivePath + ":" + strings.TrimPrefix(escapeEntryPath(loc.Path()), "/")
|
||||
entries, ok := archiveEntries(loc.Coordinates)
|
||||
if !ok {
|
||||
return accessPath
|
||||
}
|
||||
segments := strings.Split(entries, file.ArchivePathSeparator)
|
||||
segments[len(segments)-1] = strings.TrimPrefix(accessPath, ContainerPath(loc.Coordinates)+file.ArchivePathSeparator)
|
||||
for i, s := range segments {
|
||||
segments[i] = strings.TrimPrefix(escapeEntryPath(s), "/")
|
||||
}
|
||||
return loc.RealPath + ":" + strings.Join(segments, ":")
|
||||
}
|
||||
|
||||
var (
|
||||
entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A")
|
||||
entryPathUnescaper = strings.NewReplacer("%3A", ":", "%25", "%")
|
||||
)
|
||||
var entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A")
|
||||
|
||||
func escapeEntryPath(p string) string {
|
||||
return entryPathEscaper.Replace(p)
|
||||
}
|
||||
|
||||
// UnescapeEntryPath reverses the escaping VirtualPath applies to one segment of the chain.
|
||||
func UnescapeEntryPath(p string) string {
|
||||
return entryPathUnescaper.Replace(p)
|
||||
}
|
||||
|
||||
type traversalCtxKey struct{}
|
||||
|
||||
func WithTraversal(ctx context.Context, t *Traversal) context.Context {
|
||||
|
||||
@@ -2,7 +2,6 @@ package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -18,16 +17,24 @@ func TestTraversalContextRoundTrip(t *testing.T) {
|
||||
ctx := WithTraversal(context.Background(), trav)
|
||||
require.Same(t, trav, TraversalFromContext(ctx))
|
||||
|
||||
child := &Traversal{Location: locationIn("app.war", "/WEB-INF/lib/dep.jar")}
|
||||
child := &Traversal{Location: locationIn("app.war", "WEB-INF/lib/dep.jar")}
|
||||
ctx = WithTraversal(ctx, child)
|
||||
require.Same(t, child, TraversalFromContext(ctx))
|
||||
}
|
||||
|
||||
// locationIn is a location as the Index returns it: an entry path stamped with the containing chain.
|
||||
func locationIn(archivePath, entryPath string) file.Location {
|
||||
loc := file.NewLocation(entryPath)
|
||||
loc.ArchivePath = archivePath
|
||||
return loc
|
||||
// locationIn is a location as the Resolver returns it: the outermost archive's real path, and the chain
|
||||
// of entry paths within it.
|
||||
func locationIn(realPath, archivePath string) file.Location {
|
||||
return file.NewLocationFromCoordinates(file.Coordinates{RealPath: realPath, FullPath: realPath + file.ArchivePathSeparator + archivePath})
|
||||
}
|
||||
|
||||
func TestContainerPath(t *testing.T) {
|
||||
assert.Equal(t, "", ContainerPath(file.NewCoordinates("/app.war", "")))
|
||||
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war"}), "a top-level file whose FullPath is set")
|
||||
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/odd!/name"}), "a separator in a top-level real path is not an archive boundary")
|
||||
assert.Equal(t, "/app.war", ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar").Coordinates))
|
||||
assert.Equal(t, "/app.war!/WEB-INF/lib/dep.jar",
|
||||
ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF").Coordinates))
|
||||
}
|
||||
|
||||
func TestVirtualPath(t *testing.T) {
|
||||
@@ -46,6 +53,11 @@ func TestVirtualPath(t *testing.T) {
|
||||
location: file.NewLocation("some/dir/example.jar"),
|
||||
expected: "some/dir/example.jar",
|
||||
},
|
||||
{
|
||||
name: "outside an archive with FullPath set returns the path unchanged",
|
||||
location: file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/odd!/name.jar", FullPath: "/odd!/name.jar"}),
|
||||
expected: "/odd!/name.jar",
|
||||
},
|
||||
{
|
||||
name: "single level trims leading slash",
|
||||
location: locationIn("app.war", "/WEB-INF/lib/dep.jar"),
|
||||
@@ -58,7 +70,7 @@ func TestVirtualPath(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "multi-level chain",
|
||||
location: locationIn("app.war:WEB-INF/lib/dep.jar", "/inner/most.jar"),
|
||||
location: locationIn("app.war", "WEB-INF/lib/dep.jar!/inner/most.jar"),
|
||||
expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar",
|
||||
},
|
||||
{
|
||||
@@ -72,11 +84,23 @@ func TestVirtualPath(t *testing.T) {
|
||||
location: locationIn("app.war", "lib/a:b:c.jar"),
|
||||
expected: "app.war:lib/a%3Ab%3Ac.jar",
|
||||
},
|
||||
{
|
||||
// or the literal text %3A would collide with an escaped colon
|
||||
name: "the escape character is itself escaped",
|
||||
location: locationIn("outer.zip", "a%3Ab.jar"),
|
||||
expected: "outer.zip:a%253Ab.jar",
|
||||
},
|
||||
{
|
||||
name: "access path is preferred over real path",
|
||||
location: file.NewVirtualLocation("/real/dep.jar", "/link/dep.jar"),
|
||||
expected: "/link/dep.jar",
|
||||
},
|
||||
{
|
||||
name: "access path within an archive replaces the last entry",
|
||||
location: file.NewVirtualLocationFromCoordinates(
|
||||
file.Coordinates{RealPath: "app.war", FullPath: "app.war!/WEB-INF/lib/real.jar"}, "app.war!/WEB-INF/lib/link.jar"),
|
||||
expected: "app.war:WEB-INF/lib/link.jar",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -84,27 +108,3 @@ func TestVirtualPath(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualPath_theEscapeCharacterIsItselfEscaped(t *testing.T) {
|
||||
// an entry path holding the literal text %3A must not collide with one holding a colon, or their
|
||||
// contents share an ArchivePath and overwrite each other's file metadata
|
||||
fromColon := VirtualPath(locationIn("outer.zip", "a:b.jar"))
|
||||
fromLiteral := VirtualPath(locationIn("outer.zip", "a%3Ab.jar"))
|
||||
|
||||
assert.Equal(t, "outer.zip:a%3Ab.jar", fromColon)
|
||||
assert.Equal(t, "outer.zip:a%253Ab.jar", fromLiteral)
|
||||
}
|
||||
|
||||
func TestVirtualPath_escapingComposesAcrossNestingLevels(t *testing.T) {
|
||||
level2 := VirtualPath(locationIn("outer.zip", "lib/mid:dle.war"))
|
||||
require.Equal(t, "outer.zip:lib/mid%3Adle.war", level2)
|
||||
|
||||
final := VirtualPath(locationIn(level2, "WEB-INF/lib/inn:er.jar"))
|
||||
require.Equal(t, "outer.zip:lib/mid%3Adle.war:WEB-INF/lib/inn%3Aer.jar", final)
|
||||
|
||||
assert.Equal(t,
|
||||
[]string{"outer.zip", "lib/mid%3Adle.war", "WEB-INF/lib/inn%3Aer.jar"},
|
||||
strings.Split(final, ":"),
|
||||
"splitting on ':' must recover exactly the three real archive boundaries",
|
||||
)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user