fix: keep RealPath as top-level, use FullPath and utility functions FullNestedPath, LocalPath, Path

Signed-off-by: Keith Zantow <kzantow@gmail.com>
This commit is contained in:
Keith Zantow
2026-10-19 15:44:37 -04:00
parent 17d109e8b5
commit 3e4b036e22
165 changed files with 1349 additions and 1970 deletions
+6 -12
View File
@@ -8,7 +8,6 @@ import (
"fmt"
"io"
"path"
"strings"
"github.com/mholt/archives"
@@ -22,11 +21,12 @@ import (
// ErrDiskLimitReached when the archive's own bytes cannot be placed within the disk limit. The caller
// must Cleanup the result.
//
// fileSystemID and archivePath are stamped onto every location the resolver returns: the filesystem
// the archive was found on, and the colon-delimited chain of archives from the scan root to this one.
func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string, limiter *Limiter, exclusions Exclusions) (*Resolver, error) {
// archiveCoordinates are those of the archive file itself; every location the resolver returns
// extends them with its entry path (see file.Coordinates).
func Extract(ctx context.Context, r io.Reader, archiveCoordinates file.Coordinates, limiter *Limiter, exclusions Exclusions) (*Resolver, error) {
charge := limiter.charge()
resolver := newResolver(ctx, fileSystemID, archivePath, charge)
resolver := newResolver(ctx, archiveCoordinates, charge)
archivePath := archiveCoordinates.FullNestedPath()
// deferred so that a panic in a decoder still gives back the charge and removes any spill file
succeeded := false
@@ -43,7 +43,7 @@ func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string,
// the archive's own bytes are needed only until they are extracted and digested
defer release()
format := identifyFormat(ctx, archiveFileName(archivePath), content)
format := identifyFormat(ctx, path.Base(archiveCoordinates.LocalPath()), content)
if format == nil {
return nil, nil
}
@@ -75,12 +75,6 @@ func (r *Resolver) acquire(content io.Reader) (ReaderAtSeeker, func(), error) {
return r.open(&b), func() { r.discard(&b) }, nil
}
// archiveFileName returns the archive's own file name from the end of its archive path, which format
// identification reads for its extension.
func archiveFileName(archivePath string) string {
return path.Base(archivePath[strings.LastIndexByte(archivePath, ':')+1:])
}
// identifyFormat returns the format able to extract this content, or nil when it is not an archive.
//
// Identification from the head of the stream wins: a tar holding one jar ends in that jar's central
+53 -78
View File
@@ -118,28 +118,6 @@ func TestResolver_extract_diskLimitBitesMidEntry(t *testing.T) {
assert.Equal(t, r.written, info.Size())
}
func TestResolver_extract_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) {
data := zipBytes(t, map[string]string{"a.txt": strings.Repeat("a", 200)})
oneArchive := approxIndexBytes(tar.Header{Name: "a.txt"}) + 200
limiter := NewLimiter(Limits{MaxDiskBytes: oneArchive * 3 / 2})
first, _ := extractBytes(t, data, "test.zip", limiter.charge())
require.NoError(t, first.Truncated)
require.Equal(t, int64(200), first.written)
second, _ := extractBytes(t, data, "test.zip", limiter.charge())
assert.Error(t, second.Truncated, "a second archive is bounded while the first is still held")
second.Cleanup()
first.Cleanup()
_, disk := limiter.InUse()
require.Zero(t, disk)
third, _ := extractBytes(t, data, "test.zip", limiter.charge())
assert.NoError(t, third.Truncated, "and extracts in full once the first is released")
assert.Equal(t, int64(200), third.written)
}
func TestResolver_extract_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) {
var entries []testEntry
for i := range 50 {
@@ -262,7 +240,7 @@ func TestResolver_extract_excludedEntriesAreNeverStored(t *testing.T) {
func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
extracted, err := Extract(context.Background(), bytes.NewReader(data), "parentFS", "app.war:some/path/app.zip", nil, nil)
extracted, err := Extract(context.Background(), bytes.NewReader(data), file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip"}, nil, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
@@ -274,7 +252,7 @@ func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
locations, err := extracted.FilesByGlob("**/*.txt")
require.NoError(t, err)
require.Len(t, locations, 1)
assert.Equal(t, file.Coordinates{RealPath: "dir/hello.txt", FileSystemID: "parentFS", ArchivePath: "app.war:some/path/app.zip"},
assert.Equal(t, file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip!/dir/hello.txt"},
locations[0].Coordinates)
reader, err := extracted.FileContentsByLocation(locations[0])
@@ -284,39 +262,13 @@ func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
assert.Equal(t, "hello world", string(body))
}
func TestExtract_notAnArchive(t *testing.T) {
extracted, err := Extract(context.Background(), strings.NewReader("this is not an archive"), "", "notes.txt", nil, nil)
require.NoError(t, err)
assert.Nil(t, extracted)
}
func TestExtract_spillsUnderTheScansTempRootAndCleansUp(t *testing.T) {
ctx, root := scanContext(t)
limiter := NewLimiter(spillingLimits)
extracted, err := Extract(ctx, bytes.NewReader(zipBytes(t, sampleFiles)), "", "app.zip", limiter, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
require.Len(t, filesIn(t, root), 1)
_, disk := limiter.InUse()
assert.Positive(t, disk)
extracted.Cleanup()
assert.Empty(t, filesIn(t, root))
_, disk = limiter.InUse()
assert.Zero(t, disk, "cleanup releases everything the archive held")
extracted.Cleanup()
}
func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) {
// the archive's own bytes are only needed to extract and digest it; afterwards the limiter holds
// the entries and the index estimate alone
data := zipBytes(t, map[string]string{"hello.txt": "hi"})
limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil)
extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
require.NoError(t, err)
t.Cleanup(extracted.Cleanup)
@@ -332,7 +284,7 @@ func TestExtract_heldContentWrittenToDiskStaysUntilCleanup(t *testing.T) {
data := zipBytes(t, map[string]string{"hello.txt": "hi"})
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil)
extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
require.NoError(t, err)
t.Cleanup(extracted.Cleanup)
@@ -354,7 +306,7 @@ func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) {
data := tarGzBytes(t, map[string]string{"a.txt": "first", "b.bin": string(noise)})
cut := data[:len(data)/2]
extracted, err := Extract(context.Background(), bytes.NewReader(cut), "", "app.tar.gz", NewLimiter(unboundedLimits), nil)
extracted, err := Extract(context.Background(), bytes.NewReader(cut), file.NewCoordinates("app.tar.gz", ""), NewLimiter(unboundedLimits), nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
@@ -362,7 +314,7 @@ func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) {
assert.Error(t, extracted.Truncated)
locations, err := extracted.FilesByGlob("**")
require.NoError(t, err)
assert.Equal(t, []string{"a.txt"}, realPaths(locations))
assert.Equal(t, []string{"a.txt"}, localPaths(locations))
}
func TestExtract_aPanicStillReleasesEverything(t *testing.T) {
@@ -371,7 +323,7 @@ func TestExtract_aPanicStillReleasesEverything(t *testing.T) {
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
require.Panics(t, func() {
_, _ = Extract(ctx, &panicsAfter{n: 4096}, "", "app.zip", limiter, nil)
_, _ = Extract(ctx, &panicsAfter{n: 4096}, file.NewCoordinates("app.zip", ""), limiter, nil)
})
mem, disk := limiter.InUse()
@@ -397,11 +349,11 @@ func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) {
data := zipBytes(t, sampleFiles)
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0})
_, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil)
_, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
assert.ErrorIs(t, err, ErrDiskLimitReached)
// read in place, the same bytes cost nothing; only the entries are refused
extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "app.zip", limiter, nil)
extracted, err := Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("app.zip", ""), limiter, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
@@ -411,24 +363,6 @@ func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) {
assert.Empty(t, locations)
}
func TestExtract_aZipBehindALauncherScript(t *testing.T) {
// a Spring Boot executable jar sniffs as a shell script; archive/zip still finds the central directory
data := append([]byte("#!/bin/bash\necho launcher\nexit 0\n"), zipBytes(t, map[string]string{
"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n",
"BOOT-INF/lib/dep.jar": "not really a jar, just bytes",
})...)
extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "/opt/app.jar", nil, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
assert.NoError(t, extracted.Truncated)
locations, err := extracted.FilesByGlob("**/*")
require.NoError(t, err)
assert.Len(t, locations, 2)
}
func Test_HasZipEndOfCentralDirectory(t *testing.T) {
tests := []struct {
name string
@@ -521,7 +455,7 @@ func TestExtract_decompressionBombIsBoundedByTheLimits(t *testing.T) {
ctx, root := scanContext(t)
limiter := NewLimiter(limits)
r, err := Extract(ctx, bytes.NewReader(data), "", name, limiter, nil)
r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates(name, ""), limiter, nil)
require.NoError(t, err)
require.NotNil(t, r)
t.Cleanup(r.Cleanup)
@@ -547,7 +481,7 @@ func TestExtract_entryCountBombIsBoundedByTheMemoryLimit(t *testing.T) {
limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}
limiter := NewLimiter(limits)
r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), "", "many.zip", limiter, nil)
r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), file.NewCoordinates("many.zip", ""), limiter, nil)
require.NoError(t, err)
t.Cleanup(r.Cleanup)
@@ -575,7 +509,7 @@ func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) {
limits := Limits{MaxMemoryBytes: 8 << 20, MaxDiskBytes: -1}
limiter := NewLimiter(limits)
r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), "", "deep.zip", limiter, nil)
r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), file.NewCoordinates("deep.zip", ""), limiter, nil)
require.NoError(t, err)
t.Cleanup(r.Cleanup)
@@ -584,3 +518,44 @@ func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) {
peakMemory, _ := limiter.Peak()
assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes)
}
func TestExtract_locationsKeepTheTopLevelRealPath(t *testing.T) {
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
// a top-level archive's coordinates, with FullPath unset or set to RealPath, name entries alike
for _, archive := range []file.Coordinates{
{RealPath: "/app.zip"},
{RealPath: "/app.zip", FullPath: "/app.zip"},
} {
extracted, err := Extract(context.Background(), bytes.NewReader(data), archive, nil, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
locations, err := extracted.FilesByPath("dir/hello.txt")
require.NoError(t, err)
require.Len(t, locations, 1)
assert.Equal(t, file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}, locations[0].Coordinates)
assert.Equal(t, "dir/hello.txt", locations[0].LocalPath())
for _, loc := range []file.Location{
locations[0],
file.NewLocation("dir/hello.txt"),
file.NewLocation("/dir/hello.txt"),
file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}),
} {
reader, err := extracted.FileContentsByLocation(loc)
require.NoError(t, err, "location %s", loc)
body, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "hello world", string(body))
metadata, err := extracted.FileMetadataByLocation(loc)
require.NoError(t, err, "location %s", loc)
assert.Equal(t, "/app.zip!/dir/hello.txt", metadata.Path)
}
_, err = extracted.FileContentsByLocation(file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/other.zip", FullPath: "/other.zip!/dir/hello.txt"}))
assert.Error(t, err, "an entry of another archive is not read from this one")
}
}
+7 -32
View File
@@ -10,7 +10,6 @@ import (
"io"
"io/fs"
"os"
"path"
"sort"
"testing"
@@ -179,7 +178,7 @@ func resolverIn(t testing.TB, charge *charge) (*Resolver, string) {
t.Helper()
root := t.TempDir()
ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))
r := newResolver(ctx, "", "test-archive", charge)
r := newResolver(ctx, file.NewCoordinates("test-archive", ""), charge)
t.Cleanup(r.Cleanup)
return r, root
}
@@ -197,7 +196,7 @@ func addAll(t testing.TB, r *Resolver, entries []testEntry) {
func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Resolver {
t.Helper()
r, _ := resolverIn(t, memCharge(maxInMemory))
r.archivePath = "outer.jar"
r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
addAll(t, r, fileEntries(files))
return r
}
@@ -207,35 +206,11 @@ func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Res
func resolverFrom(t testing.TB, entries ...testEntry) *Resolver {
t.Helper()
r, _ := resolverIn(t, memCharge(1<<20))
r.archivePath = "outer.jar"
r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
addAll(t, r, entries)
return r
}
// indexCostOf is what add charges for the given entries in name order: every listed node and every
// directory their paths imply.
func indexCostOf(entries map[string]string) int64 {
names := make([]string, 0, len(entries))
for name := range entries {
names = append(names, name)
}
sort.Strings(names)
seen := map[string]bool{}
var total int64
for _, name := range names {
p := path.Clean("/" + name)
for i := 1; i < len(p); i++ {
if p[i] == '/' && !seen[p[:i]] {
seen[p[:i]] = true
total += approxIndexBytesPerEntry
}
}
seen[p] = true
total += approxIndexBytes(tar.Header{Name: name})
}
return total
}
// extractBytes extracts an archive into a resolver built over the given charge.
func extractBytes(t testing.TB, data []byte, name string, charge *charge) (*Resolver, string) {
t.Helper()
@@ -316,11 +291,11 @@ func diskCharge(maxDisk int64) *charge {
return NewLimiter(Limits{MaxDiskBytes: maxDisk}).charge()
}
// realPaths lists the real path of every location, sorted.
func realPaths(locations []file.Location) []string {
// localPaths lists the local entry path of every location (rather than full nested archive path), sorted.
func localPaths(locations []file.Location) []string {
var out []string
for _, loc := range locations {
out = append(out, loc.RealPath)
out = append(out, loc.LocalPath())
}
sort.Strings(out)
return out
@@ -342,7 +317,7 @@ func extractedResolver(t testing.TB, data []byte, limits Limits) (*Resolver, str
t.Helper()
ctx, root := scanContext(t)
r, err := Extract(ctx, bytes.NewReader(data), "", "app.zip", NewLimiter(limits), nil)
r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates("app.zip", ""), NewLimiter(limits), nil)
require.NoError(t, err)
require.NotNil(t, r)
t.Cleanup(r.Cleanup)
+78 -43
View File
@@ -28,7 +28,8 @@ import (
// into it, holding their content within the scan's limits. Entries are indexed in a stereoscope file
// tree, the same index the directory and image resolvers search.
//
// Paths are archive-relative with no leading slash.
// Locations it returns keep the outermost archive's RealPath and carry the entry's FullPath. Queries
// may name an entry either by its full path or by its path within the archive (see rootedPath).
type Resolver struct {
// Digests are of the archive file itself.
Digests []file.Digest
@@ -37,10 +38,9 @@ type Resolver struct {
// or the decompression budget stopped extraction early, or an entry was too large or unreadable.
Truncated error
fileSystemID string
archivePath string
charge *charge
budget *Budget
archiveCoordinates file.Coordinates
charge *charge
budget *Budget
// content is held in memory while the memory limit admits it and in one spill file once it does
// not. The file is created on first use, so an archive that stays in memory never touches the
@@ -61,18 +61,17 @@ type Resolver struct {
var _ file.Resolver = (*Resolver)(nil)
// newResolver returns an empty resolver whose locations carry the given fileSystemID and archivePath,
// spilling into the scan's temp root when ctx carries one (see internal/tmpdir).
func newResolver(ctx context.Context, fileSystemID, archivePath string, charge *charge) *Resolver {
// newResolver returns an empty resolver whose locations extend the archive's coordinates, spilling into
// the scan's temp root when ctx carries one (see internal/tmpdir).
func newResolver(ctx context.Context, archiveCoordinates file.Coordinates, charge *charge) *Resolver {
return &Resolver{
fileSystemID: fileSystemID,
archivePath: archivePath,
charge: charge,
budget: budgetFromContext(ctx),
tempDir: tmpdir.FromContext(ctx),
tree: filetree.New(),
index: filetree.NewIndex(),
content: map[stereoscopeFile.ID]*blob{},
archiveCoordinates: archiveCoordinates,
charge: charge,
budget: budgetFromContext(ctx),
tempDir: tmpdir.FromContext(ctx),
tree: filetree.New(),
index: filetree.NewIndex(),
content: map[stereoscopeFile.ID]*blob{},
}
}
@@ -160,13 +159,13 @@ func (r *Resolver) add(hdr tar.Header, content io.Reader) error {
return nil
}
func (r *Resolver) store(entryPath string, hdr tar.Header, b *blob) error {
ref, err := r.addToTree(entryPath, hdr)
func (r *Resolver) store(rooted string, hdr tar.Header, b *blob) error {
ref, err := r.addToTree(rooted, hdr)
if err != nil {
return err
}
metadata := stereoscopeFile.NewMetadata(hdr, r.open(b))
metadata.Path = reportedPath(entryPath)
metadata.Path = entryPath(rooted)
r.index.Add(*ref, metadata)
r.content[ref.ID()] = b
return nil
@@ -217,34 +216,63 @@ func (r *Resolver) finish() {
r.search = filetree.NewSearchContext(r.tree, r.index)
}
// reportedPath is how paths leave the resolver: archive-relative with no leading slash.
func reportedPath(rooted string) string {
// linkDestination is the rooted path a link points at inside the archive. A hard link names its
// target from the archive root, as does an absolute symlink, so "/etc/passwd" is an entry of this
// archive and never the host's. A relative symlink is joined onto the link's own directory, and
// cleaning against the root keeps "../../x" inside the archive.
func linkDestination(linkPath, linkname string, hard bool) string {
if hard || path.IsAbs(linkname) {
return path.Join("/", linkname)
}
return path.Join(path.Dir(linkPath), linkname)
}
// entryPath is a node's path within the archive: archive-relative with no leading slash.
func entryPath(rooted string) string {
return strings.TrimPrefix(rooted, "/")
}
func rootedPath(userPath string) string {
return path.Clean("/" + userPath)
// fullPath is how paths leave the resolver: the archive's own full path, then the entry path.
func (r *Resolver) fullPath(rooted string) string {
return r.archiveCoordinates.FullNestedPath() + "!/" + entryPath(rooted)
}
// location names a resolution: a link is reported at its target's path with the link's own path as
// the access path.
func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location {
coordinates := file.Coordinates{
RealPath: reportedPath(string(res.RealPath)),
FileSystemID: r.fileSystemID,
ArchivePath: r.archivePath,
// rootedPath is the tree path a query names, which is either form a path takes: a full path is trimmed to
// the entry path within this archive, and anything else (a LocalPath) is taken as archive-relative
// already. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root)
// names the root.
func (r *Resolver) rootedPath(userPath string) string {
archivePath := r.archiveCoordinates.FullNestedPath() + "!"
if userPath == archivePath {
return "/"
}
accessPath := reportedPath(string(res.RequestPath))
if accessPath == "" || accessPath == coordinates.RealPath {
return path.Clean("/" + strings.TrimPrefix(userPath, archivePath+"/"))
}
// coordinates name an entry by the outermost archive's real path, which never includes a path inside an
// archive, and its full path through every archive.
func (r *Resolver) coordinates(rooted string) file.Coordinates {
return file.Coordinates{
RealPath: r.archiveCoordinates.RealPath,
FileSystemID: r.archiveCoordinates.FileSystemID,
FullPath: r.fullPath(rooted),
}
}
// location names a resolution: a link is reported at its target's path with the link's own full path
// as the access path.
func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location {
coordinates := r.coordinates(string(res.RealPath))
if res.RequestPath == "" || res.RequestPath == res.RealPath {
return file.NewLocationFromCoordinates(coordinates)
}
return file.NewVirtualLocationFromCoordinates(coordinates, accessPath)
return file.NewVirtualLocationFromCoordinates(coordinates, r.fullPath(string(res.RequestPath)))
}
// resolve finds the file a path names, following links, and reports false for directories and
// paths the archive never listed.
func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool) {
res, err := r.search.SearchByPath(rootedPath(userPath), filetree.FollowBasenameLinks)
res, err := r.search.SearchByPath(r.rootedPath(userPath), filetree.FollowBasenameLinks)
if err != nil || res == nil || !res.HasReference() {
return nil, false
}
@@ -256,27 +284,34 @@ func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool)
}
func (r *Resolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
res, ok := r.resolve(location.RealPath)
res, ok := r.resolve(location.FullNestedPath())
if !ok {
return nil, fmt.Errorf("no archive entry for path=%q", location.RealPath)
return nil, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
}
return entryReader{r.open(r.content[res.ID()])}, nil
}
func (r *Resolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
exists, res, err := r.tree.File(stereoscopeFile.Path(rootedPath(location.RealPath)))
rooted := r.rootedPath(location.FullNestedPath())
exists, res, err := r.tree.File(stereoscopeFile.Path(rooted))
if err != nil || !exists || !res.HasReference() {
return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.RealPath)
return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
}
entry, err := r.index.Get(*res.Reference)
if err != nil {
return file.Metadata{}, err
}
return entry.Metadata, nil
// the index holds entry paths; callers get full paths, as with locations
metadata := entry.Metadata
metadata.Path = r.fullPath(rooted)
if metadata.LinkDestination != "" {
metadata.LinkDestination = r.fullPath(linkDestination(rooted, metadata.LinkDestination, metadata.Type == stereoscopeFile.TypeHardLink))
}
return metadata, nil
}
func (r *Resolver) HasPath(userPath string) bool {
return r.tree.HasPath(stereoscopeFile.Path(rootedPath(userPath)))
return r.tree.HasPath(stereoscopeFile.Path(r.rootedPath(userPath)))
}
func (r *Resolver) FilesByPath(paths ...string) ([]file.Location, error) {
@@ -297,7 +332,7 @@ func (r *Resolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
var out []file.Location
seen := stereoscopeFile.NewFileReferenceSet()
for _, pattern := range patterns {
results, err := r.searchByGlob(rootedPath(pattern))
results, err := r.searchByGlob(r.rootedPath(pattern))
if err != nil {
return nil, fmt.Errorf("invalid glob %q: %w", pattern, err)
}
@@ -330,8 +365,8 @@ func (r *Resolver) searchByGlob(pattern string) ([]stereoscopeFile.Resolution, e
// merge into the SBOM.
func sortLocations(locations []file.Location) {
sort.Slice(locations, func(i, j int) bool {
if locations[i].RealPath != locations[j].RealPath {
return locations[i].RealPath < locations[j].RealPath
if a, b := locations[i].FullNestedPath(), locations[j].FullNestedPath(); a != b {
return a < b
}
return locations[i].AccessPath < locations[j].AccessPath
})
+35 -87
View File
@@ -4,8 +4,8 @@ import (
"archive/tar"
"bytes"
"context"
"fmt"
"io"
"path"
"sort"
"strings"
"testing"
@@ -55,31 +55,6 @@ func TestResolver_add_directoryEntriesHoldNoContent(t *testing.T) {
assert.Empty(t, readStore(t, r)["lib"].body)
}
func TestResolver_add_indexCostIsChargedEvenForEmptyContent(t *testing.T) {
// the index estimate is what bounds an archive of many empty entries
limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1})
r, _ := resolverIn(t, limiter.charge())
hdr := regularHeader("a.txt", 0)
require.NoError(t, r.add(hdr, bytes.NewReader(nil)))
mem, _ := limiter.InUse()
assert.Equal(t, approxIndexBytes(hdr), mem)
assert.Zero(t, r.heldInMemory())
}
func TestResolver_add_indexCostFallsBackToDiskWhenMemoryIsZero(t *testing.T) {
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
r, _ := resolverIn(t, limiter.charge())
hdr := regularHeader("a.txt", 0)
require.NoError(t, r.add(hdr, bytes.NewReader(nil)))
mem, disk := limiter.InUse()
assert.Zero(t, mem)
assert.Equal(t, approxIndexBytes(hdr), disk)
}
func TestResolver_add_indexDoesNotFallBackToDiskWhenMemoryIsBounded(t *testing.T) {
// the index lives in memory, so a bounded memory limit bounds it; only a zero limit defers to disk
limiter := NewLimiter(Limits{MaxMemoryBytes: 1024, MaxDiskBytes: -1})
@@ -108,21 +83,6 @@ func TestResolver_add_indexMakesRoomBySpillingHeldContent(t *testing.T) {
assert.Equal(t, strings.Repeat("a", 100), readStore(t, r)["a.txt"].body)
}
func TestResolver_add_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) {
room := approxIndexBytes(regularHeader("x0", 0)) * 3
r, _ := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).charge())
var err error
admitted := 0
for i := 0; i < 100 && err == nil; i++ {
if err = r.add(regularHeader(fmt.Sprintf("x%d", i), 0), bytes.NewReader(nil)); err == nil {
admitted++
}
}
require.ErrorIs(t, err, ErrDiskLimitReached)
assert.Equal(t, 3, admitted)
}
func TestResolver_add_firstEntryAtAPathWins(t *testing.T) {
r, _ := resolverIn(t, memCharge(1<<20))
@@ -153,22 +113,6 @@ func TestResolver_pathsAndContents(t *testing.T) {
assert.Equal(t, "Manifest-Version: 1.0\n", string(body))
}
func TestResolver_contentsSurviveTheMoveToDisk(t *testing.T) {
// room for the index but only a few bytes of content, so the rest moves to disk
r := resolverOver(t, indexCostOf(jarEntriesFixture)+8, jarEntriesFixture)
locations, err := r.FilesByGlob("**/*.jar")
require.NoError(t, err)
require.Len(t, locations, 1)
reader, err := r.FileContentsByLocation(locations[0])
require.NoError(t, err)
body, err := io.ReadAll(reader)
require.NoError(t, err)
require.NoError(t, reader.Close())
assert.Equal(t, "PK\x03\x04 not really", string(body))
}
func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) {
r := resolverOver(t, 1<<20, map[string]string{
"notes.txt": "just some text",
@@ -177,26 +121,7 @@ func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) {
locations, err := r.FilesByMIMEType("text/x-shellscript")
require.NoError(t, err)
assert.Equal(t, []string{"run.sh"}, realPaths(locations))
}
func TestResolver_allLocationsCoversEveryFile(t *testing.T) {
r := resolverOver(t, 1<<20, jarEntriesFixture)
var got []string
for loc := range r.AllLocations(context.Background()) {
got = append(got, loc.RealPath)
}
sort.Strings(got)
assert.Equal(t, []string{
"BOOT-INF/lib/inner.jar",
"LICENSE",
"META-INF/MANIFEST.MF",
"META-INF/maven/com.example/lib/pom.properties",
"META-INF/maven/com.example/lib/pom.xml",
"com/example/Thing.class",
}, got)
assert.Equal(t, []string{"run.sh"}, localPaths(locations))
}
func TestResolver_answersAreOrdered(t *testing.T) {
@@ -227,14 +152,14 @@ func TestResolver_answersAreOrdered(t *testing.T) {
var all []string
for loc := range r.AllLocations(context.Background()) {
all = append(all, loc.RealPath)
all = append(all, loc.LocalPath())
}
assert.True(t, sort.StringsAreSorted(all))
}
var paths []string
for _, loc := range first {
paths = append(paths, loc.RealPath)
paths = append(paths, loc.LocalPath())
}
assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable")
}
@@ -279,7 +204,7 @@ func globPaths(t *testing.T, r *Resolver, pattern string) []string {
t.Helper()
locations, err := r.FilesByGlob(pattern)
require.NoError(t, err)
return realPaths(locations)
return localPaths(locations)
}
// several paths to one file answer as one location: the file's own path when it matched, otherwise
@@ -399,8 +324,8 @@ func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) {
require.NoError(t, err)
require.Len(t, locations, 1)
assert.Equal(t, "opt/real.bin", locations[0].RealPath, "content comes from the file")
assert.Equal(t, "opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched")
assert.Equal(t, "opt/real.bin", locations[0].LocalPath(), "content comes from the file")
assert.Equal(t, "outer.jar!/opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched")
reader, err := r.FileContentsByLocation(locations[0])
require.NoError(t, err)
@@ -412,6 +337,28 @@ func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) {
"a location standing for a link must still read the file's bytes, not the link's none")
}
func TestResolver_fullPathsAreTrimmedToEntryPaths(t *testing.T) {
// catalogers build queries from the full paths the resolver hands out, e.g. path.Dir plus a sibling
r := resolverOver(t, 1<<20, map[string]string{"pom.xml": "a", "lib/dep.jar": "b"})
locations, err := r.FilesByPath("outer.jar!/lib/dep.jar")
require.NoError(t, err)
require.Len(t, locations, 1)
assert.Equal(t, "outer.jar!/lib/dep.jar", locations[0].AccessPath)
locations, err = r.FilesByPath(path.Join(path.Dir(locations[0].AccessPath), "../pom.xml"))
require.NoError(t, err)
require.Len(t, locations, 1, "a sibling at the archive root is reached through the full path")
assert.Equal(t, "outer.jar!/pom.xml", locations[0].FullNestedPath())
assert.True(t, r.HasPath(path.Dir("outer.jar!/pom.xml")), "the directory of a root entry is the root")
assert.True(t, r.HasPath("lib/dep.jar"), "an entry path is still accepted")
locations, err = r.FilesByGlob("outer.jar!/**/*.jar")
require.NoError(t, err)
require.Len(t, locations, 1)
}
func TestResolver_danglingLinkHasNoContent(t *testing.T) {
r := resolverFrom(t, testEntry{name: "opt/gone.jar", link: "missing.jar"})
@@ -420,14 +367,15 @@ func TestResolver_danglingLinkHasNoContent(t *testing.T) {
locations = append(locations, l)
}
require.Len(t, locations, 1)
assert.Equal(t, "opt/gone.jar", locations[0].RealPath)
assert.Equal(t, "opt/gone.jar", locations[0].LocalPath())
_, err := r.FileContentsByLocation(locations[0])
assert.Error(t, err, "a link to nothing has no content to read")
metadata, err := r.FileMetadataByLocation(locations[0])
require.NoError(t, err)
assert.Equal(t, "missing.jar", metadata.LinkDestination, "where it pointed is still reported, as the archive named it")
assert.Equal(t, "outer.jar!/opt/missing.jar", metadata.LinkDestination,
"where it pointed is still reported, resolved against the link's own directory")
}
func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) {
@@ -443,7 +391,7 @@ func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) {
t.Helper()
locations, err := r.FilesByMIMEType(types...)
require.NoError(t, err)
return realPaths(locations)
return localPaths(locations)
}
assert.Equal(t, []string{"opt/real.jar"}, byMIME("application/zip"),
@@ -465,7 +413,7 @@ func TestResolver_allLocationsKeepsEveryPath(t *testing.T) {
var got []string
for l := range r.AllLocations(context.Background()) {
got = append(got, l.RealPath)
got = append(got, l.LocalPath())
}
assert.Equal(t, []string{"opt/gone.jar", "opt/real.jar", "opt/sym.jar"}, got,
"every path the archive names, links included")
@@ -489,7 +437,7 @@ func TestResolver_add_anEntryOverTheCapIsSkippedAndReadNoFurther(t *testing.T) {
assert.ErrorContains(t, r.Truncated, "per-entry size cap of 1.0 KiB")
locations, err := r.FilesByGlob("**")
require.NoError(t, err)
assert.Equal(t, []string{"ok.txt"}, realPaths(locations))
assert.Equal(t, []string{"ok.txt"}, localPaths(locations))
}
type countingReader struct {
+2 -2
View File
@@ -109,7 +109,7 @@ func (r *Resolver) spill() error {
if err := r.openFile(); err != nil {
return err
}
log.WithFields("archive", r.archivePath, "held", len(r.held), "bytes", r.written).
log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "held", len(r.held), "bytes", r.written).
Debug("archive content does not fit in memory; writing it to disk")
for len(r.held) > 0 {
@@ -188,7 +188,7 @@ func (r *Resolver) releaseStorage() {
return
}
if err := r.file.Close(); err != nil {
log.WithFields("archive", r.archivePath, "error", err).Trace("unable to close archive temp file")
log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "error", err).Trace("unable to close archive temp file")
}
r.remove()
r.charge.refundDisk(r.written)
+2 -2
View File
@@ -223,7 +223,7 @@ func TestResolver_put_discardRefundsWhatIsHeldInMemory(t *testing.T) {
assert.Equal(t, "keep", readEntry(t, s.open(&keep)))
}
func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
func TestResolver_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
small, smallRoot := extractedResolver(t, manyEntryZip(t, 2), spillingLimits)
large, largeRoot := extractedResolver(t, manyEntryZip(t, 5000), spillingLimits)
@@ -238,7 +238,7 @@ func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
assert.Equal(t, int64(5000), large.written)
}
func TestStorage_anEntryAtTheEndIsReadBySeeking(t *testing.T) {
func TestResolver_anEntryAtTheEndIsReadBySeeking(t *testing.T) {
// the reader over an entry is a bounded window at its offset, not a re-read of what precedes it
const filler = 64 * 1024
+45 -23
View File
@@ -13,9 +13,8 @@ import (
// such as java can describe the archive it is inside without owning the recursion.
type Traversal struct {
// Location is the archive file's location within its parent filesystem, including the access path
// it was reached by. Its ArchivePath names the archives above it, so VirtualPath(Location) is the
// full chain from the scan root as seen through that access path. The ArchivePath of the files
// inside it is built from the real path instead.
// it was reached by, so VirtualPath(Location) is the chain from the scan root as seen through that
// access path. The files inside it extend its real coordinates instead (see ContainerPath).
Location file.Location
// Digests are of the archive file itself, taken during extraction; a cataloger running inside the
@@ -31,41 +30,64 @@ type Traversal struct {
Packages []pkg.Package
}
// ContentsArchivePath is the ArchivePath every file inside the archive carries: the chain to it, built
// from real paths. Outside any archive (a nil Traversal) it is blank, as for files at the scan root.
func (t *Traversal) ContentsArchivePath() string {
// FullTraversalArchivePath is the archive's own full path, which is the ContainerPath of every file inside it. Outside
// any archive (a nil Traversal) it is blank, as for files on the scanned filesystem.
func (t *Traversal) FullTraversalArchivePath() string {
if t == nil {
return ""
}
return VirtualPath(file.NewLocationFromCoordinates(t.Location.Coordinates))
return t.Location.FullNestedPath()
}
// ContainerPath returns the full path of the archive directly holding the file at c, or blank for a
// file on the scanned filesystem, whether or not its FullPath is set.
func ContainerPath(c file.Coordinates) string {
i := strings.LastIndex(c.FullPath, file.ArchivePathSeparator)
if i < 0 {
return ""
}
return c.FullPath[:i]
}
// archiveEntries is the "!/"-delimited chain of entry paths after RealPath, reporting false for a file
// not inside an archive. RealPath is split off first, so a "!/" in it is not taken for a boundary.
func archiveEntries(c file.Coordinates) (string, bool) {
remain, ok := strings.CutPrefix(c.FullNestedPath(), c.RealPath)
if !ok {
return "", false
}
return strings.CutPrefix(remain, file.ArchivePathSeparator)
}
// VirtualPath returns the colon-delimited chain of archives from the scan root to loc, ending in loc
// itself: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path unchanged. In an
// entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':' recovers the archive
// boundaries and no two entry names share an escaped form. The leading slash is dropped to match the
// java cataloger, which joins slash-less zip entry names onto the containing archive's path.
// itself as accessed: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path
// unchanged. In an entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':'
// recovers the archive boundaries and no two entry names share an escaped form. Leading slashes are
// dropped to match the java cataloger, which joins slash-less zip entry names onto the containing
// archive's path.
func VirtualPath(loc file.Location) string {
if loc.ArchivePath == "" {
return loc.Path()
accessPath := loc.AccessPath
if accessPath == "" {
accessPath = loc.FullNestedPath()
}
return loc.ArchivePath + ":" + strings.TrimPrefix(escapeEntryPath(loc.Path()), "/")
entries, ok := archiveEntries(loc.Coordinates)
if !ok {
return accessPath
}
segments := strings.Split(entries, file.ArchivePathSeparator)
segments[len(segments)-1] = strings.TrimPrefix(accessPath, ContainerPath(loc.Coordinates)+file.ArchivePathSeparator)
for i, s := range segments {
segments[i] = strings.TrimPrefix(escapeEntryPath(s), "/")
}
return loc.RealPath + ":" + strings.Join(segments, ":")
}
var (
entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A")
entryPathUnescaper = strings.NewReplacer("%3A", ":", "%25", "%")
)
var entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A")
func escapeEntryPath(p string) string {
return entryPathEscaper.Replace(p)
}
// UnescapeEntryPath reverses the escaping VirtualPath applies to one segment of the chain.
func UnescapeEntryPath(p string) string {
return entryPathUnescaper.Replace(p)
}
type traversalCtxKey struct{}
func WithTraversal(ctx context.Context, t *Traversal) context.Context {
+32 -32
View File
@@ -2,7 +2,6 @@ package archive
import (
"context"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -18,16 +17,24 @@ func TestTraversalContextRoundTrip(t *testing.T) {
ctx := WithTraversal(context.Background(), trav)
require.Same(t, trav, TraversalFromContext(ctx))
child := &Traversal{Location: locationIn("app.war", "/WEB-INF/lib/dep.jar")}
child := &Traversal{Location: locationIn("app.war", "WEB-INF/lib/dep.jar")}
ctx = WithTraversal(ctx, child)
require.Same(t, child, TraversalFromContext(ctx))
}
// locationIn is a location as the Index returns it: an entry path stamped with the containing chain.
func locationIn(archivePath, entryPath string) file.Location {
loc := file.NewLocation(entryPath)
loc.ArchivePath = archivePath
return loc
// locationIn is a location as the Resolver returns it: the outermost archive's real path, and the chain
// of entry paths within it.
func locationIn(realPath, archivePath string) file.Location {
return file.NewLocationFromCoordinates(file.Coordinates{RealPath: realPath, FullPath: realPath + file.ArchivePathSeparator + archivePath})
}
func TestContainerPath(t *testing.T) {
assert.Equal(t, "", ContainerPath(file.NewCoordinates("/app.war", "")))
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war"}), "a top-level file whose FullPath is set")
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/odd!/name"}), "a separator in a top-level real path is not an archive boundary")
assert.Equal(t, "/app.war", ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar").Coordinates))
assert.Equal(t, "/app.war!/WEB-INF/lib/dep.jar",
ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF").Coordinates))
}
func TestVirtualPath(t *testing.T) {
@@ -46,6 +53,11 @@ func TestVirtualPath(t *testing.T) {
location: file.NewLocation("some/dir/example.jar"),
expected: "some/dir/example.jar",
},
{
name: "outside an archive with FullPath set returns the path unchanged",
location: file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/odd!/name.jar", FullPath: "/odd!/name.jar"}),
expected: "/odd!/name.jar",
},
{
name: "single level trims leading slash",
location: locationIn("app.war", "/WEB-INF/lib/dep.jar"),
@@ -58,7 +70,7 @@ func TestVirtualPath(t *testing.T) {
},
{
name: "multi-level chain",
location: locationIn("app.war:WEB-INF/lib/dep.jar", "/inner/most.jar"),
location: locationIn("app.war", "WEB-INF/lib/dep.jar!/inner/most.jar"),
expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar",
},
{
@@ -72,11 +84,23 @@ func TestVirtualPath(t *testing.T) {
location: locationIn("app.war", "lib/a:b:c.jar"),
expected: "app.war:lib/a%3Ab%3Ac.jar",
},
{
// or the literal text %3A would collide with an escaped colon
name: "the escape character is itself escaped",
location: locationIn("outer.zip", "a%3Ab.jar"),
expected: "outer.zip:a%253Ab.jar",
},
{
name: "access path is preferred over real path",
location: file.NewVirtualLocation("/real/dep.jar", "/link/dep.jar"),
expected: "/link/dep.jar",
},
{
name: "access path within an archive replaces the last entry",
location: file.NewVirtualLocationFromCoordinates(
file.Coordinates{RealPath: "app.war", FullPath: "app.war!/WEB-INF/lib/real.jar"}, "app.war!/WEB-INF/lib/link.jar"),
expected: "app.war:WEB-INF/lib/link.jar",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -84,27 +108,3 @@ func TestVirtualPath(t *testing.T) {
})
}
}
func TestVirtualPath_theEscapeCharacterIsItselfEscaped(t *testing.T) {
// an entry path holding the literal text %3A must not collide with one holding a colon, or their
// contents share an ArchivePath and overwrite each other's file metadata
fromColon := VirtualPath(locationIn("outer.zip", "a:b.jar"))
fromLiteral := VirtualPath(locationIn("outer.zip", "a%3Ab.jar"))
assert.Equal(t, "outer.zip:a%3Ab.jar", fromColon)
assert.Equal(t, "outer.zip:a%253Ab.jar", fromLiteral)
}
func TestVirtualPath_escapingComposesAcrossNestingLevels(t *testing.T) {
level2 := VirtualPath(locationIn("outer.zip", "lib/mid:dle.war"))
require.Equal(t, "outer.zip:lib/mid%3Adle.war", level2)
final := VirtualPath(locationIn(level2, "WEB-INF/lib/inn:er.jar"))
require.Equal(t, "outer.zip:lib/mid%3Adle.war:WEB-INF/lib/inn%3Aer.jar", final)
assert.Equal(t,
[]string{"outer.zip", "lib/mid%3Adle.war", "WEB-INF/lib/inn%3Aer.jar"},
strings.Split(final, ":"),
"splitting on ':' must recover exactly the three real archive boundaries",
)
}