fix: keep RealPath as top-level, use FullPath and utility functions FullNestedPath, LocalPath, Path

Signed-off-by: Keith Zantow <kzantow@gmail.com>
This commit is contained in:
Keith Zantow
2026-10-19 15:44:37 -04:00
parent 17d109e8b5
commit 3e4b036e22
165 changed files with 1349 additions and 1970 deletions
+6 -12
View File
@@ -8,7 +8,6 @@ import (
"fmt" "fmt"
"io" "io"
"path" "path"
"strings"
"github.com/mholt/archives" "github.com/mholt/archives"
@@ -22,11 +21,12 @@ import (
// ErrDiskLimitReached when the archive's own bytes cannot be placed within the disk limit. The caller // ErrDiskLimitReached when the archive's own bytes cannot be placed within the disk limit. The caller
// must Cleanup the result. // must Cleanup the result.
// //
// fileSystemID and archivePath are stamped onto every location the resolver returns: the filesystem // archiveCoordinates are those of the archive file itself; every location the resolver returns
// the archive was found on, and the colon-delimited chain of archives from the scan root to this one. // extends them with its entry path (see file.Coordinates).
func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string, limiter *Limiter, exclusions Exclusions) (*Resolver, error) { func Extract(ctx context.Context, r io.Reader, archiveCoordinates file.Coordinates, limiter *Limiter, exclusions Exclusions) (*Resolver, error) {
charge := limiter.charge() charge := limiter.charge()
resolver := newResolver(ctx, fileSystemID, archivePath, charge) resolver := newResolver(ctx, archiveCoordinates, charge)
archivePath := archiveCoordinates.FullNestedPath()
// deferred so that a panic in a decoder still gives back the charge and removes any spill file // deferred so that a panic in a decoder still gives back the charge and removes any spill file
succeeded := false succeeded := false
@@ -43,7 +43,7 @@ func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string,
// the archive's own bytes are needed only until they are extracted and digested // the archive's own bytes are needed only until they are extracted and digested
defer release() defer release()
format := identifyFormat(ctx, archiveFileName(archivePath), content) format := identifyFormat(ctx, path.Base(archiveCoordinates.LocalPath()), content)
if format == nil { if format == nil {
return nil, nil return nil, nil
} }
@@ -75,12 +75,6 @@ func (r *Resolver) acquire(content io.Reader) (ReaderAtSeeker, func(), error) {
return r.open(&b), func() { r.discard(&b) }, nil return r.open(&b), func() { r.discard(&b) }, nil
} }
// archiveFileName returns the archive's own file name from the end of its archive path, which format
// identification reads for its extension.
func archiveFileName(archivePath string) string {
return path.Base(archivePath[strings.LastIndexByte(archivePath, ':')+1:])
}
// identifyFormat returns the format able to extract this content, or nil when it is not an archive. // identifyFormat returns the format able to extract this content, or nil when it is not an archive.
// //
// Identification from the head of the stream wins: a tar holding one jar ends in that jar's central // Identification from the head of the stream wins: a tar holding one jar ends in that jar's central
+53 -78
View File
@@ -118,28 +118,6 @@ func TestResolver_extract_diskLimitBitesMidEntry(t *testing.T) {
assert.Equal(t, r.written, info.Size()) assert.Equal(t, r.written, info.Size())
} }
func TestResolver_extract_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) {
data := zipBytes(t, map[string]string{"a.txt": strings.Repeat("a", 200)})
oneArchive := approxIndexBytes(tar.Header{Name: "a.txt"}) + 200
limiter := NewLimiter(Limits{MaxDiskBytes: oneArchive * 3 / 2})
first, _ := extractBytes(t, data, "test.zip", limiter.charge())
require.NoError(t, first.Truncated)
require.Equal(t, int64(200), first.written)
second, _ := extractBytes(t, data, "test.zip", limiter.charge())
assert.Error(t, second.Truncated, "a second archive is bounded while the first is still held")
second.Cleanup()
first.Cleanup()
_, disk := limiter.InUse()
require.Zero(t, disk)
third, _ := extractBytes(t, data, "test.zip", limiter.charge())
assert.NoError(t, third.Truncated, "and extracts in full once the first is released")
assert.Equal(t, int64(200), third.written)
}
func TestResolver_extract_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) { func TestResolver_extract_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) {
var entries []testEntry var entries []testEntry
for i := range 50 { for i := range 50 {
@@ -262,7 +240,7 @@ func TestResolver_extract_excludedEntriesAreNeverStored(t *testing.T) {
func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) { func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"}) data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
extracted, err := Extract(context.Background(), bytes.NewReader(data), "parentFS", "app.war:some/path/app.zip", nil, nil) extracted, err := Extract(context.Background(), bytes.NewReader(data), file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip"}, nil, nil)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, extracted) require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup) t.Cleanup(extracted.Cleanup)
@@ -274,7 +252,7 @@ func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
locations, err := extracted.FilesByGlob("**/*.txt") locations, err := extracted.FilesByGlob("**/*.txt")
require.NoError(t, err) require.NoError(t, err)
require.Len(t, locations, 1) require.Len(t, locations, 1)
assert.Equal(t, file.Coordinates{RealPath: "dir/hello.txt", FileSystemID: "parentFS", ArchivePath: "app.war:some/path/app.zip"}, assert.Equal(t, file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip!/dir/hello.txt"},
locations[0].Coordinates) locations[0].Coordinates)
reader, err := extracted.FileContentsByLocation(locations[0]) reader, err := extracted.FileContentsByLocation(locations[0])
@@ -284,39 +262,13 @@ func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
assert.Equal(t, "hello world", string(body)) assert.Equal(t, "hello world", string(body))
} }
func TestExtract_notAnArchive(t *testing.T) {
extracted, err := Extract(context.Background(), strings.NewReader("this is not an archive"), "", "notes.txt", nil, nil)
require.NoError(t, err)
assert.Nil(t, extracted)
}
func TestExtract_spillsUnderTheScansTempRootAndCleansUp(t *testing.T) {
ctx, root := scanContext(t)
limiter := NewLimiter(spillingLimits)
extracted, err := Extract(ctx, bytes.NewReader(zipBytes(t, sampleFiles)), "", "app.zip", limiter, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
require.Len(t, filesIn(t, root), 1)
_, disk := limiter.InUse()
assert.Positive(t, disk)
extracted.Cleanup()
assert.Empty(t, filesIn(t, root))
_, disk = limiter.InUse()
assert.Zero(t, disk, "cleanup releases everything the archive held")
extracted.Cleanup()
}
func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) { func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) {
// the archive's own bytes are only needed to extract and digest it; afterwards the limiter holds // the archive's own bytes are only needed to extract and digest it; afterwards the limiter holds
// the entries and the index estimate alone // the entries and the index estimate alone
data := zipBytes(t, map[string]string{"hello.txt": "hi"}) data := zipBytes(t, map[string]string{"hello.txt": "hi"})
limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}) limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil) extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
require.NoError(t, err) require.NoError(t, err)
t.Cleanup(extracted.Cleanup) t.Cleanup(extracted.Cleanup)
@@ -332,7 +284,7 @@ func TestExtract_heldContentWrittenToDiskStaysUntilCleanup(t *testing.T) {
data := zipBytes(t, map[string]string{"hello.txt": "hi"}) data := zipBytes(t, map[string]string{"hello.txt": "hi"})
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil) extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
require.NoError(t, err) require.NoError(t, err)
t.Cleanup(extracted.Cleanup) t.Cleanup(extracted.Cleanup)
@@ -354,7 +306,7 @@ func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) {
data := tarGzBytes(t, map[string]string{"a.txt": "first", "b.bin": string(noise)}) data := tarGzBytes(t, map[string]string{"a.txt": "first", "b.bin": string(noise)})
cut := data[:len(data)/2] cut := data[:len(data)/2]
extracted, err := Extract(context.Background(), bytes.NewReader(cut), "", "app.tar.gz", NewLimiter(unboundedLimits), nil) extracted, err := Extract(context.Background(), bytes.NewReader(cut), file.NewCoordinates("app.tar.gz", ""), NewLimiter(unboundedLimits), nil)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, extracted) require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup) t.Cleanup(extracted.Cleanup)
@@ -362,7 +314,7 @@ func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) {
assert.Error(t, extracted.Truncated) assert.Error(t, extracted.Truncated)
locations, err := extracted.FilesByGlob("**") locations, err := extracted.FilesByGlob("**")
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, []string{"a.txt"}, realPaths(locations)) assert.Equal(t, []string{"a.txt"}, localPaths(locations))
} }
func TestExtract_aPanicStillReleasesEverything(t *testing.T) { func TestExtract_aPanicStillReleasesEverything(t *testing.T) {
@@ -371,7 +323,7 @@ func TestExtract_aPanicStillReleasesEverything(t *testing.T) {
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}) limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
require.Panics(t, func() { require.Panics(t, func() {
_, _ = Extract(ctx, &panicsAfter{n: 4096}, "", "app.zip", limiter, nil) _, _ = Extract(ctx, &panicsAfter{n: 4096}, file.NewCoordinates("app.zip", ""), limiter, nil)
}) })
mem, disk := limiter.InUse() mem, disk := limiter.InUse()
@@ -397,11 +349,11 @@ func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) {
data := zipBytes(t, sampleFiles) data := zipBytes(t, sampleFiles)
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}) limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0})
_, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil) _, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
assert.ErrorIs(t, err, ErrDiskLimitReached) assert.ErrorIs(t, err, ErrDiskLimitReached)
// read in place, the same bytes cost nothing; only the entries are refused // read in place, the same bytes cost nothing; only the entries are refused
extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "app.zip", limiter, nil) extracted, err := Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("app.zip", ""), limiter, nil)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, extracted) require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup) t.Cleanup(extracted.Cleanup)
@@ -411,24 +363,6 @@ func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) {
assert.Empty(t, locations) assert.Empty(t, locations)
} }
func TestExtract_aZipBehindALauncherScript(t *testing.T) {
// a Spring Boot executable jar sniffs as a shell script; archive/zip still finds the central directory
data := append([]byte("#!/bin/bash\necho launcher\nexit 0\n"), zipBytes(t, map[string]string{
"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n",
"BOOT-INF/lib/dep.jar": "not really a jar, just bytes",
})...)
extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "/opt/app.jar", nil, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
assert.NoError(t, extracted.Truncated)
locations, err := extracted.FilesByGlob("**/*")
require.NoError(t, err)
assert.Len(t, locations, 2)
}
func Test_HasZipEndOfCentralDirectory(t *testing.T) { func Test_HasZipEndOfCentralDirectory(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -521,7 +455,7 @@ func TestExtract_decompressionBombIsBoundedByTheLimits(t *testing.T) {
ctx, root := scanContext(t) ctx, root := scanContext(t)
limiter := NewLimiter(limits) limiter := NewLimiter(limits)
r, err := Extract(ctx, bytes.NewReader(data), "", name, limiter, nil) r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates(name, ""), limiter, nil)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, r) require.NotNil(t, r)
t.Cleanup(r.Cleanup) t.Cleanup(r.Cleanup)
@@ -547,7 +481,7 @@ func TestExtract_entryCountBombIsBoundedByTheMemoryLimit(t *testing.T) {
limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1} limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}
limiter := NewLimiter(limits) limiter := NewLimiter(limits)
r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), "", "many.zip", limiter, nil) r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), file.NewCoordinates("many.zip", ""), limiter, nil)
require.NoError(t, err) require.NoError(t, err)
t.Cleanup(r.Cleanup) t.Cleanup(r.Cleanup)
@@ -575,7 +509,7 @@ func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) {
limits := Limits{MaxMemoryBytes: 8 << 20, MaxDiskBytes: -1} limits := Limits{MaxMemoryBytes: 8 << 20, MaxDiskBytes: -1}
limiter := NewLimiter(limits) limiter := NewLimiter(limits)
r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), "", "deep.zip", limiter, nil) r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), file.NewCoordinates("deep.zip", ""), limiter, nil)
require.NoError(t, err) require.NoError(t, err)
t.Cleanup(r.Cleanup) t.Cleanup(r.Cleanup)
@@ -584,3 +518,44 @@ func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) {
peakMemory, _ := limiter.Peak() peakMemory, _ := limiter.Peak()
assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes) assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes)
} }
func TestExtract_locationsKeepTheTopLevelRealPath(t *testing.T) {
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
// a top-level archive's coordinates, with FullPath unset or set to RealPath, name entries alike
for _, archive := range []file.Coordinates{
{RealPath: "/app.zip"},
{RealPath: "/app.zip", FullPath: "/app.zip"},
} {
extracted, err := Extract(context.Background(), bytes.NewReader(data), archive, nil, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
locations, err := extracted.FilesByPath("dir/hello.txt")
require.NoError(t, err)
require.Len(t, locations, 1)
assert.Equal(t, file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}, locations[0].Coordinates)
assert.Equal(t, "dir/hello.txt", locations[0].LocalPath())
for _, loc := range []file.Location{
locations[0],
file.NewLocation("dir/hello.txt"),
file.NewLocation("/dir/hello.txt"),
file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}),
} {
reader, err := extracted.FileContentsByLocation(loc)
require.NoError(t, err, "location %s", loc)
body, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "hello world", string(body))
metadata, err := extracted.FileMetadataByLocation(loc)
require.NoError(t, err, "location %s", loc)
assert.Equal(t, "/app.zip!/dir/hello.txt", metadata.Path)
}
_, err = extracted.FileContentsByLocation(file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/other.zip", FullPath: "/other.zip!/dir/hello.txt"}))
assert.Error(t, err, "an entry of another archive is not read from this one")
}
}
+7 -32
View File
@@ -10,7 +10,6 @@ import (
"io" "io"
"io/fs" "io/fs"
"os" "os"
"path"
"sort" "sort"
"testing" "testing"
@@ -179,7 +178,7 @@ func resolverIn(t testing.TB, charge *charge) (*Resolver, string) {
t.Helper() t.Helper()
root := t.TempDir() root := t.TempDir()
ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)) ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))
r := newResolver(ctx, "", "test-archive", charge) r := newResolver(ctx, file.NewCoordinates("test-archive", ""), charge)
t.Cleanup(r.Cleanup) t.Cleanup(r.Cleanup)
return r, root return r, root
} }
@@ -197,7 +196,7 @@ func addAll(t testing.TB, r *Resolver, entries []testEntry) {
func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Resolver { func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Resolver {
t.Helper() t.Helper()
r, _ := resolverIn(t, memCharge(maxInMemory)) r, _ := resolverIn(t, memCharge(maxInMemory))
r.archivePath = "outer.jar" r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
addAll(t, r, fileEntries(files)) addAll(t, r, fileEntries(files))
return r return r
} }
@@ -207,35 +206,11 @@ func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Res
func resolverFrom(t testing.TB, entries ...testEntry) *Resolver { func resolverFrom(t testing.TB, entries ...testEntry) *Resolver {
t.Helper() t.Helper()
r, _ := resolverIn(t, memCharge(1<<20)) r, _ := resolverIn(t, memCharge(1<<20))
r.archivePath = "outer.jar" r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
addAll(t, r, entries) addAll(t, r, entries)
return r return r
} }
// indexCostOf is what add charges for the given entries in name order: every listed node and every
// directory their paths imply.
func indexCostOf(entries map[string]string) int64 {
names := make([]string, 0, len(entries))
for name := range entries {
names = append(names, name)
}
sort.Strings(names)
seen := map[string]bool{}
var total int64
for _, name := range names {
p := path.Clean("/" + name)
for i := 1; i < len(p); i++ {
if p[i] == '/' && !seen[p[:i]] {
seen[p[:i]] = true
total += approxIndexBytesPerEntry
}
}
seen[p] = true
total += approxIndexBytes(tar.Header{Name: name})
}
return total
}
// extractBytes extracts an archive into a resolver built over the given charge. // extractBytes extracts an archive into a resolver built over the given charge.
func extractBytes(t testing.TB, data []byte, name string, charge *charge) (*Resolver, string) { func extractBytes(t testing.TB, data []byte, name string, charge *charge) (*Resolver, string) {
t.Helper() t.Helper()
@@ -316,11 +291,11 @@ func diskCharge(maxDisk int64) *charge {
return NewLimiter(Limits{MaxDiskBytes: maxDisk}).charge() return NewLimiter(Limits{MaxDiskBytes: maxDisk}).charge()
} }
// realPaths lists the real path of every location, sorted. // localPaths lists the local entry path of every location (rather than full nested archive path), sorted.
func realPaths(locations []file.Location) []string { func localPaths(locations []file.Location) []string {
var out []string var out []string
for _, loc := range locations { for _, loc := range locations {
out = append(out, loc.RealPath) out = append(out, loc.LocalPath())
} }
sort.Strings(out) sort.Strings(out)
return out return out
@@ -342,7 +317,7 @@ func extractedResolver(t testing.TB, data []byte, limits Limits) (*Resolver, str
t.Helper() t.Helper()
ctx, root := scanContext(t) ctx, root := scanContext(t)
r, err := Extract(ctx, bytes.NewReader(data), "", "app.zip", NewLimiter(limits), nil) r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates("app.zip", ""), NewLimiter(limits), nil)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, r) require.NotNil(t, r)
t.Cleanup(r.Cleanup) t.Cleanup(r.Cleanup)
+70 -35
View File
@@ -28,7 +28,8 @@ import (
// into it, holding their content within the scan's limits. Entries are indexed in a stereoscope file // into it, holding their content within the scan's limits. Entries are indexed in a stereoscope file
// tree, the same index the directory and image resolvers search. // tree, the same index the directory and image resolvers search.
// //
// Paths are archive-relative with no leading slash. // Locations it returns keep the outermost archive's RealPath and carry the entry's FullPath. Queries
// may name an entry either by its full path or by its path within the archive (see rootedPath).
type Resolver struct { type Resolver struct {
// Digests are of the archive file itself. // Digests are of the archive file itself.
Digests []file.Digest Digests []file.Digest
@@ -37,8 +38,7 @@ type Resolver struct {
// or the decompression budget stopped extraction early, or an entry was too large or unreadable. // or the decompression budget stopped extraction early, or an entry was too large or unreadable.
Truncated error Truncated error
fileSystemID string archiveCoordinates file.Coordinates
archivePath string
charge *charge charge *charge
budget *Budget budget *Budget
@@ -61,12 +61,11 @@ type Resolver struct {
var _ file.Resolver = (*Resolver)(nil) var _ file.Resolver = (*Resolver)(nil)
// newResolver returns an empty resolver whose locations carry the given fileSystemID and archivePath, // newResolver returns an empty resolver whose locations extend the archive's coordinates, spilling into
// spilling into the scan's temp root when ctx carries one (see internal/tmpdir). // the scan's temp root when ctx carries one (see internal/tmpdir).
func newResolver(ctx context.Context, fileSystemID, archivePath string, charge *charge) *Resolver { func newResolver(ctx context.Context, archiveCoordinates file.Coordinates, charge *charge) *Resolver {
return &Resolver{ return &Resolver{
fileSystemID: fileSystemID, archiveCoordinates: archiveCoordinates,
archivePath: archivePath,
charge: charge, charge: charge,
budget: budgetFromContext(ctx), budget: budgetFromContext(ctx),
tempDir: tmpdir.FromContext(ctx), tempDir: tmpdir.FromContext(ctx),
@@ -160,13 +159,13 @@ func (r *Resolver) add(hdr tar.Header, content io.Reader) error {
return nil return nil
} }
func (r *Resolver) store(entryPath string, hdr tar.Header, b *blob) error { func (r *Resolver) store(rooted string, hdr tar.Header, b *blob) error {
ref, err := r.addToTree(entryPath, hdr) ref, err := r.addToTree(rooted, hdr)
if err != nil { if err != nil {
return err return err
} }
metadata := stereoscopeFile.NewMetadata(hdr, r.open(b)) metadata := stereoscopeFile.NewMetadata(hdr, r.open(b))
metadata.Path = reportedPath(entryPath) metadata.Path = entryPath(rooted)
r.index.Add(*ref, metadata) r.index.Add(*ref, metadata)
r.content[ref.ID()] = b r.content[ref.ID()] = b
return nil return nil
@@ -217,34 +216,63 @@ func (r *Resolver) finish() {
r.search = filetree.NewSearchContext(r.tree, r.index) r.search = filetree.NewSearchContext(r.tree, r.index)
} }
// reportedPath is how paths leave the resolver: archive-relative with no leading slash. // linkDestination is the rooted path a link points at inside the archive. A hard link names its
func reportedPath(rooted string) string { // target from the archive root, as does an absolute symlink, so "/etc/passwd" is an entry of this
// archive and never the host's. A relative symlink is joined onto the link's own directory, and
// cleaning against the root keeps "../../x" inside the archive.
func linkDestination(linkPath, linkname string, hard bool) string {
if hard || path.IsAbs(linkname) {
return path.Join("/", linkname)
}
return path.Join(path.Dir(linkPath), linkname)
}
// entryPath is a node's path within the archive: archive-relative with no leading slash.
func entryPath(rooted string) string {
return strings.TrimPrefix(rooted, "/") return strings.TrimPrefix(rooted, "/")
} }
func rootedPath(userPath string) string { // fullPath is how paths leave the resolver: the archive's own full path, then the entry path.
return path.Clean("/" + userPath) func (r *Resolver) fullPath(rooted string) string {
return r.archiveCoordinates.FullNestedPath() + "!/" + entryPath(rooted)
} }
// location names a resolution: a link is reported at its target's path with the link's own path as // rootedPath is the tree path a query names, which is either form a path takes: a full path is trimmed to
// the access path. // the entry path within this archive, and anything else (a LocalPath) is taken as archive-relative
func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location { // already. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root)
coordinates := file.Coordinates{ // names the root.
RealPath: reportedPath(string(res.RealPath)), func (r *Resolver) rootedPath(userPath string) string {
FileSystemID: r.fileSystemID, archivePath := r.archiveCoordinates.FullNestedPath() + "!"
ArchivePath: r.archivePath, if userPath == archivePath {
return "/"
} }
accessPath := reportedPath(string(res.RequestPath)) return path.Clean("/" + strings.TrimPrefix(userPath, archivePath+"/"))
if accessPath == "" || accessPath == coordinates.RealPath { }
// coordinates name an entry by the outermost archive's real path, which never includes a path inside an
// archive, and its full path through every archive.
func (r *Resolver) coordinates(rooted string) file.Coordinates {
return file.Coordinates{
RealPath: r.archiveCoordinates.RealPath,
FileSystemID: r.archiveCoordinates.FileSystemID,
FullPath: r.fullPath(rooted),
}
}
// location names a resolution: a link is reported at its target's path with the link's own full path
// as the access path.
func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location {
coordinates := r.coordinates(string(res.RealPath))
if res.RequestPath == "" || res.RequestPath == res.RealPath {
return file.NewLocationFromCoordinates(coordinates) return file.NewLocationFromCoordinates(coordinates)
} }
return file.NewVirtualLocationFromCoordinates(coordinates, accessPath) return file.NewVirtualLocationFromCoordinates(coordinates, r.fullPath(string(res.RequestPath)))
} }
// resolve finds the file a path names, following links, and reports false for directories and // resolve finds the file a path names, following links, and reports false for directories and
// paths the archive never listed. // paths the archive never listed.
func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool) { func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool) {
res, err := r.search.SearchByPath(rootedPath(userPath), filetree.FollowBasenameLinks) res, err := r.search.SearchByPath(r.rootedPath(userPath), filetree.FollowBasenameLinks)
if err != nil || res == nil || !res.HasReference() { if err != nil || res == nil || !res.HasReference() {
return nil, false return nil, false
} }
@@ -256,27 +284,34 @@ func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool)
} }
func (r *Resolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { func (r *Resolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
res, ok := r.resolve(location.RealPath) res, ok := r.resolve(location.FullNestedPath())
if !ok { if !ok {
return nil, fmt.Errorf("no archive entry for path=%q", location.RealPath) return nil, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
} }
return entryReader{r.open(r.content[res.ID()])}, nil return entryReader{r.open(r.content[res.ID()])}, nil
} }
func (r *Resolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) { func (r *Resolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
exists, res, err := r.tree.File(stereoscopeFile.Path(rootedPath(location.RealPath))) rooted := r.rootedPath(location.FullNestedPath())
exists, res, err := r.tree.File(stereoscopeFile.Path(rooted))
if err != nil || !exists || !res.HasReference() { if err != nil || !exists || !res.HasReference() {
return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.RealPath) return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
} }
entry, err := r.index.Get(*res.Reference) entry, err := r.index.Get(*res.Reference)
if err != nil { if err != nil {
return file.Metadata{}, err return file.Metadata{}, err
} }
return entry.Metadata, nil // the index holds entry paths; callers get full paths, as with locations
metadata := entry.Metadata
metadata.Path = r.fullPath(rooted)
if metadata.LinkDestination != "" {
metadata.LinkDestination = r.fullPath(linkDestination(rooted, metadata.LinkDestination, metadata.Type == stereoscopeFile.TypeHardLink))
}
return metadata, nil
} }
func (r *Resolver) HasPath(userPath string) bool { func (r *Resolver) HasPath(userPath string) bool {
return r.tree.HasPath(stereoscopeFile.Path(rootedPath(userPath))) return r.tree.HasPath(stereoscopeFile.Path(r.rootedPath(userPath)))
} }
func (r *Resolver) FilesByPath(paths ...string) ([]file.Location, error) { func (r *Resolver) FilesByPath(paths ...string) ([]file.Location, error) {
@@ -297,7 +332,7 @@ func (r *Resolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
var out []file.Location var out []file.Location
seen := stereoscopeFile.NewFileReferenceSet() seen := stereoscopeFile.NewFileReferenceSet()
for _, pattern := range patterns { for _, pattern := range patterns {
results, err := r.searchByGlob(rootedPath(pattern)) results, err := r.searchByGlob(r.rootedPath(pattern))
if err != nil { if err != nil {
return nil, fmt.Errorf("invalid glob %q: %w", pattern, err) return nil, fmt.Errorf("invalid glob %q: %w", pattern, err)
} }
@@ -330,8 +365,8 @@ func (r *Resolver) searchByGlob(pattern string) ([]stereoscopeFile.Resolution, e
// merge into the SBOM. // merge into the SBOM.
func sortLocations(locations []file.Location) { func sortLocations(locations []file.Location) {
sort.Slice(locations, func(i, j int) bool { sort.Slice(locations, func(i, j int) bool {
if locations[i].RealPath != locations[j].RealPath { if a, b := locations[i].FullNestedPath(), locations[j].FullNestedPath(); a != b {
return locations[i].RealPath < locations[j].RealPath return a < b
} }
return locations[i].AccessPath < locations[j].AccessPath return locations[i].AccessPath < locations[j].AccessPath
}) })
+35 -87
View File
@@ -4,8 +4,8 @@ import (
"archive/tar" "archive/tar"
"bytes" "bytes"
"context" "context"
"fmt"
"io" "io"
"path"
"sort" "sort"
"strings" "strings"
"testing" "testing"
@@ -55,31 +55,6 @@ func TestResolver_add_directoryEntriesHoldNoContent(t *testing.T) {
assert.Empty(t, readStore(t, r)["lib"].body) assert.Empty(t, readStore(t, r)["lib"].body)
} }
func TestResolver_add_indexCostIsChargedEvenForEmptyContent(t *testing.T) {
// the index estimate is what bounds an archive of many empty entries
limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1})
r, _ := resolverIn(t, limiter.charge())
hdr := regularHeader("a.txt", 0)
require.NoError(t, r.add(hdr, bytes.NewReader(nil)))
mem, _ := limiter.InUse()
assert.Equal(t, approxIndexBytes(hdr), mem)
assert.Zero(t, r.heldInMemory())
}
func TestResolver_add_indexCostFallsBackToDiskWhenMemoryIsZero(t *testing.T) {
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
r, _ := resolverIn(t, limiter.charge())
hdr := regularHeader("a.txt", 0)
require.NoError(t, r.add(hdr, bytes.NewReader(nil)))
mem, disk := limiter.InUse()
assert.Zero(t, mem)
assert.Equal(t, approxIndexBytes(hdr), disk)
}
func TestResolver_add_indexDoesNotFallBackToDiskWhenMemoryIsBounded(t *testing.T) { func TestResolver_add_indexDoesNotFallBackToDiskWhenMemoryIsBounded(t *testing.T) {
// the index lives in memory, so a bounded memory limit bounds it; only a zero limit defers to disk // the index lives in memory, so a bounded memory limit bounds it; only a zero limit defers to disk
limiter := NewLimiter(Limits{MaxMemoryBytes: 1024, MaxDiskBytes: -1}) limiter := NewLimiter(Limits{MaxMemoryBytes: 1024, MaxDiskBytes: -1})
@@ -108,21 +83,6 @@ func TestResolver_add_indexMakesRoomBySpillingHeldContent(t *testing.T) {
assert.Equal(t, strings.Repeat("a", 100), readStore(t, r)["a.txt"].body) assert.Equal(t, strings.Repeat("a", 100), readStore(t, r)["a.txt"].body)
} }
func TestResolver_add_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) {
room := approxIndexBytes(regularHeader("x0", 0)) * 3
r, _ := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).charge())
var err error
admitted := 0
for i := 0; i < 100 && err == nil; i++ {
if err = r.add(regularHeader(fmt.Sprintf("x%d", i), 0), bytes.NewReader(nil)); err == nil {
admitted++
}
}
require.ErrorIs(t, err, ErrDiskLimitReached)
assert.Equal(t, 3, admitted)
}
func TestResolver_add_firstEntryAtAPathWins(t *testing.T) { func TestResolver_add_firstEntryAtAPathWins(t *testing.T) {
r, _ := resolverIn(t, memCharge(1<<20)) r, _ := resolverIn(t, memCharge(1<<20))
@@ -153,22 +113,6 @@ func TestResolver_pathsAndContents(t *testing.T) {
assert.Equal(t, "Manifest-Version: 1.0\n", string(body)) assert.Equal(t, "Manifest-Version: 1.0\n", string(body))
} }
func TestResolver_contentsSurviveTheMoveToDisk(t *testing.T) {
// room for the index but only a few bytes of content, so the rest moves to disk
r := resolverOver(t, indexCostOf(jarEntriesFixture)+8, jarEntriesFixture)
locations, err := r.FilesByGlob("**/*.jar")
require.NoError(t, err)
require.Len(t, locations, 1)
reader, err := r.FileContentsByLocation(locations[0])
require.NoError(t, err)
body, err := io.ReadAll(reader)
require.NoError(t, err)
require.NoError(t, reader.Close())
assert.Equal(t, "PK\x03\x04 not really", string(body))
}
func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) { func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) {
r := resolverOver(t, 1<<20, map[string]string{ r := resolverOver(t, 1<<20, map[string]string{
"notes.txt": "just some text", "notes.txt": "just some text",
@@ -177,26 +121,7 @@ func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) {
locations, err := r.FilesByMIMEType("text/x-shellscript") locations, err := r.FilesByMIMEType("text/x-shellscript")
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, []string{"run.sh"}, realPaths(locations)) assert.Equal(t, []string{"run.sh"}, localPaths(locations))
}
func TestResolver_allLocationsCoversEveryFile(t *testing.T) {
r := resolverOver(t, 1<<20, jarEntriesFixture)
var got []string
for loc := range r.AllLocations(context.Background()) {
got = append(got, loc.RealPath)
}
sort.Strings(got)
assert.Equal(t, []string{
"BOOT-INF/lib/inner.jar",
"LICENSE",
"META-INF/MANIFEST.MF",
"META-INF/maven/com.example/lib/pom.properties",
"META-INF/maven/com.example/lib/pom.xml",
"com/example/Thing.class",
}, got)
} }
func TestResolver_answersAreOrdered(t *testing.T) { func TestResolver_answersAreOrdered(t *testing.T) {
@@ -227,14 +152,14 @@ func TestResolver_answersAreOrdered(t *testing.T) {
var all []string var all []string
for loc := range r.AllLocations(context.Background()) { for loc := range r.AllLocations(context.Background()) {
all = append(all, loc.RealPath) all = append(all, loc.LocalPath())
} }
assert.True(t, sort.StringsAreSorted(all)) assert.True(t, sort.StringsAreSorted(all))
} }
var paths []string var paths []string
for _, loc := range first { for _, loc := range first {
paths = append(paths, loc.RealPath) paths = append(paths, loc.LocalPath())
} }
assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable") assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable")
} }
@@ -279,7 +204,7 @@ func globPaths(t *testing.T, r *Resolver, pattern string) []string {
t.Helper() t.Helper()
locations, err := r.FilesByGlob(pattern) locations, err := r.FilesByGlob(pattern)
require.NoError(t, err) require.NoError(t, err)
return realPaths(locations) return localPaths(locations)
} }
// several paths to one file answer as one location: the file's own path when it matched, otherwise // several paths to one file answer as one location: the file's own path when it matched, otherwise
@@ -399,8 +324,8 @@ func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
require.Len(t, locations, 1) require.Len(t, locations, 1)
assert.Equal(t, "opt/real.bin", locations[0].RealPath, "content comes from the file") assert.Equal(t, "opt/real.bin", locations[0].LocalPath(), "content comes from the file")
assert.Equal(t, "opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched") assert.Equal(t, "outer.jar!/opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched")
reader, err := r.FileContentsByLocation(locations[0]) reader, err := r.FileContentsByLocation(locations[0])
require.NoError(t, err) require.NoError(t, err)
@@ -412,6 +337,28 @@ func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) {
"a location standing for a link must still read the file's bytes, not the link's none") "a location standing for a link must still read the file's bytes, not the link's none")
} }
func TestResolver_fullPathsAreTrimmedToEntryPaths(t *testing.T) {
// catalogers build queries from the full paths the resolver hands out, e.g. path.Dir plus a sibling
r := resolverOver(t, 1<<20, map[string]string{"pom.xml": "a", "lib/dep.jar": "b"})
locations, err := r.FilesByPath("outer.jar!/lib/dep.jar")
require.NoError(t, err)
require.Len(t, locations, 1)
assert.Equal(t, "outer.jar!/lib/dep.jar", locations[0].AccessPath)
locations, err = r.FilesByPath(path.Join(path.Dir(locations[0].AccessPath), "../pom.xml"))
require.NoError(t, err)
require.Len(t, locations, 1, "a sibling at the archive root is reached through the full path")
assert.Equal(t, "outer.jar!/pom.xml", locations[0].FullNestedPath())
assert.True(t, r.HasPath(path.Dir("outer.jar!/pom.xml")), "the directory of a root entry is the root")
assert.True(t, r.HasPath("lib/dep.jar"), "an entry path is still accepted")
locations, err = r.FilesByGlob("outer.jar!/**/*.jar")
require.NoError(t, err)
require.Len(t, locations, 1)
}
func TestResolver_danglingLinkHasNoContent(t *testing.T) { func TestResolver_danglingLinkHasNoContent(t *testing.T) {
r := resolverFrom(t, testEntry{name: "opt/gone.jar", link: "missing.jar"}) r := resolverFrom(t, testEntry{name: "opt/gone.jar", link: "missing.jar"})
@@ -420,14 +367,15 @@ func TestResolver_danglingLinkHasNoContent(t *testing.T) {
locations = append(locations, l) locations = append(locations, l)
} }
require.Len(t, locations, 1) require.Len(t, locations, 1)
assert.Equal(t, "opt/gone.jar", locations[0].RealPath) assert.Equal(t, "opt/gone.jar", locations[0].LocalPath())
_, err := r.FileContentsByLocation(locations[0]) _, err := r.FileContentsByLocation(locations[0])
assert.Error(t, err, "a link to nothing has no content to read") assert.Error(t, err, "a link to nothing has no content to read")
metadata, err := r.FileMetadataByLocation(locations[0]) metadata, err := r.FileMetadataByLocation(locations[0])
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "missing.jar", metadata.LinkDestination, "where it pointed is still reported, as the archive named it") assert.Equal(t, "outer.jar!/opt/missing.jar", metadata.LinkDestination,
"where it pointed is still reported, resolved against the link's own directory")
} }
func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) { func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) {
@@ -443,7 +391,7 @@ func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) {
t.Helper() t.Helper()
locations, err := r.FilesByMIMEType(types...) locations, err := r.FilesByMIMEType(types...)
require.NoError(t, err) require.NoError(t, err)
return realPaths(locations) return localPaths(locations)
} }
assert.Equal(t, []string{"opt/real.jar"}, byMIME("application/zip"), assert.Equal(t, []string{"opt/real.jar"}, byMIME("application/zip"),
@@ -465,7 +413,7 @@ func TestResolver_allLocationsKeepsEveryPath(t *testing.T) {
var got []string var got []string
for l := range r.AllLocations(context.Background()) { for l := range r.AllLocations(context.Background()) {
got = append(got, l.RealPath) got = append(got, l.LocalPath())
} }
assert.Equal(t, []string{"opt/gone.jar", "opt/real.jar", "opt/sym.jar"}, got, assert.Equal(t, []string{"opt/gone.jar", "opt/real.jar", "opt/sym.jar"}, got,
"every path the archive names, links included") "every path the archive names, links included")
@@ -489,7 +437,7 @@ func TestResolver_add_anEntryOverTheCapIsSkippedAndReadNoFurther(t *testing.T) {
assert.ErrorContains(t, r.Truncated, "per-entry size cap of 1.0 KiB") assert.ErrorContains(t, r.Truncated, "per-entry size cap of 1.0 KiB")
locations, err := r.FilesByGlob("**") locations, err := r.FilesByGlob("**")
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, []string{"ok.txt"}, realPaths(locations)) assert.Equal(t, []string{"ok.txt"}, localPaths(locations))
} }
type countingReader struct { type countingReader struct {
+2 -2
View File
@@ -109,7 +109,7 @@ func (r *Resolver) spill() error {
if err := r.openFile(); err != nil { if err := r.openFile(); err != nil {
return err return err
} }
log.WithFields("archive", r.archivePath, "held", len(r.held), "bytes", r.written). log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "held", len(r.held), "bytes", r.written).
Debug("archive content does not fit in memory; writing it to disk") Debug("archive content does not fit in memory; writing it to disk")
for len(r.held) > 0 { for len(r.held) > 0 {
@@ -188,7 +188,7 @@ func (r *Resolver) releaseStorage() {
return return
} }
if err := r.file.Close(); err != nil { if err := r.file.Close(); err != nil {
log.WithFields("archive", r.archivePath, "error", err).Trace("unable to close archive temp file") log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "error", err).Trace("unable to close archive temp file")
} }
r.remove() r.remove()
r.charge.refundDisk(r.written) r.charge.refundDisk(r.written)
+2 -2
View File
@@ -223,7 +223,7 @@ func TestResolver_put_discardRefundsWhatIsHeldInMemory(t *testing.T) {
assert.Equal(t, "keep", readEntry(t, s.open(&keep))) assert.Equal(t, "keep", readEntry(t, s.open(&keep)))
} }
func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) { func TestResolver_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
small, smallRoot := extractedResolver(t, manyEntryZip(t, 2), spillingLimits) small, smallRoot := extractedResolver(t, manyEntryZip(t, 2), spillingLimits)
large, largeRoot := extractedResolver(t, manyEntryZip(t, 5000), spillingLimits) large, largeRoot := extractedResolver(t, manyEntryZip(t, 5000), spillingLimits)
@@ -238,7 +238,7 @@ func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
assert.Equal(t, int64(5000), large.written) assert.Equal(t, int64(5000), large.written)
} }
func TestStorage_anEntryAtTheEndIsReadBySeeking(t *testing.T) { func TestResolver_anEntryAtTheEndIsReadBySeeking(t *testing.T) {
// the reader over an entry is a bounded window at its offset, not a re-read of what precedes it // the reader over an entry is a bounded window at its offset, not a re-read of what precedes it
const filler = 64 * 1024 const filler = 64 * 1024
+45 -23
View File
@@ -13,9 +13,8 @@ import (
// such as java can describe the archive it is inside without owning the recursion. // such as java can describe the archive it is inside without owning the recursion.
type Traversal struct { type Traversal struct {
// Location is the archive file's location within its parent filesystem, including the access path // Location is the archive file's location within its parent filesystem, including the access path
// it was reached by. Its ArchivePath names the archives above it, so VirtualPath(Location) is the // it was reached by, so VirtualPath(Location) is the chain from the scan root as seen through that
// full chain from the scan root as seen through that access path. The ArchivePath of the files // access path. The files inside it extend its real coordinates instead (see ContainerPath).
// inside it is built from the real path instead.
Location file.Location Location file.Location
// Digests are of the archive file itself, taken during extraction; a cataloger running inside the // Digests are of the archive file itself, taken during extraction; a cataloger running inside the
@@ -31,41 +30,64 @@ type Traversal struct {
Packages []pkg.Package Packages []pkg.Package
} }
// ContentsArchivePath is the ArchivePath every file inside the archive carries: the chain to it, built // FullTraversalArchivePath is the archive's own full path, which is the ContainerPath of every file inside it. Outside
// from real paths. Outside any archive (a nil Traversal) it is blank, as for files at the scan root. // any archive (a nil Traversal) it is blank, as for files on the scanned filesystem.
func (t *Traversal) ContentsArchivePath() string { func (t *Traversal) FullTraversalArchivePath() string {
if t == nil { if t == nil {
return "" return ""
} }
return VirtualPath(file.NewLocationFromCoordinates(t.Location.Coordinates)) return t.Location.FullNestedPath()
}
// ContainerPath returns the full path of the archive directly holding the file at c, or blank for a
// file on the scanned filesystem, whether or not its FullPath is set.
func ContainerPath(c file.Coordinates) string {
i := strings.LastIndex(c.FullPath, file.ArchivePathSeparator)
if i < 0 {
return ""
}
return c.FullPath[:i]
}
// archiveEntries is the "!/"-delimited chain of entry paths after RealPath, reporting false for a file
// not inside an archive. RealPath is split off first, so a "!/" in it is not taken for a boundary.
func archiveEntries(c file.Coordinates) (string, bool) {
remain, ok := strings.CutPrefix(c.FullNestedPath(), c.RealPath)
if !ok {
return "", false
}
return strings.CutPrefix(remain, file.ArchivePathSeparator)
} }
// VirtualPath returns the colon-delimited chain of archives from the scan root to loc, ending in loc // VirtualPath returns the colon-delimited chain of archives from the scan root to loc, ending in loc
// itself: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path unchanged. In an // itself as accessed: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path
// entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':' recovers the archive // unchanged. In an entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':'
// boundaries and no two entry names share an escaped form. The leading slash is dropped to match the // recovers the archive boundaries and no two entry names share an escaped form. Leading slashes are
// java cataloger, which joins slash-less zip entry names onto the containing archive's path. // dropped to match the java cataloger, which joins slash-less zip entry names onto the containing
// archive's path.
func VirtualPath(loc file.Location) string { func VirtualPath(loc file.Location) string {
if loc.ArchivePath == "" { accessPath := loc.AccessPath
return loc.Path() if accessPath == "" {
accessPath = loc.FullNestedPath()
} }
return loc.ArchivePath + ":" + strings.TrimPrefix(escapeEntryPath(loc.Path()), "/") entries, ok := archiveEntries(loc.Coordinates)
if !ok {
return accessPath
}
segments := strings.Split(entries, file.ArchivePathSeparator)
segments[len(segments)-1] = strings.TrimPrefix(accessPath, ContainerPath(loc.Coordinates)+file.ArchivePathSeparator)
for i, s := range segments {
segments[i] = strings.TrimPrefix(escapeEntryPath(s), "/")
}
return loc.RealPath + ":" + strings.Join(segments, ":")
} }
var ( var entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A")
entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A")
entryPathUnescaper = strings.NewReplacer("%3A", ":", "%25", "%")
)
func escapeEntryPath(p string) string { func escapeEntryPath(p string) string {
return entryPathEscaper.Replace(p) return entryPathEscaper.Replace(p)
} }
// UnescapeEntryPath reverses the escaping VirtualPath applies to one segment of the chain.
func UnescapeEntryPath(p string) string {
return entryPathUnescaper.Replace(p)
}
type traversalCtxKey struct{} type traversalCtxKey struct{}
func WithTraversal(ctx context.Context, t *Traversal) context.Context { func WithTraversal(ctx context.Context, t *Traversal) context.Context {
+32 -32
View File
@@ -2,7 +2,6 @@ package archive
import ( import (
"context" "context"
"strings"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -18,16 +17,24 @@ func TestTraversalContextRoundTrip(t *testing.T) {
ctx := WithTraversal(context.Background(), trav) ctx := WithTraversal(context.Background(), trav)
require.Same(t, trav, TraversalFromContext(ctx)) require.Same(t, trav, TraversalFromContext(ctx))
child := &Traversal{Location: locationIn("app.war", "/WEB-INF/lib/dep.jar")} child := &Traversal{Location: locationIn("app.war", "WEB-INF/lib/dep.jar")}
ctx = WithTraversal(ctx, child) ctx = WithTraversal(ctx, child)
require.Same(t, child, TraversalFromContext(ctx)) require.Same(t, child, TraversalFromContext(ctx))
} }
// locationIn is a location as the Index returns it: an entry path stamped with the containing chain. // locationIn is a location as the Resolver returns it: the outermost archive's real path, and the chain
func locationIn(archivePath, entryPath string) file.Location { // of entry paths within it.
loc := file.NewLocation(entryPath) func locationIn(realPath, archivePath string) file.Location {
loc.ArchivePath = archivePath return file.NewLocationFromCoordinates(file.Coordinates{RealPath: realPath, FullPath: realPath + file.ArchivePathSeparator + archivePath})
return loc }
func TestContainerPath(t *testing.T) {
assert.Equal(t, "", ContainerPath(file.NewCoordinates("/app.war", "")))
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war"}), "a top-level file whose FullPath is set")
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/odd!/name"}), "a separator in a top-level real path is not an archive boundary")
assert.Equal(t, "/app.war", ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar").Coordinates))
assert.Equal(t, "/app.war!/WEB-INF/lib/dep.jar",
ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF").Coordinates))
} }
func TestVirtualPath(t *testing.T) { func TestVirtualPath(t *testing.T) {
@@ -46,6 +53,11 @@ func TestVirtualPath(t *testing.T) {
location: file.NewLocation("some/dir/example.jar"), location: file.NewLocation("some/dir/example.jar"),
expected: "some/dir/example.jar", expected: "some/dir/example.jar",
}, },
{
name: "outside an archive with FullPath set returns the path unchanged",
location: file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/odd!/name.jar", FullPath: "/odd!/name.jar"}),
expected: "/odd!/name.jar",
},
{ {
name: "single level trims leading slash", name: "single level trims leading slash",
location: locationIn("app.war", "/WEB-INF/lib/dep.jar"), location: locationIn("app.war", "/WEB-INF/lib/dep.jar"),
@@ -58,7 +70,7 @@ func TestVirtualPath(t *testing.T) {
}, },
{ {
name: "multi-level chain", name: "multi-level chain",
location: locationIn("app.war:WEB-INF/lib/dep.jar", "/inner/most.jar"), location: locationIn("app.war", "WEB-INF/lib/dep.jar!/inner/most.jar"),
expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar", expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar",
}, },
{ {
@@ -72,11 +84,23 @@ func TestVirtualPath(t *testing.T) {
location: locationIn("app.war", "lib/a:b:c.jar"), location: locationIn("app.war", "lib/a:b:c.jar"),
expected: "app.war:lib/a%3Ab%3Ac.jar", expected: "app.war:lib/a%3Ab%3Ac.jar",
}, },
{
// or the literal text %3A would collide with an escaped colon
name: "the escape character is itself escaped",
location: locationIn("outer.zip", "a%3Ab.jar"),
expected: "outer.zip:a%253Ab.jar",
},
{ {
name: "access path is preferred over real path", name: "access path is preferred over real path",
location: file.NewVirtualLocation("/real/dep.jar", "/link/dep.jar"), location: file.NewVirtualLocation("/real/dep.jar", "/link/dep.jar"),
expected: "/link/dep.jar", expected: "/link/dep.jar",
}, },
{
name: "access path within an archive replaces the last entry",
location: file.NewVirtualLocationFromCoordinates(
file.Coordinates{RealPath: "app.war", FullPath: "app.war!/WEB-INF/lib/real.jar"}, "app.war!/WEB-INF/lib/link.jar"),
expected: "app.war:WEB-INF/lib/link.jar",
},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
@@ -84,27 +108,3 @@ func TestVirtualPath(t *testing.T) {
}) })
} }
} }
func TestVirtualPath_theEscapeCharacterIsItselfEscaped(t *testing.T) {
// an entry path holding the literal text %3A must not collide with one holding a colon, or their
// contents share an ArchivePath and overwrite each other's file metadata
fromColon := VirtualPath(locationIn("outer.zip", "a:b.jar"))
fromLiteral := VirtualPath(locationIn("outer.zip", "a%3Ab.jar"))
assert.Equal(t, "outer.zip:a%3Ab.jar", fromColon)
assert.Equal(t, "outer.zip:a%253Ab.jar", fromLiteral)
}
func TestVirtualPath_escapingComposesAcrossNestingLevels(t *testing.T) {
level2 := VirtualPath(locationIn("outer.zip", "lib/mid:dle.war"))
require.Equal(t, "outer.zip:lib/mid%3Adle.war", level2)
final := VirtualPath(locationIn(level2, "WEB-INF/lib/inn:er.jar"))
require.Equal(t, "outer.zip:lib/mid%3Adle.war:WEB-INF/lib/inn%3Aer.jar", final)
assert.Equal(t,
[]string{"outer.zip", "lib/mid%3Adle.war", "WEB-INF/lib/inn%3Aer.jar"},
strings.Split(final, ":"),
"splitting on ':' must recover exactly the three real archive boundaries",
)
}
+1 -1
View File
@@ -69,5 +69,5 @@ func locationSorter(a, b file.Location) int {
return strings.Compare(a.AccessPath, b.AccessPath) return strings.Compare(a.AccessPath, b.AccessPath)
} }
return strings.Compare(a.RealPath, b.RealPath) return strings.Compare(a.FullNestedPath(), b.FullNestedPath())
} }
+1 -1
View File
@@ -14,5 +14,5 @@ func DefaultLocationComparer(x, y file.Location) bool {
} }
func LocationComparerWithoutLayer(x, y file.Location) bool { func LocationComparerWithoutLayer(x, y file.Location) bool {
return cmp.Equal(x.RealPath, y.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath) return cmp.Equal(x.FullNestedPath(), y.FullNestedPath()) && cmp.Equal(x.AccessPath, y.AccessPath)
} }
+6 -6
View File
@@ -49,7 +49,7 @@ func findRhelFeatures(resolver file.Resolver, release *linux.Release) error {
for _, contentManifestFile := range contentManifestFiles { for _, contentManifestFile := range contentManifestFiles {
found, err := fileMatchesAny(resolver, contentManifestFile, rhelEUSPatterns...) found, err := fileMatchesAny(resolver, contentManifestFile, rhelEUSPatterns...)
if err != nil { if err != nil {
return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.RealPath, err) return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.FullNestedPath(), err)
} }
if found { if found {
release.ExtendedSupport = true release.ExtendedSupport = true
@@ -163,9 +163,9 @@ func hasActiveUbuntuESMService(resolver file.Resolver) (bool, error) {
func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location) (bool, error) { func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location) (bool, error) {
contents, err := resolver.FileContentsByLocation(location) contents, err := resolver.FileContentsByLocation(location)
if err != nil { if err != nil {
return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.RealPath, err) return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.FullNestedPath(), err)
} }
defer internal.CloseAndLogError(contents, location.RealPath) defer internal.CloseAndLogError(contents, location.FullNestedPath())
var status struct { var status struct {
Services []struct { Services []struct {
@@ -174,7 +174,7 @@ func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location
} `json:"services"` } `json:"services"`
} }
if err := json.NewDecoder(contents).Decode(&status); err != nil { if err := json.NewDecoder(contents).Decode(&status); err != nil {
return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.RealPath, err) return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.FullNestedPath(), err)
} }
// pro status values are enabled/disabled/n-a/warning/—; only "enabled" proves the stream is active. // pro status values are enabled/disabled/n-a/warning/—; only "enabled" proves the stream is active.
@@ -200,8 +200,8 @@ func hasInstalledESMPackage(packages *pkg.Collection) bool {
func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) { func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) {
contents, err := resolver.FileContentsByLocation(location) contents, err := resolver.FileContentsByLocation(location)
if err != nil { if err != nil {
return false, fmt.Errorf("unable to read %s: %w", location.RealPath, err) return false, fmt.Errorf("unable to read %s: %w", location.FullNestedPath(), err)
} }
defer internal.CloseAndLogError(contents, location.RealPath) defer internal.CloseAndLogError(contents, location.FullNestedPath())
return internal.MatchAnyFromReader(contents, patterns...) return internal.MatchAnyFromReader(contents, patterns...)
} }
@@ -71,7 +71,7 @@ func compareElfBinaryPackages(s *sbom.SBOM) []artifact.ID {
pkgsToDelete := make([]artifact.ID, 0) pkgsToDelete := make([]artifact.ID, 0)
for _, elfPkg := range allElfPackages(s) { for _, elfPkg := range allElfPackages(s) {
for _, loc := range onlyPrimaryEvidenceLocations(elfPkg) { for _, loc := range onlyPrimaryEvidenceLocations(elfPkg) {
for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.RealPath) { for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.FullNestedPath()) {
// we only care about comparing binary packages to each other (not other types) // we only care about comparing binary packages to each other (not other types)
if otherPkg.Type != pkg.BinaryPkg { if otherPkg.Type != pkg.BinaryPkg {
continue continue
@@ -130,7 +130,7 @@ func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver
for _, loc := range libLocations { for _, loc := range libLocations {
// are you in our index? // are you in our index?
realBaseName := path.Base(loc.RealPath) realBaseName := path.Base(loc.FullNestedPath())
pkgCollection := index.owningLibraryPackage(realBaseName) pkgCollection := index.owningLibraryPackage(realBaseName)
if pkgCollection.PackageCount() < 1 { if pkgCollection.PackageCount() < 1 {
addRelationship( addRelationship(
@@ -3,6 +3,7 @@ package binary
import ( import (
"path" "path"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
@@ -63,7 +64,7 @@ func locationsThatProvideLibraries(accessor sbomsync.Accessor) (map[string]file.
continue continue
} }
basename := path.Base(coord.RealPath) basename := path.Base(coord.FullNestedPath())
set := libLocationsByBasename[basename] set := libLocationsByBasename[basename]
set.Add(coord) set.Add(coord)
allLibLocations.Add(coord) allLibLocations.Add(coord)
@@ -82,6 +83,11 @@ func packagesWithLibraryOwnership(resolver file.Resolver, accessor sbomsync.Acce
var ownedFilePaths []string var ownedFilePaths []string
if p.Type == pkg.BinaryPkg { if p.Type == pkg.BinaryPkg {
for _, loc := range p.Locations.ToSlice() { for _, loc := range p.Locations.ToSlice() {
if archive.ContainerPath(loc.Coordinates) != "" {
// a path within an archive cannot be resolved from the scan root; the location is the library
addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p)
continue
}
ownedFilePaths = append(ownedFilePaths, loc.Path()) ownedFilePaths = append(ownedFilePaths, loc.Path())
} }
} else { } else {
@@ -114,18 +120,19 @@ func populatePackagesByLibraryPath(
} }
for _, loc := range ownedLocation { for _, loc := range ownedLocation {
// if the location is a library, add the package to the set of packages that own the library addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p)
if !allLibLocations.Contains(loc.Coordinates) {
continue
}
if _, ok := packagesByLibraryPath[loc.Coordinates]; !ok {
packagesByLibraryPath[loc.Coordinates] = pkg.NewCollection()
}
// we have a library path, add the package to the set of packages that own the library
packagesByLibraryPath[loc.Coordinates].Add(p)
} }
} }
return packagesByLibraryPath return packagesByLibraryPath
} }
// addLibraryOwner records p as an owner of the file at coordinates when that file is a library.
func addLibraryOwner(packagesByLibraryPath map[file.Coordinates]*pkg.Collection, allLibLocations file.CoordinateSet, coordinates file.Coordinates, p pkg.Package) {
if !allLibLocations.Contains(coordinates) {
return
}
if _, ok := packagesByLibraryPath[coordinates]; !ok {
packagesByLibraryPath[coordinates] = pkg.NewCollection()
}
packagesByLibraryPath[coordinates].Add(p)
}
+20 -2
View File
@@ -6,6 +6,7 @@ import (
"github.com/bmatcuk/doublestar/v4" "github.com/bmatcuk/doublestar/v4"
"github.com/scylladb/go-set/strset" "github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
@@ -100,7 +101,7 @@ func findOwnershipByFilesRelationships(resolver file.Resolver, catalog *pkg.Coll
// Now establish relationships, considering symlinks // Now establish relationships, considering symlinks
for _, candidateOwnerPkg := range catalog.Sorted() { for _, candidateOwnerPkg := range catalog.Sorted() {
id := candidateOwnerPkg.ID() id := candidateOwnerPkg.ID()
if candidateOwnerPkg.Metadata == nil { if candidateOwnerPkg.Metadata == nil || withinArchive(candidateOwnerPkg) {
continue continue
} }
@@ -164,7 +165,7 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set {
// First, identify direct ownership of all files // First, identify direct ownership of all files
for _, p := range catalog.Sorted() { for _, p := range catalog.Sorted() {
if p.Metadata == nil { if p.Metadata == nil || withinArchive(p) {
continue continue
} }
@@ -192,6 +193,21 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set {
return directOwnership return directOwnership
} }
// withinArchive reports whether every location of p is inside an archive. Such a package's owned files
// are paths within its archive, not the scan root, so it is not treated as a file owner.
func withinArchive(p pkg.Package) bool {
locations := p.Locations.ToSlice()
if len(locations) == 0 {
return false
}
for _, l := range locations {
if archive.ContainerPath(l.Coordinates) == "" {
return false
}
}
return true
}
func resolvePaths(ownedFilePath string, resolver file.Resolver) []string { func resolvePaths(ownedFilePath string, resolver file.Resolver) []string {
// though we have a string path, we need to resolve symlinks and other filesystem oddities since we cannot assume this is a real path // though we have a string path, we need to resolve symlinks and other filesystem oddities since we cannot assume this is a real path
var locs []file.Location var locs []file.Location
@@ -206,6 +222,8 @@ func resolvePaths(ownedFilePath string, resolver file.Resolver) []string {
ownedFilePaths := strset.New(ownedFilePath) ownedFilePaths := strset.New(ownedFilePath)
for _, loc := range locs { for _, loc := range locs {
// Owned file paths are currently based on the top-level resolver, which is always the RealPath and includes
// all files found within archives at that location
ownedFilePaths.Add(loc.RealPath) ownedFilePaths.Add(loc.RealPath)
} }
return ownedFilePaths.List() return ownedFilePaths.List()
@@ -78,6 +78,61 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
return []pkg.Package{parent, child}, []artifact.Relationship{relationship} return []pkg.Package{parent, child}, []artifact.Relationship{relationship}
}, },
}, },
{
name: "package-within-archive-is-not-a-file-owner",
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
nestedOwner := pkg.Package{
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/rootfs.tar",
FullPath: "/rootfs.tar!/var/lib/rpm/Packages",
})),
Type: pkg.RpmPkg,
Metadata: pkg.RpmDBEntry{
Files: []pkg.RpmFileRecord{{Path: "/d/path"}},
},
}
nestedOwner.SetID()
child := pkg.Package{
Locations: file.NewLocationSet(file.NewLocation("/d/path")),
Type: pkg.BinaryPkg,
}
child.SetID()
return []pkg.Package{nestedOwner, child}, nil
},
},
{
name: "top-level-owner-of-an-archive-owns-packages-within-it",
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
parent := pkg.Package{
Locations: file.NewLocationSet(file.NewLocation("/var/lib/rpm/Packages")),
Type: pkg.RpmPkg,
Metadata: pkg.RpmDBEntry{
Files: []pkg.RpmFileRecord{{Path: "/opt/bundle.tar"}},
},
}
parent.SetID()
child := pkg.Package{
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/opt/bundle.tar",
FullPath: "/opt/bundle.tar!/lib/thing.jar",
})),
Type: pkg.JavaPkg,
}
child.SetID()
relationship := artifact.Relationship{
From: parent,
To: child,
Type: artifact.OwnershipByFileOverlapRelationship,
Data: ownershipByFilesMetadata{Files: []string{"/opt/bundle.tar"}},
}
return []pkg.Package{parent, child}, []artifact.Relationship{relationship}
},
},
{ {
name: "only-real-file-owner-gets-ownership-relationship-not-symlink-owner", name: "only-real-file-owner-gets-ownership-relationship-not-symlink-owner",
resolver: mockFR{ resolver: mockFR{
+4 -16
View File
@@ -15,18 +15,9 @@ func TestRelationshipsEvidentBy(t *testing.T) {
c := pkg.NewCollection() c := pkg.NewCollection()
coordA := file.Coordinates{ coordA := file.NewCoordinates("/somewhere/real", "abc")
RealPath: "/somewhere/real", coordC := file.NewCoordinates("/somewhere/real", "abc")
FileSystemID: "abc", coordD := file.NewCoordinates("/somewhere/real", "abc")
}
coordC := file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
}
coordD := file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
}
pkgA := pkg.Package{ pkgA := pkg.Package{
Locations: file.NewLocationSet( Locations: file.NewLocationSet(
// added! // added!
@@ -39,10 +30,7 @@ func TestRelationshipsEvidentBy(t *testing.T) {
pkgA.SetID() pkgA.SetID()
c.Add(pkgA) c.Add(pkgA)
coordB := file.Coordinates{ coordB := file.NewCoordinates("/somewhere-else/real", "def")
RealPath: "/somewhere-else/real",
FileSystemID: "def",
}
pkgB := pkg.Package{ pkgB := pkg.Package{
Locations: file.NewLocationSet( Locations: file.NewLocationSet(
// added! // added!
+10 -22
View File
@@ -21,12 +21,8 @@ func Test_Index(t *testing.T) {
p3 := pkg.Package{ p3 := pkg.Package{
Name: "pkg-3", Name: "pkg-3",
} }
c1 := file.Coordinates{ c1 := file.NewCoordinates("/coords/1", "")
RealPath: "/coords/1", c2 := file.NewCoordinates("/coords/2", "")
}
c2 := file.Coordinates{
RealPath: "/coords/2",
}
for _, p := range []*pkg.Package{&p1, &p2, &p3} { for _, p := range []*pkg.Package{&p1, &p2, &p3} {
p.SetID() p.SetID()
@@ -156,18 +152,10 @@ func Test_Coordinates(t *testing.T) {
p3 := pkg.Package{ p3 := pkg.Package{
Name: "pkg-3", Name: "pkg-3",
} }
c1 := file.Coordinates{ c1 := file.NewCoordinates("/coords/1", "")
RealPath: "/coords/1", c2 := file.NewCoordinates("/coords/2", "")
} c3 := file.NewCoordinates("/coords/3", "")
c2 := file.Coordinates{ c4 := file.NewCoordinates("/coords/4", "")
RealPath: "/coords/2",
}
c3 := file.Coordinates{
RealPath: "/coords/3",
}
c4 := file.Coordinates{
RealPath: "/coords/4",
}
for _, p := range []*pkg.Package{&p1, &p2, &p3} { for _, p := range []*pkg.Package{&p1, &p2, &p3} {
p.SetID() p.SetID()
@@ -237,10 +225,10 @@ func TestRemove(t *testing.T) {
p1 := pkg.Package{Name: "pkg-1"} p1 := pkg.Package{Name: "pkg-1"}
p2 := pkg.Package{Name: "pkg-2"} p2 := pkg.Package{Name: "pkg-2"}
p3 := pkg.Package{Name: "pkg-3"} p3 := pkg.Package{Name: "pkg-3"}
c1 := file.Coordinates{RealPath: "/coords/1"} c1 := file.NewCoordinates("/coords/1", "")
c2 := file.Coordinates{RealPath: "/coords/2"} c2 := file.NewCoordinates("/coords/2", "")
c3 := file.Coordinates{RealPath: "/coords/3"} c3 := file.NewCoordinates("/coords/3", "")
c4 := file.Coordinates{RealPath: "/coords/4"} c4 := file.NewCoordinates("/coords/4", "")
for _, p := range []*pkg.Package{&p1, &p2, &p3} { for _, p := range []*pkg.Package{&p1, &p2, &p3} {
p.SetID() p.SetID()
+57 -170
View File
@@ -18,6 +18,7 @@ import (
"github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/internal/testutils"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
@@ -28,7 +29,7 @@ import (
// This file covers nesting across archive families. The other nested-archive fixtures are all zips, // This file covers nesting across archive families. The other nested-archive fixtures are all zips,
// so a zip-specific mechanism would pass them; the requirement here is that an archive of one family // so a zip-specific mechanism would pass them; the requirement here is that an archive of one family
// inside another behaves as it would inside its own family - same packages, same filesystem-id chain, // inside another behaves as it would inside its own family - same packages, same container-path chain,
// same CONTAINS edges, same depth accounting. // same CONTAINS edges, same depth accounting.
// //
// Chains are built in the test rather than committed as binaries, so each case can vary which family // Chains are built in the test rather than committed as binaries, so each case can vary which family
@@ -71,8 +72,8 @@ type nestedArchive struct {
// paths is each archive's path within its parent, outermost first; paths[0] is the outermost archive // paths is each archive's path within its parent, outermost first; paths[0] is the outermost archive
// as the scan's resolver reports it. // as the scan's resolver reports it.
paths []string paths []string
// fileSystemIDs is the composed chain each archive's own contents carry, outermost first. // containerPaths is each archive's full path, the container path its own contents carry, outermost first.
fileSystemIDs []string containerPaths []string
// virtualPaths is the same chain in java's colon-joined form, outermost first. // virtualPaths is the same chain in java's colon-joined form, outermost first.
virtualPaths []string virtualPaths []string
} }
@@ -87,6 +88,7 @@ type nestingRow struct {
// nestingVisit is one filesystem the archive cataloger's sub-pipeline was run against. // nestingVisit is one filesystem the archive cataloger's sub-pipeline was run against.
type nestingVisit struct { type nestingVisit struct {
virtualPath string virtualPath string
fullPath string
depth int depth int
// heldFiles are the spill files live at this moment, sampled only when the probe was given a temp // heldFiles are the spill files live at this moment, sampled only when the probe was given a temp
// dir to look in. A nested archive's own bytes are read where they lie, so one appears per archive // dir to look in. A nested archive's own bytes are read where they lie, so one appears per archive
@@ -120,12 +122,12 @@ func Test_mixedFamilyNesting_leafIsCatalogedOnceWithTheFullChain(t *testing.T) {
locs := leafLocations(s) locs := leafLocations(s)
require.Len(t, locs, 1, "the leaf must be reached by exactly one path") require.Len(t, locs, 1, "the leaf must be reached by exactly one path")
// exact, never a substring: a partially composed chain passes a substring check // exact, never a substring: a partially composed chain passes a substring check
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath, assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates),
"the chain must name all three archives outer-to-inner") "the chain must name all three archives outer-to-inner")
assert.Equal(t, "nested/marker.txt", locs[0].RealPath, assert.Equal(t, nested.leafContainerPath()+"!/nested/marker.txt", locs[0].FullNestedPath(),
"the path stays relative to the archive the file came from") "the path stays relative to the archive the file came from")
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(), assert.Equal(t, nested.containerPaths, probe.containerPaths(),
"every level must be cataloged, outer to inner, whatever families they are") "every level must be cataloged, outer to inner, whatever families they are")
}) })
} }
@@ -145,11 +147,7 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) {
// directory, so it carries no filesystem id of its own // directory, so it carries no filesystem id of its own
archiveCoords := make([]file.Coordinates, len(row.families)) archiveCoords := make([]file.Coordinates, len(row.families))
for i := range row.families { for i := range row.families {
var parentFSID string archiveCoords[i] = nested.coordinates(i)
if i > 0 {
parentFSID = nested.fileSystemIDs[i-1]
}
archiveCoords[i] = file.Coordinates{RealPath: nested.paths[i], ArchivePath: parentFSID}
} }
fileEdges := strset.New() fileEdges := strset.New()
@@ -185,47 +183,25 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) {
for _, rel := range s.Relationships { for _, rel := range s.Relationships {
from, _ := rel.From.(file.Coordinates) from, _ := rel.From.(file.Coordinates)
to, _ := rel.To.(file.Coordinates) to, _ := rel.To.(file.Coordinates)
leafEdge = leafEdge || (from == innermost && to == file.Coordinates{RealPath: "nested/marker.txt", ArchivePath: nested.leafFileSystemID()}) leafEdge = leafEdge || (from == innermost && to == file.Coordinates{RealPath: innermost.RealPath, FullPath: innermost.FullNestedPath() + "!/nested/marker.txt"})
} }
assert.True(t, leafEdge, "expected a CONTAINS edge from the innermost archive to the leaf file") assert.True(t, leafEdge, "expected a CONTAINS edge from the innermost archive to the leaf file")
}) })
} }
} }
func Test_mixedFamilyNesting_javaLeafVirtualPathIsTheColonJoinedChain(t *testing.T) { func Test_mixedFamilyNesting_everyJarIsCatalogedOnceUnderItsChain(t *testing.T) {
// these colon-delimited strings are in already-published SBOMs, so a jar reached through three
// levels of mixed families must still be reported under the chain of all three
for _, row := range nestingRows() {
if row.families[len(row.families)-1] != familyJar {
continue
}
t.Run(row.name, func(t *testing.T) {
scanDir := t.TempDir()
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), javaTask())
// the innermost archive is the jar, so its own package is the java leaf and the chain of
// all three archives is exactly its virtual path
leaf := fmt.Sprintf("level%d", len(row.families)-1)
assert.Equal(t, 1, packageCount(s, leaf), "the leaf jar must be cataloged exactly once")
assert.Equal(t, nested.virtualPaths[len(row.families)-1], javaVirtualPaths(s)[leaf])
})
}
}
func Test_mixedFamilyNesting_jarIsExercisedAsAContainer(t *testing.T) {
// two mechanisms meet a jar: the java cataloger catalogs it as a package, the archive task descends // two mechanisms meet a jar: the java cataloger catalogs it as a package, the archive task descends
// into it. A jar mid-chain must do both, exactly once each; the failure mode is a duplicate, not an // into it. A jar at any level must do both, exactly once each; the failure mode is a duplicate, not an
// error. // error. The colon-joined virtual paths are in already-published SBOMs, so they must name the chain.
for _, row := range nestingRows() { for _, row := range nestingRows() {
var containerLevels []int var jarLevels []int
for i, family := range row.families { for i, family := range row.families {
if family == familyJar && i < len(row.families)-1 { if family == familyJar {
containerLevels = append(containerLevels, i) jarLevels = append(jarLevels, i)
} }
} }
if len(containerLevels) == 0 { if len(jarLevels) == 0 {
continue continue
} }
@@ -236,19 +212,18 @@ func Test_mixedFamilyNesting_jarIsExercisedAsAContainer(t *testing.T) {
probe := &nestingProbe{} probe := &nestingProbe{}
s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), javaTask(), probe.task()) s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), javaTask(), probe.task())
for _, level := range containerLevels { for _, level := range jarLevels {
name := fmt.Sprintf("level%d", level) name := fmt.Sprintf("level%d", level)
assert.Equal(t, 1, packageCount(s, name), assert.Equal(t, 1, packageCount(s, name), "the jar's own package must be cataloged exactly once")
"the container jar's own package must be cataloged exactly once")
assert.Equal(t, nested.virtualPaths[level], javaVirtualPaths(s)[name], assert.Equal(t, nested.virtualPaths[level], javaVirtualPaths(s)[name],
"the container jar's virtual path must be its own place in the chain") "the jar's virtual path must be its own place in the chain")
} }
// and the archive task still descended through it, so the chain is complete below it // and the archive task still descended through every level
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs()) assert.Equal(t, nested.containerPaths, probe.containerPaths())
locs := leafLocations(s) locs := leafLocations(s)
require.Len(t, locs, 1) require.Len(t, locs, 1)
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath) assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates))
}) })
} }
} }
@@ -264,16 +239,13 @@ func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) {
probe := &nestingProbe{} probe := &nestingProbe{}
s := runNesting(t, scanDir, 2, defaultLimits(), markerTask(), fileMetadataTask(), probe.task()) s := runNesting(t, scanDir, 2, defaultLimits(), markerTask(), fileMetadataTask(), probe.task())
assert.Equal(t, nested.fileSystemIDs[:2], probe.archiveFileSystemIDs(), assert.Equal(t, nested.containerPaths[:2], probe.containerPaths(),
"at depth 2 the outer and middle archives are cataloged and the innermost is not") "at depth 2 the outer and middle archives are cataloged and the innermost is not")
assert.Zero(t, packageCount(s, "marker-pkg"), assert.Zero(t, packageCount(s, "marker-pkg"),
"the leaf is one level past the bound, so its package must be absent") "the leaf is one level past the bound, so its package must be absent")
// the innermost archive is still cataloged as a file, inside the middle archive // the innermost archive is still cataloged as a file, inside the middle archive
innermost := file.Coordinates{ innermost := nested.coordinates(2)
RealPath: nested.paths[2],
ArchivePath: nested.fileSystemIDs[1],
}
_, ok := s.Artifacts.FileMetadata[innermost] _, ok := s.Artifacts.FileMetadata[innermost]
assert.True(t, ok, "the undescended archive must still be recorded as a file at %s", coordKey(innermost)) assert.True(t, ok, "the undescended archive must still be recorded as a file at %s", coordKey(innermost))
@@ -285,64 +257,6 @@ func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) {
} }
} }
func Test_mixedFamilyNesting_defaultBoundsCatalogEveryRow(t *testing.T) {
// the default bounds, pinned along with the fact that a three-level chain of any family mix sits
// well inside them
bounds := cataloging.DefaultArchiveSearchConfig()
require.Equal(t, int64(2*1024*1024*1024), bounds.MaxMemoryBytes)
require.Equal(t, int64(100*1024*1024*1024), bounds.MaxDiskBytes)
for _, row := range nestingRows() {
t.Run(row.name, func(t *testing.T) {
scanDir := t.TempDir()
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
probe := &nestingProbe{}
s := runNesting(t, scanDir, 3, archiveLimits(bounds), markerTask(), probe.task())
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(),
"every level of the chain must be cataloged at the default bounds")
locs := leafLocations(s)
require.Len(t, locs, 1)
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
})
}
}
func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing.T) {
// entries that will not fit in memory are written to disk rather than refused
scanDir := t.TempDir()
tempDir := isolatedTempDir(t)
nested := writeNestedArchive(t, scanDir, gradedChain())
t.Run("with a generous memory limit nothing is written to disk", func(t *testing.T) {
probe := &nestingProbe{tempDir: tempDir}
bounds := defaultLimits()
bounds.MaxMemoryBytes = int64(2 * (nested.sizes[0] + nested.sizes[1] + nested.sizes[2]))
runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
deepest := probe.deepestVisit()
require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held")
assert.Empty(t, deepest.heldFiles)
})
t.Run("with a memory limit the outermost archive's entries do not fit in", func(t *testing.T) {
// room for every level's index estimate, which must stay in memory, but not the outer entries
probe := &nestingProbe{tempDir: tempDir}
bounds := archive.Limits{MaxMemoryBytes: int64(nested.sizes[2] + 32*1024), MaxDiskBytes: -1}
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
deepest := probe.deepestVisit()
require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held")
assert.Contains(t, deepest.heldFiles, "archive-spill", "entries that do not fit in memory are written to disk")
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(), "and every level is still cataloged")
locs := leafLocations(s)
require.Len(t, locs, 1)
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
})
}
func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) { func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) {
// an archive's own bytes are read where they lie: a scanned file from disk, a nested archive from // an archive's own bytes are read where they lie: a scanned file from disk, a nested archive from
// its parent's entries. Nothing copies them, so at most one spill file per level appears whatever // its parent's entries. Nothing copies them, so at most one spill file per level appears whatever
@@ -376,10 +290,10 @@ func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) {
require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held") require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held")
assert.Equal(t, tc.wantHeldFiles, deepest.heldFiles) assert.Equal(t, tc.wantHeldFiles, deepest.heldFiles)
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs()) assert.Equal(t, nested.containerPaths, probe.containerPaths())
locs := leafLocations(s) locs := leafLocations(s)
require.Len(t, locs, 1) require.Len(t, locs, 1)
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath) assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates))
}) })
} }
} }
@@ -395,7 +309,7 @@ func Test_mixedFamilyNesting_diskLimitTruncatesTheLevelItCannotAdmit(t *testing.
// a sibling of a different family, processed once the chain has been released, so cataloged in full // a sibling of a different family, processed once the chain has been released, so cataloged in full
require.NoError(t, os.WriteFile( require.NoError(t, os.WriteFile(
filepath.Join(scanDir, "sibling.tgz"), filepath.Join(scanDir, "sibling.tgz"),
makeTarGz(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}), testutils.TarGz(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}),
0o644, 0o644,
)) ))
@@ -408,50 +322,29 @@ func Test_mixedFamilyNesting_diskLimitTruncatesTheLevelItCannotAdmit(t *testing.
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task()) s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
want := append(append([]string{}, nested.fileSystemIDs...), "/sibling.tgz") want := append(append([]string{}, nested.containerPaths...), "/sibling.tgz")
assert.ElementsMatch(t, want, probe.archiveFileSystemIDs(), assert.ElementsMatch(t, want, probe.containerPaths(),
"every level is visited, the truncated one with nothing inside it") "every level is visited, the truncated one with nothing inside it")
fsIDs := strset.New() containers := strset.New()
for _, loc := range leafLocations(s) { for _, loc := range leafLocations(s) {
fsIDs.Add(loc.ArchivePath) containers.Add(archive.ContainerPath(loc.Coordinates))
} }
assert.False(t, fsIDs.Has(nested.leafFileSystemID()), "the truncated level's contents must be absent") assert.False(t, containers.Has(nested.leafContainerPath()), "the truncated level's contents must be absent")
assert.True(t, fsIDs.Has("/sibling.tgz"), "the sibling of a different family must be cataloged fully") assert.True(t, containers.Has("/sibling.tgz"), "the sibling of a different family must be cataloged fully")
var truncated []string var truncated []string
for coords, reasons := range s.Artifacts.Unknowns { for coords, reasons := range s.Artifacts.Unknowns {
for _, reason := range reasons { for _, reason := range reasons {
if strings.Contains(reason, "disk limit") { if strings.Contains(reason, "disk limit") {
truncated = append(truncated, coords.ArchivePath+":"+coords.RealPath) truncated = append(truncated, coords.FullNestedPath())
} }
} }
} }
assert.Equal(t, []string{nested.fileSystemIDs[1] + ":" + nested.paths[2]}, truncated, assert.Equal(t, []string{nested.coordinates(2).FullNestedPath()}, truncated,
"the truncation is recorded against the archive that reached the limit") "the truncation is recorded against the archive that reached the limit")
} }
func Test_mixedFamilyNesting_unboundedLimitsEnforceNothing(t *testing.T) {
// a caller can opt out of one bound without the other, and opting out of both must still find the
// deepest leaf. Opting out is a negative value; zero forbids the resource.
for _, row := range nestingRows() {
t.Run(row.name, func(t *testing.T) {
scanDir := t.TempDir()
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
probe := &nestingProbe{}
bounds := archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs())
locs := leafLocations(s)
require.Len(t, locs, 1)
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
})
}
}
func defaultLimits() archive.Limits { func defaultLimits() archive.Limits {
return archiveLimits(cataloging.DefaultArchiveSearchConfig()) return archiveLimits(cataloging.DefaultArchiveSearchConfig())
} }
@@ -524,37 +417,30 @@ func (f archiveFamily) pack(t *testing.T, base string, entries map[string][]byte
t.Helper() t.Helper()
switch f { switch f {
case familyZip: case familyZip:
return makeZip(t, entries) return testutils.Zip(t, entries)
case familyJar: case familyJar:
// a manifest makes it a real jar to the java cataloger at any level; the package java reports is // a manifest makes it a real jar to the java cataloger at any level; the package java reports is
// named after the file, not the manifest title // named after the file, not the manifest title
return makeJar(t, base, "1.0", entries) return testutils.Jar(t, base, "1.0", entries)
case familyTar: case familyTar:
return makeTar(t, entries) return testutils.Tar(t, entries)
case familyTarGz, familyTgz: case familyTarGz, familyTgz:
return makeTarGz(t, entries) return testutils.TarGz(t, entries)
} }
t.Fatalf("unknown archive family %q", f) t.Fatalf("unknown archive family %q", f)
return nil return nil
} }
func makeJar(t *testing.T, title, version string, extra map[string][]byte) []byte { // leafContainerPath is the container path the innermost archive's contents carry: every archive, outer to
t.Helper() // inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check.
entries := map[string][]byte{ func (n nestedArchive) leafContainerPath() string {
"META-INF/MANIFEST.MF": []byte( return n.containerPaths[len(n.containerPaths)-1]
"Manifest-Version: 1.0\nImplementation-Title: " + title + "\nImplementation-Version: " + version + "\n",
),
}
for k, v := range extra {
entries[k] = v
}
return makeZip(t, entries)
} }
// leafFileSystemID is the chain the innermost archive's contents carry: every archive, outer to // coordinates are those of the archive at the given level: the outermost archive's path, and the chain
// inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check. // of entries within it.
func (n nestedArchive) leafFileSystemID() string { func (n nestedArchive) coordinates(level int) file.Coordinates {
return n.fileSystemIDs[len(n.fileSystemIDs)-1] return file.Coordinates{RealPath: n.paths[0], FullPath: strings.Join(n.paths[:level+1], "!/")}
} }
// buildNestedArchive builds one archive per family given, each holding the next, with leaf inside // buildNestedArchive builds one archive per family given, each holding the next, with leaf inside
@@ -566,7 +452,7 @@ func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive {
out := nestedArchive{ out := nestedArchive{
sizes: make([]int, len(plan.families)), sizes: make([]int, len(plan.families)),
paths: make([]string, len(plan.families)), paths: make([]string, len(plan.families)),
fileSystemIDs: make([]string, len(plan.families)), containerPaths: make([]string, len(plan.families)),
virtualPaths: make([]string, len(plan.families)), virtualPaths: make([]string, len(plan.families)),
} }
@@ -585,11 +471,11 @@ func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive {
for i := range plan.families { for i := range plan.families {
if i == 0 { if i == 0 {
out.fileSystemIDs[i] = out.paths[0] out.containerPaths[i] = out.paths[0]
out.virtualPaths[i] = out.paths[0] out.virtualPaths[i] = out.paths[0]
continue continue
} }
out.fileSystemIDs[i] = out.fileSystemIDs[i-1] + ":" + out.paths[i] out.containerPaths[i] = out.containerPaths[i-1] + "!/" + out.paths[i]
out.virtualPaths[i] = out.virtualPaths[i-1] + ":" + out.paths[i] out.virtualPaths[i] = out.virtualPaths[i-1] + ":" + out.paths[i]
} }
@@ -667,6 +553,7 @@ func (p *nestingProbe) task() Task {
visit := nestingVisit{} visit := nestingVisit{}
if trav != nil { if trav != nil {
visit.virtualPath = archive.VirtualPath(trav.Location) visit.virtualPath = archive.VirtualPath(trav.Location)
visit.fullPath = trav.FullTraversalArchivePath()
visit.depth = strings.Count(archive.VirtualPath(trav.Location), ":") + 1 visit.depth = strings.Count(archive.VirtualPath(trav.Location), ":") + 1
} }
if p.tempDir != "" { if p.tempDir != "" {
@@ -680,16 +567,16 @@ func (p *nestingProbe) task() Task {
}) })
} }
// archiveFileSystemIDs is the chain of every archive whose contents were cataloged, in visit order. // containerPaths is the chain of every archive whose contents were cataloged, in visit order.
func (p *nestingProbe) archiveFileSystemIDs() []string { func (p *nestingProbe) containerPaths() []string {
p.mu.Lock() p.mu.Lock()
defer p.mu.Unlock() defer p.mu.Unlock()
var out []string var out []string
for _, v := range p.visits { for _, v := range p.visits {
if v.virtualPath == "" { if v.fullPath == "" {
continue continue
} }
out = append(out, v.virtualPath) out = append(out, v.fullPath)
} }
return out return out
} }
@@ -770,7 +657,7 @@ func packageCount(s *sbom.SBOM, name string) int {
} }
func coordKey(c file.Coordinates) string { func coordKey(c file.Coordinates) string {
return c.ArchivePath + "|" + c.RealPath return c.FullNestedPath()
} }
// gradedChain is the headline row with each level padded so the three archives have clearly different // gradedChain is the headline row with each level padded so the three archives have clearly different
+7 -7
View File
@@ -194,12 +194,12 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi
ctx = archive.WithBudget(ctx, archive.NewBudget(archiveTreeBudget(parentResolver, location))) ctx = archive.WithBudget(ctx, archive.NewBudget(archiveTreeBudget(parentResolver, location)))
} }
// extracted files keep the filesystem ID of where the archive was found; the nesting chain is // extracted files keep the real path and filesystem ID of the outermost archive; the chain of entries
// carried separately as the archive path. The chain names real paths, not the link the archive was // to each is carried as its archive path. The chain names real paths, not the link the archive was
// reached by, so it is stable and joins to the archive's package location by prefix. // reached by, so it is stable and joins to the archive's package location by prefix.
traversal := &archive.Traversal{Location: location, Parent: archive.TraversalFromContext(ctx)} traversal := &archive.Traversal{Location: location, Parent: archive.TraversalFromContext(ctx)}
archivePath := traversal.ContentsArchivePath() archivePath := location.FullNestedPath()
extracted, err := archive.Extract(ctx, archiveContent, location.FileSystemID, archivePath, c.limiter, c.exclusions) extracted, err := archive.Extract(ctx, archiveContent, location.Coordinates, c.limiter, c.exclusions)
if errors.Is(err, archive.ErrDiskLimitReached) { if errors.Is(err, archive.ErrDiskLimitReached) {
// nothing is released while this archive waits, so skip it rather than block the scan // nothing is released while this archive waits, so skip it rather than block the scan
recordArchiveUnknown(builder, location.Coordinates, "archive skipped: its content would exceed the disk limit") recordArchiveUnknown(builder, location.Coordinates, "archive skipped: its content would exceed the disk limit")
@@ -370,7 +370,7 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan
if i, dup := seen[loc.Coordinates]; dup { if i, dup := seen[loc.Coordinates]; dup {
// a MIME hit is the bare real path; the name it is reached by (a link such as // a MIME hit is the bare real path; the name it is reached by (a link such as
// lib/dep.jar -> ../store/3f9a1c) is what name-based catalogers like java key off // lib/dep.jar -> ../store/3f9a1c) is what name-based catalogers like java key off
if prev := candidates[i].location; prev.AccessPath == "" || prev.AccessPath == prev.RealPath { if prev := candidates[i].location; prev.AccessPath == "" || prev.AccessPath == prev.FullNestedPath() {
candidates[i].location = loc candidates[i].location = loc
} }
continue continue
@@ -384,7 +384,7 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan
// mergeArchiveResults copies the throwaway SBOM's packages, files and relationships into the shared // mergeArchiveResults copies the throwaway SBOM's packages, files and relationships into the shared
// SBOM, adding a CONTAINS relationship from the archive to everything found inside it. // SBOM, adding a CONTAINS relationship from the archive to everything found inside it.
// //
// Files inside the archive are those whose ArchivePath is the archive's own chain, which leaves out the // Files inside the archive are those whose container is the archive's own full path, which leaves out the
// archive itself (a java package or an unknown recorded at its coordinates would otherwise become a // archive itself (a java package or an unknown recorded at its coordinates would otherwise become a
// self-loop). The nested archives are always included, so the chain of archives is navigable whatever // self-loop). The nested archives are always included, so the chain of archives is navigable whatever
// file selection is configured. // file selection is configured.
@@ -409,7 +409,7 @@ func mergeArchiveResults(archiveCoordinates file.Coordinates, archivePath string
contained.Add(child.location.Coordinates) contained.Add(child.location.Coordinates)
} }
for _, coordinates := range scratch.AllCoordinates() { for _, coordinates := range scratch.AllCoordinates() {
if coordinates.ArchivePath == archivePath { if archive.ContainerPath(coordinates) == archivePath {
contained.Add(coordinates) contained.Add(coordinates)
} }
} }
+49 -268
View File
@@ -1,10 +1,7 @@
package task package task
import ( import (
"archive/tar"
"archive/zip"
"bytes" "bytes"
"compress/gzip"
"context" "context"
"crypto/sha1" "crypto/sha1"
"encoding/hex" "encoding/hex"
@@ -14,7 +11,6 @@ import (
"io/fs" "io/fs"
"os" "os"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -31,6 +27,7 @@ import (
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/mimetype" "github.com/anchore/syft/internal/mimetype"
"github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/internal/testutils"
"github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/cataloging"
@@ -87,7 +84,7 @@ func (panicReader) Read([]byte) (int, error) { panic("decoder blew up") }
func Test_archiveCataloger_aPanicCostsOnlyThatArchive(t *testing.T) { func Test_archiveCataloger_aPanicCostsOnlyThatArchive(t *testing.T) {
rootDir := t.TempDir() rootDir := t.TempDir()
for _, name := range []string{"bad.zip", "good.zip"} { for _, name := range []string{"bad.zip", "good.zip"} {
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600))
} }
var seen []string var seen []string
@@ -147,35 +144,10 @@ func capturingTask(t *testing.T, seen *[]string, fileCounts *map[string]int) Tas
}) })
} }
func Test_archiveCataloger_traversalThreading(t *testing.T) {
manifest := []byte("Manifest-Version: 1.0\nImplementation-Title: example\nImplementation-Version: 1.0.0\n")
innerJar := makeZip(t, map[string][]byte{"META-INF/MANIFEST.MF": manifest})
innerZip := makeZip(t, map[string][]byte{"README.txt": []byte("hi")})
outerZip := makeZip(t, map[string][]byte{
"lib/inner.jar": innerJar,
"nested/inner.zip": innerZip,
})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
var seen []string
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3), capturingTask(t, &seen, nil))
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())))
// java archives are traversed like any other archive when the task is enabled
assert.ElementsMatch(t, []string{
"/outer.zip",
"/outer.zip:lib/inner.jar",
"/outer.zip:nested/inner.zip",
}, seen)
}
func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) { func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) {
// reaching a limit is a truncation, not a failure: the sub-pipeline still runs over what was stored // reaching a limit is a truncation, not a failure: the sub-pipeline still runs over what was stored
big := bytes.Repeat([]byte("x"), 4096) big := bytes.Repeat([]byte("x"), 4096)
outerZip := makeZip(t, map[string][]byte{ outerZip := testutils.Zip(t, map[string][]byte{
"a/small.txt": []byte("small"), "a/small.txt": []byte("small"),
"b/big.txt": big, "b/big.txt": big,
}) })
@@ -197,7 +169,7 @@ func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) {
assert.Equal(t, []string{"/outer.zip"}, seen, "the sub-pipeline must run over the partial contents") assert.Equal(t, []string{"/outer.zip"}, seen, "the sub-pipeline must run over the partial contents")
assert.Equal(t, 1, fileCounts["/outer.zip"], "only the entry stored before the limit is visible") assert.Equal(t, 1, fileCounts["/outer.zip"], "only the entry stored before the limit is visible")
reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
require.True(t, ok, "the truncation must be recorded against the archive; got %v", s.Artifacts.Unknowns) require.True(t, ok, "the truncation must be recorded against the archive; got %v", s.Artifacts.Unknowns)
require.Len(t, reasons, 1) require.Len(t, reasons, 1)
assert.Contains(t, reasons[0], "archive cataloged from part of its contents") assert.Contains(t, reasons[0], "archive cataloged from part of its contents")
@@ -208,7 +180,7 @@ func Test_archiveCataloger_aDamagedEntryCostsOnlyThatEntry(t *testing.T) {
rootDir := t.TempDir() rootDir := t.TempDir()
// entry data overwritten but headers and end-of-central-directory intact: detected as an archive, // entry data overwritten but headers and end-of-central-directory intact: detected as an archive,
// and data.bin fails when it is read. ok.txt comes after it and is intact. // and data.bin fails when it is read. ok.txt comes after it and is intact.
corrupt := makeZip(t, map[string][]byte{"data.bin": incompressible(4096), "ok.txt": []byte("ok")}) corrupt := testutils.Zip(t, map[string][]byte{"data.bin": incompressible(4096), "ok.txt": []byte("ok")})
require.Greater(t, len(corrupt), 1024) require.Greater(t, len(corrupt), 1024)
for i := 100; i < 600; i++ { for i := 100; i < 600; i++ {
corrupt[i] ^= 0xff corrupt[i] ^= 0xff
@@ -224,7 +196,7 @@ func Test_archiveCataloger_aDamagedEntryCostsOnlyThatEntry(t *testing.T) {
assert.Equal(t, []string{"/corrupt.zip"}, seen, "the archive is still cataloged") assert.Equal(t, []string{"/corrupt.zip"}, seen, "the archive is still cataloged")
assert.Equal(t, 1, fileCounts["/corrupt.zip"], "from the entries that could be read") assert.Equal(t, 1, fileCounts["/corrupt.zip"], "from the entries that could be read")
reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/corrupt.zip"}] reasons := s.Artifacts.Unknowns[file.NewCoordinates("/corrupt.zip", "")]
require.Len(t, reasons, 1) require.Len(t, reasons, 1)
assert.Contains(t, reasons[0], `entry "data.bin" could not be read`) assert.Contains(t, reasons[0], `entry "data.bin" could not be read`)
} }
@@ -275,31 +247,11 @@ func Test_NewArchiveCatalogerTask_gating(t *testing.T) {
} }
} }
func Test_archiveCataloger_depthBoundIsExact(t *testing.T) {
innerZip := makeZip(t, map[string][]byte{"deep.txt": []byte("deep")})
outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
run := func(t *testing.T, depth int) []string {
t.Helper()
var seen []string
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth), capturingTask(t, &seen, nil))
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())),
"reaching the depth bound must not be an error")
return seen
}
assert.Equal(t, []string{"/outer.zip"}, run(t, 1), "depth 1 catalogs the top-level archive and does not descend")
assert.ElementsMatch(t, []string{"/outer.zip", "/outer.zip:nested/inner.zip"}, run(t, 2), "depth 2 descends one level")
}
func Test_archiveCataloger_negativeDepthStopsAtTheCeiling(t *testing.T) { func Test_archiveCataloger_negativeDepthStopsAtTheCeiling(t *testing.T) {
// "unbounded" must still end, or a zip quine or recursive bomb runs forever // "unbounded" must still end, or a zip quine or recursive bomb runs forever
chain := makeZip(t, map[string][]byte{"deep.txt": []byte("deep")}) chain := testutils.Zip(t, map[string][]byte{"deep.txt": []byte("deep")})
for range unboundedArchiveDepth + 4 { for range unboundedArchiveDepth + 4 {
chain = makeZip(t, map[string][]byte{"n.zip": chain}) chain = testutils.Zip(t, map[string][]byte{"n.zip": chain})
} }
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), chain, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), chain, 0o600))
@@ -314,7 +266,7 @@ func Test_archiveCataloger_negativeDepthStopsAtTheCeiling(t *testing.T) {
func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing.T) { func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing.T) {
// a zip quine contains itself. Building one is impractical, so the walk is started as if it were // a zip quine contains itself. Building one is impractical, so the walk is started as if it were
// already inside an archive with these exact bytes, which is the state a quine reaches one level down // already inside an archive with these exact bytes, which is the state a quine reaches one level down
outerZip := makeZip(t, map[string][]byte{"a.txt": []byte("a")}) outerZip := testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")})
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
@@ -327,7 +279,7 @@ func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(s))) require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(s)))
assert.Zero(t, ran, "the sub-pipeline must not run inside a repeat of an ancestor") assert.Zero(t, ran, "the sub-pipeline must not run inside a repeat of an ancestor")
reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] reasons := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
require.Len(t, reasons, 1) require.Len(t, reasons, 1)
assert.Contains(t, reasons[0], "identical to an archive that contains it") assert.Contains(t, reasons[0], "identical to an archive that contains it")
assert.Len(t, c.ancestors, 1, "the walk leaves the ancestor stack as it found it") assert.Len(t, c.ancestors, 1, "the walk leaves the ancestor stack as it found it")
@@ -335,7 +287,7 @@ func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing
func Test_archiveCataloger_stopsWhenTheContextIsCancelled(t *testing.T) { func Test_archiveCataloger_stopsWhenTheContextIsCancelled(t *testing.T) {
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), makeZip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600))
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() cancel()
@@ -356,10 +308,10 @@ func Test_archiveCataloger_decompressionBudgetIsSharedByATreeAndFreshPerTopLevel
t.Cleanup(func() { archiveTreeRatio, archiveTreeFloorBytes = origRatio, origFloor }) t.Cleanup(func() { archiveTreeRatio, archiveTreeFloorBytes = origRatio, origFloor })
zeros := make([]byte, 60_000) zeros := make([]byte, 60_000)
nested := makeZip(t, map[string][]byte{"zeros.bin": zeros}) nested := testutils.Zip(t, map[string][]byte{"zeros.bin": zeros})
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bomb.zip"), makeZip(t, map[string][]byte{"a.zip": nested, "b.zip": nested}), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bomb.zip"), testutils.Zip(t, map[string][]byte{"a.zip": nested, "b.zip": nested}), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "sibling.zip"), makeZip(t, map[string][]byte{"zeros.bin": zeros}), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "sibling.zip"), testutils.Zip(t, map[string][]byte{"zeros.bin": zeros}), 0o600))
var seen []string var seen []string
fileCounts := map[string]int{} fileCounts := map[string]int{}
@@ -369,7 +321,7 @@ func Test_archiveCataloger_decompressionBudgetIsSharedByATreeAndFreshPerTopLevel
assert.Equal(t, 1, fileCounts["/bomb.zip:a.zip"], "the first nested archive fits the tree's budget") assert.Equal(t, 1, fileCounts["/bomb.zip:a.zip"], "the first nested archive fits the tree's budget")
assert.Equal(t, 0, fileCounts["/bomb.zip:b.zip"], "the second does not: the budget is shared, not per archive") assert.Equal(t, 0, fileCounts["/bomb.zip:b.zip"], "the second does not: the budget is shared, not per archive")
reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "b.zip", ArchivePath: "/bomb.zip"}] reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/bomb.zip", FullPath: "/bomb.zip!/b.zip"}]
require.Len(t, reasons, 1) require.Len(t, reasons, 1)
assert.Contains(t, reasons[0], "possible decompression bomb") assert.Contains(t, reasons[0], "possible decompression bomb")
assert.Equal(t, 1, fileCounts["/sibling.zip"], "another top-level archive starts with a fresh budget") assert.Equal(t, 1, fileCounts["/sibling.zip"], "another top-level archive starts with a fresh budget")
@@ -379,14 +331,14 @@ func Test_archiveCataloger_noSelfContainsEdge(t *testing.T) {
// a jar java cannot identify records an unknown at its own coordinates; that must not turn into // a jar java cannot identify records an unknown at its own coordinates; that must not turn into
// "x.jar CONTAINS x.jar", which a graph walker would loop on // "x.jar CONTAINS x.jar", which a graph walker would loop on
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "x.jar"), makeZip(t, map[string][]byte{"readme.txt": []byte("no manifest")}), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "x.jar"), testutils.Zip(t, map[string][]byte{"readme.txt": []byte("no manifest")}), 0o600))
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), javaTask(), fileMetadataTask()) tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), javaTask(), fileMetadataTask())
s := newTestSBOM() s := newTestSBOM()
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
_ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) _ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))
require.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "/x.jar"}, "the fixture must reproduce the unknown") require.Contains(t, s.Artifacts.Unknowns, file.NewCoordinates("/x.jar", ""), "the fixture must reproduce the unknown")
var contains int var contains int
for _, rel := range s.Relationships { for _, rel := range s.Relationships {
if rel.Type != artifact.ContainsRelationship { if rel.Type != artifact.ContainsRelationship {
@@ -402,8 +354,8 @@ func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) {
// a copy of itself in the sub-pipeline would process every nesting level twice // a copy of itself in the sub-pipeline would process every nesting level twice
cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2) cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)
innerZip := makeZip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")})
outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip}) outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip})
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
@@ -424,11 +376,11 @@ func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) {
}) })
} }
func Test_archiveCataloger_chainStartsAtTheArchivesOwnFileSystemID(t *testing.T) { func Test_archiveCataloger_nestedArchivesKeepTheLayerFileSystemID(t *testing.T) {
// extracted files keep the layer digest as their FileSystemID at every level; the nesting chain // extracted files keep the layer digest as their FileSystemID at every level; the nesting chain
// rides on the traversal's VirtualPath and the coordinate's ArchivePath // rides on the traversal's VirtualPath and the coordinate's FullPath
innerZip := makeZip(t, map[string][]byte{"leaf.txt": []byte("leaf")}) innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")})
outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip}) outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip})
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
@@ -463,8 +415,8 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T
resolver := twoLayerResolver{ resolver := twoLayerResolver{
path: "/app/bundle.zip", path: "/app/bundle.zip",
contentBy: map[string][]byte{ contentBy: map[string][]byte{
"layer-one": makeZip(t, map[string][]byte{"lib/config.json": []byte(`{"v":1}`)}), "layer-one": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":1}`)}),
"layer-two": makeZip(t, map[string][]byte{"lib/config.json": []byte(`{"v":2}`)}), "layer-two": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":2}`)}),
}, },
} }
@@ -501,7 +453,7 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T
func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) { func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) {
// runSubPipeline bypasses RunTask, where the coordinate-error to unknown conversion lives // runSubPipeline bypasses RunTask, where the coordinate-error to unknown conversion lives
outerZip := makeZip(t, map[string][]byte{"lib/broken.json": []byte("{")}) outerZip := testutils.Zip(t, map[string][]byte{"lib/broken.json": []byte("{")})
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
@@ -526,8 +478,8 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T)
require.Len(t, s.Artifacts.Unknowns, 1) require.Len(t, s.Artifacts.Unknowns, 1)
for coords, reasons := range s.Artifacts.Unknowns { for coords, reasons := range s.Artifacts.Unknowns {
assert.Equal(t, "lib/broken.json", coords.RealPath, "the inner coordinates must survive the merge") assert.Equal(t, "/outer.zip!/lib/broken.json", coords.FullNestedPath(), "the inner coordinates must survive the merge")
assert.Equal(t, "/outer.zip", coords.ArchivePath, "and must be addressable per archive") assert.Equal(t, "/outer.zip", coords.RealPath, "and must be addressable per archive")
assert.Equal(t, []string{"failing-cataloger: unable to parse"}, reasons) assert.Equal(t, []string{"failing-cataloger: unable to parse"}, reasons)
} }
}) })
@@ -542,7 +494,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T)
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
require.Len(t, s.Artifacts.Unknowns, 1) require.Len(t, s.Artifacts.Unknowns, 1)
reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
require.True(t, ok, "an error with no location of its own belongs to the containing archive") require.True(t, ok, "an error with no location of its own belongs to the containing archive")
assert.Equal(t, []string{"bare-error-cataloger: cataloger blew up"}, reasons) assert.Equal(t, []string{"bare-error-cataloger: cataloger blew up"}, reasons)
}) })
@@ -557,7 +509,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T)
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)), require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)),
"a panic inside an archive must not fail the scan") "a panic inside an archive must not fail the scan")
reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
require.True(t, ok) require.True(t, ok)
require.Len(t, reasons, 1) require.Len(t, reasons, 1)
assert.Equal(t, "panicking-cataloger: panic: nope", reasons[0]) assert.Equal(t, "panicking-cataloger: panic: nope", reasons[0])
@@ -565,11 +517,11 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T)
} }
func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) { func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) {
outerZip := makeZip(t, map[string][]byte{"lib/thing.json": []byte("{}")}) outerZip := testutils.Zip(t, map[string][]byte{"lib/thing.json": []byte("{}")})
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
preexisting := file.Coordinates{RealPath: "lib/thing.json", ArchivePath: "/outer.zip"} preexisting := file.Coordinates{RealPath: "/outer.zip", FullPath: "/outer.zip!/lib/thing.json"}
writer := NewTask("scratch-writer", func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error { writer := NewTask("scratch-writer", func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error {
builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) { builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) {
if s.Artifacts.Unknowns == nil { if s.Artifacts.Unknowns == nil {
@@ -588,39 +540,7 @@ func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T)
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))) require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
assert.Equal(t, []string{"recorded by the cataloger itself"}, s.Artifacts.Unknowns[preexisting]) assert.Equal(t, []string{"recorded by the cataloger itself"}, s.Artifacts.Unknowns[preexisting])
assert.Equal(t, []string{"failing-cataloger: boom"}, s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}]) assert.Equal(t, []string{"failing-cataloger: boom"}, s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")])
}
func Test_archiveCataloger_manySmallArchivesAreEachCataloged(t *testing.T) {
// every archive is released before the next begins, so a scan whose archives sum well past a limit
// never holds more than one at a time
entry := bytes.Repeat([]byte("z"), 400)
entries := map[string][]byte{}
for i := range 5 {
entries[fmt.Sprintf("e%d.bin", i)] = entry
}
rootDir := t.TempDir()
names := []string{"one.zip", "two.zip", "three.zip", "four.zip", "five.zip"}
for _, name := range names {
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, entries), 0o600))
}
// room for one archive's entries and their index cost, not for two
limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: 20000}
var seen []string
fileCounts := map[string]int{}
tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts))
s := newTestSBOM()
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
assert.Len(t, seen, len(names), "every archive must be cataloged: no two were ever held at once")
for _, archivePath := range seen {
assert.Equal(t, 5, fileCounts[archivePath], "%s must extract in full", archivePath)
}
assert.Empty(t, s.Artifacts.Unknowns)
} }
func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) { func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) {
@@ -632,7 +552,7 @@ func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) {
rootDir := t.TempDir() rootDir := t.TempDir()
for _, name := range []string{"one.zip", "two.zip"} { for _, name := range []string{"one.zip", "two.zip"} {
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, entries), 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, entries), 0o600))
} }
// room in memory for one archive's entries and their index cost, not for two at once // room in memory for one archive's entries and their index cost, not for two at once
@@ -675,8 +595,8 @@ func newTestCataloger(maxDepth int, limiter *archive.Limiter, subPipeline ...Tas
func Test_archiveCataloger_archiveWhoseOwnBytesExceedTheDiskLimitIsSkipped(t *testing.T) { func Test_archiveCataloger_archiveWhoseOwnBytesExceedTheDiskLimitIsSkipped(t *testing.T) {
// a streamed archive's own bytes must be placed before it can be opened; when they do not fit on // a streamed archive's own bytes must be placed before it can be opened; when they do not fit on
// disk the archive is skipped rather than waited for, and a sibling that fits is cataloged in full // disk the archive is skipped rather than waited for, and a sibling that fits is cataloged in full
oversized := makeZip(t, map[string][]byte{"payload.bin": incompressible(8000)}) oversized := testutils.Zip(t, map[string][]byte{"payload.bin": incompressible(8000)})
small := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")}) small := testutils.Zip(t, map[string][]byte{"ok.txt": []byte("ok")})
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600))
@@ -699,84 +619,20 @@ func Test_archiveCataloger_archiveWhoseOwnBytesExceedTheDiskLimitIsSkipped(t *te
assert.Equal(t, []string{"/small.zip"}, seen, "the oversized archive is skipped and the sibling is still cataloged") assert.Equal(t, []string{"/small.zip"}, seen, "the oversized archive is skipped and the sibling is still cataloged")
assert.Equal(t, 1, fileCounts["/small.zip"]) assert.Equal(t, 1, fileCounts["/small.zip"])
reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/oversized.zip"}] reasons := s.Artifacts.Unknowns[file.NewCoordinates("/oversized.zip", "")]
require.Len(t, reasons, 1, "the skip must be recorded against the archive; got %v", s.Artifacts.Unknowns) require.Len(t, reasons, 1, "the skip must be recorded against the archive; got %v", s.Artifacts.Unknowns)
assert.Contains(t, reasons[0], "archive skipped") assert.Contains(t, reasons[0], "archive skipped")
} }
func Test_archiveCataloger_zeroDiskLimitTruncatesWhatWillNotFitInMemory(t *testing.T) {
// with a zero disk limit there is nowhere to overflow, so an archive's entries are stored exactly
// while they fit in memory
oversized := makeZip(t, map[string][]byte{"payload.bin": incompressible(8000)})
small := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600))
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "small.zip"), small, 0o600))
// room for the small archive's entry and its index cost, not for the oversized one's
limits := archive.Limits{MaxMemoryBytes: 3000, MaxDiskBytes: 0}
var seen []string
fileCounts := map[string]int{}
tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts))
s := newTestSBOM()
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)),
"reaching the disk limit must not fail the scan")
assert.ElementsMatch(t, []string{"/oversized.zip", "/small.zip"}, seen)
assert.Equal(t, 1, fileCounts["/small.zip"], "the archive that fits in memory needs no disk at all")
assert.Equal(t, 0, fileCounts["/oversized.zip"], "the entry that does not fit is refused")
assert.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "/oversized.zip"})
assert.NotContains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "/small.zip"})
}
func Test_archiveCataloger_nestedArchiveExceedingALimitIsTruncatedNotBlocked(t *testing.T) {
// the walk descends before it unwinds, so nothing is released while a child waits; a child that does
// not fit is truncated and the scan finishes
inner := makeZip(t, map[string][]byte{"payload.bin": incompressible(3000)})
outer := makeZip(t, map[string][]byte{"inner.zip": inner})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outer, 0o600))
// enough for the outer archive's one entry and its index cost, plus the inner's index cost, and not
// the inner's payload on top
diskLimit := int64(len(inner)) + 2*2100 + 500
require.Greater(t, 3000, 500, "the inner payload must not fit in the slack")
limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: diskLimit}
var seen []string
fileCounts := map[string]int{}
tsk := newLimitedTestTask(t, 2, limits, capturingTask(t, &seen, &fileCounts))
s := newTestSBOM()
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
assert.Equal(t, []string{"/outer.zip", "/outer.zip:inner.zip"}, seen)
assert.Equal(t, 1, fileCounts["/outer.zip"], "the parent is cataloged in full")
assert.Equal(t, 0, fileCounts["/outer.zip:inner.zip"], "the child it cannot afford is truncated")
assert.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "inner.zip", ArchivePath: "/outer.zip"})
}
// allFilesResolver is dirTestResolver without a filesystem id.
type allFilesResolver struct {
file.Resolver
dir string
}
func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) { func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) {
marker := map[string][]byte{"nested/marker.txt": []byte("hello")} marker := map[string][]byte{"nested/marker.txt": []byte("hello")}
scanDir := t.TempDir() scanDir := t.TempDir()
for name, body := range map[string][]byte{ for name, body := range map[string][]byte{
"bundle.tar.gz": makeTarGz(t, marker), "bundle.tar.gz": testutils.TarGz(t, marker),
"bundle.tgz": makeTarGz(t, marker), "bundle.tgz": testutils.TarGz(t, marker),
"bundle.tar": makeTar(t, marker), "bundle.tar": testutils.Tar(t, marker),
"bundle.zip": makeZip(t, marker), "bundle.zip": testutils.Zip(t, marker),
"notes.txt": []byte("not an archive at all"), "notes.txt": []byte("not an archive at all"),
} { } {
require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), body, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), body, 0o600))
@@ -785,7 +641,7 @@ func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) {
var seen []string var seen []string
fileCounts := map[string]int{} fileCounts := map[string]int{}
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, &fileCounts)) tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, &fileCounts))
require.NoError(t, tsk.Execute(context.Background(), allFilesResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM()))) require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM())))
assert.ElementsMatch(t, []string{"/bundle.tar.gz", "/bundle.tgz", "/bundle.tar", "/bundle.zip"}, seen, assert.ElementsMatch(t, []string{"/bundle.tar.gz", "/bundle.tgz", "/bundle.tar", "/bundle.zip"}, seen,
"every tar-family archive must be reached, and the plain text file must not be") "every tar-family archive must be reached, and the plain text file must not be")
@@ -807,7 +663,7 @@ type recordingPublisher struct {
func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) { func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) {
// a consumer keying rows by ID (as the syft CLI does) replaces a republished row and never sees it // a consumer keying rows by ID (as the syft CLI does) replaces a republished row and never sees it
// complete, so each cataloger keeps to the one row it already owns // complete, so each cataloger keeps to the one row it already owns
inner := makeZip(t, map[string][]byte{"inner/file.txt": []byte("hello")}) inner := testutils.Zip(t, map[string][]byte{"inner/file.txt": []byte("hello")})
scanDir := t.TempDir() scanDir := t.TempDir()
for _, name := range []string{"one.zip", "two.zip"} { for _, name := range []string{"one.zip", "two.zip"} {
require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), inner, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), inner, 0o600))
@@ -865,8 +721,8 @@ func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T)
func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) { func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) {
// self time excludes nested archives, or the outer archive would be named every scan // self time excludes nested archives, or the outer archive would be named every scan
innerJar := makeZip(t, map[string][]byte{"README.txt": []byte("inner")}) innerJar := testutils.Zip(t, map[string][]byte{"README.txt": []byte("inner")})
outerZip := makeZip(t, map[string][]byte{"lib/inner.jar": innerJar}) outerZip := testutils.Zip(t, map[string][]byte{"lib/inner.jar": innerJar})
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
@@ -883,7 +739,7 @@ func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) {
c := newTestCataloger(3, limiter, slow) c := newTestCataloger(3, limiter, slow)
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM()))) require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM())))
assert.Equal(t, "/outer.zip:lib/inner.jar", c.slowestPath, "the slowest archive is named by its full chain") assert.Equal(t, "/outer.zip!/lib/inner.jar", c.slowestPath, "the slowest archive is named by its full chain")
assert.GreaterOrEqual(t, c.slowest, innerWork) assert.GreaterOrEqual(t, c.slowest, innerWork)
assert.Equal(t, int64(2), c.progress.Current()) assert.Equal(t, int64(2), c.progress.Current())
@@ -926,9 +782,9 @@ func Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError(t *testing.T) {
func Test_archiveCataloger_catalogsAZipBehindALauncherScript(t *testing.T) { func Test_archiveCataloger_catalogsAZipBehindALauncherScript(t *testing.T) {
// the Spring Boot executable jar shape: content sniffing types it as a shell script, so it reaches // the Spring Boot executable jar shape: content sniffing types it as a shell script, so it reaches
// the walk only by name and qualifies only by its end-of-central-directory record // the walk only by name and qualifies only by its end-of-central-directory record
inner := makeZip(t, map[string][]byte{"nested/marker.txt": []byte("found me")}) inner := testutils.Zip(t, map[string][]byte{"nested/marker.txt": []byte("found me")})
prefixed := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"), prefixed := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"),
makeZip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...) testutils.Zip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...)
rootDir := t.TempDir() rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "app.jar"), prefixed, 0o600)) require.NoError(t, os.WriteFile(filepath.Join(rootDir, "app.jar"), prefixed, 0o600))
@@ -983,28 +839,6 @@ func (d dirTestResolver) FileMetadataByLocation(loc file.Location) (file.Metadat
return file.Metadata{FileInfo: info}, err return file.Metadata{FileInfo: info}, err
} }
// makeZip builds a zip in sorted entry-name order: limits are enforced as the walk proceeds, so entry
// order decides which entries land before a truncation.
func makeZip(t *testing.T, entries map[string][]byte) []byte {
t.Helper()
names := make([]string, 0, len(entries))
for name := range entries {
names = append(names, name)
}
sort.Strings(names)
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for _, name := range names {
w, err := zw.Create(name)
require.NoError(t, err)
_, err = w.Write(entries[name])
require.NoError(t, err)
}
require.NoError(t, zw.Close())
return buf.Bytes()
}
// matchesAnyGlob matches the resolver-relative path with a leading slash, the form a real resolver // matchesAnyGlob matches the resolver-relative path with a leading slash, the form a real resolver
// globs over. No patterns matches everything. // globs over. No patterns matches everything.
func matchesAnyGlob(root, path string, patterns []string) bool { func matchesAnyGlob(root, path string, patterns []string) bool {
@@ -1024,10 +858,6 @@ func matchesAnyGlob(root, path string, patterns []string) bool {
return false return false
} }
func (a allFilesResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
return dirTestResolver{dir: a.dir}.FilesByGlob(patterns...)
}
func (r twoLayerResolver) FilesByGlob(_ ...string) ([]file.Location, error) { func (r twoLayerResolver) FilesByGlob(_ ...string) ([]file.Location, error) {
return nil, nil return nil, nil
} }
@@ -1055,7 +885,7 @@ func (d dirTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error
func (r twoLayerResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) { func (r twoLayerResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) {
var locs []file.Location var locs []file.Location
for _, fsid := range []string{"layer-one", "layer-two"} { for _, fsid := range []string{"layer-one", "layer-two"} {
locs = append(locs, file.NewLocationFromCoordinates(file.Coordinates{RealPath: r.path, FileSystemID: fsid})) locs = append(locs, file.NewLocationFromCoordinates(file.NewCoordinates(r.path, fsid)))
} }
return locs, nil return locs, nil
} }
@@ -1085,55 +915,6 @@ func incompressible(n int) []byte {
return b return b
} }
func makeTarGz(t *testing.T, entries map[string][]byte) []byte {
t.Helper()
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
require.NoError(t, writeTar(t, gw, entries))
require.NoError(t, gw.Close())
return buf.Bytes()
}
func makeTar(t *testing.T, entries map[string][]byte) []byte {
t.Helper()
var buf bytes.Buffer
require.NoError(t, writeTar(t, &buf, entries))
return buf.Bytes()
}
func writeTar(t *testing.T, w io.Writer, entries map[string][]byte) error {
t.Helper()
names := make([]string, 0, len(entries))
for name := range entries {
names = append(names, name)
}
sort.Strings(names)
tw := tar.NewWriter(w)
for _, name := range names {
body := entries[name]
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil {
return err
}
if _, err := tw.Write(body); err != nil {
return err
}
}
return tw.Close()
}
func (a allFilesResolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
return filesByMIMEType(a.dir, "", types...)
}
func (a allFilesResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) {
return os.Open(filepath.Join(a.dir, filepath.FromSlash(loc.RealPath)))
}
func (a allFilesResolver) FileMetadataByLocation(loc file.Location) (file.Metadata, error) {
return dirTestResolver{dir: a.dir}.FileMetadataByLocation(loc)
}
func (p *recordingPublisher) Publish(e partybus.Event) { func (p *recordingPublisher) Publish(e partybus.Event) {
p.events = append(p.events, e) p.events = append(p.events, e)
} }
+2 -2
View File
@@ -175,10 +175,10 @@ func coordinatesForSelection(ctx context.Context, selection file.Selection, acce
} }
}) })
frame := archive.TraversalFromContext(ctx).ContentsArchivePath() frame := archive.TraversalFromContext(ctx).FullTraversalArchivePath()
var coords []file.Coordinates var coords []file.Coordinates
for _, c := range coordinates.ToSlice() { for _, c := range coordinates.ToSlice() {
if c.ArchivePath == frame { if archive.ContainerPath(c) == frame {
coords = append(coords, c) coords = append(coords, c)
} }
} }
+6 -18
View File
@@ -43,19 +43,13 @@ func Test_coordinatesForSelection(t *testing.T) {
Relationships: []artifact.Relationship{ Relationships: []artifact.Relationship{
{ {
From: pkg.Package{}, From: pkg.Package{},
To: file.Coordinates{ To: file.NewCoordinates("path", "fs"),
RealPath: "path",
FileSystemID: "fs",
},
Type: artifact.ContainsRelationship, Type: artifact.ContainsRelationship,
}, },
}, },
}, },
files: []file.Coordinates{ files: []file.Coordinates{
{ file.NewCoordinates("path", "fs"),
RealPath: "path",
FileSystemID: "fs",
},
}, },
ok: true, ok: true,
}, },
@@ -66,10 +60,7 @@ func Test_coordinatesForSelection(t *testing.T) {
Relationships: []artifact.Relationship{ Relationships: []artifact.Relationship{
{ {
From: pkg.Package{}, From: pkg.Package{},
To: file.Coordinates{ To: file.NewCoordinates("path", "fs"),
RealPath: "path",
FileSystemID: "fs",
},
// wrong type // wrong type
Type: artifact.DependencyOfRelationship, Type: artifact.DependencyOfRelationship,
}, },
@@ -85,10 +76,7 @@ func Test_coordinatesForSelection(t *testing.T) {
Relationships: []artifact.Relationship{ Relationships: []artifact.Relationship{
{ {
From: file.Coordinates{}, // wrong type From: file.Coordinates{}, // wrong type
To: file.Coordinates{ To: file.NewCoordinates("path", "fs"),
RealPath: "path",
FileSystemID: "fs",
},
Type: artifact.ContainsRelationship, Type: artifact.ContainsRelationship,
}, },
}, },
@@ -109,8 +97,8 @@ func Test_coordinatesForSelection(t *testing.T) {
func Test_coordinatesForSelection_onlyTheResolversFrame(t *testing.T) { func Test_coordinatesForSelection_onlyTheResolversFrame(t *testing.T) {
// a package located at an archive in its parent's frame must not send the file catalogers looking up // a package located at an archive in its parent's frame must not send the file catalogers looking up
// that path inside the archive's own resolver // that path inside the archive's own resolver
atRoot := file.Coordinates{RealPath: "/app.war"} atRoot := file.NewCoordinates("/app.war", "")
inWar := file.Coordinates{RealPath: "WEB-INF/lib/dep.jar", ArchivePath: "/app.war"} inWar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/dep.jar"}
s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection( s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(
pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(atRoot))}, pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(atRoot))},
pkg.Package{Name: "dep", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(inWar))}, pkg.Package{Name: "dep", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(inWar))},
+2 -2
View File
@@ -167,7 +167,7 @@ func applyComplianceRules(p *pkg.Package, cfg cataloging.ComplianceConfig) (bool
loc := "unknown" loc := "unknown"
locs := p.Locations.ToSlice() locs := p.Locations.ToSlice()
if len(locs) > 0 { if len(locs) > 0 {
loc = locs[0].Path() loc = locs[0].FullNestedPath()
} }
switch action { switch action {
case cataloging.ComplianceActionDrop: case cataloging.ComplianceActionDrop:
@@ -251,7 +251,7 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
for _, ref := range pathRefs { for _, ref := range pathRefs {
if oldRef, ok := locations[ref.ID()]; ok { if oldRef, ok := locations[ref.ID()]; ok {
log.Debugf("found path duplicate of %s", oldRef.RealPath) log.Debugf("found path duplicate of %s", oldRef.FullNestedPath())
} }
locations[ref.ID()] = ref locations[ref.ID()] = ref
} }
+2 -2
View File
@@ -64,8 +64,8 @@ func TestApplyCompliance(t *testing.T) {
p2 := pkg.Package{Name: "", Version: "1.0"} // missing name p2 := pkg.Package{Name: "", Version: "1.0"} // missing name
p3 := pkg.Package{Name: "pkg-3", Version: ""} // missing version p3 := pkg.Package{Name: "pkg-3", Version: ""} // missing version
p4 := pkg.Package{Name: "pkg-4", Version: ""} // missing version p4 := pkg.Package{Name: "pkg-4", Version: ""} // missing version
c1 := file.Coordinates{RealPath: "/coords/1"} c1 := file.NewCoordinates("/coords/1", "")
c2 := file.Coordinates{RealPath: "/coords/2"} c2 := file.NewCoordinates("/coords/2", "")
for _, p := range []*pkg.Package{&p1, &p2, &p3, &p4} { for _, p := range []*pkg.Package{&p1, &p2, &p3, &p4} {
p.SetID() p.SetID()
+4 -3
View File
@@ -40,10 +40,11 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID {
} }
case "": case "":
if isPrimaryEvidence { if isPrimaryEvidence {
if exists := filterDuplicates[getKey(p, l)]; exists { k := getKey(p, l)
if exists := filterDuplicates[k]; exists {
break break
} }
filterDuplicates[getKey(p, l)] = true filterDuplicates[k] = true
noPrimary = false noPrimary = false
break break
} }
@@ -59,5 +60,5 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID {
} }
func getKey(pkg pkg.Package, loc file.Location) string { func getKey(pkg pkg.Package, loc file.Location) string {
return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.RealPath, loc.AccessPath) return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.FullNestedPath(), loc.AccessPath)
} }
+1 -1
View File
@@ -59,7 +59,7 @@ func (c unknownsLabelerTask) finalize(resolver file.Resolver, s *sbom.SBOM) {
if c.IncludeUnexpandedArchives { if c.IncludeUnexpandedArchives {
ctx := context.Background() ctx := context.Background()
for coords := range s.Artifacts.FileMetadata { for coords := range s.Artifacts.FileMetadata {
format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.RealPath, nil) format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.FullNestedPath(), nil)
if format != nil && notArchiveErr == nil && !hasPackageReference(coords) { if format != nil && notArchiveErr == nil && !hasPackageReference(coords) {
s.Artifacts.Unknowns[coords] = append(s.Artifacts.Unknowns[coords], "archive not cataloged") s.Artifacts.Unknowns[coords] = append(s.Artifacts.Unknowns[coords], "archive not cataloged")
} }
+84
View File
@@ -0,0 +1,84 @@
package testutils
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"fmt"
"io"
"sort"
"testing"
"github.com/stretchr/testify/require"
)
// Zip builds a zip of the entries in sorted name order: archive limits bite as entries are read, so
// order decides which land before a truncation.
func Zip[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte {
t.Helper()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for _, name := range sortedNames(entries) {
w, err := zw.Create(name)
require.NoError(t, err)
_, err = w.Write([]byte(entries[name]))
require.NoError(t, err)
}
require.NoError(t, zw.Close())
return buf.Bytes()
}
// Tar builds a tar of the entries in sorted name order.
func Tar[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte {
t.Helper()
var buf bytes.Buffer
writeTar(t, &buf, entries)
return buf.Bytes()
}
// TarGz builds a gzipped tar of the entries in sorted name order.
func TarGz[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte {
t.Helper()
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
writeTar(t, gw, entries)
require.NoError(t, gw.Close())
return buf.Bytes()
}
// Jar is the smallest jar the java cataloger identifies, a manifest naming the artifact, plus extra.
func Jar(t testing.TB, title, version string, extra map[string][]byte) []byte {
t.Helper()
entries := map[string][]byte{"META-INF/MANIFEST.MF": Manifest(title, version)}
for name, body := range extra {
entries[name] = body
}
return Zip(t, entries)
}
// Manifest is a jar manifest naming the artifact.
func Manifest(title, version string) []byte {
return fmt.Appendf(nil, "Manifest-Version: 1.0\nImplementation-Title: %s\nImplementation-Version: %s\n", title, version)
}
func writeTar[T ~string | ~[]byte](t testing.TB, w io.Writer, entries map[string]T) {
t.Helper()
tw := tar.NewWriter(w)
for _, name := range sortedNames(entries) {
body := []byte(entries[name])
require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}))
_, err := tw.Write(body)
require.NoError(t, err)
}
require.NoError(t, tw.Close())
}
func sortedNames[T any](entries map[string]T) []string {
names := make([]string, 0, len(entries))
for name := range entries {
names = append(names, name)
}
sort.Strings(names)
return names
}
+2 -2
View File
@@ -23,9 +23,9 @@ var _ error = (*CoordinateError)(nil)
func (u *CoordinateError) Error() string { func (u *CoordinateError) Error() string {
if u.Coordinates.FileSystemID == "" { if u.Coordinates.FileSystemID == "" {
return fmt.Sprintf("%s: %v", u.Coordinates.RealPath, u.Reason) return fmt.Sprintf("%s: %v", u.Coordinates.FullNestedPath(), u.Reason)
} }
return fmt.Sprintf("%s (%s): %v", u.Coordinates.RealPath, u.Coordinates.FileSystemID, u.Reason) return fmt.Sprintf("%s (%s): %v", u.Coordinates.FullNestedPath(), u.Coordinates.FileSystemID, u.Reason)
} }
// New returns a new CoordinateError unless the reason is a CoordinateError itself, in which case // New returns a new CoordinateError unless the reason is a CoordinateError itself, in which case
+1 -3
View File
@@ -169,9 +169,7 @@ func Test_Join(t *testing.T) {
} }
func Test_flatten(t *testing.T) { func Test_flatten(t *testing.T) {
coords := file.Coordinates{ coords := file.NewCoordinates("real/path", "")
RealPath: "real/path",
}
e1 := fmt.Errorf("e1") e1 := fmt.Errorf("e1")
e2 := fmt.Errorf("e2") e2 := fmt.Errorf("e2")
c1 := New(coords, fmt.Errorf("c1")) c1 := New(coords, fmt.Errorf("c1"))
+3 -9
View File
@@ -17,27 +17,21 @@ func Test_ProcessPathErrors(t *testing.T) {
{ {
errorText: `prefix path="/var/lib/thing" suffix`, errorText: `prefix path="/var/lib/thing" suffix`,
expected: &CoordinateError{ expected: &CoordinateError{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/var/lib/thing", ""),
RealPath: "/var/lib/thing",
},
Reason: fmt.Errorf(`prefix path="/var/lib/thing" suffix`), Reason: fmt.Errorf(`prefix path="/var/lib/thing" suffix`),
}, },
}, },
{ {
errorText: `prefix path="/var/lib/thing"`, errorText: `prefix path="/var/lib/thing"`,
expected: &CoordinateError{ expected: &CoordinateError{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/var/lib/thing", ""),
RealPath: "/var/lib/thing",
},
Reason: fmt.Errorf(`prefix path="/var/lib/thing"`), Reason: fmt.Errorf(`prefix path="/var/lib/thing"`),
}, },
}, },
{ {
errorText: `path="/var/lib/thing" suffix`, errorText: `path="/var/lib/thing" suffix`,
expected: &CoordinateError{ expected: &CoordinateError{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/var/lib/thing", ""),
RealPath: "/var/lib/thing",
},
Reason: fmt.Errorf(`path="/var/lib/thing" suffix`), Reason: fmt.Errorf(`path="/var/lib/thing" suffix`),
}, },
}, },
+7 -7
View File
@@ -716,15 +716,15 @@
"properties": { "properties": {
"path": { "path": {
"type": "string", "type": "string",
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)." "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem."
}, },
"layerID": { "layerID": {
"type": "string", "type": "string",
"description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in."
}, },
"archivePath": { "fullPath": {
"type": "string", "type": "string",
"description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it." "description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM."
} }
}, },
"type": "object", "type": "object",
@@ -2548,19 +2548,19 @@
"properties": { "properties": {
"path": { "path": {
"type": "string", "type": "string",
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)." "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem."
}, },
"layerID": { "layerID": {
"type": "string", "type": "string",
"description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in."
}, },
"archivePath": { "fullPath": {
"type": "string", "type": "string",
"description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it." "description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM."
}, },
"accessPath": { "accessPath": {
"type": "string", "type": "string",
"description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path)" "description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.RealFullPath)."
}, },
"annotations": { "annotations": {
"additionalProperties": { "additionalProperties": {
+7 -7
View File
@@ -716,15 +716,15 @@
"properties": { "properties": {
"path": { "path": {
"type": "string", "type": "string",
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)." "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem."
}, },
"layerID": { "layerID": {
"type": "string", "type": "string",
"description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in."
}, },
"archivePath": { "fullPath": {
"type": "string", "type": "string",
"description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it." "description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM."
} }
}, },
"type": "object", "type": "object",
@@ -2548,19 +2548,19 @@
"properties": { "properties": {
"path": { "path": {
"type": "string", "type": "string",
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)." "description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem."
}, },
"layerID": { "layerID": {
"type": "string", "type": "string",
"description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in." "description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in."
}, },
"archivePath": { "fullPath": {
"type": "string", "type": "string",
"description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it." "description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM."
}, },
"accessPath": { "accessPath": {
"type": "string", "type": "string",
"description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path)" "description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.RealFullPath)."
}, },
"annotations": { "annotations": {
"additionalProperties": { "additionalProperties": {
+77 -151
View File
@@ -2,7 +2,6 @@ package syft
import ( import (
"archive/tar" "archive/tar"
"archive/zip"
"bytes" "bytes"
"compress/gzip" "compress/gzip"
"context" "context"
@@ -18,8 +17,10 @@ import (
"github.com/wagoodman/go-partybus" "github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress" "github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/task" "github.com/anchore/syft/internal/task"
"github.com/anchore/syft/internal/testutils"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/filecataloging" "github.com/anchore/syft/syft/cataloging/filecataloging"
@@ -59,38 +60,37 @@ var fixtureExclusions = []string{"./nested/marker.txt", "**/excluded.txt", "**/i
func writeFixture(t *testing.T) string { func writeFixture(t *testing.T) string {
t.Helper() t.Helper()
dir := t.TempDir() dir := t.TempDir()
marker := map[string]string{"nested/marker.txt": "hello"} write := func(name string, data []byte) {
markerZip := buildZipBytes(t, marker) require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, name), data, 0o644))
}
markerZip := testutils.Zip(t, map[string]string{"nested/marker.txt": "hello"})
require.NoError(t, os.MkdirAll(filepath.Join(dir, "nested"), 0o755)) write("loose.txt", []byte("outside"))
require.NoError(t, os.MkdirAll(filepath.Join(dir, "app"), 0o755)) write("nested/marker.txt", []byte("at the root"))
require.NoError(t, os.WriteFile(filepath.Join(dir, "loose.txt"), []byte("outside"), 0o600)) write("app.zip", testutils.Zip(t, map[string]string{
require.NoError(t, os.WriteFile(filepath.Join(dir, "nested", "marker.txt"), []byte("at the root"), 0o644))
writeTestZip(t, filepath.Join(dir, "app.zip"), map[string]string{
"nested/marker.txt": "hello", "nested/marker.txt": "hello",
"nested/keep.txt": "hello", "nested/keep.txt": "hello",
"nested/excluded.txt": "hello", "nested/excluded.txt": "hello",
}) }))
writeTestZipRaw(t, filepath.Join(dir, "app", "data.zip"), map[string][]byte{"nested/marker.txt": []byte("hello")}) write("app/data.zip", markerZip)
writeTestZipRaw(t, filepath.Join(dir, "outer.zip"), map[string][]byte{ write("outer.zip", testutils.Zip(t, map[string][]byte{
"outer/marker.txt": []byte("hello"), "outer/marker.txt": []byte("hello"),
"deeper/inner.zip": markerZip, "deeper/inner.zip": markerZip,
"skip/inner-excluded.zip": markerZip, "skip/inner-excluded.zip": markerZip,
"日本語/café-🎉/unicode.zip": markerZip, "日本語/café-🎉/unicode.zip": markerZip,
}) }))
require.NoError(t, os.WriteFile(filepath.Join(dir, "pkg.rpm"), markerZip, 0o644)) write("pkg.rpm", markerZip)
require.NoError(t, os.WriteFile(filepath.Join(dir, "KEPT.RPM"), markerZip, 0o644)) write("KEPT.RPM", markerZip)
write("app.war", testutils.Zip(t, map[string][]byte{
writeTestZipRaw(t, filepath.Join(dir, "app.war"), map[string][]byte{ "WEB-INF/lib/war-lib-2.0.jar": testutils.Jar(t, "war-lib", "2.0", nil),
"WEB-INF/lib/war-lib-2.0.jar": jarBytes(t, "war-lib", "2.0"), }))
}) write("bundle.zip", testutils.Zip(t, map[string][]byte{
writeTestZipRaw(t, filepath.Join(dir, "bundle.zip"), map[string][]byte{ "lib/zip-lib-1.2.jar": testutils.Jar(t, "zip-lib", "1.2", nil),
"lib/zip-lib-1.2.jar": jarBytes(t, "zip-lib", "1.2"), }))
}) write("bundle.tar.gz", testutils.TarGz(t, map[string][]byte{
require.NoError(t, os.WriteFile(filepath.Join(dir, "bundle.tar.gz"), buildTarGzBytes(t, map[string][]byte{ "lib/tgz-lib-1.2.jar": testutils.Jar(t, "tgz-lib", "1.2", nil),
"lib/tgz-lib-1.2.jar": jarBytes(t, "tgz-lib", "1.2"), }))
}), 0o644))
return dir return dir
} }
@@ -134,21 +134,21 @@ func TestArchiveCataloger(t *testing.T) {
s := scanDirWithExclusions(t, scanDir, cfg, fixtureExclusions...) s := scanDirWithExclusions(t, scanDir, cfg, fixtureExclusions...)
t.Run("markers are found at every depth under archive-relative paths", func(t *testing.T) { t.Run("markers are found at every depth under archive-relative paths", func(t *testing.T) {
// the location's ArchivePath is the chain of archives; RealPath is relative to the innermost // the location's RealPath is the outermost archive; FullPath adds the chain of entries within it
assert.Equal(t, []string{ assert.Equal(t, []string{
"KEPT.RPM|nested/marker.txt", "KEPT.RPM!/nested/marker.txt",
"app.zip|nested/marker.txt", "app.zip!/nested/marker.txt",
"app/data.zip|nested/marker.txt", "app/data.zip!/nested/marker.txt",
"outer.zip:deeper/inner.zip|nested/marker.txt", "outer.zip!/deeper/inner.zip!/nested/marker.txt",
"outer.zip:日本語/café-🎉/unicode.zip|nested/marker.txt", "outer.zip!/outer/marker.txt",
"outer.zip|outer/marker.txt", "outer.zip!/日本語/café-🎉/unicode.zip!/nested/marker.txt",
}, packageLocations(s, "marker-pkg")) }, packageLocations(s, "marker-pkg"))
}) })
t.Run("archive contents do not leak into the parent tree", func(t *testing.T) { t.Run("archive contents do not leak into the parent tree", func(t *testing.T) {
for _, loc := range packageLocations(s, "marker-pkg") { for _, loc := range packageLocations(s, "marker-pkg") {
assert.NotContains(t, loc, ".zip/", "a joined path smears the archive's contents into the parent's tree") assert.NotContains(t, loc, ".zip/", "a joined path smears the archive's contents into the parent's tree")
assert.NotContains(t, loc, "|/", "a nested path must not be rooted") assert.NotContains(t, loc, "!//", "a nested path must not be rooted")
} }
}) })
@@ -169,18 +169,18 @@ func TestArchiveCataloger(t *testing.T) {
continue continue
} }
if coords, ok := rel.From.(file.Coordinates); ok { if coords, ok := rel.From.(file.Coordinates); ok {
containers = append(containers, coords.ArchivePath+"|"+coords.RealPath) containers = append(containers, coords.FullNestedPath())
} }
} }
} }
sort.Strings(containers) sort.Strings(containers)
assert.Equal(t, []string{ assert.Equal(t, []string{
"outer.zip|deeper/inner.zip", "KEPT.RPM",
"outer.zip|日本語/café-🎉/unicode.zip", "app.zip",
"|KEPT.RPM", "app/data.zip",
"|app.zip", "outer.zip",
"|app/data.zip", "outer.zip!/deeper/inner.zip",
"|outer.zip", "outer.zip!/日本語/café-🎉/unicode.zip",
}, containers, "the archive file, named in its own parent, contains the package") }, containers, "the archive file, named in its own parent, contains the package")
}) })
@@ -189,16 +189,16 @@ func TestArchiveCataloger(t *testing.T) {
// reaches inside an archive, and an excluded entry is absent from the archive's filesystem // reaches inside an archive, and an excluded entry is absent from the archive's filesystem
// rather than present and skipped // rather than present and skipped
markers := packageLocations(s, "marker-pkg") markers := packageLocations(s, "marker-pkg")
assert.NotContains(t, markers, "|nested/marker.txt", "a root-anchored pattern excludes the file at the scan root") assert.NotContains(t, markers, "nested/marker.txt", "a root-anchored pattern excludes the file at the scan root")
assert.Contains(t, markers, "app.zip|nested/marker.txt", "and says nothing about the same path inside an archive") assert.Contains(t, markers, "app.zip!/nested/marker.txt", "and says nothing about the same path inside an archive")
assert.NotContains(t, markers, "pkg.rpm|nested/marker.txt", "an archive whose own path is excluded is not extracted") assert.NotContains(t, markers, "pkg.rpm!/nested/marker.txt", "an archive whose own path is excluded is not extracted")
assert.Contains(t, markers, "KEPT.RPM|nested/marker.txt", "a lower-case pattern does not match an upper-case name") assert.Contains(t, markers, "KEPT.RPM!/nested/marker.txt", "a lower-case pattern does not match an upper-case name")
assert.NotContains(t, markers, "outer.zip:skip/inner-excluded.zip|nested/marker.txt", "a nested archive is excluded at its own level") assert.NotContains(t, markers, "outer.zip!/skip/inner-excluded.zip!/nested/marker.txt", "a nested archive is excluded at its own level")
assert.Contains(t, seen, "nested/keep.txt") assert.Contains(t, seen, "app.zip!/nested/keep.txt")
assert.NotContains(t, seen, "nested/excluded.txt", "a **/ pattern excludes inside an archive, from the index itself") assert.NotContains(t, seen, "app.zip!/nested/excluded.txt", "a **/ pattern excludes inside an archive, from the index itself")
assert.Contains(t, seen, "deeper/inner.zip", "the probe ran inside outer.zip") assert.Contains(t, seen, "outer.zip!/deeper/inner.zip", "the probe ran inside outer.zip")
assert.NotContains(t, seen, "skip/inner-excluded.zip", "the excluded inner archive is absent from outer's filesystem") assert.NotContains(t, seen, "outer.zip!/skip/inner-excluded.zip", "the excluded inner archive is absent from outer's filesystem")
}) })
t.Run("a jar inside an archive is cataloged exactly once, under the virtual path java gives it", func(t *testing.T) { t.Run("a jar inside an archive is cataloged exactly once, under the virtual path java gives it", func(t *testing.T) {
@@ -249,28 +249,28 @@ func TestArchiveCataloger(t *testing.T) {
// output would fail it // output would fail it
var got []string var got []string
for coords := range s.Artifacts.FileMetadata { for coords := range s.Artifacts.FileMetadata {
if coords.ArchivePath == "" { if archive.ContainerPath(coords) == "" {
continue continue
} }
got = append(got, coords.ArchivePath+"|"+coords.RealPath) got = append(got, coords.FullNestedPath())
} }
sort.Strings(got) sort.Strings(got)
assert.Equal(t, []string{ assert.Equal(t, []string{
"KEPT.RPM|nested/marker.txt", "KEPT.RPM!/nested/marker.txt",
"app.war:WEB-INF/lib/war-lib-2.0.jar|META-INF/MANIFEST.MF", "app.war!/WEB-INF/lib/war-lib-2.0.jar",
"app.war|WEB-INF/lib/war-lib-2.0.jar", "app.war!/WEB-INF/lib/war-lib-2.0.jar!/META-INF/MANIFEST.MF",
"app.zip|nested/keep.txt", "app.zip!/nested/keep.txt",
"app.zip|nested/marker.txt", "app.zip!/nested/marker.txt",
"app/data.zip|nested/marker.txt", "app/data.zip!/nested/marker.txt",
"bundle.tar.gz:lib/tgz-lib-1.2.jar|META-INF/MANIFEST.MF", "bundle.tar.gz!/lib/tgz-lib-1.2.jar",
"bundle.tar.gz|lib/tgz-lib-1.2.jar", "bundle.tar.gz!/lib/tgz-lib-1.2.jar!/META-INF/MANIFEST.MF",
"bundle.zip:lib/zip-lib-1.2.jar|META-INF/MANIFEST.MF", "bundle.zip!/lib/zip-lib-1.2.jar",
"bundle.zip|lib/zip-lib-1.2.jar", "bundle.zip!/lib/zip-lib-1.2.jar!/META-INF/MANIFEST.MF",
"outer.zip:deeper/inner.zip|nested/marker.txt", "outer.zip!/deeper/inner.zip",
"outer.zip:日本語/café-🎉/unicode.zip|nested/marker.txt", "outer.zip!/deeper/inner.zip!/nested/marker.txt",
"outer.zip|deeper/inner.zip", "outer.zip!/outer/marker.txt",
"outer.zip|outer/marker.txt", "outer.zip!/日本語/café-🎉/unicode.zip",
"outer.zip|日本語/café-🎉/unicode.zip", "outer.zip!/日本語/café-🎉/unicode.zip!/nested/marker.txt",
}, got) }, got)
}) })
@@ -338,7 +338,7 @@ func TestArchiveCataloger_jarReachedThroughALinkIsIdentified(t *testing.T) {
var buf bytes.Buffer var buf bytes.Buffer
gw := gzip.NewWriter(&buf) gw := gzip.NewWriter(&buf)
tw := tar.NewWriter(gw) tw := tar.NewWriter(gw)
jar := jarBytes(t, "json-simple", "1.1.1") jar := testutils.Jar(t, "json-simple", "1.1.1", nil)
require.NoError(t, tw.WriteHeader(&tar.Header{Name: "store/3f9a1c", Mode: 0o644, Size: int64(len(jar))})) require.NoError(t, tw.WriteHeader(&tar.Header{Name: "store/3f9a1c", Mode: 0o644, Size: int64(len(jar))}))
_, err := tw.Write(jar) _, err := tw.Write(jar)
require.NoError(t, err) require.NoError(t, err)
@@ -355,10 +355,10 @@ func TestArchiveCataloger_jarReachedThroughALinkIsIdentified(t *testing.T) {
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)) WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2))
s := scanDirWithExclusions(t, dir, cfg) s := scanDirWithExclusions(t, dir, cfg)
assert.Equal(t, []string{"bundle.tar.gz|store/3f9a1c"}, packageLocations(s, "json-simple")) assert.Equal(t, []string{"bundle.tar.gz!/store/3f9a1c"}, packageLocations(s, "json-simple"))
// the jar's own files are addressed by its real path, the same one its package location names // the jar's own files are addressed by its real path, the same one its package location names
manifest := file.Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "bundle.tar.gz:store/3f9a1c"} manifest := file.Coordinates{RealPath: "bundle.tar.gz", FullPath: "bundle.tar.gz!/store/3f9a1c!/META-INF/MANIFEST.MF"}
assert.Contains(t, s.Artifacts.FileMetadata, manifest) assert.Contains(t, s.Artifacts.FileMetadata, manifest)
} }
@@ -366,7 +366,7 @@ func TestArchiveCataloger_defaultFileSelectionCoversRootJars(t *testing.T) {
// with the default owned-by-package selection, a jar at the root is described by the archive task, // with the default owned-by-package selection, a jar at the root is described by the archive task,
// so the root file catalogers must run after it or the jar gets no file entries at all // so the root file catalogers must run after it or the jar gets no file entries at all
dir := t.TempDir() dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "app.jar"), jarBytes(t, "app", "1.0"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "app.jar"), testutils.Jar(t, "app", "1.0", nil), 0o644))
for name, depth := range map[string]int{"feature off": 0, "feature on": 1} { for name, depth := range map[string]int{"feature off": 0, "feature on": 1} {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
@@ -375,8 +375,8 @@ func TestArchiveCataloger_defaultFileSelectionCoversRootJars(t *testing.T) {
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth)) WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth))
s := scanDirWithExclusions(t, dir, cfg) s := scanDirWithExclusions(t, dir, cfg)
require.Equal(t, []string{"|app.jar"}, packageLocations(s, "app")) require.Equal(t, []string{"app.jar"}, packageLocations(s, "app"))
jar := file.Coordinates{RealPath: "app.jar"} jar := file.NewCoordinates("app.jar", "")
assert.Contains(t, s.Artifacts.FileMetadata, jar) assert.Contains(t, s.Artifacts.FileMetadata, jar)
assert.Contains(t, s.Artifacts.FileDigests, jar) assert.Contains(t, s.Artifacts.FileDigests, jar)
}) })
@@ -387,9 +387,8 @@ func TestArchiveCataloger_javaParentLinkageMatchesTheLegacyPath(t *testing.T) {
// a consumer walking java DEPENDENCY_OF edges (the only structure SPDX and CycloneDX keep) must see // a consumer walking java DEPENDENCY_OF edges (the only structure SPDX and CycloneDX keep) must see
// the same war-to-jar linkage whether or not the archive task does the recursion // the same war-to-jar linkage whether or not the archive task does the recursion
dir := t.TempDir() dir := t.TempDir()
war := buildZipBytesRaw(t, map[string][]byte{ war := testutils.Jar(t, "app", "1.0", map[string][]byte{
"META-INF/MANIFEST.MF": []byte("Manifest-Version: 1.0\nImplementation-Title: app\nImplementation-Version: 1.0\n"), "WEB-INF/lib/lib-2.0.jar": testutils.Jar(t, "lib", "2.0", nil),
"WEB-INF/lib/lib-2.0.jar": jarBytes(t, "lib", "2.0"),
}) })
require.NoError(t, os.WriteFile(filepath.Join(dir, "app.war"), war, 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "app.war"), war, 0o644))
@@ -458,7 +457,7 @@ func TestArchiveCataloger_javaArchiveDepthIsAConfigError(t *testing.T) {
} }
} }
// packageLocations lists "archivePath|realPath" for every location of every package with the name. // packageLocations lists the full path of every location of every package with the name.
func packageLocations(s *sbom.SBOM, name string) []string { func packageLocations(s *sbom.SBOM, name string) []string {
var out []string var out []string
for _, p := range s.Artifacts.Packages.Sorted() { for _, p := range s.Artifacts.Packages.Sorted() {
@@ -466,7 +465,7 @@ func packageLocations(s *sbom.SBOM, name string) []string {
continue continue
} }
for _, loc := range p.Locations.ToSlice() { for _, loc := range p.Locations.ToSlice() {
out = append(out, loc.ArchivePath+"|"+loc.RealPath) out = append(out, loc.FullNestedPath())
} }
} }
sort.Strings(out) sort.Strings(out)
@@ -526,7 +525,7 @@ func (c resolverProbeCataloger) Catalog(_ context.Context, resolver file.Resolve
} }
var paths []string var paths []string
for _, loc := range locations { for _, loc := range locations {
paths = append(paths, loc.RealPath) paths = append(paths, loc.FullNestedPath())
} }
c.record(paths) c.record(paths)
return nil, nil, nil return nil, nil, nil
@@ -559,76 +558,3 @@ func scanDirWithExclusions(t *testing.T, dir string, cfg *CreateSBOMConfig, excl
require.NoError(t, err) require.NoError(t, err)
return s return s
} }
func writeTestZip(t *testing.T, path string, files map[string]string) {
t.Helper()
require.NoError(t, os.WriteFile(path, buildZipBytes(t, files), 0o644))
}
func writeTestZipRaw(t *testing.T, path string, files map[string][]byte) {
t.Helper()
require.NoError(t, os.WriteFile(path, buildZipBytesRaw(t, files), 0o644))
}
func buildZipBytes(t *testing.T, files map[string]string) []byte {
t.Helper()
raw := make(map[string][]byte, len(files))
for k, v := range files {
raw[k] = []byte(v)
}
return buildZipBytesRaw(t, raw)
}
// buildZipBytesRaw writes entries in sorted name order rather than map order: limits are enforced as
// the walk proceeds, so entry order decides which entries land before a truncation.
func buildZipBytesRaw(t *testing.T, files map[string][]byte) []byte {
t.Helper()
names := make([]string, 0, len(files))
for name := range files {
names = append(names, name)
}
sort.Strings(names)
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for _, name := range names {
w, err := zw.Create(name)
require.NoError(t, err)
_, err = w.Write(files[name])
require.NoError(t, err)
}
require.NoError(t, zw.Close())
return buf.Bytes()
}
func buildTarGzBytes(t *testing.T, files map[string][]byte) []byte {
t.Helper()
names := make([]string, 0, len(files))
for name := range files {
names = append(names, name)
}
sort.Strings(names)
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
tw := tar.NewWriter(gw)
for _, name := range names {
body := files[name]
require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}))
_, err := tw.Write(body)
require.NoError(t, err)
}
require.NoError(t, tw.Close())
require.NoError(t, gw.Close())
return buf.Bytes()
}
// jarBytes is the smallest jar the java cataloger identifies: a manifest naming the artifact.
func jarBytes(t *testing.T, title, version string) []byte {
t.Helper()
return buildZipBytesRaw(t, map[string][]byte{
"META-INF/MANIFEST.MF": []byte(
"Manifest-Version: 1.0\nImplementation-Title: " + title + "\nImplementation-Version: " + version + "\n",
),
})
}
+11 -11
View File
@@ -68,7 +68,7 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map
results := make(map[file.Coordinates]file.Executable) results := make(map[file.Coordinates]file.Executable)
errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(locs), func(loc file.Location) (*file.Executable, error) { errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(locs), func(loc file.Location) (*file.Executable, error) {
prog.AtomicStage.Set(loc.Path()) prog.AtomicStage.Set(loc.FullNestedPath())
exec, err := processExecutableLocation(loc, resolver) exec, err := processExecutableLocation(loc, resolver)
if err != nil { if err != nil {
@@ -93,15 +93,15 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map
func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec *file.Executable, err error) { func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec *file.Executable, err error) {
reader, err := resolver.FileContentsByLocation(loc) reader, err := resolver.FileContentsByLocation(loc)
if err != nil { if err != nil {
log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get file contents") log.WithFields("error", err, "path", loc.FullNestedPath()).Debug("unable to get file contents")
return nil, fmt.Errorf("unable to get file contents: %w", err) return nil, fmt.Errorf("unable to get file contents: %w", err)
} }
defer internal.CloseAndLogError(reader, loc.RealPath) defer internal.CloseAndLogError(reader, loc.FullNestedPath())
defer func() { defer func() {
if r := recover(); r != nil { if r := recover(); r != nil {
// a malformed binary must not take down the whole SBOM; the caller records this as an unknown // a malformed binary must not take down the whole SBOM; the caller records this as an unknown
log.WithFields("path", loc.RealPath, "panic", r).Debug("recovered from panic while reading executable") log.WithFields("path", loc.FullNestedPath(), "panic", r).Debug("recovered from panic while reading executable")
log.Tracef("executable panic stack:\n%s", debug.Stack()) log.Tracef("executable panic stack:\n%s", debug.Stack())
exec, err = nil, fmt.Errorf("recovered from panic while reading executable: %v", r) exec, err = nil, fmt.Errorf("recovered from panic while reading executable: %v", r)
} }
@@ -109,7 +109,7 @@ func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec
uReader, err := unionreader.GetUnionReader(reader) uReader, err := unionreader.GetUnionReader(reader)
if err != nil { if err != nil {
log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get union reader") log.WithFields("error", err, "path", loc.FullNestedPath()).Debug("unable to get union reader")
return nil, fmt.Errorf("unable to get union reader: %w", err) return nil, fmt.Errorf("unable to get union reader: %w", err)
} }
@@ -146,7 +146,7 @@ func filterByGlobs(locs []file.Location, globs []string) ([]file.Location, error
func locationMatchesGlob(loc file.Location, globs []string) (bool, error) { func locationMatchesGlob(loc file.Location, globs []string) (bool, error) {
for _, glob := range globs { for _, glob := range globs {
for _, path := range []string{loc.RealPath, loc.AccessPath} { for _, path := range []string{loc.LocalPath(), loc.AccessPath} {
if path == "" { if path == "" {
continue continue
} }
@@ -169,13 +169,13 @@ func processExecutable(loc file.Location, reader unionreader.UnionReader) (*file
format, err := findExecutableFormat(reader) format, err := findExecutableFormat(reader)
if err != nil { if err != nil {
log.Debugf("unable to determine executable kind for %v: %v", loc.RealPath, err) log.Debugf("unable to determine executable kind for %v: %v", loc.FullNestedPath(), err)
return nil, fmt.Errorf("unable to determine executable kind: %w", err) return nil, fmt.Errorf("unable to determine executable kind: %w", err)
} }
if format == "" { if format == "" {
// this is not an "unknown", so just log -- this binary does not have parseable data in it // this is not an "unknown", so just log -- this binary does not have parseable data in it
log.Debugf("unable to determine executable format for %q", loc.RealPath) log.Debugf("unable to determine executable format for %q", loc.FullNestedPath())
return nil, nil return nil, nil
} }
@@ -184,17 +184,17 @@ func processExecutable(loc file.Location, reader unionreader.UnionReader) (*file
switch format { switch format {
case file.ELF: case file.ELF:
if err = findELFFeatures(&data, reader); err != nil { if err = findELFFeatures(&data, reader); err != nil {
log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine ELF features") log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine ELF features")
err = fmt.Errorf("unable to determine ELF features: %w", err) err = fmt.Errorf("unable to determine ELF features: %w", err)
} }
case file.PE: case file.PE:
if err = findPEFeatures(&data, reader); err != nil { if err = findPEFeatures(&data, reader); err != nil {
log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine PE features") log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine PE features")
err = fmt.Errorf("unable to determine PE features: %w", err) err = fmt.Errorf("unable to determine PE features: %w", err)
} }
case file.MachO: case file.MachO:
if err = findMachoFeatures(&data, reader); err != nil { if err = findMachoFeatures(&data, reader); err != nil {
log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine Macho features") log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine Macho features")
err = fmt.Errorf("unable to determine Macho features: %w", err) err = fmt.Errorf("unable to determine Macho features: %w", err)
} }
} }
+2 -2
View File
@@ -57,7 +57,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) (map[fi
prog := catalogingProgress(ctx, int64(len(locations))) prog := catalogingProgress(ctx, int64(len(locations)))
for _, location := range locations { for _, location := range locations {
prog.AtomicStage.Set(location.Path()) prog.AtomicStage.Set(location.FullNestedPath())
metadata, err := resolver.FileMetadataByLocation(location) metadata, err := resolver.FileMetadataByLocation(location)
if err != nil { if err != nil {
@@ -103,7 +103,7 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
encoder := base64.NewEncoder(base64.StdEncoding, buf) encoder := base64.NewEncoder(base64.StdEncoding, buf)
if _, err = io.Copy(encoder, contentReader); err != nil { if _, err = io.Copy(encoder, contentReader); err != nil {
return "", internal.ErrPath{Context: "content-cataloger", Path: location.RealPath, Err: err} return "", internal.ErrPath{Context: "content-cataloger", Path: location.FullNestedPath(), Err: err}
} }
// note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes // note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes
if err := encoder.Close(); err != nil { if err := encoder.Close(); err != nil {
+6 -6
View File
@@ -42,9 +42,9 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
locations = intCataloger.AllRegularFiles(ctx, resolver) locations = intCataloger.AllRegularFiles(ctx, resolver)
} else { } else {
for _, c := range coordinates { for _, c := range coordinates {
locs, err := resolver.FilesByPath(c.RealPath) locs, err := resolver.FilesByPath(c.LocalPath())
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to get file locations for path %q: %w", c.RealPath, err) return nil, fmt.Errorf("unable to get file locations for path %q: %w", c.FullNestedPath(), err)
} }
locations = append(locations, locs...) locations = append(locations, locs...)
} }
@@ -59,10 +59,10 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
return nil, nil return nil, nil
} }
prog.AtomicStage.Set(location.Path()) prog.AtomicStage.Set(location.FullNestedPath())
if internal.IsErrPathPermission(err) { if internal.IsErrPathPermission(err) {
log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err) log.Debugf("file digests cataloger skipping %q: %+v", location.FullNestedPath(), err)
return nil, unknown.New(location, err) return nil, unknown.New(location, err)
} }
@@ -108,7 +108,7 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver,
defer func() { defer func() {
if r := recover(); r != nil { if r := recover(); r != nil {
// the caller records this as an unknown for the location instead of failing the whole SBOM // the caller records this as an unknown for the location instead of failing the whole SBOM
log.WithFields("path", location.RealPath, "panic", r).Debug("recovered from panic while digesting file") log.WithFields("path", location.FullNestedPath(), "panic", r).Debug("recovered from panic while digesting file")
log.Tracef("file digest panic stack:\n%s", debug.Stack()) log.Tracef("file digest panic stack:\n%s", debug.Stack())
digests, err = nil, fmt.Errorf("recovered from panic while digesting file: %v", r) digests, err = nil, fmt.Errorf("recovered from panic while digesting file: %v", r)
} }
@@ -116,7 +116,7 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver,
digests, err = intFile.NewDigestsFromFile(ctx, contentReader, i.hashes) digests, err = intFile.NewDigestsFromFile(ctx, contentReader, i.hashes)
if err != nil { if err != nil {
return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err} return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.FullNestedPath(), Err: err}
} }
return digests, nil return digests, nil
@@ -34,7 +34,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
go func() { go func() {
defer close(ch) defer close(ch)
for _, c := range coordinates { for _, c := range coordinates {
locs, err := resolver.FilesByPath(c.RealPath) locs, err := resolver.FilesByPath(c.LocalPath())
if err != nil { if err != nil {
errs = unknown.Append(errs, c, err) errs = unknown.Append(errs, c, err)
continue continue
@@ -55,7 +55,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
prog := catalogingProgress(ctx, -1) prog := catalogingProgress(ctx, -1)
for location := range locations { for location := range locations {
prog.AtomicStage.Set(location.Path()) prog.AtomicStage.Set(location.FullNestedPath())
metadata, err := resolver.FileMetadataByLocation(location) metadata, err := resolver.FileMetadataByLocation(location)
if err != nil { if err != nil {
@@ -12,7 +12,7 @@ func AllRegularFiles(ctx context.Context, resolver file.Resolver) (locations []f
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
for location := range resolver.AllLocations(ctx) { for location := range resolver.AllLocations(ctx) {
resolvedLocations, err := resolver.FilesByPath(location.RealPath) resolvedLocations, err := resolver.FilesByPath(location.LocalPath())
if err != nil { if err != nil {
log.Debugf("unable to resolve %+v: %+v", location, err) log.Debugf("unable to resolve %+v: %+v", location, err)
continue continue
+1 -1
View File
@@ -79,7 +79,7 @@ func (s CoordinateSet) ToSlice(sorters ...func(a, b Coordinates) int) []Coordina
sort.SliceStable(coordinates, func(i, j int) bool { sort.SliceStable(coordinates, func(i, j int) bool {
if coordinates[i].FileSystemID == coordinates[j].FileSystemID { if coordinates[i].FileSystemID == coordinates[j].FileSystemID {
if coordinates[i].RealPath == coordinates[j].RealPath { if coordinates[i].RealPath == coordinates[j].RealPath {
return coordinates[i].ArchivePath < coordinates[j].ArchivePath return coordinates[i].FullNestedPath() < coordinates[j].FullNestedPath()
} }
return coordinates[i].RealPath < coordinates[j].RealPath return coordinates[i].RealPath < coordinates[j].RealPath
} }
+30 -8
View File
@@ -2,21 +2,42 @@ package file
import ( import (
"fmt" "fmt"
"strings"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
) )
const ArchivePathSeparator = "!/"
// Coordinates contains the minimal information needed to describe how to find a file within any possible source object (e.g. image and directory sources) // Coordinates contains the minimal information needed to describe how to find a file within any possible source object (e.g. image and directory sources)
type Coordinates struct { type Coordinates struct {
// RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). // RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem.
RealPath string `json:"path" cyclonedx:"path"` RealPath string `json:"path" cyclonedx:"path"`
// FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in. // FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in.
FileSystemID string `json:"layerID,omitempty" cyclonedx:"layerID"` FileSystemID string `json:"layerID,omitempty" cyclonedx:"layerID"`
// ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. "app.war:WEB-INF/lib/dep.jar"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as "%25" and ':' as "%3A", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it. // FullPath is the full path to the file: RealPath, then for a file within an archive a "!/"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. "/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it is blank, so coordinates outside archives are unchanged by the field (in JSON, equality and ID).
ArchivePath string `json:"archivePath,omitempty" cyclonedx:"archivePath"` FullPath string `json:"fullPath,omitempty" cyclonedx:"fullPath"`
}
// FullNestedPath is FullPath, or RealPath where FullPath is blank.
func (c Coordinates) FullNestedPath() string {
if c.FullPath == "" {
return c.RealPath
}
return c.FullPath
}
// LocalPath is the path within the resolver holding the file: the segment of FullNestedPath after the
// last archive separator, or RealPath for a file not within an archive.
func (c Coordinates) LocalPath() string {
fullPath := c.FullNestedPath()
if i := strings.LastIndex(fullPath, ArchivePathSeparator); i >= 0 {
return fullPath[i+len(ArchivePathSeparator):]
}
return fullPath
} }
func NewCoordinates(realPath, fsID string) Coordinates { func NewCoordinates(realPath, fsID string) Coordinates {
@@ -43,8 +64,9 @@ func (c Coordinates) String() string {
if c.FileSystemID != "" { if c.FileSystemID != "" {
str += fmt.Sprintf(" Layer=%q", c.FileSystemID) str += fmt.Sprintf(" Layer=%q", c.FileSystemID)
} }
if c.ArchivePath != "" {
str += fmt.Sprintf(" Archive=%q", c.ArchivePath) if c.FullNestedPath() != c.RealPath {
str += fmt.Sprintf(" FullPath=%q", c.FullNestedPath())
} }
return fmt.Sprintf("Location<%s>", str) return fmt.Sprintf("Location<%s>", str)
} }
@@ -53,10 +75,10 @@ func (c Coordinates) GetCoordinates() Coordinates {
return c return c
} }
// HashInclude keeps an empty ArchivePath out of the artifact ID hash (see artifact.IDByHash), so // HashInclude keeps a FullPath that adds nothing to RealPath out of the artifact ID hash (see
// coordinates outside archives keep the IDs they had before the field existed. // artifact.IDByHash), so coordinates outside archives keep the IDs they had before the field existed.
func (c Coordinates) HashInclude(field string, _ any) (bool, error) { func (c Coordinates) HashInclude(field string, _ any) (bool, error) {
if field == "ArchivePath" && c.ArchivePath == "" { if field == "FullPath" && c.FullNestedPath() == c.RealPath {
return false, nil return false, nil
} }
return true, nil return true, nil
+9 -9
View File
@@ -33,7 +33,7 @@ type LocationData struct {
// note: it is IMPORTANT to ignore anything but the coordinates for a Location when considering the ID (hash value) // note: it is IMPORTANT to ignore anything but the coordinates for a Location when considering the ID (hash value)
// since the coordinates are the minimally correct ID for a location (symlinks should not come into play) // since the coordinates are the minimally correct ID for a location (symlinks should not come into play)
// AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path) // AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.FullPath).
AccessPath string `hash:"ignore" json:"accessPath"` AccessPath string `hash:"ignore" json:"accessPath"`
// ref is the stereoscope file reference relative to the stereoscope.FileCatalog that has more information about this location. // ref is the stereoscope file reference relative to the stereoscope.FileCatalog that has more information about this location.
@@ -110,12 +110,12 @@ func NewVirtualLocation(realPath, accessPath string) Location {
}} }}
} }
// NewLocationFromCoordinates creates a new location for the given Coordinates. // NewLocationFromCoordinates creates a new location for the given Coordinates, accessed by their full path.
func NewLocationFromCoordinates(coordinates Coordinates) Location { func NewLocationFromCoordinates(coordinates Coordinates) Location {
return Location{ return Location{
LocationData: LocationData{ LocationData: LocationData{
Coordinates: coordinates, Coordinates: coordinates,
AccessPath: coordinates.RealPath, AccessPath: coordinates.FullNestedPath(),
}, },
LocationMetadata: LocationMetadata{ LocationMetadata: LocationMetadata{
Annotations: map[string]string{}, Annotations: map[string]string{},
@@ -205,11 +205,12 @@ func NewVirtualLocationFromDirectory(responsePath, responseAccessPath string, re
} }
} }
// Path is the path this location was accessed by, or its local path (see Coordinates.LocalPath).
func (l Location) Path() string { func (l Location) Path() string {
if l.AccessPath != "" { if l.AccessPath != "" {
return l.AccessPath return l.AccessPath
} }
return l.RealPath return l.LocalPath()
} }
func (l Location) String() string { func (l Location) String() string {
@@ -220,12 +221,12 @@ func (l Location) String() string {
str += fmt.Sprintf("RealPath=%q", l.RealPath) str += fmt.Sprintf("RealPath=%q", l.RealPath)
if l.AccessPath != "" && l.AccessPath != l.RealPath { if l.AccessPath != "" {
str += fmt.Sprintf(" AccessPath=%q", l.AccessPath) str += fmt.Sprintf(" AccessPath=%q", l.AccessPath)
} }
if l.ArchivePath != "" { if l.FullNestedPath() != l.RealPath {
str += fmt.Sprintf(" ArchivePath=%q", l.ArchivePath) str += fmt.Sprintf(" FullPath=%q", l.FullNestedPath())
} }
if l.FileSystemID != "" { if l.FileSystemID != "" {
@@ -235,8 +236,7 @@ func (l Location) String() string {
} }
func (l Location) Equals(other Location) bool { func (l Location) Equals(other Location) bool {
return l.RealPath == other.RealPath && return l.FullNestedPath() == other.FullNestedPath() &&
l.AccessPath == other.AccessPath && l.AccessPath == other.AccessPath &&
l.ArchivePath == other.ArchivePath &&
l.FileSystemID == other.FileSystemID l.FileSystemID == other.FileSystemID
} }
+8 -2
View File
@@ -5,6 +5,7 @@ import (
"sort" "sort"
"github.com/gohugoio/hashstructure" "github.com/gohugoio/hashstructure"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
) )
@@ -112,8 +113,13 @@ func (s *LocationSet) Empty() bool {
} }
func (s LocationSet) Hash() (uint64, error) { func (s LocationSet) Hash() (uint64, error) {
// access paths and filesystem IDs are not considered when hashing a location set, only the real paths // access paths and filesystem IDs are not considered when hashing a location set, only the full paths
return hashstructure.Hash(s.CoordinateSet().Paths(), &hashstructure.HashOptions{ // (unlike CoordinateSet.Paths, which ownership-by-overlap needs as real paths only)
paths := strset.New()
for _, c := range s.CoordinateSet().ToUnorderedSlice() {
paths.Add(c.FullNestedPath())
}
return hashstructure.Hash(paths.List(), &hashstructure.HashOptions{
ZeroNil: true, ZeroNil: true,
SlicesAsSets: true, SlicesAsSets: true,
}) })
+27
View File
@@ -4,6 +4,7 @@ import (
"sort" "sort"
"testing" "testing"
"github.com/gohugoio/hashstructure"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -255,6 +256,10 @@ func TestLocationSet_Hash(t *testing.T) {
}, },
} }
inA := NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FullPath: "/app.war!/a.jar!/package.json"})
inB := NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FullPath: "/app.war!/b.jar!/package.json"})
etcFull := Location{LocationData: LocationData{Coordinates: Coordinates{RealPath: "/etc/hosts", FileSystemID: "a", FullPath: "/etc/hosts"}}}
tests := []struct { tests := []struct {
name string name string
setA LocationSet setA LocationSet
@@ -297,6 +302,18 @@ func TestLocationSet_Hash(t *testing.T) {
setB: NewLocationSet(binB, etcB), setB: NewLocationSet(binB, etcB),
want: assert.Equal, want: assert.Equal,
}, },
{
name: "same outer archive but different nested entries have different hashes",
setA: NewLocationSet(inA),
setB: NewLocationSet(inB),
want: assert.NotEqual,
},
{
name: "a full path equal to the real path has the same hash as a blank one",
setA: NewLocationSet(etcA),
setB: NewLocationSet(etcFull),
want: assert.Equal,
},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
@@ -308,3 +325,13 @@ func TestLocationSet_Hash(t *testing.T) {
}) })
} }
} }
// IDs outside archives must not change from when only real paths were hashed
func TestLocationSet_Hash_unchangedOutsideArchives(t *testing.T) {
want, err := hashstructure.Hash([]string{"/etc/hosts", "/bin"}, &hashstructure.HashOptions{ZeroNil: true, SlicesAsSets: true})
require.NoError(t, err)
got, err := NewLocationSet(NewLocation("/etc/hosts"), NewVirtualLocation("/bin", "/usr/bin")).Hash()
require.NoError(t, err)
assert.Equal(t, want, got)
}
+54 -4
View File
@@ -1,9 +1,11 @@
package file package file
import ( import (
"encoding/json"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/file"
) )
@@ -49,11 +51,59 @@ func TestLocation_ID(t *testing.T) {
} }
func TestLocation_archivePathIsIdentity(t *testing.T) { func TestLocation_fullPathIsIdentity(t *testing.T) {
a := NewLocationFromCoordinates(Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/a.jar"}) a := NewLocationFromCoordinates(Coordinates{RealPath: "/a.jar", FullPath: "/a.jar!/META-INF/MANIFEST.MF"})
b := NewLocationFromCoordinates(Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/b.jar"}) b := NewLocationFromCoordinates(Coordinates{RealPath: "/b.jar", FullPath: "/b.jar!/META-INF/MANIFEST.MF"})
assert.False(t, a.Equals(b)) assert.False(t, a.Equals(b))
assert.NotEqual(t, a.String(), b.String()) assert.NotEqual(t, a.String(), b.String())
assert.Contains(t, a.String(), `ArchivePath="/a.jar"`) assert.Contains(t, a.String(), `RealPath="/a.jar"`)
}
func TestLocation_JSON(t *testing.T) {
tests := []struct {
name string
location Location
expected string
}{
{
name: "a file outside an archive writes no fullPath",
location: NewVirtualLocationFromDirectory("/etc/os-release", "/etc/link", file.Reference{}).WithAnnotation("k", "v"),
expected: `{"path":"/etc/os-release","accessPath":"/etc/link","annotations":{"k":"v"}}`,
},
{
name: "a file within an archive writes its full path",
location: NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/META-INF/MANIFEST.MF"}),
expected: `{"path":"/app.war","layerID":"layer","fullPath":"/app.war!/META-INF/MANIFEST.MF","accessPath":"/app.war!/META-INF/MANIFEST.MF"}`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
data, err := json.Marshal(tt.location)
require.NoError(t, err)
assert.JSONEq(t, tt.expected, string(data))
var decoded Location
require.NoError(t, json.Unmarshal(data, &decoded))
assert.Equal(t, tt.location.Coordinates, decoded.Coordinates, "decoding fills FullPath back in")
assert.Equal(t, tt.location.AccessPath, decoded.AccessPath)
data, err = json.Marshal(tt.location.Coordinates)
require.NoError(t, err)
var coordinates Coordinates
require.NoError(t, json.Unmarshal(data, &coordinates))
assert.Equal(t, tt.location.Coordinates, coordinates)
})
}
}
func TestCoordinates_ID_unchangedByTopLevelFullRealPath(t *testing.T) {
assert.Equal(t, Coordinates{RealPath: "/a"}.ID(), NewCoordinates("/a", "").ID())
assert.NotEqual(t, NewCoordinates("/a", "").ID(), Coordinates{RealPath: "/a", FullPath: "/a!/b"}.ID())
}
func TestCoordinates_LocalPath(t *testing.T) {
assert.Equal(t, "/a.jar", NewCoordinates("/a.jar", "").LocalPath())
assert.Equal(t, "/a.jar", Coordinates{RealPath: "/a.jar"}.LocalPath())
assert.Equal(t, "META-INF/MANIFEST.MF", Coordinates{RealPath: "/a.war", FullPath: "/a.war!/lib/b.jar!/META-INF/MANIFEST.MF"}.LocalPath())
} }
+2 -2
View File
@@ -88,7 +88,7 @@ func LocationSorter(layers []string) func(a, b Location) int { //nolint:gocognit
} }
// the same path in different archives is a different file // the same path in different archives is a different file
return strings.Compare(a.ArchivePath, b.ArchivePath) return strings.Compare(a.FullNestedPath(), b.FullNestedPath())
} }
} }
@@ -128,6 +128,6 @@ func CoordinatesSorter(layers []string) func(a, b Coordinates) int {
} }
// the same path in different archives is a different file // the same path in different archives is a different file
return strings.Compare(a.ArchivePath, b.ArchivePath) return strings.Compare(a.FullNestedPath(), b.FullNestedPath())
} }
} }
+7 -7
View File
@@ -557,23 +557,23 @@ func TestLocationAndCoordinatesSorters(t *testing.T) {
} }
} }
func TestSorters_archivePathBreaksTies(t *testing.T) { func TestSorters_fullPathBreaksTies(t *testing.T) {
// the same path in two archives must sort the same way whatever order the inputs arrive in, or // the same path in two archives must sort the same way whatever order the inputs arrive in, or
// output built from map iteration is not reproducible // output built from map iteration is not reproducible
a := Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/app.war:WEB-INF/lib/a.jar"} a := Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"}
b := Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/app.war:WEB-INF/lib/b.jar"} b := Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/META-INF/MANIFEST.MF"}
want := []string{a.ArchivePath, b.ArchivePath} want := []string{a.FullNestedPath(), b.FullNestedPath()}
for _, in := range [][]Coordinates{{a, b}, {b, a}} { for _, in := range [][]Coordinates{{a, b}, {b, a}} {
coords := slices.Clone(in) coords := slices.Clone(in)
slices.SortFunc(coords, CoordinatesSorter(nil)) slices.SortFunc(coords, CoordinatesSorter(nil))
assert.Equal(t, want, []string{coords[0].ArchivePath, coords[1].ArchivePath}, "CoordinatesSorter") assert.Equal(t, want, []string{coords[0].FullNestedPath(), coords[1].FullNestedPath()}, "CoordinatesSorter")
locs := []Location{NewLocationFromCoordinates(in[0]), NewLocationFromCoordinates(in[1])} locs := []Location{NewLocationFromCoordinates(in[0]), NewLocationFromCoordinates(in[1])}
slices.SortFunc(locs, LocationSorter(nil)) slices.SortFunc(locs, LocationSorter(nil))
assert.Equal(t, want, []string{locs[0].ArchivePath, locs[1].ArchivePath}, "LocationSorter") assert.Equal(t, want, []string{locs[0].FullNestedPath(), locs[1].FullNestedPath()}, "LocationSorter")
got := NewCoordinateSet(in...).ToSlice() got := NewCoordinateSet(in...).ToSlice()
assert.Equal(t, want, []string{got[0].ArchivePath, got[1].ArchivePath}, "CoordinateSet.ToSlice") assert.Equal(t, want, []string{got[0].FullNestedPath(), got[1].FullNestedPath()}, "CoordinateSet.ToSlice")
} }
} }
+7 -7
View File
@@ -60,9 +60,9 @@ func NewMockResolverForPathsWithMetadata(metadata map[Coordinates]Metadata) *Moc
l := NewLocationFromCoordinates(c) l := NewLocationFromCoordinates(c)
locations = append(locations, l) locations = append(locations, l)
mimeTypeIndex[m.MIMEType] = append(mimeTypeIndex[m.MIMEType], l) mimeTypeIndex[m.MIMEType] = append(mimeTypeIndex[m.MIMEType], l)
ext := path.Ext(l.RealPath) ext := path.Ext(l.FullNestedPath())
extension[ext] = append(extension[ext], l) extension[ext] = append(extension[ext], l)
bn := path.Base(l.RealPath) bn := path.Base(l.FullNestedPath())
basename[bn] = append(basename[bn], l) basename[bn] = append(basename[bn], l)
} }
@@ -97,9 +97,9 @@ func NewMockResolverForMediaTypes(mediaTypes map[string][]Location) *MockResolve
mediaTypeIndex[mediaType] = append(mediaTypeIndex[mediaType], locs...) mediaTypeIndex[mediaType] = append(mediaTypeIndex[mediaType], locs...)
for _, l := range locs { for _, l := range locs {
locations = append(locations, l) locations = append(locations, l)
ext := path.Ext(l.RealPath) ext := path.Ext(l.FullNestedPath())
extension[ext] = append(extension[ext], l) extension[ext] = append(extension[ext], l)
bn := path.Base(l.RealPath) bn := path.Base(l.FullNestedPath())
basename[bn] = append(basename[bn], l) basename[bn] = append(basename[bn], l)
} }
} }
@@ -121,7 +121,7 @@ func (r MockResolver) ImageReference() string {
// HasPath indicates if the given path exists in the underlying source. // HasPath indicates if the given path exists in the underlying source.
func (r MockResolver) HasPath(path string) bool { func (r MockResolver) HasPath(path string) bool {
for _, l := range r.locations { for _, l := range r.locations {
if l.RealPath == path { if l.FullNestedPath() == path {
return true return true
} }
} }
@@ -150,7 +150,7 @@ func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) {
var results []Location var results []Location
for _, p := range paths { for _, p := range paths {
for _, location := range r.locations { for _, location := range r.locations {
if p == location.RealPath { if p == location.LocalPath() {
results = append(results, NewLocation(p)) results = append(results, NewLocation(p))
} }
} }
@@ -164,7 +164,7 @@ func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) {
var results []Location var results []Location
for _, pattern := range patterns { for _, pattern := range patterns {
for _, location := range r.locations { for _, location := range r.locations {
matches, err := doublestar.Match(pattern, location.RealPath) matches, err := doublestar.Match(pattern, location.LocalPath())
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -15,7 +15,6 @@ import (
"github.com/spdx/tools-golang/spdx" "github.com/spdx/tools-golang/spdx"
"github.com/anchore/packageurl-go" "github.com/anchore/packageurl-go"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/mimetype" "github.com/anchore/syft/internal/mimetype"
"github.com/anchore/syft/internal/relationship" "github.com/anchore/syft/internal/relationship"
@@ -356,7 +355,7 @@ func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID {
return spdx.ElementID(helpers.SanitizeElementID(id)) return spdx.ElementID(helpers.SanitizeElementID(id))
} }
p := "" p := ""
parts := strings.Split(it.RealPath, "/") parts := strings.Split(it.FullNestedPath(), "/")
for i := len(parts); i > 0; i-- { for i := len(parts); i > 0; i-- {
part := parts[i-1] part := parts[i-1]
if len(part) == 0 { if len(part) == 0 {
@@ -748,13 +747,13 @@ func toFiles(s sbom.SBOM) (results []*spdx.File) {
comments = append(comments, layerIDCommentPrefix+c.FileSystemID) comments = append(comments, layerIDCommentPrefix+c.FileSystemID)
} }
// a file inside an archive is named by its full chain ("app.war:WEB-INF/lib/a.jar:META-INF/MANIFEST.MF"), // a file inside an archive is named by its full path ("/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"),
// so a consumer reading only the name does not mistake it for a file at that path in the scan // so a consumer reading only the name does not mistake it for a file at that path in the scan
// root. The archive path is also kept in the comment so decoding need not guess where it ends. // root. The entry chain after the real path is also kept in the comment so decoding need not guess
name := c.RealPath // where it starts.
if c.ArchivePath != "" { name := c.FullNestedPath()
name = archive.VirtualPath(file.NewLocationFromCoordinates(c)) if entries, ok := strings.CutPrefix(c.FullNestedPath(), c.RealPath+file.ArchivePathSeparator); ok {
comments = append(comments, archivePathCommentPrefix+c.ArchivePath) comments = append(comments, archivePathCommentPrefix+entries)
} }
relativePath, err := convertAbsoluteToRelative(name) relativePath, err := convertAbsoluteToRelative(name)
@@ -762,10 +762,7 @@ func Test_fileIDsForPackage(t *testing.T) {
Name: "bogus", Name: "bogus",
} }
c := file.Coordinates{ c := file.NewCoordinates("/path", "nowhere")
RealPath: "/path",
FileSystemID: "nowhere",
}
docElementId := func(identifiable artifact.Identifiable) spdx.DocElementID { docElementId := func(identifiable artifact.Identifiable) spdx.DocElementID {
return spdx.DocElementID{ return spdx.DocElementID{
@@ -1028,16 +1025,12 @@ func Test_toSPDXID(t *testing.T) {
}{ }{
{ {
name: "short filename", name: "short filename",
it: file.Coordinates{ it: file.NewCoordinates("/short/path/file.txt", ""),
RealPath: "/short/path/file.txt",
},
expected: "File-short-path-file.txt", expected: "File-short-path-file.txt",
}, },
{ {
name: "long filename", name: "long filename",
it: file.Coordinates{ it: file.NewCoordinates("/some/long/path/with/a/lot/of-text/that-contains-a/file.txt", ""),
RealPath: "/some/long/path/with/a/lot/of-text/that-contains-a/file.txt",
},
expected: "File-...a-lot-of-text-that-contains-a-file.txt", expected: "File-...a-lot-of-text-that-contains-a-file.txt",
}, },
{ {
@@ -1166,9 +1159,9 @@ func Test_otherLicenses(t *testing.T) {
func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) { func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) {
// two archives' META-INF/MANIFEST.MF must not come out as two files with the same name and nothing // two archives' META-INF/MANIFEST.MF must not come out as two files with the same name and nothing
// saying which archive each is in // saying which archive each is in
root := file.Coordinates{RealPath: "/etc/os-release", FileSystemID: "sha256:layer"} root := file.NewCoordinates("/etc/os-release", "sha256:layer")
inA := file.Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/app.war:WEB-INF/lib/a.jar", FileSystemID: "sha256:layer"} inA := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", FileSystemID: "sha256:layer"}
inB := file.Coordinates{RealPath: "odd:na%me.txt", ArchivePath: "/app.war:WEB-INF/lib/b.jar"} inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"}
s := sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{root: {}, inA: {}, inB: {}}}} s := sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{root: {}, inA: {}, inB: {}}}}
byName := map[string]*spdx.File{} byName := map[string]*spdx.File{}
@@ -1178,15 +1171,23 @@ func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) {
require.Len(t, byName, 3) require.Len(t, byName, 3)
assert.Equal(t, "layerID: sha256:layer", byName["etc/os-release"].FileComment, "files outside archives are unchanged") assert.Equal(t, "layerID: sha256:layer", byName["etc/os-release"].FileComment, "files outside archives are unchanged")
assert.Equal(t, file.NewCoordinates("etc/os-release", "sha256:layer"), toSyftCoordinates(byName["etc/os-release"]))
a := byName["app.war:WEB-INF/lib/a.jar:META-INF/MANIFEST.MF"] // the name is made relative like any other file's, so the decoded top path is too
require.NotNil(t, a) a := byName["app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"]
assert.Equal(t, "layerID: sha256:layer\narchivePath: /app.war:WEB-INF/lib/a.jar", a.FileComment) require.NotNil(t, a, "names: %v", byName)
assert.Equal(t, inA, toSyftCoordinates(a)) assert.Equal(t, "layerID: sha256:layer\narchivePath: WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", a.FileComment)
wantA := inA
wantA.RealPath = "app.war"
wantA.FullPath = "app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"
assert.Equal(t, wantA, toSyftCoordinates(a))
b := byName["app.war:WEB-INF/lib/b.jar:odd%3Ana%25me.txt"] b := byName["app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"]
require.NotNil(t, b, "names: %v", byName) require.NotNil(t, b, "names: %v", byName)
assert.Equal(t, inB, toSyftCoordinates(b), "the entry path's escaping is undone on decode") wantB := inB
wantB.RealPath = "app.war"
wantB.FullPath = "app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"
assert.Equal(t, wantB, toSyftCoordinates(b))
} }
func Test_toRootPackage_name(t *testing.T) { func Test_toRootPackage_name(t *testing.T) {
@@ -15,7 +15,6 @@ import (
"github.com/spdx/tools-golang/spdx/v2/common" "github.com/spdx/tools-golang/spdx/v2/common"
"github.com/anchore/packageurl-go" "github.com/anchore/packageurl-go"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/spdxlicense" "github.com/anchore/syft/internal/spdxlicense"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
@@ -531,28 +530,25 @@ func collectPackageFileRelationships(spdxIDMap map[string]any, doc *spdx.Documen
} }
func toSyftCoordinates(f *spdx.File) file.Coordinates { func toSyftCoordinates(f *spdx.File) file.Coordinates {
c := file.Coordinates{RealPath: f.FileName} c := file.NewCoordinates(f.FileName, "")
for _, line := range strings.Split(f.FileComment, "\n") { for _, line := range strings.Split(f.FileComment, "\n") {
if v, ok := strings.CutPrefix(line, layerIDCommentPrefix); ok { if v, ok := strings.CutPrefix(line, layerIDCommentPrefix); ok {
c.FileSystemID = v c.FileSystemID = v
} }
if v, ok := strings.CutPrefix(line, archivePathCommentPrefix); ok { if v, ok := strings.CutPrefix(line, archivePathCommentPrefix); ok {
c.ArchivePath = v // the name is the full path; the real path is what precedes the entry chain
c.RealPath = strings.TrimSuffix(f.FileName, file.ArchivePathSeparator+v)
c.FullPath = f.FileName
} }
} }
if v, ok := strings.CutPrefix(string(f.FileSPDXIdentifier), layerIDCommentPrefix); ok { if v, ok := strings.CutPrefix(string(f.FileSPDXIdentifier), layerIDCommentPrefix); ok {
c.FileSystemID = v c.FileSystemID = v
} }
if c.ArchivePath != "" {
// the name is the chain; the file's own path is its last, escaped, segment
c.RealPath = archive.UnescapeEntryPath(f.FileName[strings.LastIndexByte(f.FileName, ':')+1:])
}
return c return c
} }
func toSyftLocation(f *spdx.File) file.Location { func toSyftLocation(f *spdx.File) file.Location {
c := toSyftCoordinates(f) return file.NewLocationFromCoordinates(toSyftCoordinates(f))
return file.NewVirtualLocationFromCoordinates(c, c.RealPath)
} }
func requireAndTrimPrefix(val any, prefix string) string { func requireAndTrimPrefix(val any, prefix string) string {
@@ -344,10 +344,7 @@ func Test_toSyftRelationships(t *testing.T) {
} }
pkg3.SetID() pkg3.SetID()
loc1 := file.NewLocationFromCoordinates(file.Coordinates{ loc1 := file.NewLocationFromCoordinates(file.NewCoordinates("/somewhere/real", "abc"))
RealPath: "/somewhere/real",
FileSystemID: "abc",
})
tests := []struct { tests := []struct {
name string name string
@@ -694,10 +691,7 @@ func Test_directPackageFiles(t *testing.T) {
p.OverrideID("1") // the same as the spdxID on the package element p.OverrideID("1") // the same as the spdxID on the package element
f := file.Location{ f := file.Location{
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("some-file", ""),
RealPath: "some-file",
FileSystemID: "",
},
AccessPath: "some-file", AccessPath: "some-file",
}, },
LocationMetadata: file.LocationMetadata{ LocationMetadata: file.LocationMetadata{
@@ -401,10 +401,7 @@ func v3toSyftCoordinates(f spdx.AnyFile) file.Coordinates {
if strings.HasPrefix(f.GetID(), layerIDPrefix) { if strings.HasPrefix(f.GetID(), layerIDPrefix) {
fileSystemID = strings.TrimPrefix(f.GetID(), layerIDPrefix) fileSystemID = strings.TrimPrefix(f.GetID(), layerIDPrefix)
} }
return file.Coordinates{ return file.NewCoordinates(f.GetName(), fileSystemID)
RealPath: f.GetName(),
FileSystemID: fileSystemID,
}
} }
func v3toSyftLocation(f spdx.AnyFile) file.Location { func v3toSyftLocation(f spdx.AnyFile) file.Location {
@@ -193,14 +193,14 @@ func TestToSyftModelV3_documentConversion(t *testing.T) {
assert.Equal(t, "BSD-3-Clause AND GPL-2.0+", pkg2Concluded[0].Value) assert.Equal(t, "BSD-3-Clause AND GPL-2.0+", pkg2Concluded[0].Value)
// Verify files // Verify files
coords1 := file.Coordinates{RealPath: "/src/main.go"} coords1 := file.NewCoordinates("/src/main.go", "")
digests, ok := result.Artifacts.FileDigests[coords1] digests, ok := result.Artifacts.FileDigests[coords1]
require.True(t, ok, "file1 digests not found") require.True(t, ok, "file1 digests not found")
require.Len(t, digests, 1) require.Len(t, digests, 1)
assert.Equal(t, "sha256", digests[0].Algorithm) assert.Equal(t, "sha256", digests[0].Algorithm)
assert.Equal(t, "abc123", digests[0].Value) assert.Equal(t, "abc123", digests[0].Value)
coords2 := file.Coordinates{RealPath: "/src/util.go"} coords2 := file.NewCoordinates("/src/util.go", "")
_, ok = result.Artifacts.FileMetadata[coords2] _, ok = result.Artifacts.FileMetadata[coords2]
assert.True(t, ok, "file2 metadata not found") assert.True(t, ok, "file2 metadata not found")
+3 -3
View File
@@ -113,9 +113,9 @@ func toPath(s source.Description, p pkg.Package) string {
locations := p.Locations.ToSlice() locations := p.Locations.ToSlice()
if len(locations) > 0 { if len(locations) > 0 {
location := locations[0] location := locations[0]
packagePath := location.RealPath packagePath := location.AccessPath
if location.AccessPath != "" { if packagePath == "" {
packagePath = location.AccessPath packagePath = location.FullNestedPath()
} }
packagePath = strings.TrimPrefix(packagePath, "/") packagePath = strings.TrimPrefix(packagePath, "/")
switch metadata := s.Metadata.(type) { switch metadata := s.Metadata.(type) {
@@ -41,30 +41,21 @@ func sbomFixture() sbom.SBOM {
Name: "pkg-1", Name: "pkg-1",
Version: "1.0.1", Version: "1.0.1",
Locations: file.NewLocationSet( Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{ file.NewLocationFromCoordinates(file.NewCoordinates("/usr/lib", "fsid-1")),
RealPath: "/usr/lib",
FileSystemID: "fsid-1",
}),
), ),
}, },
{ {
Name: "pkg-2", Name: "pkg-2",
Version: "2.0.2", Version: "2.0.2",
Locations: file.NewLocationSet( Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{ file.NewLocationFromCoordinates(file.NewCoordinates("/usr/lib", "fsid-1")),
RealPath: "/usr/lib",
FileSystemID: "fsid-1",
}),
), ),
}, },
{ {
Name: "pkg-3", Name: "pkg-3",
Version: "3.0.3", Version: "3.0.3",
Locations: file.NewLocationSet( Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{ file.NewLocationFromCoordinates(file.NewCoordinates("/etc", "fsid-1")),
RealPath: "/etc",
FileSystemID: "fsid-1",
}),
), ),
}, },
} { } {
@@ -29,7 +29,7 @@ func Test_encodeComponentProperties(t *testing.T) {
input: pkg.Package{ input: pkg.Package{
FoundBy: "cataloger", FoundBy: "cataloger",
Locations: file.NewLocationSet( Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{RealPath: "test"}), file.NewLocationFromCoordinates(file.NewCoordinates("test", "")),
), ),
Metadata: pkg.ApkDBEntry{ Metadata: pkg.ApkDBEntry{
Package: "libc-utils", Package: "libc-utils",
@@ -98,7 +98,7 @@ func SourceInfo(p pkg.Package) string {
} }
var paths []string var paths []string
for _, l := range p.Locations.ToSlice() { for _, l := range p.Locations.ToSlice() {
paths = append(paths, l.RealPath) paths = append(paths, l.FullNestedPath())
} }
return answer + ": " + strings.Join(paths, ", ") return answer + ": " + strings.Join(paths, ", ")
@@ -20,7 +20,7 @@ func AddSampleFileRelationships(s *sbom.SBOM) {
for _, f := range files { for _, f := range files {
meta := file.Metadata{} meta := file.Metadata{}
coords := file.Coordinates{RealPath: f} coords := file.NewCoordinates(f, "")
s.Artifacts.FileMetadata[coords] = meta s.Artifacts.FileMetadata[coords] = meta
s.Relationships = append(s.Relationships, artifact.Relationship{ s.Relationships = append(s.Relationships, artifact.Relationship{
+1 -4
View File
@@ -123,10 +123,7 @@ func TestSPDX22JSONRequredProperties(t *testing.T) {
enc, err := NewFormatEncoderWithConfig(cfg) enc, err := NewFormatEncoderWithConfig(cfg)
require.NoError(t, err) require.NoError(t, err)
coords := file.Coordinates{ coords := file.NewCoordinates("/some/file", "ac897d978b6c38749a1")
RealPath: "/some/file",
FileSystemID: "ac897d978b6c38749a1",
}
p1 := pkg.Package{ p1 := pkg.Package{
Name: "files-analyzed-true", Name: "files-analyzed-true",
+2 -8
View File
@@ -137,10 +137,7 @@ func Test_encodeDecodeFileMetadata(t *testing.T) {
FoundBy: "something", FoundBy: "something",
Locations: file.NewLocationSet(file.Location{ Locations: file.NewLocationSet(file.Location{
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/somewhere", "id"),
RealPath: "/somewhere",
FileSystemID: "id",
},
}, },
LocationMetadata: file.LocationMetadata{ LocationMetadata: file.LocationMetadata{
Annotations: map[string]string{ Annotations: map[string]string{
@@ -174,10 +171,7 @@ func Test_encodeDecodeFileMetadata(t *testing.T) {
} }
p.SetID() p.SetID()
c := file.Coordinates{ c := file.NewCoordinates("some-file", "some-fs-id")
RealPath: "some-file",
FileSystemID: "some-fs-id",
}
s := sbom.SBOM{ s := sbom.SBOM{
Artifacts: sbom.Artifacts{ Artifacts: sbom.Artifacts{
+3 -7
View File
@@ -98,7 +98,7 @@ func TestEncode_sameFileInManyArchivesIsReproducible(t *testing.T) {
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
p := pkg.Package{Name: "shared", Version: "1.0"} p := pkg.Package{Name: "shared", Version: "1.0"}
for i := range 20 { for i := range 20 {
c := file.Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: fmt.Sprintf("/app.war:WEB-INF/lib/%02d.jar", i)} c := file.Coordinates{RealPath: "/app.war", FullPath: fmt.Sprintf("/app.war!/WEB-INF/lib/%02d.jar!/META-INF/MANIFEST.MF", i)}
s.Artifacts.FileMetadata[c] = file.Metadata{} s.Artifacts.FileMetadata[c] = file.Metadata{}
p.Locations.Add(file.NewLocationFromCoordinates(c)) p.Locations.Add(file.NewLocationFromCoordinates(c))
} }
@@ -168,9 +168,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
Name: "package-1", Name: "package-1",
Version: "1.0.1", Version: "1.0.1",
Locations: file.NewLocationSet( Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{ file.NewLocationFromCoordinates(file.NewCoordinates("/a/place/a", "")),
RealPath: "/a/place/a",
}),
), ),
Type: pkg.PythonPkg, Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1", FoundBy: "the-cataloger-1",
@@ -191,9 +189,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
Name: "package-2", Name: "package-2",
Version: "2.0.1", Version: "2.0.1",
Locations: file.NewLocationSet( Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{ file.NewLocationFromCoordinates(file.NewCoordinates("/b/place/b", "")),
RealPath: "/b/place/b",
}),
), ),
Type: pkg.DebPkg, Type: pkg.DebPkg,
FoundBy: "the-cataloger-2", FoundBy: "the-cataloger-2",
+4 -4
View File
@@ -172,17 +172,17 @@ func unpackPkgMetadata(p *Package, unpacker packageMetadataUnpacker) error {
switch unpacker.MetadataType { switch unpacker.MetadataType {
case "HackageMetadataType": case "HackageMetadataType":
for _, l := range p.Locations { for _, l := range p.Locations {
if strings.HasSuffix(l.RealPath, ".yaml.lock") { if strings.HasSuffix(l.FullNestedPath(), ".yaml.lock") {
ty = "haskell-hackage-stack-lock-entry" ty = "haskell-hackage-stack-lock-entry"
break break
} else if strings.HasSuffix(l.RealPath, ".yaml") { } else if strings.HasSuffix(l.FullNestedPath(), ".yaml") {
ty = "haskell-hackage-stack-entry" ty = "haskell-hackage-stack-entry"
break break
} }
} }
case "RpmMetadata": case "RpmMetadata":
for _, l := range p.Locations { for _, l := range p.Locations {
if strings.HasSuffix(l.RealPath, ".rpm") { if strings.HasSuffix(l.FullNestedPath(), ".rpm") {
ty = "rpm-archive" ty = "rpm-archive"
break break
} }
@@ -190,7 +190,7 @@ func unpackPkgMetadata(p *Package, unpacker packageMetadataUnpacker) error {
case "RustCargoPackageMetadata": case "RustCargoPackageMetadata":
var found bool var found bool
for _, l := range p.Locations { for _, l := range p.Locations {
if strings.HasSuffix(strings.ToLower(l.RealPath), "cargo.lock") { if strings.HasSuffix(strings.ToLower(l.FullNestedPath()), "cargo.lock") {
ty = "rust-cargo-lock-entry" ty = "rust-cargo-lock-entry"
found = true found = true
break break
+67 -268
View File
@@ -352,10 +352,7 @@ func Test_toFileType(t *testing.T) {
} }
func Test_toFileMetadataEntry(t *testing.T) { func Test_toFileMetadataEntry(t *testing.T) {
coords := file.Coordinates{ coords := file.NewCoordinates("/path", "x")
RealPath: "/path",
FileSystemID: "x",
}
tests := []struct { tests := []struct {
name string name string
metadata *file.Metadata metadata *file.Metadata
@@ -453,41 +450,17 @@ func Test_toPackageModel_layerOrdering(t *testing.T) {
Licenses: pkg.NewLicenseSet(pkg.License{ Licenses: pkg.NewLicenseSet(pkg.License{
Value: "MIT", Value: "MIT",
Locations: file.NewLocationSet( Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{ file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-3")),
RealPath: "/lic-a", file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-1")),
FileSystemID: "fsid-3", file.NewLocationFromCoordinates(file.NewCoordinates("/lic-b", "fsid-0")),
}), file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-2")),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/lic-b",
FileSystemID: "fsid-0",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-2",
}),
), ),
}), }),
Locations: file.NewLocationSet( Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{ file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-3")),
RealPath: "/a", file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")),
FileSystemID: "fsid-3", file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-0")),
}), file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-2")),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-0",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
}),
), ),
}, },
layerOrder: []string{ layerOrder: []string{
@@ -505,37 +478,25 @@ func Test_toPackageModel_layerOrdering(t *testing.T) {
Locations: []file.Location{ Locations: []file.Location{
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/lic-b", "fsid-0"), // important!
RealPath: "/lic-b",
FileSystemID: "fsid-0", // important!
},
AccessPath: "/lic-b", AccessPath: "/lic-b",
}, },
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/lic-a", "fsid-1"), // important!
RealPath: "/lic-a",
FileSystemID: "fsid-1", // important!
},
AccessPath: "/lic-a", AccessPath: "/lic-a",
}, },
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/lic-a", "fsid-2"), // important!
RealPath: "/lic-a",
FileSystemID: "fsid-2", // important!
},
AccessPath: "/lic-a", AccessPath: "/lic-a",
}, },
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/lic-a", "fsid-3"), // important!
RealPath: "/lic-a",
FileSystemID: "fsid-3", // important!
},
AccessPath: "/lic-a", AccessPath: "/lic-a",
}, },
}, },
@@ -545,37 +506,25 @@ func Test_toPackageModel_layerOrdering(t *testing.T) {
Locations: []file.Location{ Locations: []file.Location{
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/b", "fsid-0"), // important!
RealPath: "/b",
FileSystemID: "fsid-0", // important!
},
AccessPath: "/b", AccessPath: "/b",
}, },
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/a", "fsid-1"), // important!
RealPath: "/a",
FileSystemID: "fsid-1", // important!
},
AccessPath: "/a", AccessPath: "/a",
}, },
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/a", "fsid-2"), // important!
RealPath: "/a",
FileSystemID: "fsid-2", // important!
},
AccessPath: "/a", AccessPath: "/a",
}, },
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/a", "fsid-3"), // important!
RealPath: "/a",
FileSystemID: "fsid-3", // important!
},
AccessPath: "/a", AccessPath: "/a",
}, },
}, },
@@ -609,33 +558,21 @@ func Test_toLocationModel(t *testing.T) {
{ {
name: "nil layer order map", name: "nil layer order map",
locations: file.NewLocationSet( locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{ file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")),
RealPath: "/a", file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-2")),
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-2",
}),
), ),
layers: nil, // please don't panic! layers: nil, // please don't panic!
want: []file.Location{ want: []file.Location{
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
AccessPath: "/a", AccessPath: "/a",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/b", "fsid-2"),
RealPath: "/b",
FileSystemID: "fsid-2",
},
AccessPath: "/b", AccessPath: "/b",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
@@ -645,18 +582,9 @@ func Test_toLocationModel(t *testing.T) {
{ {
name: "go case", name: "go case",
locations: file.NewLocationSet( locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{ file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-3")),
RealPath: "/a", file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")),
FileSystemID: "fsid-3", file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-2")),
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
}),
), ),
layers: []string{ layers: []string{
"fsid-1", "fsid-1",
@@ -666,30 +594,21 @@ func Test_toLocationModel(t *testing.T) {
want: []file.Location{ want: []file.Location{
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/b", "fsid-1"),
RealPath: "/b",
FileSystemID: "fsid-1",
},
AccessPath: "/b", AccessPath: "/b",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/c", "fsid-2"),
RealPath: "/c",
FileSystemID: "fsid-2",
},
AccessPath: "/c", AccessPath: "/c",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/a", "fsid-3"),
RealPath: "/a",
FileSystemID: "fsid-3",
},
AccessPath: "/a", AccessPath: "/a",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
@@ -699,18 +618,9 @@ func Test_toLocationModel(t *testing.T) {
{ {
name: "same layer different paths", // prove we can sort by path irrespective of layer name: "same layer different paths", // prove we can sort by path irrespective of layer
locations: file.NewLocationSet( locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{ file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-1")),
RealPath: "/c", file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")),
FileSystemID: "fsid-1", file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")),
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
}),
), ),
layers: []string{ layers: []string{
"fsid-1", "fsid-1",
@@ -718,30 +628,21 @@ func Test_toLocationModel(t *testing.T) {
want: []file.Location{ want: []file.Location{
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
AccessPath: "/a", AccessPath: "/a",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/b", "fsid-1"),
RealPath: "/b",
FileSystemID: "fsid-1",
},
AccessPath: "/b", AccessPath: "/b",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/c", "fsid-1"),
RealPath: "/c",
FileSystemID: "fsid-1",
},
AccessPath: "/c", AccessPath: "/c",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
@@ -751,22 +652,10 @@ func Test_toLocationModel(t *testing.T) {
{ {
name: "mixed layers and paths", name: "mixed layers and paths",
locations: file.NewLocationSet( locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{ file.NewLocationFromCoordinates(file.NewCoordinates("/z", "fsid-3")),
RealPath: "/z", file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-2")),
FileSystemID: "fsid-3", file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")),
}), file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-2")),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
}),
), ),
layers: []string{ layers: []string{
"fsid-1", "fsid-1",
@@ -776,40 +665,28 @@ func Test_toLocationModel(t *testing.T) {
want: []file.Location{ want: []file.Location{
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/b", "fsid-1"),
RealPath: "/b",
FileSystemID: "fsid-1",
},
AccessPath: "/b", AccessPath: "/b",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/a", "fsid-2"),
RealPath: "/a",
FileSystemID: "fsid-2",
},
AccessPath: "/a", AccessPath: "/a",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/c", "fsid-2"),
RealPath: "/c",
FileSystemID: "fsid-2",
},
AccessPath: "/c", AccessPath: "/c",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
}, },
{ {
LocationData: file.LocationData{ LocationData: file.LocationData{
Coordinates: file.Coordinates{ Coordinates: file.NewCoordinates("/z", "fsid-3"),
RealPath: "/z",
FileSystemID: "fsid-3",
},
AccessPath: "/z", AccessPath: "/z",
}, },
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}}, LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
@@ -846,34 +723,22 @@ func Test_sortFiles(t *testing.T) {
files: []model.File{ files: []model.File{
{ {
ID: "file-1", ID: "file-1",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-2", ID: "file-2",
Location: file.Coordinates{ Location: file.NewCoordinates("/b", "fsid-2"),
RealPath: "/b",
FileSystemID: "fsid-2",
},
}, },
}, },
layers: nil, layers: nil,
want: []model.File{ want: []model.File{
{ {
ID: "file-1", ID: "file-1",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-2", ID: "file-2",
Location: file.Coordinates{ Location: file.NewCoordinates("/b", "fsid-2"),
RealPath: "/b",
FileSystemID: "fsid-2",
},
}, },
}, },
}, },
@@ -882,24 +747,15 @@ func Test_sortFiles(t *testing.T) {
files: []model.File{ files: []model.File{
{ {
ID: "file-1", ID: "file-1",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-3"),
RealPath: "/a",
FileSystemID: "fsid-3",
},
}, },
{ {
ID: "file-2", ID: "file-2",
Location: file.Coordinates{ Location: file.NewCoordinates("/b", "fsid-1"),
RealPath: "/b",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-3", ID: "file-3",
Location: file.Coordinates{ Location: file.NewCoordinates("/c", "fsid-2"),
RealPath: "/c",
FileSystemID: "fsid-2",
},
}, },
}, },
layers: []string{ layers: []string{
@@ -910,24 +766,15 @@ func Test_sortFiles(t *testing.T) {
want: []model.File{ want: []model.File{
{ {
ID: "file-2", ID: "file-2",
Location: file.Coordinates{ Location: file.NewCoordinates("/b", "fsid-1"),
RealPath: "/b",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-3", ID: "file-3",
Location: file.Coordinates{ Location: file.NewCoordinates("/c", "fsid-2"),
RealPath: "/c",
FileSystemID: "fsid-2",
},
}, },
{ {
ID: "file-1", ID: "file-1",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-3"),
RealPath: "/a",
FileSystemID: "fsid-3",
},
}, },
}, },
}, },
@@ -936,24 +783,15 @@ func Test_sortFiles(t *testing.T) {
files: []model.File{ files: []model.File{
{ {
ID: "file-1", ID: "file-1",
Location: file.Coordinates{ Location: file.NewCoordinates("/c", "fsid-1"),
RealPath: "/c",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-2", ID: "file-2",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-3", ID: "file-3",
Location: file.Coordinates{ Location: file.NewCoordinates("/b", "fsid-1"),
RealPath: "/b",
FileSystemID: "fsid-1",
},
}, },
}, },
layers: []string{ layers: []string{
@@ -962,24 +800,15 @@ func Test_sortFiles(t *testing.T) {
want: []model.File{ want: []model.File{
{ {
ID: "file-2", ID: "file-2",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-3", ID: "file-3",
Location: file.Coordinates{ Location: file.NewCoordinates("/b", "fsid-1"),
RealPath: "/b",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-1", ID: "file-1",
Location: file.Coordinates{ Location: file.NewCoordinates("/c", "fsid-1"),
RealPath: "/c",
FileSystemID: "fsid-1",
},
}, },
}, },
}, },
@@ -988,24 +817,15 @@ func Test_sortFiles(t *testing.T) {
files: []model.File{ files: []model.File{
{ {
ID: "file-1", ID: "file-1",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-2", ID: "file-2",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-3", ID: "file-3",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
}, },
}, },
layers: []string{ layers: []string{
@@ -1014,24 +834,15 @@ func Test_sortFiles(t *testing.T) {
want: []model.File{ want: []model.File{
{ {
ID: "file-1", ID: "file-1",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-2", ID: "file-2",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
}, },
{ {
ID: "file-3", ID: "file-3",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-1"),
RealPath: "/a",
FileSystemID: "fsid-1",
},
}, },
}, },
}, },
@@ -1040,10 +851,7 @@ func Test_sortFiles(t *testing.T) {
files: []model.File{ files: []model.File{
{ {
ID: "file-1", ID: "file-1",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-2"),
RealPath: "/a",
FileSystemID: "fsid-2",
},
Metadata: &model.FileMetadataEntry{ Metadata: &model.FileMetadataEntry{
Mode: 0644, Mode: 0644,
Type: "file", Type: "file",
@@ -1062,10 +870,7 @@ func Test_sortFiles(t *testing.T) {
}, },
{ {
ID: "file-2", ID: "file-2",
Location: file.Coordinates{ Location: file.NewCoordinates("/b", "fsid-1"),
RealPath: "/b",
FileSystemID: "fsid-1",
},
Metadata: &model.FileMetadataEntry{ Metadata: &model.FileMetadataEntry{
Mode: 0755, Mode: 0755,
Type: "directory", Type: "directory",
@@ -1083,10 +888,7 @@ func Test_sortFiles(t *testing.T) {
want: []model.File{ want: []model.File{
{ {
ID: "file-2", ID: "file-2",
Location: file.Coordinates{ Location: file.NewCoordinates("/b", "fsid-1"),
RealPath: "/b",
FileSystemID: "fsid-1",
},
Metadata: &model.FileMetadataEntry{ Metadata: &model.FileMetadataEntry{
Mode: 0755, Mode: 0755,
Type: "directory", Type: "directory",
@@ -1098,10 +900,7 @@ func Test_sortFiles(t *testing.T) {
}, },
{ {
ID: "file-1", ID: "file-1",
Location: file.Coordinates{ Location: file.NewCoordinates("/a", "fsid-2"),
RealPath: "/a",
FileSystemID: "fsid-2",
},
Metadata: &model.FileMetadataEntry{ Metadata: &model.FileMetadataEntry{
Mode: 0644, Mode: 0644,
Type: "file", Type: "file",
+2 -2
View File
@@ -90,11 +90,11 @@ func toSyftFiles(files []model.File) sbom.Artifacts {
ret.FileMetadata[coord] = file.Metadata{ ret.FileMetadata[coord] = file.Metadata{
FileInfo: stereoscopeFile.ManualInfo{ FileInfo: stereoscopeFile.ManualInfo{
NameValue: path.Base(coord.RealPath), NameValue: path.Base(coord.FullNestedPath()),
SizeValue: f.Metadata.Size, SizeValue: f.Metadata.Size,
ModeValue: fm, ModeValue: fm,
}, },
Path: coord.RealPath, Path: coord.FullNestedPath(),
LinkDestination: f.Metadata.LinkDestination, LinkDestination: f.Metadata.LinkDestination,
UserID: f.Metadata.UserID, UserID: f.Metadata.UserID,
GroupID: f.Metadata.GroupID, GroupID: f.Metadata.GroupID,
+1 -4
View File
@@ -295,10 +295,7 @@ func Test_idsHaveChanged(t *testing.T) {
} }
func Test_toSyftFiles(t *testing.T) { func Test_toSyftFiles(t *testing.T) {
coord := file.Coordinates{ coord := file.NewCoordinates("/somerwhere/place", "abc")
RealPath: "/somerwhere/place",
FileSystemID: "abc",
}
tests := []struct { tests := []struct {
name string name string
@@ -197,7 +197,7 @@ func (r *ContainerImageAllLayers) RelativeFileByPath(location file.Location, pat
func (r *ContainerImageAllLayers) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { func (r *ContainerImageAllLayers) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
entry, err := r.img.FileCatalog.Get(location.Reference()) entry, err := r.img.FileCatalog.Get(location.Reference())
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err) return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.FullNestedPath(), err)
} }
switch entry.Type { switch entry.Type {
@@ -1104,15 +1104,9 @@ func TestContainerImageDeepSquash_MergeLocationStreams_FunCases(t *testing.T) {
} }
func makeLocation(path string, layer int) file.Location { func makeLocation(path string, layer int) file.Location {
return file.NewLocationFromCoordinates(file.Coordinates{ return file.NewLocationFromCoordinates(file.NewCoordinates(path, fmt.Sprintf("layer-%d", layer)))
RealPath: path,
FileSystemID: fmt.Sprintf("layer-%d", layer),
})
} }
func makeVirtualLocation(path, access string, layer int) file.Location { func makeVirtualLocation(path, access string, layer int) file.Location {
return file.NewVirtualLocationFromCoordinates(file.Coordinates{ return file.NewVirtualLocationFromCoordinates(file.NewCoordinates(path, fmt.Sprintf("layer-%d", layer)), access)
RealPath: path,
FileSystemID: fmt.Sprintf("layer-%d", layer),
}, access)
} }
@@ -148,7 +148,7 @@ func (r *ContainerImageSquash) RelativeFileByPath(_ file.Location, path string)
func (r *ContainerImageSquash) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { func (r *ContainerImageSquash) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
entry, err := r.img.FileCatalog.Get(location.Reference()) entry, err := r.img.FileCatalog.Get(location.Reference())
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err) return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.FullNestedPath(), err)
} }
switch entry.Type { switch entry.Type {
+3 -3
View File
@@ -28,14 +28,14 @@ func NewExcludingDecorator(delegate file.Resolver, excludeFn excludeFn) file.Res
func (r *excluding) FileContentsByLocation(location file.Location) (io.ReadCloser, error) { func (r *excluding) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
if locationMatches(&location, r.excludeFn) { if locationMatches(&location, r.excludeFn) {
return nil, fmt.Errorf("no such location: %+v", location.RealPath) return nil, fmt.Errorf("no such location: %+v", location.FullNestedPath())
} }
return r.delegate.FileContentsByLocation(location) return r.delegate.FileContentsByLocation(location)
} }
func (r *excluding) FileMetadataByLocation(location file.Location) (file.Metadata, error) { func (r *excluding) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
if locationMatches(&location, r.excludeFn) { if locationMatches(&location, r.excludeFn) {
return file.Metadata{}, fmt.Errorf("no such location: %+v", location.RealPath) return file.Metadata{}, fmt.Errorf("no such location: %+v", location.FullNestedPath())
} }
return r.delegate.FileMetadataByLocation(location) return r.delegate.FileMetadataByLocation(location)
} }
@@ -89,7 +89,7 @@ func (r *excluding) AllLocations(ctx context.Context) <-chan file.Location {
} }
func locationMatches(location *file.Location, exclusionFn excludeFn) bool { func locationMatches(location *file.Location, exclusionFn excludeFn) bool {
return exclusionFn(location.RealPath) || exclusionFn(location.AccessPath) return exclusionFn(location.LocalPath()) || exclusionFn(location.AccessPath)
} }
func filterLocations(locations []file.Location, err error, exclusionFn excludeFn) ([]file.Location, error) { func filterLocations(locations []file.Location, err error, exclusionFn excludeFn) ([]file.Location, error) {
@@ -89,7 +89,7 @@ func (u UnindexedDirectory) FileContentsByLocation(location file.Location) (io.R
return nil, err return nil, err
} }
if fi.IsDir() { if fi.IsDir() {
return nil, fmt.Errorf("unable to get contents of directory: %s", location.RealPath) return nil, fmt.Errorf("unable to get contents of directory: %s", location.FullNestedPath())
} }
return f, nil return f, nil
} }
+3 -3
View File
@@ -87,20 +87,20 @@ func IdentifyRelease(resolver file.Resolver) *Release {
func tryParseReleaseInfo(resolver file.Resolver, location file.Location, logger logger.Logger, entry parseEntry) *Release { func tryParseReleaseInfo(resolver file.Resolver, location file.Location, logger logger.Logger, entry parseEntry) *Release {
contentReader, err := resolver.FileContentsByLocation(location) contentReader, err := resolver.FileContentsByLocation(location)
if err != nil { if err != nil {
logger.WithFields("error", err, "path", location.RealPath).Trace("unable to get contents") logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to get contents")
return nil return nil
} }
defer internal.CloseAndLogError(contentReader, location.AccessPath) defer internal.CloseAndLogError(contentReader, location.AccessPath)
content, err := io.ReadAll(io.LimitReader(contentReader, 5*1024*1024)) content, err := io.ReadAll(io.LimitReader(contentReader, 5*1024*1024))
if err != nil { if err != nil {
logger.WithFields("error", err, "path", location.RealPath).Trace("unable to read contents") logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to read contents")
return nil return nil
} }
release, err := entry.fn(string(content)) release, err := entry.fn(string(content))
if err != nil { if err != nil {
logger.WithFields("error", err, "path", location.RealPath).Trace("unable to parse contents") logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to parse contents")
return nil return nil
} }
+3 -3
View File
@@ -35,13 +35,13 @@ func supplementDebianVersion(resolver file.Resolver, release *Release) {
func readDebianVersionFile(resolver file.Resolver, location file.Location) string { func readDebianVersionFile(resolver file.Resolver, location file.Location) string {
rdr, err := resolver.FileContentsByLocation(location) rdr, err := resolver.FileContentsByLocation(location)
if err != nil { if err != nil {
log.Debugf("error getting contents for %s: %v", location.RealPath, err) log.Debugf("error getting contents for %s: %v", location.FullNestedPath(), err)
return "" return ""
} }
defer internal.CloseAndLogError(rdr, location.RealPath) defer internal.CloseAndLogError(rdr, location.FullNestedPath())
contents, err := io.ReadAll(io.LimitReader(rdr, 5*1024*1024)) contents, err := io.ReadAll(io.LimitReader(rdr, 5*1024*1024))
if err != nil { if err != nil {
log.Debugf("error reading %s: %v", location.RealPath, err) log.Debugf("error reading %s: %v", location.FullNestedPath(), err)
return "" return ""
} }
version := strings.TrimSpace(string(contents)) version := strings.TrimSpace(string(contents))
+4 -4
View File
@@ -32,8 +32,8 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver,
// seen wins below, so sort for a stable result across runs. For OCI // seen wins below, so sort for a stable result across runs. For OCI
// artifacts every RealPath is "/", so in practice this is digest order. // artifacts every RealPath is "/", so in practice this is digest order.
sort.Slice(modelFileLocs, func(i, j int) bool { sort.Slice(modelFileLocs, func(i, j int) bool {
if modelFileLocs[i].RealPath != modelFileLocs[j].RealPath { if modelFileLocs[i].FullNestedPath() != modelFileLocs[j].FullNestedPath() {
return modelFileLocs[i].RealPath < modelFileLocs[j].RealPath return modelFileLocs[i].FullNestedPath() < modelFileLocs[j].FullNestedPath()
} }
return modelFileLocs[i].FileSystemID < modelFileLocs[j].FileSystemID return modelFileLocs[i].FileSystemID < modelFileLocs[j].FileSystemID
}) })
@@ -128,7 +128,7 @@ func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfCo
if err != nil { if err != nil {
return nil, nil return nil, nil
} }
defer internal.CloseAndLogError(rc, loc.RealPath) defer internal.CloseAndLogError(rc, loc.FullNestedPath())
buf, err := readBounded(rc, maxModelFileLayerSize) buf, err := readBounded(rc, maxModelFileLayerSize)
if err != nil { if err != nil {
@@ -174,7 +174,7 @@ func readLicenseSPDXIDFromFrontmatter(resolver file.Resolver, loc file.Location)
if err != nil { if err != nil {
return "" return ""
} }
defer internal.CloseAndLogError(rc, loc.RealPath) defer internal.CloseAndLogError(rc, loc.FullNestedPath())
buf, err := readPrefix(rc, maxLicensePrefixSize) buf, err := readPrefix(rc, maxLicensePrefixSize)
if err != nil { if err != nil {
+1 -1
View File
@@ -23,7 +23,7 @@ import (
// parseGGUFModel parses a GGUF model file and returns the discovered package. // parseGGUFModel parses a GGUF model file and returns the discovered package.
// This implementation only reads the header portion of the file, not the entire model. // This implementation only reads the header portion of the file, not the entire model.
func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
defer internal.CloseAndLogError(reader, reader.Path()) defer internal.CloseAndLogError(reader, reader.FullNestedPath())
td := tmpdir.FromContext(ctx) td := tmpdir.FromContext(ctx)
if td == nil { if td == nil {
@@ -18,7 +18,7 @@ import (
// resolution, sibling enrichment, and cross-shard rollup are all handled by // resolution, sibling enrichment, and cross-shard rollup are all handled by
// safeTensorsMergeProcessor. // safeTensorsMergeProcessor.
func parseSafeTensorsFile(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { func parseSafeTensorsFile(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
defer internal.CloseAndLogError(reader, reader.Path()) defer internal.CloseAndLogError(reader, reader.FullNestedPath())
header, err := readSafeTensorsHeader(&io.LimitedReader{R: reader, N: maxSafeTensorsHeaderSize + 8}) header, err := readSafeTensorsHeader(&io.LimitedReader{R: reader, N: maxSafeTensorsHeaderSize + 8})
if err != nil { if err != nil {
@@ -47,7 +47,7 @@ type dockerAIModelConfig struct {
// parseSafeTensorsOCIConfig decodes the Docker AI model-config blob // parseSafeTensorsOCIConfig decodes the Docker AI model-config blob
func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
defer internal.CloseAndLogError(reader, reader.Path()) defer internal.CloseAndLogError(reader, reader.FullNestedPath())
body, err := readBounded(reader, maxModelConfigBlobSize) body, err := readBounded(reader, maxModelConfigBlobSize)
if err != nil { if err != nil {
@@ -86,7 +86,7 @@ func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.En
// parseSafeTensorsOCILayer decodes the JSON header of a SafeTensors weight // parseSafeTensorsOCILayer decodes the JSON header of a SafeTensors weight
// layer fetched from an OCI model artifact // layer fetched from an OCI model artifact
func parseSafeTensorsOCILayer(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { func parseSafeTensorsOCILayer(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
defer internal.CloseAndLogError(reader, reader.Path()) defer internal.CloseAndLogError(reader, reader.FullNestedPath())
header, err := readSafeTensorsHeader(&io.LimitedReader{R: reader, N: maxSafeTensorsHeaderSize + 8}) header, err := readSafeTensorsHeader(&io.LimitedReader{R: reader, N: maxSafeTensorsHeaderSize + 8})
if err != nil { if err != nil {
+2 -2
View File
@@ -56,7 +56,7 @@ func partitionSafeTensorsPackages(pkgs []pkg.Package) (safeTensors, other []pkg.
// single scan is one source, so the first package is representative of the rest. // single scan is one source, so the first package is representative of the rest.
func fromOCIArtifact(pkgs []pkg.Package) bool { func fromOCIArtifact(pkgs []pkg.Package) bool {
loc := primaryEvidenceLocation(pkgs[0]) loc := primaryEvidenceLocation(pkgs[0])
return loc != nil && loc.RealPath == "/" return loc != nil && loc.LocalPath() == "/"
} }
// mergeOCIModel treats the whole OCI artifact as a single model: every layer // mergeOCIModel treats the whole OCI artifact as a single model: every layer
@@ -120,7 +120,7 @@ func groupByParentDir(pkgs []pkg.Package) map[string][]pkg.Package {
if loc == nil { if loc == nil {
continue continue
} }
dir := path.Dir(loc.RealPath) dir := path.Dir(loc.LocalPath())
out[dir] = append(out[dir], p) out[dir] = append(out[dir], p)
} }
return out return out
+3 -3
View File
@@ -126,7 +126,7 @@ func parseApkDB(ctx context.Context, resolver file.Resolver, env *generic.Enviro
// This should get fixed with https://gitlab.alpinelinux.org/alpine/apk-tools/-/issues/10875 // This should get fixed with https://gitlab.alpinelinux.org/alpine/apk-tools/-/issues/10875
if r == nil { if r == nil {
// find the repositories file from the relative directory of the DB file // find the repositories file from the relative directory of the DB file
releases := findReleases(resolver, reader.RealPath) releases := findReleases(resolver, reader.LocalPath())
if len(releases) > 0 { if len(releases) > 0 {
r = &releases[0] r = &releases[0]
@@ -163,7 +163,7 @@ func findReleases(resolver file.Resolver, dbPath string) []linux.Release {
log.Tracef("unable to fetch contents for APK repositories file %q: %+v", reposLocation, err) log.Tracef("unable to fetch contents for APK repositories file %q: %+v", reposLocation, err)
return nil return nil
} }
defer internal.CloseAndLogError(reposReader, location.RealPath) defer internal.CloseAndLogError(reposReader, location.FullNestedPath())
return parseReleasesFromAPKRepository(file.LocationReadCloser{ return parseReleasesFromAPKRepository(file.LocationReadCloser{
Location: location, Location: location,
@@ -176,7 +176,7 @@ func parseReleasesFromAPKRepository(reader file.LocationReadCloser) []linux.Rele
reposB, err := io.ReadAll(reader) //nolint:gocritic // regex matching requires full buffer reposB, err := io.ReadAll(reader) //nolint:gocritic // regex matching requires full buffer
if err != nil { if err != nil {
log.Tracef("unable to read APK repositories file %q: %+v", reader.RealPath, err) log.Tracef("unable to read APK repositories file %q: %+v", reader.FullNestedPath(), err)
return nil return nil
} }
+3 -3
View File
@@ -55,7 +55,7 @@ func parseAlpmDB(ctx context.Context, resolver file.Resolver, env *generic.Envir
return nil, nil, nil return nil, nil, nil
} }
base := path.Dir(reader.RealPath) base := path.Dir(reader.LocalPath())
var locs []file.Location var locs []file.Location
@@ -102,7 +102,7 @@ func fetchPkgFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord, f
if err != nil { if err != nil {
return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err)) return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err))
} }
defer internal.CloseAndLogError(reader, loc.RealPath) defer internal.CloseAndLogError(reader, loc.FullNestedPath())
pkgFiles, err := parseMtree(reader) pkgFiles, err := parseMtree(reader)
if err != nil { if err != nil {
@@ -126,7 +126,7 @@ func fetchBackupFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord
if err != nil { if err != nil {
return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err)) return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err))
} }
defer internal.CloseAndLogError(reader, loc.RealPath) defer internal.CloseAndLogError(reader, loc.FullNestedPath())
filesMetadata, err := parseAlpmDBEntry(reader) filesMetadata, err := parseAlpmDBEntry(reader)
if err != nil { if err != nil {
@@ -3410,9 +3410,7 @@ func match(classifier string, paths ...string) pkg.ClassifierMatch {
return pkg.ClassifierMatch{ return pkg.ClassifierMatch{
Classifier: classifier, Classifier: classifier,
Location: file.NewVirtualLocationFromCoordinates( Location: file.NewVirtualLocationFromCoordinates(
file.Coordinates{ file.NewCoordinates(realPath, ""),
RealPath: realPath,
},
virtualPath, virtualPath,
), ),
} }
@@ -99,7 +99,7 @@ func (c *elfPackageCataloger) Catalog(ctx context.Context, resolver file.Resolve
func parseElfPackageNotes(resolver file.Resolver, location file.Location, c *elfPackageCataloger) (*elfBinaryPackageNotes, elfPackageKey, error) { func parseElfPackageNotes(resolver file.Resolver, location file.Location, c *elfPackageCataloger) (*elfBinaryPackageNotes, elfPackageKey, error) {
reader, err := resolver.FileContentsByLocation(location) reader, err := resolver.FileContentsByLocation(location)
if err != nil { if err != nil {
return nil, elfPackageKey{}, fmt.Errorf("unable to get binary contents %q: %w", location.Path(), err) return nil, elfPackageKey{}, fmt.Errorf("unable to get binary contents %q: %w", location.FullNestedPath(), err)
} }
defer internal.CloseAndLogError(reader, location.AccessPath) defer internal.CloseAndLogError(reader, location.AccessPath)
@@ -109,7 +109,7 @@ func parseElfPackageNotes(resolver file.Resolver, location file.Location, c *elf
}) })
if err != nil { if err != nil {
log.WithFields("file", location.Path(), "error", err).Trace("unable to parse ELF notes") log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to parse ELF notes")
return nil, elfPackageKey{}, err return nil, elfPackageKey{}, err
} }
@@ -148,7 +148,7 @@ func getELFNotes(r file.LocationReadCloser) (*elfBinaryPackageNotes, error) {
f, err := elfutil.NewFile(unionReader) f, err := elfutil.NewFile(unionReader)
if f == nil || err != nil { if f == nil || err != nil {
log.WithFields("file", r.Location.Path(), "error", err).Trace("unable to parse binary as ELF") log.WithFields("file", r.Location.FullNestedPath(), "error", err).Trace("unable to parse binary as ELF")
return nil, nil return nil, nil
} }
@@ -182,7 +182,7 @@ func getELFNotes(r file.LocationReadCloser) (*elfBinaryPackageNotes, error) {
if metadata, err = unmarshalELFPackageNotesPayload(newPayload); err == nil { if metadata, err = unmarshalELFPackageNotesPayload(newPayload); err == nil {
return metadata, nil return metadata, nil
} }
log.WithFields("file", r.Location.Path(), "error", err).Trace("unable to unmarshal ELF package notes as JSON") log.WithFields("file", r.Location.FullNestedPath(), "error", err).Trace("unable to unmarshal ELF package notes as JSON")
} }
} }
+1 -1
View File
@@ -22,7 +22,7 @@ func newPEPackage(versionResources map[string]string, f file.Location) pkg.Packa
if name == "" { if name == "" {
// it's possible that the version resources are empty, so we fall back to the file name // it's possible that the version resources are empty, so we fall back to the file name
name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.RealPath), ".exe"), ".dll") name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.LocalPath()), ".exe"), ".dll")
} }
p := pkg.Package{ p := pkg.Package{
@@ -22,7 +22,7 @@ func NewPEPackageCataloger() pkg.Cataloger {
func parsePE(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { func parsePE(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
f, err := pe.Read(reader) f, err := pe.Read(reader)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("failed to parse PE file %q: %w", reader.RealPath, err) return nil, nil, fmt.Errorf("failed to parse PE file %q: %w", reader.FullNestedPath(), err)
} }
if f == nil { if f == nil {
+6 -6
View File
@@ -36,13 +36,13 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
} }
if s == nil { if s == nil {
log.WithFields("path", reader.RealPath).Trace("file is not an SBOM") log.WithFields("path", reader.FullNestedPath()).Trace("file is not an SBOM")
return nil, nil, nil return nil, nil, nil
} }
// Bitnami exclusively uses SPDX JSON SBOMs // Bitnami exclusively uses SPDX JSON SBOMs
if sFormat != "spdx-json" { if sFormat != "spdx-json" {
log.WithFields("path", reader.RealPath).Trace("file is not an SPDX JSON SBOM") log.WithFields("path", reader.FullNestedPath()).Trace("file is not an SPDX JSON SBOM")
return nil, nil, nil return nil, nil, nil
} }
@@ -73,7 +73,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
// Bitnami packages reported in a SPDX file are shipped under the same directory // Bitnami packages reported in a SPDX file are shipped under the same directory
// as the SPDX file itself. // as the SPDX file itself.
metadata.Path = path.Dir(reader.RealPath) metadata.Path = path.Dir(reader.LocalPath())
if p.ID() != mainPkgID { if p.ID() != mainPkgID {
metadata.Files = packageFiles(s.Relationships, p, metadata.Path) metadata.Files = packageFiles(s.Relationships, p, metadata.Path)
secondaryPkgsFiles = append(secondaryPkgsFiles, metadata.Files...) secondaryPkgsFiles = append(secondaryPkgsFiles, metadata.Files...)
@@ -89,12 +89,12 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
} }
// Resolve all files owned by the main package in the SBOM and update the metadata // Resolve all files owned by the main package in the SBOM and update the metadata
if mainPkgFiles, err := mainPkgFiles(resolver, reader.RealPath, secondaryPkgsFiles); err == nil { if mainPkgFiles, err := mainPkgFiles(resolver, reader.LocalPath(), secondaryPkgsFiles); err == nil {
for i, p := range pkgs { for i, p := range pkgs {
if p.ID() == mainPkgID { if p.ID() == mainPkgID {
metadata, ok := p.Metadata.(*pkg.BitnamiSBOMEntry) metadata, ok := p.Metadata.(*pkg.BitnamiSBOMEntry)
if !ok { if !ok {
log.WithFields("spdx-filepath", reader.RealPath).Trace("main package in SBOM does not have Bitnami metadata") log.WithFields("spdx-filepath", reader.FullNestedPath()).Trace("main package in SBOM does not have Bitnami metadata")
continue continue
} }
@@ -103,7 +103,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
} }
} }
} else { } else {
log.WithFields("spdx-filepath", reader.RealPath, "error", err).Trace("unable to resolve owned files for main package in SBOM") log.WithFields("spdx-filepath", reader.FullNestedPath(), "error", err).Trace("unable to resolve owned files for main package in SBOM")
} }
return pkgs, filterRelationships(s.Relationships, pkgs), nil return pkgs, filterRelationships(s.Relationships, pkgs), nil
+1 -1
View File
@@ -61,7 +61,7 @@ func packageFiles(relationships []artifact.Relationship, p pkg.Package, baseDire
// note: the file.Location is from the SBOM, and all files within the Bitnami SBOM by convention // note: the file.Location is from the SBOM, and all files within the Bitnami SBOM by convention
// are relative to the /opt/bitnami/PRODUCT directory, so we need to prepend the base directory // are relative to the /opt/bitnami/PRODUCT directory, so we need to prepend the base directory
// so that it's relative to the path found within the container image. // so that it's relative to the path found within the container image.
result = append(result, path.Join(baseDirectory, value.RealPath)) result = append(result, path.Join(baseDirectory, value.LocalPath()))
} }
} }
} }
@@ -67,7 +67,7 @@ func parseComponentsJSON(_ context.Context, _ file.Resolver, _ *generic.Environm
Revision: revision, Revision: revision,
Architecture: entry.Arch, Architecture: entry.Arch,
Distro: entry.Distro, Distro: entry.Distro,
Path: path.Join(path.Dir(reader.RealPath), name), Path: path.Join(path.Dir(reader.LocalPath()), name),
} }
p := pkg.Package{ p := pkg.Package{
+1 -1
View File
@@ -24,7 +24,7 @@ func parseCondaMetaJSON(ctx context.Context, _ file.Resolver, _ *generic.Environ
dec := json.NewDecoder(reader) dec := json.NewDecoder(reader)
var meta pkg.CondaMetaPackage var meta pkg.CondaMetaPackage
if err := dec.Decode(&meta); err != nil { if err := dec.Decode(&meta); err != nil {
return nil, nil, fmt.Errorf("failed to parse conda-meta package file at %s: %w", reader.RealPath, err) return nil, nil, fmt.Errorf("failed to parse conda-meta package file at %s: %w", reader.FullNestedPath(), err)
} }
p := pkg.Package{ p := pkg.Package{
+1 -4
View File
@@ -81,10 +81,7 @@ func parseFullRequiresLine(line string, reader file.LocationReadCloser, pkgs *[]
// - recipe revision (recipe_hash) // - recipe revision (recipe_hash)
func parseConaninfo(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { func parseConaninfo(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
// First set the base package info by checking the relative path // First set the base package info by checking the relative path
fullFilePath := string(reader.Location.LocationData.Reference().RealPath) fullFilePath := reader.LocalPath()
if len(fullFilePath) == 0 {
fullFilePath = reader.RealPath
}
mainMetadata, err := parseConanMetadataFromFilePath(fullFilePath) mainMetadata, err := parseConanMetadataFromFilePath(fullFilePath)
if err != nil { if err != nil {
@@ -132,7 +132,7 @@ func findManAtLoc(loc file.Location, resolver file.Resolver) (*vcpkg.Vcpkg, erro
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer internal.CloseAndLogError(manCont, loc.RealPath) defer internal.CloseAndLogError(manCont, loc.FullNestedPath())
manBytes, err := io.ReadAll(io.LimitReader(manCont, maxVcpkgManifestSize)) manBytes, err := io.ReadAll(io.LimitReader(manCont, maxVcpkgManifestSize))
if err != nil { if err != nil {
return nil, err return nil, err
@@ -166,7 +166,7 @@ func findVcpkgConfig(resolver file.Resolver) (*vcpkg.Config, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer internal.CloseAndLogError(cfgCont, locs[0].RealPath) defer internal.CloseAndLogError(cfgCont, locs[0].FullNestedPath())
cfgBytes, err := io.ReadAll(io.LimitReader(cfgCont, maxVcpkgManifestSize)) cfgBytes, err := io.ReadAll(io.LimitReader(cfgCont, maxVcpkgManifestSize))
if err != nil { if err != nil {
return nil, err return nil, err
+4 -4
View File
@@ -250,7 +250,7 @@ func getAdditionalFileListing(resolver file.Resolver, dbLocation file.Location,
// attach the file list // attach the file list
records, err := parseDpkgMD5Info(md5Reader) records, err := parseDpkgMD5Info(md5Reader)
if err != nil { if err != nil {
log.Debugf("dpkg md5sums file %q: %v", md5Location.RealPath, err) log.Debugf("dpkg md5sums file %q: %v", md5Location.FullNestedPath(), err)
} }
files = append(files, records...) files = append(files, records...)
@@ -265,7 +265,7 @@ func getAdditionalFileListing(resolver file.Resolver, dbLocation file.Location,
// attach the file list // attach the file list
records, err := parseDpkgConffileInfo(conffilesReader) records, err := parseDpkgConffileInfo(conffilesReader)
if err != nil { if err != nil {
log.Debugf("dpkg conffiles file %q: %v", conffilesLocation.RealPath, err) log.Debugf("dpkg conffiles file %q: %v", conffilesLocation.FullNestedPath(), err)
} }
files = append(files, records...) files = append(files, records...)
@@ -288,7 +288,7 @@ func fetchMd5Contents(resolver file.Resolver, dbLocation file.Location, m pkg.Dp
// and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed // and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed
// package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in // package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in
// the same directory // the same directory
searchPath := filepath.Dir(dbLocation.RealPath) searchPath := filepath.Dir(dbLocation.LocalPath())
if !strings.HasSuffix(searchPath, "status.d") { if !strings.HasSuffix(searchPath, "status.d") {
searchPath = path.Join(searchPath, "info") searchPath = path.Join(searchPath, "info")
@@ -327,7 +327,7 @@ func fetchConffileContents(resolver file.Resolver, dbLocation file.Location, m p
return nil, nil return nil, nil
} }
parentPath := filepath.Dir(dbLocation.RealPath) parentPath := filepath.Dir(dbLocation.LocalPath())
// look for /var/lib/dpkg/info/NAME:ARCH.conffiles // look for /var/lib/dpkg/info/NAME:ARCH.conffiles
name := md5Key(m) name := md5Key(m)
+2 -2
View File
@@ -37,7 +37,7 @@ var (
func parseDpkgDB(ctx context.Context, resolver file.Resolver, env *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { func parseDpkgDB(ctx context.Context, resolver file.Resolver, env *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
metadata, err := parseDpkgStatus(reader) metadata, err := parseDpkgStatus(reader)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("unable to catalog dpkg DB=%q: %w", reader.RealPath, err) return nil, nil, fmt.Errorf("unable to catalog dpkg DB=%q: %w", reader.FullNestedPath(), err)
} }
dbLoc := reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation) dbLoc := reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)
@@ -61,7 +61,7 @@ func findDpkgInfoFiles(name string, resolver file.Resolver, dbLocation file.Loca
// and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed // and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed
// package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in // package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in
// the same directory // the same directory
searchPath := path.Dir(dbLocation.RealPath) searchPath := path.Dir(dbLocation.LocalPath())
if !strings.HasSuffix(searchPath, "status.d") { if !strings.HasSuffix(searchPath, "status.d") {
searchPath = path.Join(searchPath, "info") searchPath = path.Join(searchPath, "info")
@@ -150,7 +150,7 @@ var runtimeDLLPathPattern = regexp.MustCompile(`/Microsoft\.NETCore\.App/(?P<ver
func isRuntimePackageLocation(loc file.Location) (string, bool) { func isRuntimePackageLocation(loc file.Location) (string, bool) {
// we should look at the realpath to see if it is a "**/Microsoft.NETCore.App/\d+.\d+.\d+/*.dll" // we should look at the realpath to see if it is a "**/Microsoft.NETCore.App/\d+.\d+.\d+/*.dll"
// and if so treat it as a runtime package // and if so treat it as a runtime package
if match := runtimeDLLPathPattern.FindStringSubmatch(loc.RealPath); match != nil { if match := runtimeDLLPathPattern.FindStringSubmatch(loc.FullNestedPath()); match != nil {
versionIndex := runtimeDLLPathPattern.SubexpIndex("version") versionIndex := runtimeDLLPathPattern.SubexpIndex("version")
if versionIndex != -1 { if versionIndex != -1 {
version := match[versionIndex] version := match[versionIndex]
@@ -177,12 +177,12 @@ func partitionPEs(depJsons []logicalDepsJSON, peFiles []logicalPE) ([]logicalDep
// sort deps.json paths from longest to shortest. This is so we are processing the most specific match first. // sort deps.json paths from longest to shortest. This is so we are processing the most specific match first.
sort.Slice(depJsons, func(i, j int) bool { sort.Slice(depJsons, func(i, j int) bool {
return depJsons[i].Location.RealPath > depJsons[j].Location.RealPath return depJsons[i].Location.FullNestedPath() > depJsons[j].Location.FullNestedPath()
}) })
// we should be processing PE files in a stable order // we should be processing PE files in a stable order
sort.Slice(peFiles, func(i, j int) bool { sort.Slice(peFiles, func(i, j int) bool {
return peFiles[i].Location.RealPath > peFiles[j].Location.RealPath return peFiles[i].Location.FullNestedPath() > peFiles[j].Location.FullNestedPath()
}) })
peFilesByPath := make(map[file.Coordinates][]logicalPE) peFilesByPath := make(map[file.Coordinates][]logicalPE)
@@ -191,7 +191,7 @@ func partitionPEs(depJsons []logicalDepsJSON, peFiles []logicalPE) ([]logicalDep
var found bool var found bool
for i := range depJsons { for i := range depJsons {
dep := &depJsons[i] dep := &depJsons[i]
if isParentOf(dep.Location.RealPath, pe.Location.RealPath) && attachAssociatedExecutables(dep, pe) { if isParentOf(dep.Location.LocalPath(), pe.Location.LocalPath()) && attachAssociatedExecutables(dep, pe) {
peFilesByPath[dep.Location.Coordinates] = append(peFilesByPath[dep.Location.Coordinates], pe) peFilesByPath[dep.Location.Coordinates] = append(peFilesByPath[dep.Location.Coordinates], pe)
found = true found = true
// note: we cannot break from the dep JSON search since the same binary could be associated with multiple packages // note: we cannot break from the dep JSON search since the same binary could be associated with multiple packages
@@ -222,8 +222,8 @@ func partitionPEs(depJsons []logicalDepsJSON, peFiles []logicalPE) ([]logicalDep
// attachAssociatedExecutables looks for PE files matching runtime or resource entries // attachAssociatedExecutables looks for PE files matching runtime or resource entries
// and attaches them to the appropriate package. // and attaches them to the appropriate package.
func attachAssociatedExecutables(dep *logicalDepsJSON, pe logicalPE) bool { func attachAssociatedExecutables(dep *logicalDepsJSON, pe logicalPE) bool {
appDir := path.Dir(dep.Location.RealPath) appDir := path.Dir(dep.Location.LocalPath())
relativeDllPath := strings.TrimPrefix(strings.TrimPrefix(pe.Location.RealPath, appDir), "/") relativeDllPath := strings.TrimPrefix(strings.TrimPrefix(pe.Location.LocalPath(), appDir), "/")
var found bool var found bool
for key, p := range dep.PackagesByNameVersion { for key, p := range dep.PackagesByNameVersion {
@@ -471,7 +471,7 @@ func readDepsJSON(resolver file.Resolver, loc file.Location) (*depsJSON, error)
if err != nil { if err != nil {
return nil, unknown.New(loc, fmt.Errorf("unable to read deps.json file: %w", err)) return nil, unknown.New(loc, fmt.Errorf("unable to read deps.json file: %w", err))
} }
defer internal.CloseAndLogError(reader, loc.RealPath) defer internal.CloseAndLogError(reader, loc.FullNestedPath())
dj, err := newDepsJSON(file.NewLocationReadCloser(loc, reader)) dj, err := newDepsJSON(file.NewLocationReadCloser(loc, reader))
if err != nil { if err != nil {
@@ -479,7 +479,7 @@ func readDepsJSON(resolver file.Resolver, loc file.Location) (*depsJSON, error)
} }
if dj == nil { if dj == nil {
return nil, unknown.New(loc, fmt.Errorf("expected to find packages in deps.json but did not: %q", loc.RealPath)) return nil, unknown.New(loc, fmt.Errorf("expected to find packages in deps.json but did not: %q", loc.FullNestedPath()))
} }
return dj, nil return dj, nil
@@ -520,7 +520,7 @@ func readPEFile(resolver file.Resolver, loc file.Location) (*logicalPE, error) {
if err != nil { if err != nil {
return nil, unknown.New(loc, fmt.Errorf("unable to read PE file: %w", err)) return nil, unknown.New(loc, fmt.Errorf("unable to read PE file: %w", err))
} }
defer internal.CloseAndLogError(reader, loc.RealPath) defer internal.CloseAndLogError(reader, loc.FullNestedPath())
ldpe, err := readLogicalPE(file.NewLocationReadCloser(loc, reader)) ldpe, err := readLogicalPE(file.NewLocationReadCloser(loc, reader))
if err != nil { if err != nil {
@@ -566,7 +566,7 @@ func readELFBundledDepsJSON(resolver file.Resolver, loc file.Location) (*logical
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer internal.CloseAndLogError(reader, loc.RealPath) defer internal.CloseAndLogError(reader, loc.FullNestedPath())
header := make([]byte, 4) header := make([]byte, 4)
if _, err := io.ReadFull(reader, header); err != nil { if _, err := io.ReadFull(reader, header); err != nil {
+1 -1
View File
@@ -161,7 +161,7 @@ type logicalDepsJSON struct {
} }
func (l logicalDepsJSON) RootPackage() (logicalDepsJSONPackage, bool) { func (l logicalDepsJSON) RootPackage() (logicalDepsJSONPackage, bool) {
rootName := getDepsJSONFilePrefix(l.Location.RealPath) rootName := getDepsJSONFilePrefix(l.Location.FullNestedPath())
if rootName == "" { if rootName == "" {
return logicalDepsJSONPackage{}, false return logicalDepsJSONPackage{}, false
} }
+2 -2
View File
@@ -87,14 +87,14 @@ func newLibmanJSON(reader file.LocationReadCloser) (*libmanJSON, error) {
} }
func findLibmanJSON(resolver file.Resolver, depsJSON file.Location) (*libmanJSON, error) { func findLibmanJSON(resolver file.Resolver, depsJSON file.Location) (*libmanJSON, error) {
parent := path.Dir(depsJSON.RealPath) parent := path.Dir(depsJSON.LocalPath())
loc := resolver.RelativeFileByPath(depsJSON, path.Join(parent, "libman.json")) loc := resolver.RelativeFileByPath(depsJSON, path.Join(parent, "libman.json"))
if loc == nil { if loc == nil {
return nil, nil return nil, nil
} }
reader, err := resolver.FileContentsByLocation(*loc) reader, err := resolver.FileContentsByLocation(*loc)
defer internal.CloseAndLogError(reader, loc.RealPath) defer internal.CloseAndLogError(reader, loc.FullNestedPath())
if err != nil { if err != nil {
return nil, err return nil, err
} }
+1 -1
View File
@@ -265,7 +265,7 @@ func newDotnetBinaryPackage(versionResources map[string]string, f file.Location)
if name == "" { if name == "" {
// older .NET runtime dlls may not have any version resources // older .NET runtime dlls may not have any version resources
name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.RealPath), ".exe"), ".dll") name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.FullNestedPath()), ".exe"), ".dll")
} }
ver := findVersionFromVersionResources(versionResources) ver := findVersionFromVersionResources(versionResources)
+1 -1
View File
@@ -83,7 +83,7 @@ func parseMixLock(_ context.Context, _ file.Resolver, _ *generic.Environment, re
} }
if name == "" { if name == "" {
log.WithFields("path", reader.RealPath).Debug("skipping empty package name from mix.lock file") log.WithFields("path", reader.FullNestedPath()).Debug("skipping empty package name from mix.lock file")
errs = unknown.Appendf(errs, reader, "skipping empty package name from mix.lock file, for line: %d: %s", lineNum, line) errs = unknown.Appendf(errs, reader, "skipping empty package name from mix.lock file, for line: %d: %s", lineNum, line)
continue continue
} }
+4 -4
View File
@@ -203,7 +203,7 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.
errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(c.selectFiles(resolver)), func(req request) (result, error) { errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(c.selectFiles(resolver)), func(req request) (result, error) {
location, parser := req.Location, req.Parser location, parser := req.Location, req.Parser
log.WithFields("path", location.RealPath).Trace("parsing file contents") log.WithFields("path", location.FullNestedPath()).Trace("parsing file contents")
discoveredPackages, discoveredRelationships, err := invokeParser(ctx, resolver, location, lgr, parser, &env) discoveredPackages, discoveredRelationships, err := invokeParser(ctx, resolver, location, lgr, parser, &env)
if err != nil { if err != nil {
@@ -243,7 +243,7 @@ func (c *Cataloger) processSafely(ctx context.Context, p processExecutor, resolv
func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) (pkgs []pkg.Package, rels []artifact.Relationship, err error) { func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) (pkgs []pkg.Package, rels []artifact.Relationship, err error) {
contentReader, err := resolver.FileContentsByLocation(location) contentReader, err := resolver.FileContentsByLocation(location)
if err != nil { if err != nil {
logger.WithFields("location", location.RealPath, "error", err).Debug("unable to fetch contents") logger.WithFields("location", location.FullNestedPath(), "error", err).Debug("unable to fetch contents")
return nil, nil, err return nil, nil, err
} }
defer internal.CloseAndLogError(contentReader, location.AccessPath) defer internal.CloseAndLogError(contentReader, location.AccessPath)
@@ -251,7 +251,7 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc
defer func() { defer func() {
if r := recover(); r != nil { if r := recover(); r != nil {
// a malformed file must not take down the whole SBOM; the caller records this as an unknown // a malformed file must not take down the whole SBOM; the caller records this as an unknown
logger.WithFields("location", location.RealPath, "panic", r).Debug("recovered from panic while parsing file") logger.WithFields("location", location.FullNestedPath(), "panic", r).Debug("recovered from panic while parsing file")
logger.Tracef("parser panic stack:\n%s", debug.Stack()) logger.Tracef("parser panic stack:\n%s", debug.Stack())
pkgs, rels, err = nil, nil, fmt.Errorf("recovered from panic while parsing file: %v", r) pkgs, rels, err = nil, nil, fmt.Errorf("recovered from panic while parsing file: %v", r)
} }
@@ -260,7 +260,7 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc
discoveredPackages, discoveredRelationships, err := parser(ctx, resolver, env, file.NewLocationReadCloser(location, contentReader)) discoveredPackages, discoveredRelationships, err := parser(ctx, resolver, env, file.NewLocationReadCloser(location, contentReader))
if err != nil { if err != nil {
// these errors are propagated up, and are likely to be coordinate errors // these errors are propagated up, and are likely to be coordinate errors
logger.WithFields("location", location.RealPath, "error", err).Trace("cataloger returned errors") logger.WithFields("location", location.FullNestedPath(), "error", err).Trace("cataloger returned errors")
} }
return discoveredPackages, discoveredRelationships, err return discoveredPackages, discoveredRelationships, err
+4 -4
View File
@@ -19,7 +19,7 @@ import (
func Test_Cataloger(t *testing.T) { func Test_Cataloger(t *testing.T) {
allParsedPaths := make(map[string]bool) allParsedPaths := make(map[string]bool)
parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
allParsedPaths[reader.Path()] = true allParsedPaths[reader.FullNestedPath()] = true
contents, err := io.ReadAll(reader) contents, err := io.ReadAll(reader)
require.NoError(t, err) require.NoError(t, err)
@@ -219,7 +219,7 @@ func TestParserPanicBecomesUnknown(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
parser := func(_ context.Context, _ file.Resolver, _ *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { parser := func(_ context.Context, _ file.Resolver, _ *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
if reader.Path() == "testdata/another-path.txt" { if reader.FullNestedPath() == "testdata/another-path.txt" {
tt.doPanic() tt.doPanic()
} }
return []pkg.Package{{Name: "good", Locations: file.NewLocationSet(reader.Location)}}, nil, nil return []pkg.Package{{Name: "good", Locations: file.NewLocationSet(reader.Location)}}, nil, nil
@@ -308,7 +308,7 @@ func TestProcessorPanicKeepsResults(t *testing.T) {
func Test_CatalogerWithParserByMediaType(t *testing.T) { func Test_CatalogerWithParserByMediaType(t *testing.T) {
allParsedPaths := make(map[string]bool) allParsedPaths := make(map[string]bool)
parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
allParsedPaths[reader.Path()] = true allParsedPaths[reader.FullNestedPath()] = true
contents, err := io.ReadAll(reader) contents, err := io.ReadAll(reader)
require.NoError(t, err) require.NoError(t, err)
@@ -358,7 +358,7 @@ func Test_genericCatalogerReturnsErrors(t *testing.T) {
genericErrorReturning := NewCataloger("error returning").WithParserByGlobs(func(ctx context.Context, resolver file.Resolver, environment *Environment, locationReader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { genericErrorReturning := NewCataloger("error returning").WithParserByGlobs(func(ctx context.Context, resolver file.Resolver, environment *Environment, locationReader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
return []pkg.Package{ return []pkg.Package{
{ {
Name: "some-package-" + locationReader.Path(), Name: "some-package-" + locationReader.FullNestedPath(),
}, },
}, nil, unknown.Newf(locationReader, "unable to read") }, nil, unknown.Newf(locationReader, "unable to read")
}, "**/*") }, "**/*")
+4 -4
View File
@@ -71,7 +71,7 @@ func extractLicenses(resolver file.Resolver, closestLocation *file.Location, rea
if err := scanner.Err(); err != nil { if err := scanner.Err(); err != nil {
fields := []any{"error", err} fields := []any{"error", err}
if closestLocation != nil { if closestLocation != nil {
fields = append(fields, "path", closestLocation.RealPath) fields = append(fields, "path", closestLocation.FullNestedPath())
} }
log.WithFields(fields...).Debug("failed to fully read portage LICENSE") log.WithFields(fields...).Debug("failed to fully read portage LICENSE")
} }
@@ -115,9 +115,9 @@ func readLicenseGroups(resolver file.Resolver, closestLocation *file.Location) m
} }
groupReader, err := resolver.FileContentsByLocation(*groupLocation) groupReader, err := resolver.FileContentsByLocation(*groupLocation)
defer internal.CloseAndLogError(groupReader, groupLocation.RealPath) defer internal.CloseAndLogError(groupReader, groupLocation.FullNestedPath())
if err != nil { if err != nil {
log.WithFields("path", groupLocation.RealPath, "error", err).Debug("failed to fetch portage LICENSE") log.WithFields("path", groupLocation.FullNestedPath(), "error", err).Debug("failed to fetch portage LICENSE")
return nil return nil
} }
@@ -127,7 +127,7 @@ func readLicenseGroups(resolver file.Resolver, closestLocation *file.Location) m
licenseGroups, err = parseLicenseGroups(groupReader) licenseGroups, err = parseLicenseGroups(groupReader)
if err != nil { if err != nil {
log.WithFields("path", groupLocation.RealPath, "error", err).Debug("failed to parse portage LICENSE") log.WithFields("path", groupLocation.FullNestedPath(), "error", err).Debug("failed to parse portage LICENSE")
} }
return licenseGroups return licenseGroups

Some files were not shown because too many files have changed in this diff Show More