oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-05 17:06:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix: keep RealPath as top-level, use FullPath and utility functions FullNestedPath, LocalPath, Path

Browse Source 
Signed-off-by: Keith Zantow <kzantow@gmail.com>
This commit is contained in:
Keith Zantow
2026-10-19 15:44:37 -04:00
parent 17d109e8b5
commit 3e4b036e22
165 changed files with 1349 additions and 1970 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/archive/extract.go
+6 -12
View File
@@ -8,7 +8,6 @@ import (
"fmt"
"io"
"path"
"strings"
"github.com/mholt/archives"
@@ -22,11 +21,12 @@ import (
// ErrDiskLimitReached when the archive's own bytes cannot be placed within the disk limit. The caller
// must Cleanup the result.
//
// fileSystemID and archivePath are stamped onto every location the resolver returns: the filesystem
// the archive was found on, and the colon-delimited chain of archives from the scan root to this one.
func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string, limiter *Limiter, exclusions Exclusions) (*Resolver, error) {
// archiveCoordinates are those of the archive file itself; every location the resolver returns
// extends them with its entry path (see file.Coordinates).
func Extract(ctx context.Context, r io.Reader, archiveCoordinates file.Coordinates, limiter *Limiter, exclusions Exclusions) (*Resolver, error) {
charge := limiter.charge()
resolver := newResolver(ctx, fileSystemID, archivePath, charge)
resolver := newResolver(ctx, archiveCoordinates, charge)
archivePath := archiveCoordinates.FullNestedPath()
// deferred so that a panic in a decoder still gives back the charge and removes any spill file
succeeded := false
@@ -43,7 +43,7 @@ func Extract(ctx context.Context, r io.Reader, fileSystemID, archivePath string,
// the archive's own bytes are needed only until they are extracted and digested
defer release()
format := identifyFormat(ctx, archiveFileName(archivePath), content)
format := identifyFormat(ctx, path.Base(archiveCoordinates.LocalPath()), content)
if format == nil {
return nil, nil
}
@@ -75,12 +75,6 @@ func (r *Resolver) acquire(content io.Reader) (ReaderAtSeeker, func(), error) {
return r.open(&b), func() { r.discard(&b) }, nil
}
// archiveFileName returns the archive's own file name from the end of its archive path, which format
// identification reads for its extension.
func archiveFileName(archivePath string) string {
return path.Base(archivePath[strings.LastIndexByte(archivePath, ':')+1:])
}
// identifyFormat returns the format able to extract this content, or nil when it is not an archive.
//
// Identification from the head of the stream wins: a tar holding one jar ends in that jar's central
internal/archive/extract_test.go
+53 -78
View File
@@ -118,28 +118,6 @@ func TestResolver_extract_diskLimitBitesMidEntry(t *testing.T) {
assert.Equal(t, r.written, info.Size())
}
func TestResolver_extract_diskLimitFallsWhenAnArchiveIsReleased(t *testing.T) {
data := zipBytes(t, map[string]string{"a.txt": strings.Repeat("a", 200)})
oneArchive := approxIndexBytes(tar.Header{Name: "a.txt"}) + 200
limiter := NewLimiter(Limits{MaxDiskBytes: oneArchive * 3 / 2})
first, _ := extractBytes(t, data, "test.zip", limiter.charge())
require.NoError(t, first.Truncated)
require.Equal(t, int64(200), first.written)
second, _ := extractBytes(t, data, "test.zip", limiter.charge())
assert.Error(t, second.Truncated, "a second archive is bounded while the first is still held")
second.Cleanup()
first.Cleanup()
_, disk := limiter.InUse()
require.Zero(t, disk)
third, _ := extractBytes(t, data, "test.zip", limiter.charge())
assert.NoError(t, third.Truncated, "and extracts in full once the first is released")
assert.Equal(t, int64(200), third.written)
}
func TestResolver_extract_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) {
var entries []testEntry
for i := range 50 {
@@ -262,7 +240,7 @@ func TestResolver_extract_excludedEntriesAreNeverStored(t *testing.T) {
func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
extracted, err := Extract(context.Background(), bytes.NewReader(data), "parentFS", "app.war:some/path/app.zip", nil, nil)
extracted, err := Extract(context.Background(), bytes.NewReader(data), file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip"}, nil, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
@@ -274,7 +252,7 @@ func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
locations, err := extracted.FilesByGlob("**/*.txt")
require.NoError(t, err)
require.Len(t, locations, 1)
assert.Equal(t, file.Coordinates{RealPath: "dir/hello.txt", FileSystemID: "parentFS", ArchivePath: "app.war:some/path/app.zip"},
assert.Equal(t, file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip!/dir/hello.txt"},
locations[0].Coordinates)
reader, err := extracted.FileContentsByLocation(locations[0])
@@ -284,39 +262,13 @@ func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
assert.Equal(t, "hello world", string(body))
}
func TestExtract_notAnArchive(t *testing.T) {
extracted, err := Extract(context.Background(), strings.NewReader("this is not an archive"), "", "notes.txt", nil, nil)
require.NoError(t, err)
assert.Nil(t, extracted)
}
func TestExtract_spillsUnderTheScansTempRootAndCleansUp(t *testing.T) {
ctx, root := scanContext(t)
limiter := NewLimiter(spillingLimits)
extracted, err := Extract(ctx, bytes.NewReader(zipBytes(t, sampleFiles)), "", "app.zip", limiter, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
require.Len(t, filesIn(t, root), 1)
_, disk := limiter.InUse()
assert.Positive(t, disk)
extracted.Cleanup()
assert.Empty(t, filesIn(t, root))
_, disk = limiter.InUse()
assert.Zero(t, disk, "cleanup releases everything the archive held")
extracted.Cleanup()
}
func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) {
// the archive's own bytes are only needed to extract and digest it; afterwards the limiter holds
// the entries and the index estimate alone
data := zipBytes(t, map[string]string{"hello.txt": "hi"})
limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil)
extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
require.NoError(t, err)
t.Cleanup(extracted.Cleanup)
@@ -332,7 +284,7 @@ func TestExtract_heldContentWrittenToDiskStaysUntilCleanup(t *testing.T) {
data := zipBytes(t, map[string]string{"hello.txt": "hi"})
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil)
extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
require.NoError(t, err)
t.Cleanup(extracted.Cleanup)
@@ -354,7 +306,7 @@ func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) {
data := tarGzBytes(t, map[string]string{"a.txt": "first", "b.bin": string(noise)})
cut := data[:len(data)/2]
extracted, err := Extract(context.Background(), bytes.NewReader(cut), "", "app.tar.gz", NewLimiter(unboundedLimits), nil)
extracted, err := Extract(context.Background(), bytes.NewReader(cut), file.NewCoordinates("app.tar.gz", ""), NewLimiter(unboundedLimits), nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
@@ -362,7 +314,7 @@ func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) {
assert.Error(t, extracted.Truncated)
locations, err := extracted.FilesByGlob("**")
require.NoError(t, err)
assert.Equal(t, []string{"a.txt"}, realPaths(locations))
assert.Equal(t, []string{"a.txt"}, localPaths(locations))
}
func TestExtract_aPanicStillReleasesEverything(t *testing.T) {
@@ -371,7 +323,7 @@ func TestExtract_aPanicStillReleasesEverything(t *testing.T) {
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
require.Panics(t, func() {
_, _ = Extract(ctx, &panicsAfter{n: 4096}, "", "app.zip", limiter, nil)
_, _ = Extract(ctx, &panicsAfter{n: 4096}, file.NewCoordinates("app.zip", ""), limiter, nil)
})
mem, disk := limiter.InUse()
@@ -397,11 +349,11 @@ func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) {
data := zipBytes(t, sampleFiles)
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0})
_, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, "", "app.zip", limiter, nil)
_, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
assert.ErrorIs(t, err, ErrDiskLimitReached)
// read in place, the same bytes cost nothing; only the entries are refused
extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "app.zip", limiter, nil)
extracted, err := Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("app.zip", ""), limiter, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
@@ -411,24 +363,6 @@ func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) {
assert.Empty(t, locations)
}
func TestExtract_aZipBehindALauncherScript(t *testing.T) {
// a Spring Boot executable jar sniffs as a shell script; archive/zip still finds the central directory
data := append([]byte("#!/bin/bash\necho launcher\nexit 0\n"), zipBytes(t, map[string]string{
"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n",
"BOOT-INF/lib/dep.jar": "not really a jar, just bytes",
})...)
extracted, err := Extract(context.Background(), bytes.NewReader(data), "", "/opt/app.jar", nil, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
assert.NoError(t, extracted.Truncated)
locations, err := extracted.FilesByGlob("**/*")
require.NoError(t, err)
assert.Len(t, locations, 2)
}
func Test_HasZipEndOfCentralDirectory(t *testing.T) {
tests := []struct {
name string
@@ -521,7 +455,7 @@ func TestExtract_decompressionBombIsBoundedByTheLimits(t *testing.T) {
ctx, root := scanContext(t)
limiter := NewLimiter(limits)
r, err := Extract(ctx, bytes.NewReader(data), "", name, limiter, nil)
r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates(name, ""), limiter, nil)
require.NoError(t, err)
require.NotNil(t, r)
t.Cleanup(r.Cleanup)
@@ -547,7 +481,7 @@ func TestExtract_entryCountBombIsBoundedByTheMemoryLimit(t *testing.T) {
limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}
limiter := NewLimiter(limits)
r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), "", "many.zip", limiter, nil)
r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), file.NewCoordinates("many.zip", ""), limiter, nil)
require.NoError(t, err)
t.Cleanup(r.Cleanup)
@@ -575,7 +509,7 @@ func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) {
limits := Limits{MaxMemoryBytes: 8 << 20, MaxDiskBytes: -1}
limiter := NewLimiter(limits)
r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), "", "deep.zip", limiter, nil)
r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), file.NewCoordinates("deep.zip", ""), limiter, nil)
require.NoError(t, err)
t.Cleanup(r.Cleanup)
@@ -584,3 +518,44 @@ func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) {
peakMemory, _ := limiter.Peak()
assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes)
}
func TestExtract_locationsKeepTheTopLevelRealPath(t *testing.T) {
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
// a top-level archive's coordinates, with FullPath unset or set to RealPath, name entries alike
for _, archive := range []file.Coordinates{
{RealPath: "/app.zip"},
{RealPath: "/app.zip", FullPath: "/app.zip"},
} {
extracted, err := Extract(context.Background(), bytes.NewReader(data), archive, nil, nil)
require.NoError(t, err)
require.NotNil(t, extracted)
t.Cleanup(extracted.Cleanup)
locations, err := extracted.FilesByPath("dir/hello.txt")
require.NoError(t, err)
require.Len(t, locations, 1)
assert.Equal(t, file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}, locations[0].Coordinates)
assert.Equal(t, "dir/hello.txt", locations[0].LocalPath())
for _, loc := range []file.Location{
locations[0],
file.NewLocation("dir/hello.txt"),
file.NewLocation("/dir/hello.txt"),
file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}),
} {
reader, err := extracted.FileContentsByLocation(loc)
require.NoError(t, err, "location %s", loc)
body, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "hello world", string(body))
metadata, err := extracted.FileMetadataByLocation(loc)
require.NoError(t, err, "location %s", loc)
assert.Equal(t, "/app.zip!/dir/hello.txt", metadata.Path)
}
_, err = extracted.FileContentsByLocation(file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/other.zip", FullPath: "/other.zip!/dir/hello.txt"}))
assert.Error(t, err, "an entry of another archive is not read from this one")
}
}
internal/archive/helpers_test.go
+7 -32
View File
@@ -10,7 +10,6 @@ import (
"io"
"io/fs"
"os"
"path"
"sort"
"testing"
@@ -179,7 +178,7 @@ func resolverIn(t testing.TB, charge *charge) (*Resolver, string) {
t.Helper()
root := t.TempDir()
ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))
r := newResolver(ctx, "", "test-archive", charge)
r := newResolver(ctx, file.NewCoordinates("test-archive", ""), charge)
t.Cleanup(r.Cleanup)
return r, root
}
@@ -197,7 +196,7 @@ func addAll(t testing.TB, r *Resolver, entries []testEntry) {
func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Resolver {
t.Helper()
r, _ := resolverIn(t, memCharge(maxInMemory))
r.archivePath = "outer.jar"
r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
addAll(t, r, fileEntries(files))
return r
}
@@ -207,35 +206,11 @@ func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Res
func resolverFrom(t testing.TB, entries ...testEntry) *Resolver {
t.Helper()
r, _ := resolverIn(t, memCharge(1<<20))
r.archivePath = "outer.jar"
r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
addAll(t, r, entries)
return r
}
// indexCostOf is what add charges for the given entries in name order: every listed node and every
// directory their paths imply.
func indexCostOf(entries map[string]string) int64 {
names := make([]string, 0, len(entries))
for name := range entries {
names = append(names, name)
}
sort.Strings(names)
seen := map[string]bool{}
var total int64
for _, name := range names {
p := path.Clean("/" + name)
for i := 1; i < len(p); i++ {
if p[i] == '/' && !seen[p[:i]] {
seen[p[:i]] = true
total += approxIndexBytesPerEntry
}
}
seen[p] = true
total += approxIndexBytes(tar.Header{Name: name})
}
return total
}
// extractBytes extracts an archive into a resolver built over the given charge.
func extractBytes(t testing.TB, data []byte, name string, charge *charge) (*Resolver, string) {
t.Helper()
@@ -316,11 +291,11 @@ func diskCharge(maxDisk int64) *charge {
return NewLimiter(Limits{MaxDiskBytes: maxDisk}).charge()
}
// realPaths lists the real path of every location, sorted.
func realPaths(locations []file.Location) []string {
// localPaths lists the local entry path of every location (rather than full nested archive path), sorted.
func localPaths(locations []file.Location) []string {
var out []string
for _, loc := range locations {
out = append(out, loc.RealPath)
out = append(out, loc.LocalPath())
}
sort.Strings(out)
return out
@@ -342,7 +317,7 @@ func extractedResolver(t testing.TB, data []byte, limits Limits) (*Resolver, str
t.Helper()
ctx, root := scanContext(t)
r, err := Extract(ctx, bytes.NewReader(data), "", "app.zip", NewLimiter(limits), nil)
r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates("app.zip", ""), NewLimiter(limits), nil)
require.NoError(t, err)
require.NotNil(t, r)
t.Cleanup(r.Cleanup)
internal/archive/resolver.go
+78 -43
View File
@@ -28,7 +28,8 @@ import (
// into it, holding their content within the scan's limits. Entries are indexed in a stereoscope file
// tree, the same index the directory and image resolvers search.
//
// Paths are archive-relative with no leading slash.
// Locations it returns keep the outermost archive's RealPath and carry the entry's FullPath. Queries
// may name an entry either by its full path or by its path within the archive (see rootedPath).
type Resolver struct {
// Digests are of the archive file itself.
Digests []file.Digest
@@ -37,10 +38,9 @@ type Resolver struct {
// or the decompression budget stopped extraction early, or an entry was too large or unreadable.
Truncated error
fileSystemID string
archivePath string
charge *charge
budget *Budget
archiveCoordinates file.Coordinates
charge *charge
budget *Budget
// content is held in memory while the memory limit admits it and in one spill file once it does
// not. The file is created on first use, so an archive that stays in memory never touches the
@@ -61,18 +61,17 @@ type Resolver struct {
var _ file.Resolver = (*Resolver)(nil)
// newResolver returns an empty resolver whose locations carry the given fileSystemID and archivePath,
// spilling into the scan's temp root when ctx carries one (see internal/tmpdir).
func newResolver(ctx context.Context, fileSystemID, archivePath string, charge *charge) *Resolver {
// newResolver returns an empty resolver whose locations extend the archive's coordinates, spilling into
// the scan's temp root when ctx carries one (see internal/tmpdir).
func newResolver(ctx context.Context, archiveCoordinates file.Coordinates, charge *charge) *Resolver {
return &Resolver{
fileSystemID: fileSystemID,
archivePath: archivePath,
charge: charge,
budget: budgetFromContext(ctx),
tempDir: tmpdir.FromContext(ctx),
tree: filetree.New(),
index: filetree.NewIndex(),
content: map[stereoscopeFile.ID]*blob{},
archiveCoordinates: archiveCoordinates,
charge: charge,
budget: budgetFromContext(ctx),
tempDir: tmpdir.FromContext(ctx),
tree: filetree.New(),
index: filetree.NewIndex(),
content: map[stereoscopeFile.ID]*blob{},
}
}
@@ -160,13 +159,13 @@ func (r *Resolver) add(hdr tar.Header, content io.Reader) error {
return nil
}
func (r *Resolver) store(entryPath string, hdr tar.Header, b *blob) error {
ref, err := r.addToTree(entryPath, hdr)
func (r *Resolver) store(rooted string, hdr tar.Header, b *blob) error {
ref, err := r.addToTree(rooted, hdr)
if err != nil {
return err
}
metadata := stereoscopeFile.NewMetadata(hdr, r.open(b))
metadata.Path = reportedPath(entryPath)
metadata.Path = entryPath(rooted)
r.index.Add(*ref, metadata)
r.content[ref.ID()] = b
return nil
@@ -217,34 +216,63 @@ func (r *Resolver) finish() {
r.search = filetree.NewSearchContext(r.tree, r.index)
}
// reportedPath is how paths leave the resolver: archive-relative with no leading slash.
func reportedPath(rooted string) string {
// linkDestination is the rooted path a link points at inside the archive. A hard link names its
// target from the archive root, as does an absolute symlink, so "/etc/passwd" is an entry of this
// archive and never the host's. A relative symlink is joined onto the link's own directory, and
// cleaning against the root keeps "../../x" inside the archive.
func linkDestination(linkPath, linkname string, hard bool) string {
if hard || path.IsAbs(linkname) {
return path.Join("/", linkname)
}
return path.Join(path.Dir(linkPath), linkname)
}
// entryPath is a node's path within the archive: archive-relative with no leading slash.
func entryPath(rooted string) string {
return strings.TrimPrefix(rooted, "/")
}
func rootedPath(userPath string) string {
return path.Clean("/" + userPath)
// fullPath is how paths leave the resolver: the archive's own full path, then the entry path.
func (r *Resolver) fullPath(rooted string) string {
return r.archiveCoordinates.FullNestedPath() + "!/" + entryPath(rooted)
}
// location names a resolution: a link is reported at its target's path with the link's own path as
// the access path.
func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location {
coordinates := file.Coordinates{
RealPath: reportedPath(string(res.RealPath)),
FileSystemID: r.fileSystemID,
ArchivePath: r.archivePath,
// rootedPath is the tree path a query names, which is either form a path takes: a full path is trimmed to
// the entry path within this archive, and anything else (a LocalPath) is taken as archive-relative
// already. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root)
// names the root.
func (r *Resolver) rootedPath(userPath string) string {
archivePath := r.archiveCoordinates.FullNestedPath() + "!"
if userPath == archivePath {
return "/"
}
accessPath := reportedPath(string(res.RequestPath))
if accessPath == "" || accessPath == coordinates.RealPath {
return path.Clean("/" + strings.TrimPrefix(userPath, archivePath+"/"))
}
// coordinates name an entry by the outermost archive's real path, which never includes a path inside an
// archive, and its full path through every archive.
func (r *Resolver) coordinates(rooted string) file.Coordinates {
return file.Coordinates{
RealPath: r.archiveCoordinates.RealPath,
FileSystemID: r.archiveCoordinates.FileSystemID,
FullPath: r.fullPath(rooted),
}
}
// location names a resolution: a link is reported at its target's path with the link's own full path
// as the access path.
func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location {
coordinates := r.coordinates(string(res.RealPath))
if res.RequestPath == "" || res.RequestPath == res.RealPath {
return file.NewLocationFromCoordinates(coordinates)
}
return file.NewVirtualLocationFromCoordinates(coordinates, accessPath)
return file.NewVirtualLocationFromCoordinates(coordinates, r.fullPath(string(res.RequestPath)))
}
// resolve finds the file a path names, following links, and reports false for directories and
// paths the archive never listed.
func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool) {
res, err := r.search.SearchByPath(rootedPath(userPath), filetree.FollowBasenameLinks)
res, err := r.search.SearchByPath(r.rootedPath(userPath), filetree.FollowBasenameLinks)
if err != nil || res == nil || !res.HasReference() {
return nil, false
}
@@ -256,27 +284,34 @@ func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool)
}
func (r *Resolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
res, ok := r.resolve(location.RealPath)
res, ok := r.resolve(location.FullNestedPath())
if !ok {
return nil, fmt.Errorf("no archive entry for path=%q", location.RealPath)
return nil, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
}
return entryReader{r.open(r.content[res.ID()])}, nil
}
func (r *Resolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
exists, res, err := r.tree.File(stereoscopeFile.Path(rootedPath(location.RealPath)))
rooted := r.rootedPath(location.FullNestedPath())
exists, res, err := r.tree.File(stereoscopeFile.Path(rooted))
if err != nil || !exists || !res.HasReference() {
return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.RealPath)
return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
}
entry, err := r.index.Get(*res.Reference)
if err != nil {
return file.Metadata{}, err
}
return entry.Metadata, nil
// the index holds entry paths; callers get full paths, as with locations
metadata := entry.Metadata
metadata.Path = r.fullPath(rooted)
if metadata.LinkDestination != "" {
metadata.LinkDestination = r.fullPath(linkDestination(rooted, metadata.LinkDestination, metadata.Type == stereoscopeFile.TypeHardLink))
}
return metadata, nil
}
func (r *Resolver) HasPath(userPath string) bool {
return r.tree.HasPath(stereoscopeFile.Path(rootedPath(userPath)))
return r.tree.HasPath(stereoscopeFile.Path(r.rootedPath(userPath)))
}
func (r *Resolver) FilesByPath(paths ...string) ([]file.Location, error) {
@@ -297,7 +332,7 @@ func (r *Resolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
var out []file.Location
seen := stereoscopeFile.NewFileReferenceSet()
for _, pattern := range patterns {
results, err := r.searchByGlob(rootedPath(pattern))
results, err := r.searchByGlob(r.rootedPath(pattern))
if err != nil {
return nil, fmt.Errorf("invalid glob %q: %w", pattern, err)
}
@@ -330,8 +365,8 @@ func (r *Resolver) searchByGlob(pattern string) ([]stereoscopeFile.Resolution, e
// merge into the SBOM.
func sortLocations(locations []file.Location) {
sort.Slice(locations, func(i, j int) bool {
if locations[i].RealPath != locations[j].RealPath {
return locations[i].RealPath < locations[j].RealPath
if a, b := locations[i].FullNestedPath(), locations[j].FullNestedPath(); a != b {
return a < b
}
return locations[i].AccessPath < locations[j].AccessPath
})
internal/archive/resolver_test.go
+35 -87
View File
@@ -4,8 +4,8 @@ import (
"archive/tar"
"bytes"
"context"
"fmt"
"io"
"path"
"sort"
"strings"
"testing"
@@ -55,31 +55,6 @@ func TestResolver_add_directoryEntriesHoldNoContent(t *testing.T) {
assert.Empty(t, readStore(t, r)["lib"].body)
}
func TestResolver_add_indexCostIsChargedEvenForEmptyContent(t *testing.T) {
// the index estimate is what bounds an archive of many empty entries
limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1})
r, _ := resolverIn(t, limiter.charge())
hdr := regularHeader("a.txt", 0)
require.NoError(t, r.add(hdr, bytes.NewReader(nil)))
mem, _ := limiter.InUse()
assert.Equal(t, approxIndexBytes(hdr), mem)
assert.Zero(t, r.heldInMemory())
}
func TestResolver_add_indexCostFallsBackToDiskWhenMemoryIsZero(t *testing.T) {
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
r, _ := resolverIn(t, limiter.charge())
hdr := regularHeader("a.txt", 0)
require.NoError(t, r.add(hdr, bytes.NewReader(nil)))
mem, disk := limiter.InUse()
assert.Zero(t, mem)
assert.Equal(t, approxIndexBytes(hdr), disk)
}
func TestResolver_add_indexDoesNotFallBackToDiskWhenMemoryIsBounded(t *testing.T) {
// the index lives in memory, so a bounded memory limit bounds it; only a zero limit defers to disk
limiter := NewLimiter(Limits{MaxMemoryBytes: 1024, MaxDiskBytes: -1})
@@ -108,21 +83,6 @@ func TestResolver_add_indexMakesRoomBySpillingHeldContent(t *testing.T) {
assert.Equal(t, strings.Repeat("a", 100), readStore(t, r)["a.txt"].body)
}
func TestResolver_add_refusesEntriesWhenNoBudgetAdmitsTheIndex(t *testing.T) {
room := approxIndexBytes(regularHeader("x0", 0)) * 3
r, _ := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: room}).charge())
var err error
admitted := 0
for i := 0; i < 100 && err == nil; i++ {
if err = r.add(regularHeader(fmt.Sprintf("x%d", i), 0), bytes.NewReader(nil)); err == nil {
admitted++
}
}
require.ErrorIs(t, err, ErrDiskLimitReached)
assert.Equal(t, 3, admitted)
}
func TestResolver_add_firstEntryAtAPathWins(t *testing.T) {
r, _ := resolverIn(t, memCharge(1<<20))
@@ -153,22 +113,6 @@ func TestResolver_pathsAndContents(t *testing.T) {
assert.Equal(t, "Manifest-Version: 1.0\n", string(body))
}
func TestResolver_contentsSurviveTheMoveToDisk(t *testing.T) {
// room for the index but only a few bytes of content, so the rest moves to disk
r := resolverOver(t, indexCostOf(jarEntriesFixture)+8, jarEntriesFixture)
locations, err := r.FilesByGlob("**/*.jar")
require.NoError(t, err)
require.Len(t, locations, 1)
reader, err := r.FileContentsByLocation(locations[0])
require.NoError(t, err)
body, err := io.ReadAll(reader)
require.NoError(t, err)
require.NoError(t, reader.Close())
assert.Equal(t, "PK\x03\x04 not really", string(body))
}
func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) {
r := resolverOver(t, 1<<20, map[string]string{
"notes.txt": "just some text",
@@ -177,26 +121,7 @@ func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) {
locations, err := r.FilesByMIMEType("text/x-shellscript")
require.NoError(t, err)
assert.Equal(t, []string{"run.sh"}, realPaths(locations))
}
func TestResolver_allLocationsCoversEveryFile(t *testing.T) {
r := resolverOver(t, 1<<20, jarEntriesFixture)
var got []string
for loc := range r.AllLocations(context.Background()) {
got = append(got, loc.RealPath)
}
sort.Strings(got)
assert.Equal(t, []string{
"BOOT-INF/lib/inner.jar",
"LICENSE",
"META-INF/MANIFEST.MF",
"META-INF/maven/com.example/lib/pom.properties",
"META-INF/maven/com.example/lib/pom.xml",
"com/example/Thing.class",
}, got)
assert.Equal(t, []string{"run.sh"}, localPaths(locations))
}
func TestResolver_answersAreOrdered(t *testing.T) {
@@ -227,14 +152,14 @@ func TestResolver_answersAreOrdered(t *testing.T) {
var all []string
for loc := range r.AllLocations(context.Background()) {
all = append(all, loc.RealPath)
all = append(all, loc.LocalPath())
}
assert.True(t, sort.StringsAreSorted(all))
}
var paths []string
for _, loc := range first {
paths = append(paths, loc.RealPath)
paths = append(paths, loc.LocalPath())
}
assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable")
}
@@ -279,7 +204,7 @@ func globPaths(t *testing.T, r *Resolver, pattern string) []string {
t.Helper()
locations, err := r.FilesByGlob(pattern)
require.NoError(t, err)
return realPaths(locations)
return localPaths(locations)
}
// several paths to one file answer as one location: the file's own path when it matched, otherwise
@@ -399,8 +324,8 @@ func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) {
require.NoError(t, err)
require.Len(t, locations, 1)
assert.Equal(t, "opt/real.bin", locations[0].RealPath, "content comes from the file")
assert.Equal(t, "opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched")
assert.Equal(t, "opt/real.bin", locations[0].LocalPath(), "content comes from the file")
assert.Equal(t, "outer.jar!/opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched")
reader, err := r.FileContentsByLocation(locations[0])
require.NoError(t, err)
@@ -412,6 +337,28 @@ func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) {
"a location standing for a link must still read the file's bytes, not the link's none")
}
func TestResolver_fullPathsAreTrimmedToEntryPaths(t *testing.T) {
// catalogers build queries from the full paths the resolver hands out, e.g. path.Dir plus a sibling
r := resolverOver(t, 1<<20, map[string]string{"pom.xml": "a", "lib/dep.jar": "b"})
locations, err := r.FilesByPath("outer.jar!/lib/dep.jar")
require.NoError(t, err)
require.Len(t, locations, 1)
assert.Equal(t, "outer.jar!/lib/dep.jar", locations[0].AccessPath)
locations, err = r.FilesByPath(path.Join(path.Dir(locations[0].AccessPath), "../pom.xml"))
require.NoError(t, err)
require.Len(t, locations, 1, "a sibling at the archive root is reached through the full path")
assert.Equal(t, "outer.jar!/pom.xml", locations[0].FullNestedPath())
assert.True(t, r.HasPath(path.Dir("outer.jar!/pom.xml")), "the directory of a root entry is the root")
assert.True(t, r.HasPath("lib/dep.jar"), "an entry path is still accepted")
locations, err = r.FilesByGlob("outer.jar!/**/*.jar")
require.NoError(t, err)
require.Len(t, locations, 1)
}
func TestResolver_danglingLinkHasNoContent(t *testing.T) {
r := resolverFrom(t, testEntry{name: "opt/gone.jar", link: "missing.jar"})
@@ -420,14 +367,15 @@ func TestResolver_danglingLinkHasNoContent(t *testing.T) {
locations = append(locations, l)
}
require.Len(t, locations, 1)
assert.Equal(t, "opt/gone.jar", locations[0].RealPath)
assert.Equal(t, "opt/gone.jar", locations[0].LocalPath())
_, err := r.FileContentsByLocation(locations[0])
assert.Error(t, err, "a link to nothing has no content to read")
metadata, err := r.FileMetadataByLocation(locations[0])
require.NoError(t, err)
assert.Equal(t, "missing.jar", metadata.LinkDestination, "where it pointed is still reported, as the archive named it")
assert.Equal(t, "outer.jar!/opt/missing.jar", metadata.LinkDestination,
"where it pointed is still reported, resolved against the link's own directory")
}
func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) {
@@ -443,7 +391,7 @@ func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) {
t.Helper()
locations, err := r.FilesByMIMEType(types...)
require.NoError(t, err)
return realPaths(locations)
return localPaths(locations)
}
assert.Equal(t, []string{"opt/real.jar"}, byMIME("application/zip"),
@@ -465,7 +413,7 @@ func TestResolver_allLocationsKeepsEveryPath(t *testing.T) {
var got []string
for l := range r.AllLocations(context.Background()) {
got = append(got, l.RealPath)
got = append(got, l.LocalPath())
}
assert.Equal(t, []string{"opt/gone.jar", "opt/real.jar", "opt/sym.jar"}, got,
"every path the archive names, links included")
@@ -489,7 +437,7 @@ func TestResolver_add_anEntryOverTheCapIsSkippedAndReadNoFurther(t *testing.T) {
assert.ErrorContains(t, r.Truncated, "per-entry size cap of 1.0 KiB")
locations, err := r.FilesByGlob("**")
require.NoError(t, err)
assert.Equal(t, []string{"ok.txt"}, realPaths(locations))
assert.Equal(t, []string{"ok.txt"}, localPaths(locations))
}
type countingReader struct {
internal/archive/storage.go
+2 -2
View File
@@ -109,7 +109,7 @@ func (r *Resolver) spill() error {
if err := r.openFile(); err != nil {
return err
}
log.WithFields("archive", r.archivePath, "held", len(r.held), "bytes", r.written).
log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "held", len(r.held), "bytes", r.written).
Debug("archive content does not fit in memory; writing it to disk")
for len(r.held) > 0 {
@@ -188,7 +188,7 @@ func (r *Resolver) releaseStorage() {
return
}
if err := r.file.Close(); err != nil {
log.WithFields("archive", r.archivePath, "error", err).Trace("unable to close archive temp file")
log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "error", err).Trace("unable to close archive temp file")
}
r.remove()
r.charge.refundDisk(r.written)
internal/archive/storage_test.go
+2 -2
View File
@@ -223,7 +223,7 @@ func TestResolver_put_discardRefundsWhatIsHeldInMemory(t *testing.T) {
assert.Equal(t, "keep", readEntry(t, s.open(&keep)))
}
func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
func TestResolver_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
small, smallRoot := extractedResolver(t, manyEntryZip(t, 2), spillingLimits)
large, largeRoot := extractedResolver(t, manyEntryZip(t, 5000), spillingLimits)
@@ -238,7 +238,7 @@ func TestStorage_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
assert.Equal(t, int64(5000), large.written)
}
func TestStorage_anEntryAtTheEndIsReadBySeeking(t *testing.T) {
func TestResolver_anEntryAtTheEndIsReadBySeeking(t *testing.T) {
// the reader over an entry is a bounded window at its offset, not a re-read of what precedes it
const filler = 64 * 1024
internal/archive/traversal.go
+45 -23
View File
@@ -13,9 +13,8 @@ import (
// such as java can describe the archive it is inside without owning the recursion.
type Traversal struct {
// Location is the archive file's location within its parent filesystem, including the access path
// it was reached by. Its ArchivePath names the archives above it, so VirtualPath(Location) is the
// full chain from the scan root as seen through that access path. The ArchivePath of the files
// inside it is built from the real path instead.
// it was reached by, so VirtualPath(Location) is the chain from the scan root as seen through that
// access path. The files inside it extend its real coordinates instead (see ContainerPath).
Location file.Location
// Digests are of the archive file itself, taken during extraction; a cataloger running inside the
@@ -31,41 +30,64 @@ type Traversal struct {
Packages []pkg.Package
}
// ContentsArchivePath is the ArchivePath every file inside the archive carries: the chain to it, built
// from real paths. Outside any archive (a nil Traversal) it is blank, as for files at the scan root.
func (t *Traversal) ContentsArchivePath() string {
// FullTraversalArchivePath is the archive's own full path, which is the ContainerPath of every file inside it. Outside
// any archive (a nil Traversal) it is blank, as for files on the scanned filesystem.
func (t *Traversal) FullTraversalArchivePath() string {
if t == nil {
return ""
}
return VirtualPath(file.NewLocationFromCoordinates(t.Location.Coordinates))
return t.Location.FullNestedPath()
}
// ContainerPath returns the full path of the archive directly holding the file at c, or blank for a
// file on the scanned filesystem, whether or not its FullPath is set.
func ContainerPath(c file.Coordinates) string {
i := strings.LastIndex(c.FullPath, file.ArchivePathSeparator)
if i < 0 {
return ""
}
return c.FullPath[:i]
}
// archiveEntries is the "!/"-delimited chain of entry paths after RealPath, reporting false for a file
// not inside an archive. RealPath is split off first, so a "!/" in it is not taken for a boundary.
func archiveEntries(c file.Coordinates) (string, bool) {
remain, ok := strings.CutPrefix(c.FullNestedPath(), c.RealPath)
if !ok {
return "", false
}
return strings.CutPrefix(remain, file.ArchivePathSeparator)
}
// VirtualPath returns the colon-delimited chain of archives from the scan root to loc, ending in loc
// itself: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path unchanged. In an
// entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':' recovers the archive
// boundaries and no two entry names share an escaped form. The leading slash is dropped to match the
// java cataloger, which joins slash-less zip entry names onto the containing archive's path.
// itself as accessed: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path
// unchanged. In an entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':'
// recovers the archive boundaries and no two entry names share an escaped form. Leading slashes are
// dropped to match the java cataloger, which joins slash-less zip entry names onto the containing
// archive's path.
func VirtualPath(loc file.Location) string {
if loc.ArchivePath == "" {
return loc.Path()
accessPath := loc.AccessPath
if accessPath == "" {
accessPath = loc.FullNestedPath()
}
return loc.ArchivePath + ":" + strings.TrimPrefix(escapeEntryPath(loc.Path()), "/")
entries, ok := archiveEntries(loc.Coordinates)
if !ok {
return accessPath
}
segments := strings.Split(entries, file.ArchivePathSeparator)
segments[len(segments)-1] = strings.TrimPrefix(accessPath, ContainerPath(loc.Coordinates)+file.ArchivePathSeparator)
for i, s := range segments {
segments[i] = strings.TrimPrefix(escapeEntryPath(s), "/")
}
return loc.RealPath + ":" + strings.Join(segments, ":")
}
var (
entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A")
entryPathUnescaper = strings.NewReplacer("%3A", ":", "%25", "%")
)
var entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A")
func escapeEntryPath(p string) string {
return entryPathEscaper.Replace(p)
}
// UnescapeEntryPath reverses the escaping VirtualPath applies to one segment of the chain.
func UnescapeEntryPath(p string) string {
return entryPathUnescaper.Replace(p)
}
type traversalCtxKey struct{}
func WithTraversal(ctx context.Context, t *Traversal) context.Context {
internal/archive/traversal_test.go
+32 -32
View File
@@ -2,7 +2,6 @@ package archive
import (
"context"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -18,16 +17,24 @@ func TestTraversalContextRoundTrip(t *testing.T) {
ctx := WithTraversal(context.Background(), trav)
require.Same(t, trav, TraversalFromContext(ctx))
child := &Traversal{Location: locationIn("app.war", "/WEB-INF/lib/dep.jar")}
child := &Traversal{Location: locationIn("app.war", "WEB-INF/lib/dep.jar")}
ctx = WithTraversal(ctx, child)
require.Same(t, child, TraversalFromContext(ctx))
}
// locationIn is a location as the Index returns it: an entry path stamped with the containing chain.
func locationIn(archivePath, entryPath string) file.Location {
loc := file.NewLocation(entryPath)
loc.ArchivePath = archivePath
return loc
// locationIn is a location as the Resolver returns it: the outermost archive's real path, and the chain
// of entry paths within it.
func locationIn(realPath, archivePath string) file.Location {
return file.NewLocationFromCoordinates(file.Coordinates{RealPath: realPath, FullPath: realPath + file.ArchivePathSeparator + archivePath})
}
func TestContainerPath(t *testing.T) {
assert.Equal(t, "", ContainerPath(file.NewCoordinates("/app.war", "")))
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war"}), "a top-level file whose FullPath is set")
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/odd!/name"}), "a separator in a top-level real path is not an archive boundary")
assert.Equal(t, "/app.war", ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar").Coordinates))
assert.Equal(t, "/app.war!/WEB-INF/lib/dep.jar",
ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF").Coordinates))
}
func TestVirtualPath(t *testing.T) {
@@ -46,6 +53,11 @@ func TestVirtualPath(t *testing.T) {
location: file.NewLocation("some/dir/example.jar"),
expected: "some/dir/example.jar",
},
{
name: "outside an archive with FullPath set returns the path unchanged",
location: file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/odd!/name.jar", FullPath: "/odd!/name.jar"}),
expected: "/odd!/name.jar",
},
{
name: "single level trims leading slash",
location: locationIn("app.war", "/WEB-INF/lib/dep.jar"),
@@ -58,7 +70,7 @@ func TestVirtualPath(t *testing.T) {
},
{
name: "multi-level chain",
location: locationIn("app.war:WEB-INF/lib/dep.jar", "/inner/most.jar"),
location: locationIn("app.war", "WEB-INF/lib/dep.jar!/inner/most.jar"),
expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar",
},
{
@@ -72,11 +84,23 @@ func TestVirtualPath(t *testing.T) {
location: locationIn("app.war", "lib/a:b:c.jar"),
expected: "app.war:lib/a%3Ab%3Ac.jar",
},
{
// or the literal text %3A would collide with an escaped colon
name: "the escape character is itself escaped",
location: locationIn("outer.zip", "a%3Ab.jar"),
expected: "outer.zip:a%253Ab.jar",
},
{
name: "access path is preferred over real path",
location: file.NewVirtualLocation("/real/dep.jar", "/link/dep.jar"),
expected: "/link/dep.jar",
},
{
name: "access path within an archive replaces the last entry",
location: file.NewVirtualLocationFromCoordinates(
file.Coordinates{RealPath: "app.war", FullPath: "app.war!/WEB-INF/lib/real.jar"}, "app.war!/WEB-INF/lib/link.jar"),
expected: "app.war:WEB-INF/lib/link.jar",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -84,27 +108,3 @@ func TestVirtualPath(t *testing.T) {
})
}
}
func TestVirtualPath_theEscapeCharacterIsItselfEscaped(t *testing.T) {
// an entry path holding the literal text %3A must not collide with one holding a colon, or their
// contents share an ArchivePath and overwrite each other's file metadata
fromColon := VirtualPath(locationIn("outer.zip", "a:b.jar"))
fromLiteral := VirtualPath(locationIn("outer.zip", "a%3Ab.jar"))
assert.Equal(t, "outer.zip:a%3Ab.jar", fromColon)
assert.Equal(t, "outer.zip:a%253Ab.jar", fromLiteral)
}
func TestVirtualPath_escapingComposesAcrossNestingLevels(t *testing.T) {
level2 := VirtualPath(locationIn("outer.zip", "lib/mid:dle.war"))
require.Equal(t, "outer.zip:lib/mid%3Adle.war", level2)
final := VirtualPath(locationIn(level2, "WEB-INF/lib/inn:er.jar"))
require.Equal(t, "outer.zip:lib/mid%3Adle.war:WEB-INF/lib/inn%3Aer.jar", final)
assert.Equal(t,
[]string{"outer.zip", "lib/mid%3Adle.war", "WEB-INF/lib/inn%3Aer.jar"},
strings.Split(final, ":"),
"splitting on ':' must recover exactly the three real archive boundaries",
)
}
internal/cmptest/common_options.go
+1 -1
View File
@@ -69,5 +69,5 @@ func locationSorter(a, b file.Location) int {
return strings.Compare(a.AccessPath, b.AccessPath)
}
return strings.Compare(a.RealPath, b.RealPath)
return strings.Compare(a.FullNestedPath(), b.FullNestedPath())
}
internal/cmptest/location.go
+1 -1
View File
@@ -14,5 +14,5 @@ func DefaultLocationComparer(x, y file.Location) bool {
}
func LocationComparerWithoutLayer(x, y file.Location) bool {
return cmp.Equal(x.RealPath, y.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath)
return cmp.Equal(x.FullNestedPath(), y.FullNestedPath()) && cmp.Equal(x.AccessPath, y.AccessPath)
}
internal/os/feature_detection.go
+6 -6
View File
@@ -49,7 +49,7 @@ func findRhelFeatures(resolver file.Resolver, release *linux.Release) error {
for _, contentManifestFile := range contentManifestFiles {
found, err := fileMatchesAny(resolver, contentManifestFile, rhelEUSPatterns...)
if err != nil {
return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.RealPath, err)
return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.FullNestedPath(), err)
}
if found {
release.ExtendedSupport = true
@@ -163,9 +163,9 @@ func hasActiveUbuntuESMService(resolver file.Resolver) (bool, error) {
func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location) (bool, error) {
contents, err := resolver.FileContentsByLocation(location)
if err != nil {
return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.RealPath, err)
return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.FullNestedPath(), err)
}
defer internal.CloseAndLogError(contents, location.RealPath)
defer internal.CloseAndLogError(contents, location.FullNestedPath())
var status struct {
Services []struct {
@@ -174,7 +174,7 @@ func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location
} `json:"services"`
}
if err := json.NewDecoder(contents).Decode(&status); err != nil {
return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.RealPath, err)
return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.FullNestedPath(), err)
}
// pro status values are enabled/disabled/n-a/warning/—; only "enabled" proves the stream is active.
@@ -200,8 +200,8 @@ func hasInstalledESMPackage(packages *pkg.Collection) bool {
func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) {
contents, err := resolver.FileContentsByLocation(location)
if err != nil {
return false, fmt.Errorf("unable to read %s: %w", location.RealPath, err)
return false, fmt.Errorf("unable to read %s: %w", location.FullNestedPath(), err)
}
defer internal.CloseAndLogError(contents, location.RealPath)
defer internal.CloseAndLogError(contents, location.FullNestedPath())
return internal.MatchAnyFromReader(contents, patterns...)
}
internal/relationship/binary/binary_dependencies.go
+2 -2
View File
@@ -71,7 +71,7 @@ func compareElfBinaryPackages(s *sbom.SBOM) []artifact.ID {
pkgsToDelete := make([]artifact.ID, 0)
for _, elfPkg := range allElfPackages(s) {
for _, loc := range onlyPrimaryEvidenceLocations(elfPkg) {
for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.RealPath) {
for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.FullNestedPath()) {
// we only care about comparing binary packages to each other (not other types)
if otherPkg.Type != pkg.BinaryPkg {
continue
@@ -130,7 +130,7 @@ func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver
for _, loc := range libLocations {
// are you in our index?
realBaseName := path.Base(loc.RealPath)
realBaseName := path.Base(loc.FullNestedPath())
pkgCollection := index.owningLibraryPackage(realBaseName)
if pkgCollection.PackageCount() < 1 {
addRelationship(
internal/relationship/binary/shared_library_index.go
+19 -12
View File
@@ -3,6 +3,7 @@ package binary
import (
"path"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/file"
@@ -63,7 +64,7 @@ func locationsThatProvideLibraries(accessor sbomsync.Accessor) (map[string]file.
continue
}
basename := path.Base(coord.RealPath)
basename := path.Base(coord.FullNestedPath())
set := libLocationsByBasename[basename]
set.Add(coord)
allLibLocations.Add(coord)
@@ -82,6 +83,11 @@ func packagesWithLibraryOwnership(resolver file.Resolver, accessor sbomsync.Acce
var ownedFilePaths []string
if p.Type == pkg.BinaryPkg {
for _, loc := range p.Locations.ToSlice() {
if archive.ContainerPath(loc.Coordinates) != "" {
// a path within an archive cannot be resolved from the scan root; the location is the library
addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p)
continue
}
ownedFilePaths = append(ownedFilePaths, loc.Path())
}
} else {
@@ -114,18 +120,19 @@ func populatePackagesByLibraryPath(
}
for _, loc := range ownedLocation {
// if the location is a library, add the package to the set of packages that own the library
if !allLibLocations.Contains(loc.Coordinates) {
continue
}
if _, ok := packagesByLibraryPath[loc.Coordinates]; !ok {
packagesByLibraryPath[loc.Coordinates] = pkg.NewCollection()
}
// we have a library path, add the package to the set of packages that own the library
packagesByLibraryPath[loc.Coordinates].Add(p)
addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p)
}
}
return packagesByLibraryPath
}
// addLibraryOwner records p as an owner of the file at coordinates when that file is a library.
func addLibraryOwner(packagesByLibraryPath map[file.Coordinates]*pkg.Collection, allLibLocations file.CoordinateSet, coordinates file.Coordinates, p pkg.Package) {
if !allLibLocations.Contains(coordinates) {
return
}
if _, ok := packagesByLibraryPath[coordinates]; !ok {
packagesByLibraryPath[coordinates] = pkg.NewCollection()
}
packagesByLibraryPath[coordinates].Add(p)
}
internal/relationship/by_file_ownership.go
+20 -2
View File
@@ -6,6 +6,7 @@ import (
"github.com/bmatcuk/doublestar/v4"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact"
@@ -100,7 +101,7 @@ func findOwnershipByFilesRelationships(resolver file.Resolver, catalog *pkg.Coll
// Now establish relationships, considering symlinks
for _, candidateOwnerPkg := range catalog.Sorted() {
id := candidateOwnerPkg.ID()
if candidateOwnerPkg.Metadata == nil {
if candidateOwnerPkg.Metadata == nil || withinArchive(candidateOwnerPkg) {
continue
}
@@ -164,7 +165,7 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set {
// First, identify direct ownership of all files
for _, p := range catalog.Sorted() {
if p.Metadata == nil {
if p.Metadata == nil || withinArchive(p) {
continue
}
@@ -192,6 +193,21 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set {
return directOwnership
}
// withinArchive reports whether every location of p is inside an archive. Such a package's owned files
// are paths within its archive, not the scan root, so it is not treated as a file owner.
func withinArchive(p pkg.Package) bool {
locations := p.Locations.ToSlice()
if len(locations) == 0 {
return false
}
for _, l := range locations {
if archive.ContainerPath(l.Coordinates) == "" {
return false
}
}
return true
}
func resolvePaths(ownedFilePath string, resolver file.Resolver) []string {
// though we have a string path, we need to resolve symlinks and other filesystem oddities since we cannot assume this is a real path
var locs []file.Location
@@ -206,6 +222,8 @@ func resolvePaths(ownedFilePath string, resolver file.Resolver) []string {
ownedFilePaths := strset.New(ownedFilePath)
for _, loc := range locs {
// Owned file paths are currently based on the top-level resolver, which is always the RealPath and includes
// all files found within archives at that location
ownedFilePaths.Add(loc.RealPath)
}
return ownedFilePaths.List()
internal/relationship/by_file_ownership_test.go
+55
View File
@@ -78,6 +78,61 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
return []pkg.Package{parent, child}, []artifact.Relationship{relationship}
},
},
{
name: "package-within-archive-is-not-a-file-owner",
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
nestedOwner := pkg.Package{
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/rootfs.tar",
FullPath: "/rootfs.tar!/var/lib/rpm/Packages",
})),
Type: pkg.RpmPkg,
Metadata: pkg.RpmDBEntry{
Files: []pkg.RpmFileRecord{{Path: "/d/path"}},
},
}
nestedOwner.SetID()
child := pkg.Package{
Locations: file.NewLocationSet(file.NewLocation("/d/path")),
Type: pkg.BinaryPkg,
}
child.SetID()
return []pkg.Package{nestedOwner, child}, nil
},
},
{
name: "top-level-owner-of-an-archive-owns-packages-within-it",
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
parent := pkg.Package{
Locations: file.NewLocationSet(file.NewLocation("/var/lib/rpm/Packages")),
Type: pkg.RpmPkg,
Metadata: pkg.RpmDBEntry{
Files: []pkg.RpmFileRecord{{Path: "/opt/bundle.tar"}},
},
}
parent.SetID()
child := pkg.Package{
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/opt/bundle.tar",
FullPath: "/opt/bundle.tar!/lib/thing.jar",
})),
Type: pkg.JavaPkg,
}
child.SetID()
relationship := artifact.Relationship{
From: parent,
To: child,
Type: artifact.OwnershipByFileOverlapRelationship,
Data: ownershipByFilesMetadata{Files: []string{"/opt/bundle.tar"}},
}
return []pkg.Package{parent, child}, []artifact.Relationship{relationship}
},
},
{
name: "only-real-file-owner-gets-ownership-relationship-not-symlink-owner",
resolver: mockFR{
internal/relationship/evident_by_test.go
+4 -16
View File
@@ -15,18 +15,9 @@ func TestRelationshipsEvidentBy(t *testing.T) {
c := pkg.NewCollection()
coordA := file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
}
coordC := file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
}
coordD := file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
}
coordA := file.NewCoordinates("/somewhere/real", "abc")
coordC := file.NewCoordinates("/somewhere/real", "abc")
coordD := file.NewCoordinates("/somewhere/real", "abc")
pkgA := pkg.Package{
Locations: file.NewLocationSet(
// added!
@@ -39,10 +30,7 @@ func TestRelationshipsEvidentBy(t *testing.T) {
pkgA.SetID()
c.Add(pkgA)
coordB := file.Coordinates{
RealPath: "/somewhere-else/real",
FileSystemID: "def",
}
coordB := file.NewCoordinates("/somewhere-else/real", "def")
pkgB := pkg.Package{
Locations: file.NewLocationSet(
// added!
internal/relationship/index_test.go
+10 -22
View File
@@ -21,12 +21,8 @@ func Test_Index(t *testing.T) {
p3 := pkg.Package{
Name: "pkg-3",
}
c1 := file.Coordinates{
RealPath: "/coords/1",
}
c2 := file.Coordinates{
RealPath: "/coords/2",
}
c1 := file.NewCoordinates("/coords/1", "")
c2 := file.NewCoordinates("/coords/2", "")
for _, p := range []*pkg.Package{&p1, &p2, &p3} {
p.SetID()
@@ -156,18 +152,10 @@ func Test_Coordinates(t *testing.T) {
p3 := pkg.Package{
Name: "pkg-3",
}
c1 := file.Coordinates{
RealPath: "/coords/1",
}
c2 := file.Coordinates{
RealPath: "/coords/2",
}
c3 := file.Coordinates{
RealPath: "/coords/3",
}
c4 := file.Coordinates{
RealPath: "/coords/4",
}
c1 := file.NewCoordinates("/coords/1", "")
c2 := file.NewCoordinates("/coords/2", "")
c3 := file.NewCoordinates("/coords/3", "")
c4 := file.NewCoordinates("/coords/4", "")
for _, p := range []*pkg.Package{&p1, &p2, &p3} {
p.SetID()
@@ -237,10 +225,10 @@ func TestRemove(t *testing.T) {
p1 := pkg.Package{Name: "pkg-1"}
p2 := pkg.Package{Name: "pkg-2"}
p3 := pkg.Package{Name: "pkg-3"}
c1 := file.Coordinates{RealPath: "/coords/1"}
c2 := file.Coordinates{RealPath: "/coords/2"}
c3 := file.Coordinates{RealPath: "/coords/3"}
c4 := file.Coordinates{RealPath: "/coords/4"}
c1 := file.NewCoordinates("/coords/1", "")
c2 := file.NewCoordinates("/coords/2", "")
c3 := file.NewCoordinates("/coords/3", "")
c4 := file.NewCoordinates("/coords/4", "")
for _, p := range []*pkg.Package{&p1, &p2, &p3} {
p.SetID()
internal/task/archive_nesting_test.go
+60 -173
View File
@@ -18,6 +18,7 @@ import (
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/internal/testutils"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/file"
@@ -28,7 +29,7 @@ import (
// This file covers nesting across archive families. The other nested-archive fixtures are all zips,
// so a zip-specific mechanism would pass them; the requirement here is that an archive of one family
// inside another behaves as it would inside its own family - same packages, same filesystem-id chain,
// inside another behaves as it would inside its own family - same packages, same container-path chain,
// same CONTAINS edges, same depth accounting.
//
// Chains are built in the test rather than committed as binaries, so each case can vary which family
@@ -71,8 +72,8 @@ type nestedArchive struct {
// paths is each archive's path within its parent, outermost first; paths[0] is the outermost archive
// as the scan's resolver reports it.
paths []string
// fileSystemIDs is the composed chain each archive's own contents carry, outermost first.
fileSystemIDs []string
// containerPaths is each archive's full path, the container path its own contents carry, outermost first.
containerPaths []string
// virtualPaths is the same chain in java's colon-joined form, outermost first.
virtualPaths []string
}
@@ -87,6 +88,7 @@ type nestingRow struct {
// nestingVisit is one filesystem the archive cataloger's sub-pipeline was run against.
type nestingVisit struct {
virtualPath string
fullPath string
depth int
// heldFiles are the spill files live at this moment, sampled only when the probe was given a temp
// dir to look in. A nested archive's own bytes are read where they lie, so one appears per archive
@@ -120,12 +122,12 @@ func Test_mixedFamilyNesting_leafIsCatalogedOnceWithTheFullChain(t *testing.T) {
locs := leafLocations(s)
require.Len(t, locs, 1, "the leaf must be reached by exactly one path")
// exact, never a substring: a partially composed chain passes a substring check
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath,
assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates),
"the chain must name all three archives outer-to-inner")
assert.Equal(t, "nested/marker.txt", locs[0].RealPath,
assert.Equal(t, nested.leafContainerPath()+"!/nested/marker.txt", locs[0].FullNestedPath(),
"the path stays relative to the archive the file came from")
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(),
assert.Equal(t, nested.containerPaths, probe.containerPaths(),
"every level must be cataloged, outer to inner, whatever families they are")
})
}
@@ -145,11 +147,7 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) {
// directory, so it carries no filesystem id of its own
archiveCoords := make([]file.Coordinates, len(row.families))
for i := range row.families {
var parentFSID string
if i > 0 {
parentFSID = nested.fileSystemIDs[i-1]
}
archiveCoords[i] = file.Coordinates{RealPath: nested.paths[i], ArchivePath: parentFSID}
archiveCoords[i] = nested.coordinates(i)
}
fileEdges := strset.New()
@@ -185,47 +183,25 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) {
for _, rel := range s.Relationships {
from, _ := rel.From.(file.Coordinates)
to, _ := rel.To.(file.Coordinates)
leafEdge = leafEdge || (from == innermost && to == file.Coordinates{RealPath: "nested/marker.txt", ArchivePath: nested.leafFileSystemID()})
leafEdge = leafEdge || (from == innermost && to == file.Coordinates{RealPath: innermost.RealPath, FullPath: innermost.FullNestedPath() + "!/nested/marker.txt"})
}
assert.True(t, leafEdge, "expected a CONTAINS edge from the innermost archive to the leaf file")
})
}
}
func Test_mixedFamilyNesting_javaLeafVirtualPathIsTheColonJoinedChain(t *testing.T) {
// these colon-delimited strings are in already-published SBOMs, so a jar reached through three
// levels of mixed families must still be reported under the chain of all three
for _, row := range nestingRows() {
if row.families[len(row.families)-1] != familyJar {
continue
}
t.Run(row.name, func(t *testing.T) {
scanDir := t.TempDir()
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), javaTask())
// the innermost archive is the jar, so its own package is the java leaf and the chain of
// all three archives is exactly its virtual path
leaf := fmt.Sprintf("level%d", len(row.families)-1)
assert.Equal(t, 1, packageCount(s, leaf), "the leaf jar must be cataloged exactly once")
assert.Equal(t, nested.virtualPaths[len(row.families)-1], javaVirtualPaths(s)[leaf])
})
}
}
func Test_mixedFamilyNesting_jarIsExercisedAsAContainer(t *testing.T) {
func Test_mixedFamilyNesting_everyJarIsCatalogedOnceUnderItsChain(t *testing.T) {
// two mechanisms meet a jar: the java cataloger catalogs it as a package, the archive task descends
// into it. A jar mid-chain must do both, exactly once each; the failure mode is a duplicate, not an
// error.
// into it. A jar at any level must do both, exactly once each; the failure mode is a duplicate, not an
// error. The colon-joined virtual paths are in already-published SBOMs, so they must name the chain.
for _, row := range nestingRows() {
var containerLevels []int
var jarLevels []int
for i, family := range row.families {
if family == familyJar && i < len(row.families)-1 {
containerLevels = append(containerLevels, i)
if family == familyJar {
jarLevels = append(jarLevels, i)
}
}
if len(containerLevels) == 0 {
if len(jarLevels) == 0 {
continue
}
@@ -236,19 +212,18 @@ func Test_mixedFamilyNesting_jarIsExercisedAsAContainer(t *testing.T) {
probe := &nestingProbe{}
s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), javaTask(), probe.task())
for _, level := range containerLevels {
for _, level := range jarLevels {
name := fmt.Sprintf("level%d", level)
assert.Equal(t, 1, packageCount(s, name),
"the container jar's own package must be cataloged exactly once")
assert.Equal(t, 1, packageCount(s, name), "the jar's own package must be cataloged exactly once")
assert.Equal(t, nested.virtualPaths[level], javaVirtualPaths(s)[name],
"the container jar's virtual path must be its own place in the chain")
"the jar's virtual path must be its own place in the chain")
}
// and the archive task still descended through it, so the chain is complete below it
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs())
// and the archive task still descended through every level
assert.Equal(t, nested.containerPaths, probe.containerPaths())
locs := leafLocations(s)
require.Len(t, locs, 1)
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates))
})
}
}
@@ -264,16 +239,13 @@ func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) {
probe := &nestingProbe{}
s := runNesting(t, scanDir, 2, defaultLimits(), markerTask(), fileMetadataTask(), probe.task())
assert.Equal(t, nested.fileSystemIDs[:2], probe.archiveFileSystemIDs(),
assert.Equal(t, nested.containerPaths[:2], probe.containerPaths(),
"at depth 2 the outer and middle archives are cataloged and the innermost is not")
assert.Zero(t, packageCount(s, "marker-pkg"),
"the leaf is one level past the bound, so its package must be absent")
// the innermost archive is still cataloged as a file, inside the middle archive
innermost := file.Coordinates{
RealPath: nested.paths[2],
ArchivePath: nested.fileSystemIDs[1],
}
innermost := nested.coordinates(2)
_, ok := s.Artifacts.FileMetadata[innermost]
assert.True(t, ok, "the undescended archive must still be recorded as a file at %s", coordKey(innermost))
@@ -285,64 +257,6 @@ func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) {
}
}
func Test_mixedFamilyNesting_defaultBoundsCatalogEveryRow(t *testing.T) {
// the default bounds, pinned along with the fact that a three-level chain of any family mix sits
// well inside them
bounds := cataloging.DefaultArchiveSearchConfig()
require.Equal(t, int64(2*1024*1024*1024), bounds.MaxMemoryBytes)
require.Equal(t, int64(100*1024*1024*1024), bounds.MaxDiskBytes)
for _, row := range nestingRows() {
t.Run(row.name, func(t *testing.T) {
scanDir := t.TempDir()
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
probe := &nestingProbe{}
s := runNesting(t, scanDir, 3, archiveLimits(bounds), markerTask(), probe.task())
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(),
"every level of the chain must be cataloged at the default bounds")
locs := leafLocations(s)
require.Len(t, locs, 1)
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
})
}
}
func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing.T) {
// entries that will not fit in memory are written to disk rather than refused
scanDir := t.TempDir()
tempDir := isolatedTempDir(t)
nested := writeNestedArchive(t, scanDir, gradedChain())
t.Run("with a generous memory limit nothing is written to disk", func(t *testing.T) {
probe := &nestingProbe{tempDir: tempDir}
bounds := defaultLimits()
bounds.MaxMemoryBytes = int64(2 * (nested.sizes[0] + nested.sizes[1] + nested.sizes[2]))
runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
deepest := probe.deepestVisit()
require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held")
assert.Empty(t, deepest.heldFiles)
})
t.Run("with a memory limit the outermost archive's entries do not fit in", func(t *testing.T) {
// room for every level's index estimate, which must stay in memory, but not the outer entries
probe := &nestingProbe{tempDir: tempDir}
bounds := archive.Limits{MaxMemoryBytes: int64(nested.sizes[2] + 32*1024), MaxDiskBytes: -1}
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
deepest := probe.deepestVisit()
require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held")
assert.Contains(t, deepest.heldFiles, "archive-spill", "entries that do not fit in memory are written to disk")
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(), "and every level is still cataloged")
locs := leafLocations(s)
require.Len(t, locs, 1)
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
})
}
func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) {
// an archive's own bytes are read where they lie: a scanned file from disk, a nested archive from
// its parent's entries. Nothing copies them, so at most one spill file per level appears whatever
@@ -376,10 +290,10 @@ func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) {
require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held")
assert.Equal(t, tc.wantHeldFiles, deepest.heldFiles)
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs())
assert.Equal(t, nested.containerPaths, probe.containerPaths())
locs := leafLocations(s)
require.Len(t, locs, 1)
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates))
})
}
}
@@ -395,7 +309,7 @@ func Test_mixedFamilyNesting_diskLimitTruncatesTheLevelItCannotAdmit(t *testing.
// a sibling of a different family, processed once the chain has been released, so cataloged in full
require.NoError(t, os.WriteFile(
filepath.Join(scanDir, "sibling.tgz"),
makeTarGz(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}),
testutils.TarGz(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}),
0o644,
))
@@ -408,50 +322,29 @@ func Test_mixedFamilyNesting_diskLimitTruncatesTheLevelItCannotAdmit(t *testing.
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
want := append(append([]string{}, nested.fileSystemIDs...), "/sibling.tgz")
assert.ElementsMatch(t, want, probe.archiveFileSystemIDs(),
want := append(append([]string{}, nested.containerPaths...), "/sibling.tgz")
assert.ElementsMatch(t, want, probe.containerPaths(),
"every level is visited, the truncated one with nothing inside it")
fsIDs := strset.New()
containers := strset.New()
for _, loc := range leafLocations(s) {
fsIDs.Add(loc.ArchivePath)
containers.Add(archive.ContainerPath(loc.Coordinates))
}
assert.False(t, fsIDs.Has(nested.leafFileSystemID()), "the truncated level's contents must be absent")
assert.True(t, fsIDs.Has("/sibling.tgz"), "the sibling of a different family must be cataloged fully")
assert.False(t, containers.Has(nested.leafContainerPath()), "the truncated level's contents must be absent")
assert.True(t, containers.Has("/sibling.tgz"), "the sibling of a different family must be cataloged fully")
var truncated []string
for coords, reasons := range s.Artifacts.Unknowns {
for _, reason := range reasons {
if strings.Contains(reason, "disk limit") {
truncated = append(truncated, coords.ArchivePath+":"+coords.RealPath)
truncated = append(truncated, coords.FullNestedPath())
}
}
}
assert.Equal(t, []string{nested.fileSystemIDs[1] + ":" + nested.paths[2]}, truncated,
assert.Equal(t, []string{nested.coordinates(2).FullNestedPath()}, truncated,
"the truncation is recorded against the archive that reached the limit")
}
func Test_mixedFamilyNesting_unboundedLimitsEnforceNothing(t *testing.T) {
// a caller can opt out of one bound without the other, and opting out of both must still find the
// deepest leaf. Opting out is a negative value; zero forbids the resource.
for _, row := range nestingRows() {
t.Run(row.name, func(t *testing.T) {
scanDir := t.TempDir()
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
probe := &nestingProbe{}
bounds := archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs())
locs := leafLocations(s)
require.Len(t, locs, 1)
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
})
}
}
func defaultLimits() archive.Limits {
return archiveLimits(cataloging.DefaultArchiveSearchConfig())
}
@@ -524,37 +417,30 @@ func (f archiveFamily) pack(t *testing.T, base string, entries map[string][]byte
t.Helper()
switch f {
case familyZip:
return makeZip(t, entries)
return testutils.Zip(t, entries)
case familyJar:
// a manifest makes it a real jar to the java cataloger at any level; the package java reports is
// named after the file, not the manifest title
return makeJar(t, base, "1.0", entries)
return testutils.Jar(t, base, "1.0", entries)
case familyTar:
return makeTar(t, entries)
return testutils.Tar(t, entries)
case familyTarGz, familyTgz:
return makeTarGz(t, entries)
return testutils.TarGz(t, entries)
}
t.Fatalf("unknown archive family %q", f)
return nil
}
func makeJar(t *testing.T, title, version string, extra map[string][]byte) []byte {
t.Helper()
entries := map[string][]byte{
"META-INF/MANIFEST.MF": []byte(
"Manifest-Version: 1.0\nImplementation-Title: " + title + "\nImplementation-Version: " + version + "\n",
),
}
for k, v := range extra {
entries[k] = v
}
return makeZip(t, entries)
// leafContainerPath is the container path the innermost archive's contents carry: every archive, outer to
// inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check.
func (n nestedArchive) leafContainerPath() string {
return n.containerPaths[len(n.containerPaths)-1]
}
// leafFileSystemID is the chain the innermost archive's contents carry: every archive, outer to
// inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check.
func (n nestedArchive) leafFileSystemID() string {
return n.fileSystemIDs[len(n.fileSystemIDs)-1]
// coordinates are those of the archive at the given level: the outermost archive's path, and the chain
// of entries within it.
func (n nestedArchive) coordinates(level int) file.Coordinates {
return file.Coordinates{RealPath: n.paths[0], FullPath: strings.Join(n.paths[:level+1], "!/")}
}
// buildNestedArchive builds one archive per family given, each holding the next, with leaf inside
@@ -564,10 +450,10 @@ func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive {
require.NotEmpty(t, plan.families, "a chain needs at least one level")
out := nestedArchive{
sizes: make([]int, len(plan.families)),
paths: make([]string, len(plan.families)),
fileSystemIDs: make([]string, len(plan.families)),
virtualPaths: make([]string, len(plan.families)),
sizes: make([]int, len(plan.families)),
paths: make([]string, len(plan.families)),
containerPaths: make([]string, len(plan.families)),
virtualPaths: make([]string, len(plan.families)),
}
// the level index goes in every name and path segment, so no two levels share a path and a chain
@@ -585,11 +471,11 @@ func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive {
for i := range plan.families {
if i == 0 {
out.fileSystemIDs[i] = out.paths[0]
out.containerPaths[i] = out.paths[0]
out.virtualPaths[i] = out.paths[0]
continue
}
out.fileSystemIDs[i] = out.fileSystemIDs[i-1] + ":" + out.paths[i]
out.containerPaths[i] = out.containerPaths[i-1] + "!/" + out.paths[i]
out.virtualPaths[i] = out.virtualPaths[i-1] + ":" + out.paths[i]
}
@@ -667,6 +553,7 @@ func (p *nestingProbe) task() Task {
visit := nestingVisit{}
if trav != nil {
visit.virtualPath = archive.VirtualPath(trav.Location)
visit.fullPath = trav.FullTraversalArchivePath()
visit.depth = strings.Count(archive.VirtualPath(trav.Location), ":") + 1
}
if p.tempDir != "" {
@@ -680,16 +567,16 @@ func (p *nestingProbe) task() Task {
})
}
// archiveFileSystemIDs is the chain of every archive whose contents were cataloged, in visit order.
func (p *nestingProbe) archiveFileSystemIDs() []string {
// containerPaths is the chain of every archive whose contents were cataloged, in visit order.
func (p *nestingProbe) containerPaths() []string {
p.mu.Lock()
defer p.mu.Unlock()
var out []string
for _, v := range p.visits {
if v.virtualPath == "" {
if v.fullPath == "" {
continue
}
out = append(out, v.virtualPath)
out = append(out, v.fullPath)
}
return out
}
@@ -770,7 +657,7 @@ func packageCount(s *sbom.SBOM, name string) int {
}
func coordKey(c file.Coordinates) string {
return c.ArchivePath + "|" + c.RealPath
return c.FullNestedPath()
}
// gradedChain is the headline row with each level padded so the three archives have clearly different
internal/task/archive_tasks.go
+7 -7
View File
@@ -194,12 +194,12 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi
ctx = archive.WithBudget(ctx, archive.NewBudget(archiveTreeBudget(parentResolver, location)))
}
// extracted files keep the filesystem ID of where the archive was found; the nesting chain is
// carried separately as the archive path. The chain names real paths, not the link the archive was
// extracted files keep the real path and filesystem ID of the outermost archive; the chain of entries
// to each is carried as its archive path. The chain names real paths, not the link the archive was
// reached by, so it is stable and joins to the archive's package location by prefix.
traversal := &archive.Traversal{Location: location, Parent: archive.TraversalFromContext(ctx)}
archivePath := traversal.ContentsArchivePath()
extracted, err := archive.Extract(ctx, archiveContent, location.FileSystemID, archivePath, c.limiter, c.exclusions)
archivePath := location.FullNestedPath()
extracted, err := archive.Extract(ctx, archiveContent, location.Coordinates, c.limiter, c.exclusions)
if errors.Is(err, archive.ErrDiskLimitReached) {
// nothing is released while this archive waits, so skip it rather than block the scan
recordArchiveUnknown(builder, location.Coordinates, "archive skipped: its content would exceed the disk limit")
@@ -370,7 +370,7 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan
if i, dup := seen[loc.Coordinates]; dup {
// a MIME hit is the bare real path; the name it is reached by (a link such as
// lib/dep.jar -> ../store/3f9a1c) is what name-based catalogers like java key off
if prev := candidates[i].location; prev.AccessPath == "" || prev.AccessPath == prev.RealPath {
if prev := candidates[i].location; prev.AccessPath == "" || prev.AccessPath == prev.FullNestedPath() {
candidates[i].location = loc
}
continue
@@ -384,7 +384,7 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan
// mergeArchiveResults copies the throwaway SBOM's packages, files and relationships into the shared
// SBOM, adding a CONTAINS relationship from the archive to everything found inside it.
//
// Files inside the archive are those whose ArchivePath is the archive's own chain, which leaves out the
// Files inside the archive are those whose container is the archive's own full path, which leaves out the
// archive itself (a java package or an unknown recorded at its coordinates would otherwise become a
// self-loop). The nested archives are always included, so the chain of archives is navigable whatever
// file selection is configured.
@@ -409,7 +409,7 @@ func mergeArchiveResults(archiveCoordinates file.Coordinates, archivePath string
contained.Add(child.location.Coordinates)
}
for _, coordinates := range scratch.AllCoordinates() {
if coordinates.ArchivePath == archivePath {
if archive.ContainerPath(coordinates) == archivePath {
contained.Add(coordinates)
}
}
internal/task/archive_tasks_test.go
+49 -268
View File
@@ -1,10 +1,7 @@
package task
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"context"
"crypto/sha1"
"encoding/hex"
@@ -14,7 +11,6 @@ import (
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
@@ -31,6 +27,7 @@ import (
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/mimetype"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/internal/testutils"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging"
@@ -87,7 +84,7 @@ func (panicReader) Read([]byte) (int, error) { panic("decoder blew up") }
func Test_archiveCataloger_aPanicCostsOnlyThatArchive(t *testing.T) {
rootDir := t.TempDir()
for _, name := range []string{"bad.zip", "good.zip"} {
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600))
}
var seen []string
@@ -147,35 +144,10 @@ func capturingTask(t *testing.T, seen *[]string, fileCounts *map[string]int) Tas
})
}
func Test_archiveCataloger_traversalThreading(t *testing.T) {
manifest := []byte("Manifest-Version: 1.0\nImplementation-Title: example\nImplementation-Version: 1.0.0\n")
innerJar := makeZip(t, map[string][]byte{"META-INF/MANIFEST.MF": manifest})
innerZip := makeZip(t, map[string][]byte{"README.txt": []byte("hi")})
outerZip := makeZip(t, map[string][]byte{
"lib/inner.jar": innerJar,
"nested/inner.zip": innerZip,
})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
var seen []string
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3), capturingTask(t, &seen, nil))
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())))
// java archives are traversed like any other archive when the task is enabled
assert.ElementsMatch(t, []string{
"/outer.zip",
"/outer.zip:lib/inner.jar",
"/outer.zip:nested/inner.zip",
}, seen)
}
func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) {
// reaching a limit is a truncation, not a failure: the sub-pipeline still runs over what was stored
big := bytes.Repeat([]byte("x"), 4096)
outerZip := makeZip(t, map[string][]byte{
outerZip := testutils.Zip(t, map[string][]byte{
"a/small.txt": []byte("small"),
"b/big.txt": big,
})
@@ -197,7 +169,7 @@ func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) {
assert.Equal(t, []string{"/outer.zip"}, seen, "the sub-pipeline must run over the partial contents")
assert.Equal(t, 1, fileCounts["/outer.zip"], "only the entry stored before the limit is visible")
reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}]
reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
require.True(t, ok, "the truncation must be recorded against the archive; got %v", s.Artifacts.Unknowns)
require.Len(t, reasons, 1)
assert.Contains(t, reasons[0], "archive cataloged from part of its contents")
@@ -208,7 +180,7 @@ func Test_archiveCataloger_aDamagedEntryCostsOnlyThatEntry(t *testing.T) {
rootDir := t.TempDir()
// entry data overwritten but headers and end-of-central-directory intact: detected as an archive,
// and data.bin fails when it is read. ok.txt comes after it and is intact.
corrupt := makeZip(t, map[string][]byte{"data.bin": incompressible(4096), "ok.txt": []byte("ok")})
corrupt := testutils.Zip(t, map[string][]byte{"data.bin": incompressible(4096), "ok.txt": []byte("ok")})
require.Greater(t, len(corrupt), 1024)
for i := 100; i < 600; i++ {
corrupt[i] ^= 0xff
@@ -224,7 +196,7 @@ func Test_archiveCataloger_aDamagedEntryCostsOnlyThatEntry(t *testing.T) {
assert.Equal(t, []string{"/corrupt.zip"}, seen, "the archive is still cataloged")
assert.Equal(t, 1, fileCounts["/corrupt.zip"], "from the entries that could be read")
reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/corrupt.zip"}]
reasons := s.Artifacts.Unknowns[file.NewCoordinates("/corrupt.zip", "")]
require.Len(t, reasons, 1)
assert.Contains(t, reasons[0], `entry "data.bin" could not be read`)
}
@@ -275,31 +247,11 @@ func Test_NewArchiveCatalogerTask_gating(t *testing.T) {
}
}
func Test_archiveCataloger_depthBoundIsExact(t *testing.T) {
innerZip := makeZip(t, map[string][]byte{"deep.txt": []byte("deep")})
outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
run := func(t *testing.T, depth int) []string {
t.Helper()
var seen []string
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth), capturingTask(t, &seen, nil))
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())),
"reaching the depth bound must not be an error")
return seen
}
assert.Equal(t, []string{"/outer.zip"}, run(t, 1), "depth 1 catalogs the top-level archive and does not descend")
assert.ElementsMatch(t, []string{"/outer.zip", "/outer.zip:nested/inner.zip"}, run(t, 2), "depth 2 descends one level")
}
func Test_archiveCataloger_negativeDepthStopsAtTheCeiling(t *testing.T) {
// "unbounded" must still end, or a zip quine or recursive bomb runs forever
chain := makeZip(t, map[string][]byte{"deep.txt": []byte("deep")})
chain := testutils.Zip(t, map[string][]byte{"deep.txt": []byte("deep")})
for range unboundedArchiveDepth + 4 {
chain = makeZip(t, map[string][]byte{"n.zip": chain})
chain = testutils.Zip(t, map[string][]byte{"n.zip": chain})
}
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), chain, 0o600))
@@ -314,7 +266,7 @@ func Test_archiveCataloger_negativeDepthStopsAtTheCeiling(t *testing.T) {
func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing.T) {
// a zip quine contains itself. Building one is impractical, so the walk is started as if it were
// already inside an archive with these exact bytes, which is the state a quine reaches one level down
outerZip := makeZip(t, map[string][]byte{"a.txt": []byte("a")})
outerZip := testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
@@ -327,7 +279,7 @@ func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(s)))
assert.Zero(t, ran, "the sub-pipeline must not run inside a repeat of an ancestor")
reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}]
reasons := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
require.Len(t, reasons, 1)
assert.Contains(t, reasons[0], "identical to an archive that contains it")
assert.Len(t, c.ancestors, 1, "the walk leaves the ancestor stack as it found it")
@@ -335,7 +287,7 @@ func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing
func Test_archiveCataloger_stopsWhenTheContextIsCancelled(t *testing.T) {
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), makeZip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600))
ctx, cancel := context.WithCancel(context.Background())
cancel()
@@ -356,10 +308,10 @@ func Test_archiveCataloger_decompressionBudgetIsSharedByATreeAndFreshPerTopLevel
t.Cleanup(func() { archiveTreeRatio, archiveTreeFloorBytes = origRatio, origFloor })
zeros := make([]byte, 60_000)
nested := makeZip(t, map[string][]byte{"zeros.bin": zeros})
nested := testutils.Zip(t, map[string][]byte{"zeros.bin": zeros})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bomb.zip"), makeZip(t, map[string][]byte{"a.zip": nested, "b.zip": nested}), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "sibling.zip"), makeZip(t, map[string][]byte{"zeros.bin": zeros}), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bomb.zip"), testutils.Zip(t, map[string][]byte{"a.zip": nested, "b.zip": nested}), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "sibling.zip"), testutils.Zip(t, map[string][]byte{"zeros.bin": zeros}), 0o600))
var seen []string
fileCounts := map[string]int{}
@@ -369,7 +321,7 @@ func Test_archiveCataloger_decompressionBudgetIsSharedByATreeAndFreshPerTopLevel
assert.Equal(t, 1, fileCounts["/bomb.zip:a.zip"], "the first nested archive fits the tree's budget")
assert.Equal(t, 0, fileCounts["/bomb.zip:b.zip"], "the second does not: the budget is shared, not per archive")
reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "b.zip", ArchivePath: "/bomb.zip"}]
reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/bomb.zip", FullPath: "/bomb.zip!/b.zip"}]
require.Len(t, reasons, 1)
assert.Contains(t, reasons[0], "possible decompression bomb")
assert.Equal(t, 1, fileCounts["/sibling.zip"], "another top-level archive starts with a fresh budget")
@@ -379,14 +331,14 @@ func Test_archiveCataloger_noSelfContainsEdge(t *testing.T) {
// a jar java cannot identify records an unknown at its own coordinates; that must not turn into
// "x.jar CONTAINS x.jar", which a graph walker would loop on
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "x.jar"), makeZip(t, map[string][]byte{"readme.txt": []byte("no manifest")}), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "x.jar"), testutils.Zip(t, map[string][]byte{"readme.txt": []byte("no manifest")}), 0o600))
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), javaTask(), fileMetadataTask())
s := newTestSBOM()
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
_ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))
require.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "/x.jar"}, "the fixture must reproduce the unknown")
require.Contains(t, s.Artifacts.Unknowns, file.NewCoordinates("/x.jar", ""), "the fixture must reproduce the unknown")
var contains int
for _, rel := range s.Relationships {
if rel.Type != artifact.ContainsRelationship {
@@ -402,8 +354,8 @@ func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) {
// a copy of itself in the sub-pipeline would process every nesting level twice
cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)
innerZip := makeZip(t, map[string][]byte{"leaf.txt": []byte("leaf")})
outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip})
innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")})
outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
@@ -424,11 +376,11 @@ func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) {
})
}
func Test_archiveCataloger_chainStartsAtTheArchivesOwnFileSystemID(t *testing.T) {
func Test_archiveCataloger_nestedArchivesKeepTheLayerFileSystemID(t *testing.T) {
// extracted files keep the layer digest as their FileSystemID at every level; the nesting chain
// rides on the traversal's VirtualPath and the coordinate's ArchivePath
innerZip := makeZip(t, map[string][]byte{"leaf.txt": []byte("leaf")})
outerZip := makeZip(t, map[string][]byte{"nested/inner.zip": innerZip})
// rides on the traversal's VirtualPath and the coordinate's FullPath
innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")})
outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
@@ -463,8 +415,8 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T
resolver := twoLayerResolver{
path: "/app/bundle.zip",
contentBy: map[string][]byte{
"layer-one": makeZip(t, map[string][]byte{"lib/config.json": []byte(`{"v":1}`)}),
"layer-two": makeZip(t, map[string][]byte{"lib/config.json": []byte(`{"v":2}`)}),
"layer-one": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":1}`)}),
"layer-two": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":2}`)}),
},
}
@@ -501,7 +453,7 @@ func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T
func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) {
// runSubPipeline bypasses RunTask, where the coordinate-error to unknown conversion lives
outerZip := makeZip(t, map[string][]byte{"lib/broken.json": []byte("{")})
outerZip := testutils.Zip(t, map[string][]byte{"lib/broken.json": []byte("{")})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
@@ -526,8 +478,8 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T)
require.Len(t, s.Artifacts.Unknowns, 1)
for coords, reasons := range s.Artifacts.Unknowns {
assert.Equal(t, "lib/broken.json", coords.RealPath, "the inner coordinates must survive the merge")
assert.Equal(t, "/outer.zip", coords.ArchivePath, "and must be addressable per archive")
assert.Equal(t, "/outer.zip!/lib/broken.json", coords.FullNestedPath(), "the inner coordinates must survive the merge")
assert.Equal(t, "/outer.zip", coords.RealPath, "and must be addressable per archive")
assert.Equal(t, []string{"failing-cataloger: unable to parse"}, reasons)
}
})
@@ -542,7 +494,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T)
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
require.Len(t, s.Artifacts.Unknowns, 1)
reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}]
reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
require.True(t, ok, "an error with no location of its own belongs to the containing archive")
assert.Equal(t, []string{"bare-error-cataloger: cataloger blew up"}, reasons)
})
@@ -557,7 +509,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T)
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)),
"a panic inside an archive must not fail the scan")
reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}]
reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
require.True(t, ok)
require.Len(t, reasons, 1)
assert.Equal(t, "panicking-cataloger: panic: nope", reasons[0])
@@ -565,11 +517,11 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T)
}
func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) {
outerZip := makeZip(t, map[string][]byte{"lib/thing.json": []byte("{}")})
outerZip := testutils.Zip(t, map[string][]byte{"lib/thing.json": []byte("{}")})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
preexisting := file.Coordinates{RealPath: "lib/thing.json", ArchivePath: "/outer.zip"}
preexisting := file.Coordinates{RealPath: "/outer.zip", FullPath: "/outer.zip!/lib/thing.json"}
writer := NewTask("scratch-writer", func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error {
builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) {
if s.Artifacts.Unknowns == nil {
@@ -588,39 +540,7 @@ func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T)
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
assert.Equal(t, []string{"recorded by the cataloger itself"}, s.Artifacts.Unknowns[preexisting])
assert.Equal(t, []string{"failing-cataloger: boom"}, s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}])
}
func Test_archiveCataloger_manySmallArchivesAreEachCataloged(t *testing.T) {
// every archive is released before the next begins, so a scan whose archives sum well past a limit
// never holds more than one at a time
entry := bytes.Repeat([]byte("z"), 400)
entries := map[string][]byte{}
for i := range 5 {
entries[fmt.Sprintf("e%d.bin", i)] = entry
}
rootDir := t.TempDir()
names := []string{"one.zip", "two.zip", "three.zip", "four.zip", "five.zip"}
for _, name := range names {
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, entries), 0o600))
}
// room for one archive's entries and their index cost, not for two
limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: 20000}
var seen []string
fileCounts := map[string]int{}
tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts))
s := newTestSBOM()
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
assert.Len(t, seen, len(names), "every archive must be cataloged: no two were ever held at once")
for _, archivePath := range seen {
assert.Equal(t, 5, fileCounts[archivePath], "%s must extract in full", archivePath)
}
assert.Empty(t, s.Artifacts.Unknowns)
assert.Equal(t, []string{"failing-cataloger: boom"}, s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")])
}
func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) {
@@ -632,7 +552,7 @@ func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) {
rootDir := t.TempDir()
for _, name := range []string{"one.zip", "two.zip"} {
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), makeZip(t, entries), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, entries), 0o600))
}
// room in memory for one archive's entries and their index cost, not for two at once
@@ -675,8 +595,8 @@ func newTestCataloger(maxDepth int, limiter *archive.Limiter, subPipeline ...Tas
func Test_archiveCataloger_archiveWhoseOwnBytesExceedTheDiskLimitIsSkipped(t *testing.T) {
// a streamed archive's own bytes must be placed before it can be opened; when they do not fit on
// disk the archive is skipped rather than waited for, and a sibling that fits is cataloged in full
oversized := makeZip(t, map[string][]byte{"payload.bin": incompressible(8000)})
small := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")})
oversized := testutils.Zip(t, map[string][]byte{"payload.bin": incompressible(8000)})
small := testutils.Zip(t, map[string][]byte{"ok.txt": []byte("ok")})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600))
@@ -699,84 +619,20 @@ func Test_archiveCataloger_archiveWhoseOwnBytesExceedTheDiskLimitIsSkipped(t *te
assert.Equal(t, []string{"/small.zip"}, seen, "the oversized archive is skipped and the sibling is still cataloged")
assert.Equal(t, 1, fileCounts["/small.zip"])
reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/oversized.zip"}]
reasons := s.Artifacts.Unknowns[file.NewCoordinates("/oversized.zip", "")]
require.Len(t, reasons, 1, "the skip must be recorded against the archive; got %v", s.Artifacts.Unknowns)
assert.Contains(t, reasons[0], "archive skipped")
}
func Test_archiveCataloger_zeroDiskLimitTruncatesWhatWillNotFitInMemory(t *testing.T) {
// with a zero disk limit there is nowhere to overflow, so an archive's entries are stored exactly
// while they fit in memory
oversized := makeZip(t, map[string][]byte{"payload.bin": incompressible(8000)})
small := makeZip(t, map[string][]byte{"ok.txt": []byte("ok")})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600))
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "small.zip"), small, 0o600))
// room for the small archive's entry and its index cost, not for the oversized one's
limits := archive.Limits{MaxMemoryBytes: 3000, MaxDiskBytes: 0}
var seen []string
fileCounts := map[string]int{}
tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts))
s := newTestSBOM()
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)),
"reaching the disk limit must not fail the scan")
assert.ElementsMatch(t, []string{"/oversized.zip", "/small.zip"}, seen)
assert.Equal(t, 1, fileCounts["/small.zip"], "the archive that fits in memory needs no disk at all")
assert.Equal(t, 0, fileCounts["/oversized.zip"], "the entry that does not fit is refused")
assert.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "/oversized.zip"})
assert.NotContains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "/small.zip"})
}
func Test_archiveCataloger_nestedArchiveExceedingALimitIsTruncatedNotBlocked(t *testing.T) {
// the walk descends before it unwinds, so nothing is released while a child waits; a child that does
// not fit is truncated and the scan finishes
inner := makeZip(t, map[string][]byte{"payload.bin": incompressible(3000)})
outer := makeZip(t, map[string][]byte{"inner.zip": inner})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outer, 0o600))
// enough for the outer archive's one entry and its index cost, plus the inner's index cost, and not
// the inner's payload on top
diskLimit := int64(len(inner)) + 2*2100 + 500
require.Greater(t, 3000, 500, "the inner payload must not fit in the slack")
limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: diskLimit}
var seen []string
fileCounts := map[string]int{}
tsk := newLimitedTestTask(t, 2, limits, capturingTask(t, &seen, &fileCounts))
s := newTestSBOM()
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
assert.Equal(t, []string{"/outer.zip", "/outer.zip:inner.zip"}, seen)
assert.Equal(t, 1, fileCounts["/outer.zip"], "the parent is cataloged in full")
assert.Equal(t, 0, fileCounts["/outer.zip:inner.zip"], "the child it cannot afford is truncated")
assert.Contains(t, s.Artifacts.Unknowns, file.Coordinates{RealPath: "inner.zip", ArchivePath: "/outer.zip"})
}
// allFilesResolver is dirTestResolver without a filesystem id.
type allFilesResolver struct {
file.Resolver
dir string
}
func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) {
marker := map[string][]byte{"nested/marker.txt": []byte("hello")}
scanDir := t.TempDir()
for name, body := range map[string][]byte{
"bundle.tar.gz": makeTarGz(t, marker),
"bundle.tgz": makeTarGz(t, marker),
"bundle.tar": makeTar(t, marker),
"bundle.zip": makeZip(t, marker),
"bundle.tar.gz": testutils.TarGz(t, marker),
"bundle.tgz": testutils.TarGz(t, marker),
"bundle.tar": testutils.Tar(t, marker),
"bundle.zip": testutils.Zip(t, marker),
"notes.txt": []byte("not an archive at all"),
} {
require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), body, 0o600))
@@ -785,7 +641,7 @@ func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) {
var seen []string
fileCounts := map[string]int{}
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, &fileCounts))
require.NoError(t, tsk.Execute(context.Background(), allFilesResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM())))
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM())))
assert.ElementsMatch(t, []string{"/bundle.tar.gz", "/bundle.tgz", "/bundle.tar", "/bundle.zip"}, seen,
"every tar-family archive must be reached, and the plain text file must not be")
@@ -807,7 +663,7 @@ type recordingPublisher struct {
func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) {
// a consumer keying rows by ID (as the syft CLI does) replaces a republished row and never sees it
// complete, so each cataloger keeps to the one row it already owns
inner := makeZip(t, map[string][]byte{"inner/file.txt": []byte("hello")})
inner := testutils.Zip(t, map[string][]byte{"inner/file.txt": []byte("hello")})
scanDir := t.TempDir()
for _, name := range []string{"one.zip", "two.zip"} {
require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), inner, 0o600))
@@ -865,8 +721,8 @@ func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T)
func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) {
// self time excludes nested archives, or the outer archive would be named every scan
innerJar := makeZip(t, map[string][]byte{"README.txt": []byte("inner")})
outerZip := makeZip(t, map[string][]byte{"lib/inner.jar": innerJar})
innerJar := testutils.Zip(t, map[string][]byte{"README.txt": []byte("inner")})
outerZip := testutils.Zip(t, map[string][]byte{"lib/inner.jar": innerJar})
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
@@ -883,7 +739,7 @@ func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) {
c := newTestCataloger(3, limiter, slow)
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM())))
assert.Equal(t, "/outer.zip:lib/inner.jar", c.slowestPath, "the slowest archive is named by its full chain")
assert.Equal(t, "/outer.zip!/lib/inner.jar", c.slowestPath, "the slowest archive is named by its full chain")
assert.GreaterOrEqual(t, c.slowest, innerWork)
assert.Equal(t, int64(2), c.progress.Current())
@@ -926,9 +782,9 @@ func Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError(t *testing.T) {
func Test_archiveCataloger_catalogsAZipBehindALauncherScript(t *testing.T) {
// the Spring Boot executable jar shape: content sniffing types it as a shell script, so it reaches
// the walk only by name and qualifies only by its end-of-central-directory record
inner := makeZip(t, map[string][]byte{"nested/marker.txt": []byte("found me")})
inner := testutils.Zip(t, map[string][]byte{"nested/marker.txt": []byte("found me")})
prefixed := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"),
makeZip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...)
testutils.Zip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...)
rootDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "app.jar"), prefixed, 0o600))
@@ -983,28 +839,6 @@ func (d dirTestResolver) FileMetadataByLocation(loc file.Location) (file.Metadat
return file.Metadata{FileInfo: info}, err
}
// makeZip builds a zip in sorted entry-name order: limits are enforced as the walk proceeds, so entry
// order decides which entries land before a truncation.
func makeZip(t *testing.T, entries map[string][]byte) []byte {
t.Helper()
names := make([]string, 0, len(entries))
for name := range entries {
names = append(names, name)
}
sort.Strings(names)
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for _, name := range names {
w, err := zw.Create(name)
require.NoError(t, err)
_, err = w.Write(entries[name])
require.NoError(t, err)
}
require.NoError(t, zw.Close())
return buf.Bytes()
}
// matchesAnyGlob matches the resolver-relative path with a leading slash, the form a real resolver
// globs over. No patterns matches everything.
func matchesAnyGlob(root, path string, patterns []string) bool {
@@ -1024,10 +858,6 @@ func matchesAnyGlob(root, path string, patterns []string) bool {
return false
}
func (a allFilesResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
return dirTestResolver{dir: a.dir}.FilesByGlob(patterns...)
}
func (r twoLayerResolver) FilesByGlob(_ ...string) ([]file.Location, error) {
return nil, nil
}
@@ -1055,7 +885,7 @@ func (d dirTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error
func (r twoLayerResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) {
var locs []file.Location
for _, fsid := range []string{"layer-one", "layer-two"} {
locs = append(locs, file.NewLocationFromCoordinates(file.Coordinates{RealPath: r.path, FileSystemID: fsid}))
locs = append(locs, file.NewLocationFromCoordinates(file.NewCoordinates(r.path, fsid)))
}
return locs, nil
}
@@ -1085,55 +915,6 @@ func incompressible(n int) []byte {
return b
}
func makeTarGz(t *testing.T, entries map[string][]byte) []byte {
t.Helper()
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
require.NoError(t, writeTar(t, gw, entries))
require.NoError(t, gw.Close())
return buf.Bytes()
}
func makeTar(t *testing.T, entries map[string][]byte) []byte {
t.Helper()
var buf bytes.Buffer
require.NoError(t, writeTar(t, &buf, entries))
return buf.Bytes()
}
func writeTar(t *testing.T, w io.Writer, entries map[string][]byte) error {
t.Helper()
names := make([]string, 0, len(entries))
for name := range entries {
names = append(names, name)
}
sort.Strings(names)
tw := tar.NewWriter(w)
for _, name := range names {
body := entries[name]
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil {
return err
}
if _, err := tw.Write(body); err != nil {
return err
}
}
return tw.Close()
}
func (a allFilesResolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
return filesByMIMEType(a.dir, "", types...)
}
func (a allFilesResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) {
return os.Open(filepath.Join(a.dir, filepath.FromSlash(loc.RealPath)))
}
func (a allFilesResolver) FileMetadataByLocation(loc file.Location) (file.Metadata, error) {
return dirTestResolver{dir: a.dir}.FileMetadataByLocation(loc)
}
func (p *recordingPublisher) Publish(e partybus.Event) {
p.events = append(p.events, e)
}
internal/task/file_tasks.go
+2 -2
View File
@@ -175,10 +175,10 @@ func coordinatesForSelection(ctx context.Context, selection file.Selection, acce
}
})
frame := archive.TraversalFromContext(ctx).ContentsArchivePath()
frame := archive.TraversalFromContext(ctx).FullTraversalArchivePath()
var coords []file.Coordinates
for _, c := range coordinates.ToSlice() {
if c.ArchivePath == frame {
if archive.ContainerPath(c) == frame {
coords = append(coords, c)
}
}
internal/task/file_tasks_test.go
+6 -18
View File
@@ -43,19 +43,13 @@ func Test_coordinatesForSelection(t *testing.T) {
Relationships: []artifact.Relationship{
{
From: pkg.Package{},
To: file.Coordinates{
RealPath: "path",
FileSystemID: "fs",
},
To: file.NewCoordinates("path", "fs"),
Type: artifact.ContainsRelationship,
},
},
},
files: []file.Coordinates{
{
RealPath: "path",
FileSystemID: "fs",
},
file.NewCoordinates("path", "fs"),
},
ok: true,
},
@@ -66,10 +60,7 @@ func Test_coordinatesForSelection(t *testing.T) {
Relationships: []artifact.Relationship{
{
From: pkg.Package{},
To: file.Coordinates{
RealPath: "path",
FileSystemID: "fs",
},
To: file.NewCoordinates("path", "fs"),
// wrong type
Type: artifact.DependencyOfRelationship,
},
@@ -85,10 +76,7 @@ func Test_coordinatesForSelection(t *testing.T) {
Relationships: []artifact.Relationship{
{
From: file.Coordinates{}, // wrong type
To: file.Coordinates{
RealPath: "path",
FileSystemID: "fs",
},
To: file.NewCoordinates("path", "fs"),
Type: artifact.ContainsRelationship,
},
},
@@ -109,8 +97,8 @@ func Test_coordinatesForSelection(t *testing.T) {
func Test_coordinatesForSelection_onlyTheResolversFrame(t *testing.T) {
// a package located at an archive in its parent's frame must not send the file catalogers looking up
// that path inside the archive's own resolver
atRoot := file.Coordinates{RealPath: "/app.war"}
inWar := file.Coordinates{RealPath: "WEB-INF/lib/dep.jar", ArchivePath: "/app.war"}
atRoot := file.NewCoordinates("/app.war", "")
inWar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/dep.jar"}
s := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(
pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(atRoot))},
pkg.Package{Name: "dep", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(inWar))},
internal/task/package_task_factory.go
+2 -2
View File
@@ -167,7 +167,7 @@ func applyComplianceRules(p *pkg.Package, cfg cataloging.ComplianceConfig) (bool
loc := "unknown"
locs := p.Locations.ToSlice()
if len(locs) > 0 {
loc = locs[0].Path()
loc = locs[0].FullNestedPath()
}
switch action {
case cataloging.ComplianceActionDrop:
@@ -251,7 +251,7 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
for _, ref := range pathRefs {
if oldRef, ok := locations[ref.ID()]; ok {
log.Debugf("found path duplicate of %s", oldRef.RealPath)
log.Debugf("found path duplicate of %s", oldRef.FullNestedPath())
}
locations[ref.ID()] = ref
}
internal/task/package_task_factory_test.go
+2 -2
View File
@@ -64,8 +64,8 @@ func TestApplyCompliance(t *testing.T) {
p2 := pkg.Package{Name: "", Version: "1.0"} // missing name
p3 := pkg.Package{Name: "pkg-3", Version: ""} // missing version
p4 := pkg.Package{Name: "pkg-4", Version: ""} // missing version
c1 := file.Coordinates{RealPath: "/coords/1"}
c2 := file.Coordinates{RealPath: "/coords/2"}
c1 := file.NewCoordinates("/coords/1", "")
c2 := file.NewCoordinates("/coords/2", "")
for _, p := range []*pkg.Package{&p1, &p2, &p3, &p4} {
p.SetID()
internal/task/scope_tasks.go
+4 -3
View File
@@ -40,10 +40,11 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID {
}
case "":
if isPrimaryEvidence {
if exists := filterDuplicates[getKey(p, l)]; exists {
k := getKey(p, l)
if exists := filterDuplicates[k]; exists {
break
}
filterDuplicates[getKey(p, l)] = true
filterDuplicates[k] = true
noPrimary = false
break
}
@@ -59,5 +60,5 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID {
}
func getKey(pkg pkg.Package, loc file.Location) string {
return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.RealPath, loc.AccessPath)
return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.FullNestedPath(), loc.AccessPath)
}
internal/task/unknowns_tasks.go
+1 -1
View File
@@ -59,7 +59,7 @@ func (c unknownsLabelerTask) finalize(resolver file.Resolver, s *sbom.SBOM) {
if c.IncludeUnexpandedArchives {
ctx := context.Background()
for coords := range s.Artifacts.FileMetadata {
format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.RealPath, nil)
format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.FullNestedPath(), nil)
if format != nil && notArchiveErr == nil && !hasPackageReference(coords) {
s.Artifacts.Unknowns[coords] = append(s.Artifacts.Unknowns[coords], "archive not cataloged")
}
internal/testutils/archives.go
+84
View File
@@ -0,0 +1,84 @@
package testutils
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"fmt"
"io"
"sort"
"testing"
"github.com/stretchr/testify/require"
)
// Zip builds a zip of the entries in sorted name order: archive limits bite as entries are read, so
// order decides which land before a truncation.
func Zip[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte {
t.Helper()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for _, name := range sortedNames(entries) {
w, err := zw.Create(name)
require.NoError(t, err)
_, err = w.Write([]byte(entries[name]))
require.NoError(t, err)
}
require.NoError(t, zw.Close())
return buf.Bytes()
}
// Tar builds a tar of the entries in sorted name order.
func Tar[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte {
t.Helper()
var buf bytes.Buffer
writeTar(t, &buf, entries)
return buf.Bytes()
}
// TarGz builds a gzipped tar of the entries in sorted name order.
func TarGz[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte {
t.Helper()
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
writeTar(t, gw, entries)
require.NoError(t, gw.Close())
return buf.Bytes()
}
// Jar is the smallest jar the java cataloger identifies, a manifest naming the artifact, plus extra.
func Jar(t testing.TB, title, version string, extra map[string][]byte) []byte {
t.Helper()
entries := map[string][]byte{"META-INF/MANIFEST.MF": Manifest(title, version)}
for name, body := range extra {
entries[name] = body
}
return Zip(t, entries)
}
// Manifest is a jar manifest naming the artifact.
func Manifest(title, version string) []byte {
return fmt.Appendf(nil, "Manifest-Version: 1.0\nImplementation-Title: %s\nImplementation-Version: %s\n", title, version)
}
func writeTar[T ~string | ~[]byte](t testing.TB, w io.Writer, entries map[string]T) {
t.Helper()
tw := tar.NewWriter(w)
for _, name := range sortedNames(entries) {
body := []byte(entries[name])
require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}))
_, err := tw.Write(body)
require.NoError(t, err)
}
require.NoError(t, tw.Close())
}
func sortedNames[T any](entries map[string]T) []string {
names := make([]string, 0, len(entries))
for name := range entries {
names = append(names, name)
}
sort.Strings(names)
return names
}
internal/unknown/coordinate_error.go
+2 -2
View File
@@ -23,9 +23,9 @@ var _ error = (*CoordinateError)(nil)
func (u *CoordinateError) Error() string {
if u.Coordinates.FileSystemID == "" {
return fmt.Sprintf("%s: %v", u.Coordinates.RealPath, u.Reason)
return fmt.Sprintf("%s: %v", u.Coordinates.FullNestedPath(), u.Reason)
}
return fmt.Sprintf("%s (%s): %v", u.Coordinates.RealPath, u.Coordinates.FileSystemID, u.Reason)
return fmt.Sprintf("%s (%s): %v", u.Coordinates.FullNestedPath(), u.Coordinates.FileSystemID, u.Reason)
}
// New returns a new CoordinateError unless the reason is a CoordinateError itself, in which case
internal/unknown/coordinate_error_test.go
+1 -3
View File
@@ -169,9 +169,7 @@ func Test_Join(t *testing.T) {
}
func Test_flatten(t *testing.T) {
coords := file.Coordinates{
RealPath: "real/path",
}
coords := file.NewCoordinates("real/path", "")
e1 := fmt.Errorf("e1")
e2 := fmt.Errorf("e2")
c1 := New(coords, fmt.Errorf("c1"))
internal/unknown/path_error_test.go
+6 -12
View File
@@ -17,28 +17,22 @@ func Test_ProcessPathErrors(t *testing.T) {
{
errorText: `prefix path="/var/lib/thing" suffix`,
expected: &CoordinateError{
Coordinates: file.Coordinates{
RealPath: "/var/lib/thing",
},
Reason: fmt.Errorf(`prefix path="/var/lib/thing" suffix`),
Coordinates: file.NewCoordinates("/var/lib/thing", ""),
Reason: fmt.Errorf(`prefix path="/var/lib/thing" suffix`),
},
},
{
errorText: `prefix path="/var/lib/thing"`,
expected: &CoordinateError{
Coordinates: file.Coordinates{
RealPath: "/var/lib/thing",
},
Reason: fmt.Errorf(`prefix path="/var/lib/thing"`),
Coordinates: file.NewCoordinates("/var/lib/thing", ""),
Reason: fmt.Errorf(`prefix path="/var/lib/thing"`),
},
},
{
errorText: `path="/var/lib/thing" suffix`,
expected: &CoordinateError{
Coordinates: file.Coordinates{
RealPath: "/var/lib/thing",
},
Reason: fmt.Errorf(`path="/var/lib/thing" suffix`),
Coordinates: file.NewCoordinates("/var/lib/thing", ""),
Reason: fmt.Errorf(`path="/var/lib/thing" suffix`),
},
},
{
schema/json/schema-16.1.12.json
+7 -7
View File
@@ -716,15 +716,15 @@
"properties": {
"path": {
"type": "string",
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)."
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem."
},
"layerID": {
"type": "string",
"description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in."
},
"archivePath": {
"fullPath": {
"type": "string",
"description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it."
"description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM."
}
},
"type": "object",
@@ -2548,19 +2548,19 @@
"properties": {
"path": {
"type": "string",
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)."
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem."
},
"layerID": {
"type": "string",
"description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in."
},
"archivePath": {
"fullPath": {
"type": "string",
"description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it."
"description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM."
},
"accessPath": {
"type": "string",
"description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path)"
"description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.RealFullPath)."
},
"annotations": {
"additionalProperties": {
schema/json/schema-latest.json
+7 -7
View File
@@ -716,15 +716,15 @@
"properties": {
"path": {
"type": "string",
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)."
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem."
},
"layerID": {
"type": "string",
"description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in."
},
"archivePath": {
"fullPath": {
"type": "string",
"description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it."
"description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM."
}
},
"type": "object",
@@ -2548,19 +2548,19 @@
"properties": {
"path": {
"type": "string",
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)."
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem."
},
"layerID": {
"type": "string",
"description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in."
},
"archivePath": {
"fullPath": {
"type": "string",
"description": "ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. \"app.war:WEB-INF/lib/dep.jar\"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as \"%25\" and ':' as \"%3A\", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it."
"description": "RealFullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it equals RealPath; it may be blank on coordinates not built by a constructor, such as those decoded from an older SBOM."
},
"accessPath": {
"type": "string",
"description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path)"
"description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.RealFullPath)."
},
"annotations": {
"additionalProperties": {
syft/archive_cataloger_test.go
+77 -151
View File
@@ -2,7 +2,6 @@ package syft
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"context"
@@ -18,8 +17,10 @@ import (
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/task"
"github.com/anchore/syft/internal/testutils"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/filecataloging"
@@ -59,38 +60,37 @@ var fixtureExclusions = []string{"./nested/marker.txt", "**/excluded.txt", "**/i
func writeFixture(t *testing.T) string {
t.Helper()
dir := t.TempDir()
marker := map[string]string{"nested/marker.txt": "hello"}
markerZip := buildZipBytes(t, marker)
write := func(name string, data []byte) {
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, name), data, 0o644))
}
markerZip := testutils.Zip(t, map[string]string{"nested/marker.txt": "hello"})
require.NoError(t, os.MkdirAll(filepath.Join(dir, "nested"), 0o755))
require.NoError(t, os.MkdirAll(filepath.Join(dir, "app"), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "loose.txt"), []byte("outside"), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(dir, "nested", "marker.txt"), []byte("at the root"), 0o644))
writeTestZip(t, filepath.Join(dir, "app.zip"), map[string]string{
write("loose.txt", []byte("outside"))
write("nested/marker.txt", []byte("at the root"))
write("app.zip", testutils.Zip(t, map[string]string{
"nested/marker.txt": "hello",
"nested/keep.txt": "hello",
"nested/excluded.txt": "hello",
})
writeTestZipRaw(t, filepath.Join(dir, "app", "data.zip"), map[string][]byte{"nested/marker.txt": []byte("hello")})
writeTestZipRaw(t, filepath.Join(dir, "outer.zip"), map[string][]byte{
}))
write("app/data.zip", markerZip)
write("outer.zip", testutils.Zip(t, map[string][]byte{
"outer/marker.txt": []byte("hello"),
"deeper/inner.zip": markerZip,
"skip/inner-excluded.zip": markerZip,
"日本語/café-🎉/unicode.zip": markerZip,
})
require.NoError(t, os.WriteFile(filepath.Join(dir, "pkg.rpm"), markerZip, 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "KEPT.RPM"), markerZip, 0o644))
writeTestZipRaw(t, filepath.Join(dir, "app.war"), map[string][]byte{
"WEB-INF/lib/war-lib-2.0.jar": jarBytes(t, "war-lib", "2.0"),
})
writeTestZipRaw(t, filepath.Join(dir, "bundle.zip"), map[string][]byte{
"lib/zip-lib-1.2.jar": jarBytes(t, "zip-lib", "1.2"),
})
require.NoError(t, os.WriteFile(filepath.Join(dir, "bundle.tar.gz"), buildTarGzBytes(t, map[string][]byte{
"lib/tgz-lib-1.2.jar": jarBytes(t, "tgz-lib", "1.2"),
}), 0o644))
}))
write("pkg.rpm", markerZip)
write("KEPT.RPM", markerZip)
write("app.war", testutils.Zip(t, map[string][]byte{
"WEB-INF/lib/war-lib-2.0.jar": testutils.Jar(t, "war-lib", "2.0", nil),
}))
write("bundle.zip", testutils.Zip(t, map[string][]byte{
"lib/zip-lib-1.2.jar": testutils.Jar(t, "zip-lib", "1.2", nil),
}))
write("bundle.tar.gz", testutils.TarGz(t, map[string][]byte{
"lib/tgz-lib-1.2.jar": testutils.Jar(t, "tgz-lib", "1.2", nil),
}))
return dir
}
@@ -134,21 +134,21 @@ func TestArchiveCataloger(t *testing.T) {
s := scanDirWithExclusions(t, scanDir, cfg, fixtureExclusions...)
t.Run("markers are found at every depth under archive-relative paths", func(t *testing.T) {
// the location's ArchivePath is the chain of archives; RealPath is relative to the innermost
// the location's RealPath is the outermost archive; FullPath adds the chain of entries within it
assert.Equal(t, []string{
"KEPT.RPM|nested/marker.txt",
"app.zip|nested/marker.txt",
"app/data.zip|nested/marker.txt",
"outer.zip:deeper/inner.zip|nested/marker.txt",
"outer.zip:日本語/café-🎉/unicode.zip|nested/marker.txt",
"outer.zip|outer/marker.txt",
"KEPT.RPM!/nested/marker.txt",
"app.zip!/nested/marker.txt",
"app/data.zip!/nested/marker.txt",
"outer.zip!/deeper/inner.zip!/nested/marker.txt",
"outer.zip!/outer/marker.txt",
"outer.zip!/日本語/café-🎉/unicode.zip!/nested/marker.txt",
}, packageLocations(s, "marker-pkg"))
})
t.Run("archive contents do not leak into the parent tree", func(t *testing.T) {
for _, loc := range packageLocations(s, "marker-pkg") {
assert.NotContains(t, loc, ".zip/", "a joined path smears the archive's contents into the parent's tree")
assert.NotContains(t, loc, "|/", "a nested path must not be rooted")
assert.NotContains(t, loc, "!//", "a nested path must not be rooted")
}
})
@@ -169,18 +169,18 @@ func TestArchiveCataloger(t *testing.T) {
continue
}
if coords, ok := rel.From.(file.Coordinates); ok {
containers = append(containers, coords.ArchivePath+"|"+coords.RealPath)
containers = append(containers, coords.FullNestedPath())
}
}
}
sort.Strings(containers)
assert.Equal(t, []string{
"outer.zip|deeper/inner.zip",
"outer.zip|日本語/café-🎉/unicode.zip",
"|KEPT.RPM",
"|app.zip",
"|app/data.zip",
"|outer.zip",
"KEPT.RPM",
"app.zip",
"app/data.zip",
"outer.zip",
"outer.zip!/deeper/inner.zip",
"outer.zip!/日本語/café-🎉/unicode.zip",
}, containers, "the archive file, named in its own parent, contains the package")
})
@@ -189,16 +189,16 @@ func TestArchiveCataloger(t *testing.T) {
// reaches inside an archive, and an excluded entry is absent from the archive's filesystem
// rather than present and skipped
markers := packageLocations(s, "marker-pkg")
assert.NotContains(t, markers, "|nested/marker.txt", "a root-anchored pattern excludes the file at the scan root")
assert.Contains(t, markers, "app.zip|nested/marker.txt", "and says nothing about the same path inside an archive")
assert.NotContains(t, markers, "pkg.rpm|nested/marker.txt", "an archive whose own path is excluded is not extracted")
assert.Contains(t, markers, "KEPT.RPM|nested/marker.txt", "a lower-case pattern does not match an upper-case name")
assert.NotContains(t, markers, "outer.zip:skip/inner-excluded.zip|nested/marker.txt", "a nested archive is excluded at its own level")
assert.NotContains(t, markers, "nested/marker.txt", "a root-anchored pattern excludes the file at the scan root")
assert.Contains(t, markers, "app.zip!/nested/marker.txt", "and says nothing about the same path inside an archive")
assert.NotContains(t, markers, "pkg.rpm!/nested/marker.txt", "an archive whose own path is excluded is not extracted")
assert.Contains(t, markers, "KEPT.RPM!/nested/marker.txt", "a lower-case pattern does not match an upper-case name")
assert.NotContains(t, markers, "outer.zip!/skip/inner-excluded.zip!/nested/marker.txt", "a nested archive is excluded at its own level")
assert.Contains(t, seen, "nested/keep.txt")
assert.NotContains(t, seen, "nested/excluded.txt", "a **/ pattern excludes inside an archive, from the index itself")
assert.Contains(t, seen, "deeper/inner.zip", "the probe ran inside outer.zip")
assert.NotContains(t, seen, "skip/inner-excluded.zip", "the excluded inner archive is absent from outer's filesystem")
assert.Contains(t, seen, "app.zip!/nested/keep.txt")
assert.NotContains(t, seen, "app.zip!/nested/excluded.txt", "a **/ pattern excludes inside an archive, from the index itself")
assert.Contains(t, seen, "outer.zip!/deeper/inner.zip", "the probe ran inside outer.zip")
assert.NotContains(t, seen, "outer.zip!/skip/inner-excluded.zip", "the excluded inner archive is absent from outer's filesystem")
})
t.Run("a jar inside an archive is cataloged exactly once, under the virtual path java gives it", func(t *testing.T) {
@@ -249,28 +249,28 @@ func TestArchiveCataloger(t *testing.T) {
// output would fail it
var got []string
for coords := range s.Artifacts.FileMetadata {
if coords.ArchivePath == "" {
if archive.ContainerPath(coords) == "" {
continue
}
got = append(got, coords.ArchivePath+"|"+coords.RealPath)
got = append(got, coords.FullNestedPath())
}
sort.Strings(got)
assert.Equal(t, []string{
"KEPT.RPM|nested/marker.txt",
"app.war:WEB-INF/lib/war-lib-2.0.jar|META-INF/MANIFEST.MF",
"app.war|WEB-INF/lib/war-lib-2.0.jar",
"app.zip|nested/keep.txt",
"app.zip|nested/marker.txt",
"app/data.zip|nested/marker.txt",
"bundle.tar.gz:lib/tgz-lib-1.2.jar|META-INF/MANIFEST.MF",
"bundle.tar.gz|lib/tgz-lib-1.2.jar",
"bundle.zip:lib/zip-lib-1.2.jar|META-INF/MANIFEST.MF",
"bundle.zip|lib/zip-lib-1.2.jar",
"outer.zip:deeper/inner.zip|nested/marker.txt",
"outer.zip:日本語/café-🎉/unicode.zip|nested/marker.txt",
"outer.zip|deeper/inner.zip",
"outer.zip|outer/marker.txt",
"outer.zip|日本語/café-🎉/unicode.zip",
"KEPT.RPM!/nested/marker.txt",
"app.war!/WEB-INF/lib/war-lib-2.0.jar",
"app.war!/WEB-INF/lib/war-lib-2.0.jar!/META-INF/MANIFEST.MF",
"app.zip!/nested/keep.txt",
"app.zip!/nested/marker.txt",
"app/data.zip!/nested/marker.txt",
"bundle.tar.gz!/lib/tgz-lib-1.2.jar",
"bundle.tar.gz!/lib/tgz-lib-1.2.jar!/META-INF/MANIFEST.MF",
"bundle.zip!/lib/zip-lib-1.2.jar",
"bundle.zip!/lib/zip-lib-1.2.jar!/META-INF/MANIFEST.MF",
"outer.zip!/deeper/inner.zip",
"outer.zip!/deeper/inner.zip!/nested/marker.txt",
"outer.zip!/outer/marker.txt",
"outer.zip!/日本語/café-🎉/unicode.zip",
"outer.zip!/日本語/café-🎉/unicode.zip!/nested/marker.txt",
}, got)
})
@@ -338,7 +338,7 @@ func TestArchiveCataloger_jarReachedThroughALinkIsIdentified(t *testing.T) {
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
tw := tar.NewWriter(gw)
jar := jarBytes(t, "json-simple", "1.1.1")
jar := testutils.Jar(t, "json-simple", "1.1.1", nil)
require.NoError(t, tw.WriteHeader(&tar.Header{Name: "store/3f9a1c", Mode: 0o644, Size: int64(len(jar))}))
_, err := tw.Write(jar)
require.NoError(t, err)
@@ -355,10 +355,10 @@ func TestArchiveCataloger_jarReachedThroughALinkIsIdentified(t *testing.T) {
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2))
s := scanDirWithExclusions(t, dir, cfg)
assert.Equal(t, []string{"bundle.tar.gz|store/3f9a1c"}, packageLocations(s, "json-simple"))
assert.Equal(t, []string{"bundle.tar.gz!/store/3f9a1c"}, packageLocations(s, "json-simple"))
// the jar's own files are addressed by its real path, the same one its package location names
manifest := file.Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "bundle.tar.gz:store/3f9a1c"}
manifest := file.Coordinates{RealPath: "bundle.tar.gz", FullPath: "bundle.tar.gz!/store/3f9a1c!/META-INF/MANIFEST.MF"}
assert.Contains(t, s.Artifacts.FileMetadata, manifest)
}
@@ -366,7 +366,7 @@ func TestArchiveCataloger_defaultFileSelectionCoversRootJars(t *testing.T) {
// with the default owned-by-package selection, a jar at the root is described by the archive task,
// so the root file catalogers must run after it or the jar gets no file entries at all
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "app.jar"), jarBytes(t, "app", "1.0"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "app.jar"), testutils.Jar(t, "app", "1.0", nil), 0o644))
for name, depth := range map[string]int{"feature off": 0, "feature on": 1} {
t.Run(name, func(t *testing.T) {
@@ -375,8 +375,8 @@ func TestArchiveCataloger_defaultFileSelectionCoversRootJars(t *testing.T) {
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth))
s := scanDirWithExclusions(t, dir, cfg)
require.Equal(t, []string{"|app.jar"}, packageLocations(s, "app"))
jar := file.Coordinates{RealPath: "app.jar"}
require.Equal(t, []string{"app.jar"}, packageLocations(s, "app"))
jar := file.NewCoordinates("app.jar", "")
assert.Contains(t, s.Artifacts.FileMetadata, jar)
assert.Contains(t, s.Artifacts.FileDigests, jar)
})
@@ -387,9 +387,8 @@ func TestArchiveCataloger_javaParentLinkageMatchesTheLegacyPath(t *testing.T) {
// a consumer walking java DEPENDENCY_OF edges (the only structure SPDX and CycloneDX keep) must see
// the same war-to-jar linkage whether or not the archive task does the recursion
dir := t.TempDir()
war := buildZipBytesRaw(t, map[string][]byte{
"META-INF/MANIFEST.MF": []byte("Manifest-Version: 1.0\nImplementation-Title: app\nImplementation-Version: 1.0\n"),
"WEB-INF/lib/lib-2.0.jar": jarBytes(t, "lib", "2.0"),
war := testutils.Jar(t, "app", "1.0", map[string][]byte{
"WEB-INF/lib/lib-2.0.jar": testutils.Jar(t, "lib", "2.0", nil),
})
require.NoError(t, os.WriteFile(filepath.Join(dir, "app.war"), war, 0o644))
@@ -458,7 +457,7 @@ func TestArchiveCataloger_javaArchiveDepthIsAConfigError(t *testing.T) {
}
}
// packageLocations lists "archivePath|realPath" for every location of every package with the name.
// packageLocations lists the full path of every location of every package with the name.
func packageLocations(s *sbom.SBOM, name string) []string {
var out []string
for _, p := range s.Artifacts.Packages.Sorted() {
@@ -466,7 +465,7 @@ func packageLocations(s *sbom.SBOM, name string) []string {
continue
}
for _, loc := range p.Locations.ToSlice() {
out = append(out, loc.ArchivePath+"|"+loc.RealPath)
out = append(out, loc.FullNestedPath())
}
}
sort.Strings(out)
@@ -526,7 +525,7 @@ func (c resolverProbeCataloger) Catalog(_ context.Context, resolver file.Resolve
}
var paths []string
for _, loc := range locations {
paths = append(paths, loc.RealPath)
paths = append(paths, loc.FullNestedPath())
}
c.record(paths)
return nil, nil, nil
@@ -559,76 +558,3 @@ func scanDirWithExclusions(t *testing.T, dir string, cfg *CreateSBOMConfig, excl
require.NoError(t, err)
return s
}
func writeTestZip(t *testing.T, path string, files map[string]string) {
t.Helper()
require.NoError(t, os.WriteFile(path, buildZipBytes(t, files), 0o644))
}
func writeTestZipRaw(t *testing.T, path string, files map[string][]byte) {
t.Helper()
require.NoError(t, os.WriteFile(path, buildZipBytesRaw(t, files), 0o644))
}
func buildZipBytes(t *testing.T, files map[string]string) []byte {
t.Helper()
raw := make(map[string][]byte, len(files))
for k, v := range files {
raw[k] = []byte(v)
}
return buildZipBytesRaw(t, raw)
}
// buildZipBytesRaw writes entries in sorted name order rather than map order: limits are enforced as
// the walk proceeds, so entry order decides which entries land before a truncation.
func buildZipBytesRaw(t *testing.T, files map[string][]byte) []byte {
t.Helper()
names := make([]string, 0, len(files))
for name := range files {
names = append(names, name)
}
sort.Strings(names)
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for _, name := range names {
w, err := zw.Create(name)
require.NoError(t, err)
_, err = w.Write(files[name])
require.NoError(t, err)
}
require.NoError(t, zw.Close())
return buf.Bytes()
}
func buildTarGzBytes(t *testing.T, files map[string][]byte) []byte {
t.Helper()
names := make([]string, 0, len(files))
for name := range files {
names = append(names, name)
}
sort.Strings(names)
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
tw := tar.NewWriter(gw)
for _, name := range names {
body := files[name]
require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}))
_, err := tw.Write(body)
require.NoError(t, err)
}
require.NoError(t, tw.Close())
require.NoError(t, gw.Close())
return buf.Bytes()
}
// jarBytes is the smallest jar the java cataloger identifies: a manifest naming the artifact.
func jarBytes(t *testing.T, title, version string) []byte {
t.Helper()
return buildZipBytesRaw(t, map[string][]byte{
"META-INF/MANIFEST.MF": []byte(
"Manifest-Version: 1.0\nImplementation-Title: " + title + "\nImplementation-Version: " + version + "\n",
),
})
}
syft/file/cataloger/executable/cataloger.go
+11 -11
View File
@@ -68,7 +68,7 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map
results := make(map[file.Coordinates]file.Executable)
errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(locs), func(loc file.Location) (*file.Executable, error) {
prog.AtomicStage.Set(loc.Path())
prog.AtomicStage.Set(loc.FullNestedPath())
exec, err := processExecutableLocation(loc, resolver)
if err != nil {
@@ -93,15 +93,15 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map
func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec *file.Executable, err error) {
reader, err := resolver.FileContentsByLocation(loc)
if err != nil {
log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get file contents")
log.WithFields("error", err, "path", loc.FullNestedPath()).Debug("unable to get file contents")
return nil, fmt.Errorf("unable to get file contents: %w", err)
}
defer internal.CloseAndLogError(reader, loc.RealPath)
defer internal.CloseAndLogError(reader, loc.FullNestedPath())
defer func() {
if r := recover(); r != nil {
// a malformed binary must not take down the whole SBOM; the caller records this as an unknown
log.WithFields("path", loc.RealPath, "panic", r).Debug("recovered from panic while reading executable")
log.WithFields("path", loc.FullNestedPath(), "panic", r).Debug("recovered from panic while reading executable")
log.Tracef("executable panic stack:\n%s", debug.Stack())
exec, err = nil, fmt.Errorf("recovered from panic while reading executable: %v", r)
}
@@ -109,7 +109,7 @@ func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec
uReader, err := unionreader.GetUnionReader(reader)
if err != nil {
log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get union reader")
log.WithFields("error", err, "path", loc.FullNestedPath()).Debug("unable to get union reader")
return nil, fmt.Errorf("unable to get union reader: %w", err)
}
@@ -146,7 +146,7 @@ func filterByGlobs(locs []file.Location, globs []string) ([]file.Location, error
func locationMatchesGlob(loc file.Location, globs []string) (bool, error) {
for _, glob := range globs {
for _, path := range []string{loc.RealPath, loc.AccessPath} {
for _, path := range []string{loc.LocalPath(), loc.AccessPath} {
if path == "" {
continue
}
@@ -169,13 +169,13 @@ func processExecutable(loc file.Location, reader unionreader.UnionReader) (*file
format, err := findExecutableFormat(reader)
if err != nil {
log.Debugf("unable to determine executable kind for %v: %v", loc.RealPath, err)
log.Debugf("unable to determine executable kind for %v: %v", loc.FullNestedPath(), err)
return nil, fmt.Errorf("unable to determine executable kind: %w", err)
}
if format == "" {
// this is not an "unknown", so just log -- this binary does not have parseable data in it
log.Debugf("unable to determine executable format for %q", loc.RealPath)
log.Debugf("unable to determine executable format for %q", loc.FullNestedPath())
return nil, nil
}
@@ -184,17 +184,17 @@ func processExecutable(loc file.Location, reader unionreader.UnionReader) (*file
switch format {
case file.ELF:
if err = findELFFeatures(&data, reader); err != nil {
log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine ELF features")
log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine ELF features")
err = fmt.Errorf("unable to determine ELF features: %w", err)
}
case file.PE:
if err = findPEFeatures(&data, reader); err != nil {
log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine PE features")
log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine PE features")
err = fmt.Errorf("unable to determine PE features: %w", err)
}
case file.MachO:
if err = findMachoFeatures(&data, reader); err != nil {
log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine Macho features")
log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine Macho features")
err = fmt.Errorf("unable to determine Macho features: %w", err)
}
}
syft/file/cataloger/filecontent/cataloger.go
+2 -2
View File
@@ -57,7 +57,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) (map[fi
prog := catalogingProgress(ctx, int64(len(locations)))
for _, location := range locations {
prog.AtomicStage.Set(location.Path())
prog.AtomicStage.Set(location.FullNestedPath())
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
@@ -103,7 +103,7 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
buf := &bytes.Buffer{}
encoder := base64.NewEncoder(base64.StdEncoding, buf)
if _, err = io.Copy(encoder, contentReader); err != nil {
return "", internal.ErrPath{Context: "content-cataloger", Path: location.RealPath, Err: err}
return "", internal.ErrPath{Context: "content-cataloger", Path: location.FullNestedPath(), Err: err}
}
// note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes
if err := encoder.Close(); err != nil {
syft/file/cataloger/filedigest/cataloger.go
+6 -6
View File
@@ -42,9 +42,9 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
locations = intCataloger.AllRegularFiles(ctx, resolver)
} else {
for _, c := range coordinates {
locs, err := resolver.FilesByPath(c.RealPath)
locs, err := resolver.FilesByPath(c.LocalPath())
if err != nil {
return nil, fmt.Errorf("unable to get file locations for path %q: %w", c.RealPath, err)
return nil, fmt.Errorf("unable to get file locations for path %q: %w", c.FullNestedPath(), err)
}
locations = append(locations, locs...)
}
@@ -59,10 +59,10 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
return nil, nil
}
prog.AtomicStage.Set(location.Path())
prog.AtomicStage.Set(location.FullNestedPath())
if internal.IsErrPathPermission(err) {
log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
log.Debugf("file digests cataloger skipping %q: %+v", location.FullNestedPath(), err)
return nil, unknown.New(location, err)
}
@@ -108,7 +108,7 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver,
defer func() {
if r := recover(); r != nil {
// the caller records this as an unknown for the location instead of failing the whole SBOM
log.WithFields("path", location.RealPath, "panic", r).Debug("recovered from panic while digesting file")
log.WithFields("path", location.FullNestedPath(), "panic", r).Debug("recovered from panic while digesting file")
log.Tracef("file digest panic stack:\n%s", debug.Stack())
digests, err = nil, fmt.Errorf("recovered from panic while digesting file: %v", r)
}
@@ -116,7 +116,7 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver,
digests, err = intFile.NewDigestsFromFile(ctx, contentReader, i.hashes)
if err != nil {
return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.FullNestedPath(), Err: err}
}
return digests, nil
syft/file/cataloger/filemetadata/cataloger.go
+2 -2
View File
@@ -34,7 +34,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
go func() {
defer close(ch)
for _, c := range coordinates {
locs, err := resolver.FilesByPath(c.RealPath)
locs, err := resolver.FilesByPath(c.LocalPath())
if err != nil {
errs = unknown.Append(errs, c, err)
continue
@@ -55,7 +55,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
prog := catalogingProgress(ctx, -1)
for location := range locations {
prog.AtomicStage.Set(location.Path())
prog.AtomicStage.Set(location.FullNestedPath())
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
syft/file/cataloger/internal/all_regular_files.go
+1 -1
View File
@@ -12,7 +12,7 @@ func AllRegularFiles(ctx context.Context, resolver file.Resolver) (locations []f
ctx, cancel := context.WithCancel(ctx)
defer cancel()
for location := range resolver.AllLocations(ctx) {
resolvedLocations, err := resolver.FilesByPath(location.RealPath)
resolvedLocations, err := resolver.FilesByPath(location.LocalPath())
if err != nil {
log.Debugf("unable to resolve %+v: %+v", location, err)
continue
syft/file/coordinate_set.go
+1 -1
View File
@@ -79,7 +79,7 @@ func (s CoordinateSet) ToSlice(sorters ...func(a, b Coordinates) int) []Coordina
sort.SliceStable(coordinates, func(i, j int) bool {
if coordinates[i].FileSystemID == coordinates[j].FileSystemID {
if coordinates[i].RealPath == coordinates[j].RealPath {
return coordinates[i].ArchivePath < coordinates[j].ArchivePath
return coordinates[i].FullNestedPath() < coordinates[j].FullNestedPath()
}
return coordinates[i].RealPath < coordinates[j].RealPath
}
syft/file/coordinates.go
+30 -8
View File
@@ -2,21 +2,42 @@ package file
import (
"fmt"
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
)
const ArchivePathSeparator = "!/"
// Coordinates contains the minimal information needed to describe how to find a file within any possible source object (e.g. image and directory sources)
type Coordinates struct {
// RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed).
// RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem.
RealPath string `json:"path" cyclonedx:"path"`
// FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in.
FileSystemID string `json:"layerID,omitempty" cyclonedx:"layerID"`
// ArchivePath is the colon-delimited chain of archive paths, from the scan root, traversed to reach this file (e.g. "app.war:WEB-INF/lib/dep.jar"). Blank for files not found within an archive. It disambiguates identically-named files in different archives on the same filesystem. Within each segment after the first, '%' is escaped as "%25" and ':' as "%3A", so splitting on ':' recovers the archive boundaries. Each segment is an archive's real path (links resolved), relative to the segment before it.
ArchivePath string `json:"archivePath,omitempty" cyclonedx:"archivePath"`
// FullPath is the full path to the file: RealPath, then for a file within an archive a "!/"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. "/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it is blank, so coordinates outside archives are unchanged by the field (in JSON, equality and ID).
FullPath string `json:"fullPath,omitempty" cyclonedx:"fullPath"`
}
// FullNestedPath is FullPath, or RealPath where FullPath is blank.
func (c Coordinates) FullNestedPath() string {
if c.FullPath == "" {
return c.RealPath
}
return c.FullPath
}
// LocalPath is the path within the resolver holding the file: the segment of FullNestedPath after the
// last archive separator, or RealPath for a file not within an archive.
func (c Coordinates) LocalPath() string {
fullPath := c.FullNestedPath()
if i := strings.LastIndex(fullPath, ArchivePathSeparator); i >= 0 {
return fullPath[i+len(ArchivePathSeparator):]
}
return fullPath
}
func NewCoordinates(realPath, fsID string) Coordinates {
@@ -43,8 +64,9 @@ func (c Coordinates) String() string {
if c.FileSystemID != "" {
str += fmt.Sprintf(" Layer=%q", c.FileSystemID)
}
if c.ArchivePath != "" {
str += fmt.Sprintf(" Archive=%q", c.ArchivePath)
if c.FullNestedPath() != c.RealPath {
str += fmt.Sprintf(" FullPath=%q", c.FullNestedPath())
}
return fmt.Sprintf("Location<%s>", str)
}
@@ -53,10 +75,10 @@ func (c Coordinates) GetCoordinates() Coordinates {
return c
}
// HashInclude keeps an empty ArchivePath out of the artifact ID hash (see artifact.IDByHash), so
// coordinates outside archives keep the IDs they had before the field existed.
// HashInclude keeps a FullPath that adds nothing to RealPath out of the artifact ID hash (see
// artifact.IDByHash), so coordinates outside archives keep the IDs they had before the field existed.
func (c Coordinates) HashInclude(field string, _ any) (bool, error) {
if field == "ArchivePath" && c.ArchivePath == "" {
if field == "FullPath" && c.FullNestedPath() == c.RealPath {
return false, nil
}
return true, nil
syft/file/location.go
+9 -9
View File
@@ -33,7 +33,7 @@ type LocationData struct {
// note: it is IMPORTANT to ignore anything but the coordinates for a Location when considering the ID (hash value)
// since the coordinates are the minimally correct ID for a location (symlinks should not come into play)
// AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path)
// AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.FullPath).
AccessPath string `hash:"ignore" json:"accessPath"`
// ref is the stereoscope file reference relative to the stereoscope.FileCatalog that has more information about this location.
@@ -110,12 +110,12 @@ func NewVirtualLocation(realPath, accessPath string) Location {
}}
}
// NewLocationFromCoordinates creates a new location for the given Coordinates.
// NewLocationFromCoordinates creates a new location for the given Coordinates, accessed by their full path.
func NewLocationFromCoordinates(coordinates Coordinates) Location {
return Location{
LocationData: LocationData{
Coordinates: coordinates,
AccessPath: coordinates.RealPath,
AccessPath: coordinates.FullNestedPath(),
},
LocationMetadata: LocationMetadata{
Annotations: map[string]string{},
@@ -205,11 +205,12 @@ func NewVirtualLocationFromDirectory(responsePath, responseAccessPath string, re
}
}
// Path is the path this location was accessed by, or its local path (see Coordinates.LocalPath).
func (l Location) Path() string {
if l.AccessPath != "" {
return l.AccessPath
}
return l.RealPath
return l.LocalPath()
}
func (l Location) String() string {
@@ -220,12 +221,12 @@ func (l Location) String() string {
str += fmt.Sprintf("RealPath=%q", l.RealPath)
if l.AccessPath != "" && l.AccessPath != l.RealPath {
if l.AccessPath != "" {
str += fmt.Sprintf(" AccessPath=%q", l.AccessPath)
}
if l.ArchivePath != "" {
str += fmt.Sprintf(" ArchivePath=%q", l.ArchivePath)
if l.FullNestedPath() != l.RealPath {
str += fmt.Sprintf(" FullPath=%q", l.FullNestedPath())
}
if l.FileSystemID != "" {
@@ -235,8 +236,7 @@ func (l Location) String() string {
}
func (l Location) Equals(other Location) bool {
return l.RealPath == other.RealPath &&
return l.FullNestedPath() == other.FullNestedPath() &&
l.AccessPath == other.AccessPath &&
l.ArchivePath == other.ArchivePath &&
l.FileSystemID == other.FileSystemID
}
syft/file/location_set.go
+8 -2
View File
@@ -5,6 +5,7 @@ import (
"sort"
"github.com/gohugoio/hashstructure"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/log"
)
@@ -112,8 +113,13 @@ func (s *LocationSet) Empty() bool {
}
func (s LocationSet) Hash() (uint64, error) {
// access paths and filesystem IDs are not considered when hashing a location set, only the real paths
return hashstructure.Hash(s.CoordinateSet().Paths(), &hashstructure.HashOptions{
// access paths and filesystem IDs are not considered when hashing a location set, only the full paths
// (unlike CoordinateSet.Paths, which ownership-by-overlap needs as real paths only)
paths := strset.New()
for _, c := range s.CoordinateSet().ToUnorderedSlice() {
paths.Add(c.FullNestedPath())
}
return hashstructure.Hash(paths.List(), &hashstructure.HashOptions{
ZeroNil: true,
SlicesAsSets: true,
})
syft/file/location_set_test.go
+27
View File
@@ -4,6 +4,7 @@ import (
"sort"
"testing"
"github.com/gohugoio/hashstructure"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -255,6 +256,10 @@ func TestLocationSet_Hash(t *testing.T) {
},
}
inA := NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FullPath: "/app.war!/a.jar!/package.json"})
inB := NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FullPath: "/app.war!/b.jar!/package.json"})
etcFull := Location{LocationData: LocationData{Coordinates: Coordinates{RealPath: "/etc/hosts", FileSystemID: "a", FullPath: "/etc/hosts"}}}
tests := []struct {
name string
setA LocationSet
@@ -297,6 +302,18 @@ func TestLocationSet_Hash(t *testing.T) {
setB: NewLocationSet(binB, etcB),
want: assert.Equal,
},
{
name: "same outer archive but different nested entries have different hashes",
setA: NewLocationSet(inA),
setB: NewLocationSet(inB),
want: assert.NotEqual,
},
{
name: "a full path equal to the real path has the same hash as a blank one",
setA: NewLocationSet(etcA),
setB: NewLocationSet(etcFull),
want: assert.Equal,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -308,3 +325,13 @@ func TestLocationSet_Hash(t *testing.T) {
})
}
}
// IDs outside archives must not change from when only real paths were hashed
func TestLocationSet_Hash_unchangedOutsideArchives(t *testing.T) {
want, err := hashstructure.Hash([]string{"/etc/hosts", "/bin"}, &hashstructure.HashOptions{ZeroNil: true, SlicesAsSets: true})
require.NoError(t, err)
got, err := NewLocationSet(NewLocation("/etc/hosts"), NewVirtualLocation("/bin", "/usr/bin")).Hash()
require.NoError(t, err)
assert.Equal(t, want, got)
}
syft/file/location_test.go
+54 -4
View File
@@ -1,9 +1,11 @@
package file
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file"
)
@@ -49,11 +51,59 @@ func TestLocation_ID(t *testing.T) {
}
func TestLocation_archivePathIsIdentity(t *testing.T) {
a := NewLocationFromCoordinates(Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/a.jar"})
b := NewLocationFromCoordinates(Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/b.jar"})
func TestLocation_fullPathIsIdentity(t *testing.T) {
a := NewLocationFromCoordinates(Coordinates{RealPath: "/a.jar", FullPath: "/a.jar!/META-INF/MANIFEST.MF"})
b := NewLocationFromCoordinates(Coordinates{RealPath: "/b.jar", FullPath: "/b.jar!/META-INF/MANIFEST.MF"})
assert.False(t, a.Equals(b))
assert.NotEqual(t, a.String(), b.String())
assert.Contains(t, a.String(), `ArchivePath="/a.jar"`)
assert.Contains(t, a.String(), `RealPath="/a.jar"`)
}
func TestLocation_JSON(t *testing.T) {
tests := []struct {
name string
location Location
expected string
}{
{
name: "a file outside an archive writes no fullPath",
location: NewVirtualLocationFromDirectory("/etc/os-release", "/etc/link", file.Reference{}).WithAnnotation("k", "v"),
expected: `{"path":"/etc/os-release","accessPath":"/etc/link","annotations":{"k":"v"}}`,
},
{
name: "a file within an archive writes its full path",
location: NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/META-INF/MANIFEST.MF"}),
expected: `{"path":"/app.war","layerID":"layer","fullPath":"/app.war!/META-INF/MANIFEST.MF","accessPath":"/app.war!/META-INF/MANIFEST.MF"}`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
data, err := json.Marshal(tt.location)
require.NoError(t, err)
assert.JSONEq(t, tt.expected, string(data))
var decoded Location
require.NoError(t, json.Unmarshal(data, &decoded))
assert.Equal(t, tt.location.Coordinates, decoded.Coordinates, "decoding fills FullPath back in")
assert.Equal(t, tt.location.AccessPath, decoded.AccessPath)
data, err = json.Marshal(tt.location.Coordinates)
require.NoError(t, err)
var coordinates Coordinates
require.NoError(t, json.Unmarshal(data, &coordinates))
assert.Equal(t, tt.location.Coordinates, coordinates)
})
}
}
func TestCoordinates_ID_unchangedByTopLevelFullRealPath(t *testing.T) {
assert.Equal(t, Coordinates{RealPath: "/a"}.ID(), NewCoordinates("/a", "").ID())
assert.NotEqual(t, NewCoordinates("/a", "").ID(), Coordinates{RealPath: "/a", FullPath: "/a!/b"}.ID())
}
func TestCoordinates_LocalPath(t *testing.T) {
assert.Equal(t, "/a.jar", NewCoordinates("/a.jar", "").LocalPath())
assert.Equal(t, "/a.jar", Coordinates{RealPath: "/a.jar"}.LocalPath())
assert.Equal(t, "META-INF/MANIFEST.MF", Coordinates{RealPath: "/a.war", FullPath: "/a.war!/lib/b.jar!/META-INF/MANIFEST.MF"}.LocalPath())
}
syft/file/locations.go
+2 -2
View File
@@ -88,7 +88,7 @@ func LocationSorter(layers []string) func(a, b Location) int { //nolint:gocognit
}
// the same path in different archives is a different file
return strings.Compare(a.ArchivePath, b.ArchivePath)
return strings.Compare(a.FullNestedPath(), b.FullNestedPath())
}
}
@@ -128,6 +128,6 @@ func CoordinatesSorter(layers []string) func(a, b Coordinates) int {
}
// the same path in different archives is a different file
return strings.Compare(a.ArchivePath, b.ArchivePath)
return strings.Compare(a.FullNestedPath(), b.FullNestedPath())
}
}
syft/file/locations_test.go
+7 -7
View File
@@ -557,23 +557,23 @@ func TestLocationAndCoordinatesSorters(t *testing.T) {
}
}
func TestSorters_archivePathBreaksTies(t *testing.T) {
func TestSorters_fullPathBreaksTies(t *testing.T) {
// the same path in two archives must sort the same way whatever order the inputs arrive in, or
// output built from map iteration is not reproducible
a := Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/app.war:WEB-INF/lib/a.jar"}
b := Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/app.war:WEB-INF/lib/b.jar"}
want := []string{a.ArchivePath, b.ArchivePath}
a := Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"}
b := Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/META-INF/MANIFEST.MF"}
want := []string{a.FullNestedPath(), b.FullNestedPath()}
for _, in := range [][]Coordinates{{a, b}, {b, a}} {
coords := slices.Clone(in)
slices.SortFunc(coords, CoordinatesSorter(nil))
assert.Equal(t, want, []string{coords[0].ArchivePath, coords[1].ArchivePath}, "CoordinatesSorter")
assert.Equal(t, want, []string{coords[0].FullNestedPath(), coords[1].FullNestedPath()}, "CoordinatesSorter")
locs := []Location{NewLocationFromCoordinates(in[0]), NewLocationFromCoordinates(in[1])}
slices.SortFunc(locs, LocationSorter(nil))
assert.Equal(t, want, []string{locs[0].ArchivePath, locs[1].ArchivePath}, "LocationSorter")
assert.Equal(t, want, []string{locs[0].FullNestedPath(), locs[1].FullNestedPath()}, "LocationSorter")
got := NewCoordinateSet(in...).ToSlice()
assert.Equal(t, want, []string{got[0].ArchivePath, got[1].ArchivePath}, "CoordinateSet.ToSlice")
assert.Equal(t, want, []string{got[0].FullNestedPath(), got[1].FullNestedPath()}, "CoordinateSet.ToSlice")
}
}
syft/file/mock_resolver.go
+7 -7
View File
@@ -60,9 +60,9 @@ func NewMockResolverForPathsWithMetadata(metadata map[Coordinates]Metadata) *Moc
l := NewLocationFromCoordinates(c)
locations = append(locations, l)
mimeTypeIndex[m.MIMEType] = append(mimeTypeIndex[m.MIMEType], l)
ext := path.Ext(l.RealPath)
ext := path.Ext(l.FullNestedPath())
extension[ext] = append(extension[ext], l)
bn := path.Base(l.RealPath)
bn := path.Base(l.FullNestedPath())
basename[bn] = append(basename[bn], l)
}
@@ -97,9 +97,9 @@ func NewMockResolverForMediaTypes(mediaTypes map[string][]Location) *MockResolve
mediaTypeIndex[mediaType] = append(mediaTypeIndex[mediaType], locs...)
for _, l := range locs {
locations = append(locations, l)
ext := path.Ext(l.RealPath)
ext := path.Ext(l.FullNestedPath())
extension[ext] = append(extension[ext], l)
bn := path.Base(l.RealPath)
bn := path.Base(l.FullNestedPath())
basename[bn] = append(basename[bn], l)
}
}
@@ -121,7 +121,7 @@ func (r MockResolver) ImageReference() string {
// HasPath indicates if the given path exists in the underlying source.
func (r MockResolver) HasPath(path string) bool {
for _, l := range r.locations {
if l.RealPath == path {
if l.FullNestedPath() == path {
return true
}
}
@@ -150,7 +150,7 @@ func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) {
var results []Location
for _, p := range paths {
for _, location := range r.locations {
if p == location.RealPath {
if p == location.LocalPath() {
results = append(results, NewLocation(p))
}
}
@@ -164,7 +164,7 @@ func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) {
var results []Location
for _, pattern := range patterns {
for _, location := range r.locations {
matches, err := doublestar.Match(pattern, location.RealPath)
matches, err := doublestar.Match(pattern, location.LocalPath())
if err != nil {
return nil, err
}
syft/format/common/spdxhelpers/to_format_model.go
+7 -8
View File
@@ -15,7 +15,6 @@ import (
"github.com/spdx/tools-golang/spdx"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/mimetype"
"github.com/anchore/syft/internal/relationship"
@@ -356,7 +355,7 @@ func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID {
return spdx.ElementID(helpers.SanitizeElementID(id))
}
p := ""
parts := strings.Split(it.RealPath, "/")
parts := strings.Split(it.FullNestedPath(), "/")
for i := len(parts); i > 0; i-- {
part := parts[i-1]
if len(part) == 0 {
@@ -748,13 +747,13 @@ func toFiles(s sbom.SBOM) (results []*spdx.File) {
comments = append(comments, layerIDCommentPrefix+c.FileSystemID)
}
// a file inside an archive is named by its full chain ("app.war:WEB-INF/lib/a.jar:META-INF/MANIFEST.MF"),
// a file inside an archive is named by its full path ("/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"),
// so a consumer reading only the name does not mistake it for a file at that path in the scan
// root. The archive path is also kept in the comment so decoding need not guess where it ends.
name := c.RealPath
if c.ArchivePath != "" {
name = archive.VirtualPath(file.NewLocationFromCoordinates(c))
comments = append(comments, archivePathCommentPrefix+c.ArchivePath)
// root. The entry chain after the real path is also kept in the comment so decoding need not guess
// where it starts.
name := c.FullNestedPath()
if entries, ok := strings.CutPrefix(c.FullNestedPath(), c.RealPath+file.ArchivePathSeparator); ok {
comments = append(comments, archivePathCommentPrefix+entries)
}
relativePath, err := convertAbsoluteToRelative(name)
syft/format/common/spdxhelpers/to_format_model_test.go
+22 -21
View File
@@ -762,10 +762,7 @@ func Test_fileIDsForPackage(t *testing.T) {
Name: "bogus",
}
c := file.Coordinates{
RealPath: "/path",
FileSystemID: "nowhere",
}
c := file.NewCoordinates("/path", "nowhere")
docElementId := func(identifiable artifact.Identifiable) spdx.DocElementID {
return spdx.DocElementID{
@@ -1027,17 +1024,13 @@ func Test_toSPDXID(t *testing.T) {
expected string
}{
{
name: "short filename",
it: file.Coordinates{
RealPath: "/short/path/file.txt",
},
name: "short filename",
it: file.NewCoordinates("/short/path/file.txt", ""),
expected: "File-short-path-file.txt",
},
{
name: "long filename",
it: file.Coordinates{
RealPath: "/some/long/path/with/a/lot/of-text/that-contains-a/file.txt",
},
name: "long filename",
it: file.NewCoordinates("/some/long/path/with/a/lot/of-text/that-contains-a/file.txt", ""),
expected: "File-...a-lot-of-text-that-contains-a-file.txt",
},
{
@@ -1166,9 +1159,9 @@ func Test_otherLicenses(t *testing.T) {
func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) {
// two archives' META-INF/MANIFEST.MF must not come out as two files with the same name and nothing
// saying which archive each is in
root := file.Coordinates{RealPath: "/etc/os-release", FileSystemID: "sha256:layer"}
inA := file.Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: "/app.war:WEB-INF/lib/a.jar", FileSystemID: "sha256:layer"}
inB := file.Coordinates{RealPath: "odd:na%me.txt", ArchivePath: "/app.war:WEB-INF/lib/b.jar"}
root := file.NewCoordinates("/etc/os-release", "sha256:layer")
inA := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", FileSystemID: "sha256:layer"}
inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"}
s := sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{root: {}, inA: {}, inB: {}}}}
byName := map[string]*spdx.File{}
@@ -1178,15 +1171,23 @@ func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) {
require.Len(t, byName, 3)
assert.Equal(t, "layerID: sha256:layer", byName["etc/os-release"].FileComment, "files outside archives are unchanged")
assert.Equal(t, file.NewCoordinates("etc/os-release", "sha256:layer"), toSyftCoordinates(byName["etc/os-release"]))
a := byName["app.war:WEB-INF/lib/a.jar:META-INF/MANIFEST.MF"]
require.NotNil(t, a)
assert.Equal(t, "layerID: sha256:layer\narchivePath: /app.war:WEB-INF/lib/a.jar", a.FileComment)
assert.Equal(t, inA, toSyftCoordinates(a))
// the name is made relative like any other file's, so the decoded top path is too
a := byName["app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"]
require.NotNil(t, a, "names: %v", byName)
assert.Equal(t, "layerID: sha256:layer\narchivePath: WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", a.FileComment)
wantA := inA
wantA.RealPath = "app.war"
wantA.FullPath = "app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"
assert.Equal(t, wantA, toSyftCoordinates(a))
b := byName["app.war:WEB-INF/lib/b.jar:odd%3Ana%25me.txt"]
b := byName["app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"]
require.NotNil(t, b, "names: %v", byName)
assert.Equal(t, inB, toSyftCoordinates(b), "the entry path's escaping is undone on decode")
wantB := inB
wantB.RealPath = "app.war"
wantB.FullPath = "app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"
assert.Equal(t, wantB, toSyftCoordinates(b))
}
func Test_toRootPackage_name(t *testing.T) {
syft/format/common/spdxhelpers/to_syft_model.go
+5 -9
View File
@@ -15,7 +15,6 @@ import (
"github.com/spdx/tools-golang/spdx/v2/common"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/spdxlicense"
"github.com/anchore/syft/syft/artifact"
@@ -531,28 +530,25 @@ func collectPackageFileRelationships(spdxIDMap map[string]any, doc *spdx.Documen
}
func toSyftCoordinates(f *spdx.File) file.Coordinates {
c := file.Coordinates{RealPath: f.FileName}
c := file.NewCoordinates(f.FileName, "")
for _, line := range strings.Split(f.FileComment, "\n") {
if v, ok := strings.CutPrefix(line, layerIDCommentPrefix); ok {
c.FileSystemID = v
}
if v, ok := strings.CutPrefix(line, archivePathCommentPrefix); ok {
c.ArchivePath = v
// the name is the full path; the real path is what precedes the entry chain
c.RealPath = strings.TrimSuffix(f.FileName, file.ArchivePathSeparator+v)
c.FullPath = f.FileName
}
}
if v, ok := strings.CutPrefix(string(f.FileSPDXIdentifier), layerIDCommentPrefix); ok {
c.FileSystemID = v
}
if c.ArchivePath != "" {
// the name is the chain; the file's own path is its last, escaped, segment
c.RealPath = archive.UnescapeEntryPath(f.FileName[strings.LastIndexByte(f.FileName, ':')+1:])
}
return c
}
func toSyftLocation(f *spdx.File) file.Location {
c := toSyftCoordinates(f)
return file.NewVirtualLocationFromCoordinates(c, c.RealPath)
return file.NewLocationFromCoordinates(toSyftCoordinates(f))
}
func requireAndTrimPrefix(val any, prefix string) string {
syft/format/common/spdxhelpers/to_syft_model_test.go
+3 -9
View File
@@ -344,10 +344,7 @@ func Test_toSyftRelationships(t *testing.T) {
}
pkg3.SetID()
loc1 := file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
})
loc1 := file.NewLocationFromCoordinates(file.NewCoordinates("/somewhere/real", "abc"))
tests := []struct {
name string
@@ -694,11 +691,8 @@ func Test_directPackageFiles(t *testing.T) {
p.OverrideID("1") // the same as the spdxID on the package element
f := file.Location{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "some-file",
FileSystemID: "",
},
AccessPath: "some-file",
Coordinates: file.NewCoordinates("some-file", ""),
AccessPath: "some-file",
},
LocationMetadata: file.LocationMetadata{
Annotations: map[string]string{},
syft/format/common/spdxhelpers/to_syft_model_v3.go
+1 -4
View File
@@ -401,10 +401,7 @@ func v3toSyftCoordinates(f spdx.AnyFile) file.Coordinates {
if strings.HasPrefix(f.GetID(), layerIDPrefix) {
fileSystemID = strings.TrimPrefix(f.GetID(), layerIDPrefix)
}
return file.Coordinates{
RealPath: f.GetName(),
FileSystemID: fileSystemID,
}
return file.NewCoordinates(f.GetName(), fileSystemID)
}
func v3toSyftLocation(f spdx.AnyFile) file.Location {
syft/format/common/spdxhelpers/to_syft_model_v3_test.go
+2 -2
View File
@@ -193,14 +193,14 @@ func TestToSyftModelV3_documentConversion(t *testing.T) {
assert.Equal(t, "BSD-3-Clause AND GPL-2.0+", pkg2Concluded[0].Value)
// Verify files
coords1 := file.Coordinates{RealPath: "/src/main.go"}
coords1 := file.NewCoordinates("/src/main.go", "")
digests, ok := result.Artifacts.FileDigests[coords1]
require.True(t, ok, "file1 digests not found")
require.Len(t, digests, 1)
assert.Equal(t, "sha256", digests[0].Algorithm)
assert.Equal(t, "abc123", digests[0].Value)
coords2 := file.Coordinates{RealPath: "/src/util.go"}
coords2 := file.NewCoordinates("/src/util.go", "")
_, ok = result.Artifacts.FileMetadata[coords2]
assert.True(t, ok, "file2 metadata not found")
syft/format/github/internal/model/model.go
+3 -3
View File
@@ -113,9 +113,9 @@ func toPath(s source.Description, p pkg.Package) string {
locations := p.Locations.ToSlice()
if len(locations) > 0 {
location := locations[0]
packagePath := location.RealPath
if location.AccessPath != "" {
packagePath = location.AccessPath
packagePath := location.AccessPath
if packagePath == "" {
packagePath = location.FullNestedPath()
}
packagePath = strings.TrimPrefix(packagePath, "/")
switch metadata := s.Metadata.(type) {
syft/format/github/internal/model/model_test.go
+3 -12
View File
@@ -41,30 +41,21 @@ func sbomFixture() sbom.SBOM {
Name: "pkg-1",
Version: "1.0.1",
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/usr/lib",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.NewCoordinates("/usr/lib", "fsid-1")),
),
},
{
Name: "pkg-2",
Version: "2.0.2",
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/usr/lib",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.NewCoordinates("/usr/lib", "fsid-1")),
),
},
{
Name: "pkg-3",
Version: "3.0.3",
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/etc",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.NewCoordinates("/etc", "fsid-1")),
),
},
} {
syft/format/internal/cyclonedxutil/helpers/component_test.go
+1 -1
View File
@@ -29,7 +29,7 @@ func Test_encodeComponentProperties(t *testing.T) {
input: pkg.Package{
FoundBy: "cataloger",
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{RealPath: "test"}),
file.NewLocationFromCoordinates(file.NewCoordinates("test", "")),
),
Metadata: pkg.ApkDBEntry{
Package: "libc-utils",
syft/format/internal/spdxutil/helpers/source_info.go
+1 -1
View File
@@ -98,7 +98,7 @@ func SourceInfo(p pkg.Package) string {
}
var paths []string
for _, l := range p.Locations.ToSlice() {
paths = append(paths, l.RealPath)
paths = append(paths, l.FullNestedPath())
}
return answer + ": " + strings.Join(paths, ", ")
syft/format/internal/testutil/file_relationships.go
+1 -1
View File
@@ -20,7 +20,7 @@ func AddSampleFileRelationships(s *sbom.SBOM) {
for _, f := range files {
meta := file.Metadata{}
coords := file.Coordinates{RealPath: f}
coords := file.NewCoordinates(f, "")
s.Artifacts.FileMetadata[coords] = meta
s.Relationships = append(s.Relationships, artifact.Relationship{
syft/format/spdxjson/encoder_test.go
+1 -4
View File
@@ -123,10 +123,7 @@ func TestSPDX22JSONRequredProperties(t *testing.T) {
enc, err := NewFormatEncoderWithConfig(cfg)
require.NoError(t, err)
coords := file.Coordinates{
RealPath: "/some/file",
FileSystemID: "ac897d978b6c38749a1",
}
coords := file.NewCoordinates("/some/file", "ac897d978b6c38749a1")
p1 := pkg.Package{
Name: "files-analyzed-true",
syft/format/syftjson/decoder_test.go
+2 -8
View File
@@ -137,10 +137,7 @@ func Test_encodeDecodeFileMetadata(t *testing.T) {
FoundBy: "something",
Locations: file.NewLocationSet(file.Location{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/somewhere",
FileSystemID: "id",
},
Coordinates: file.NewCoordinates("/somewhere", "id"),
},
LocationMetadata: file.LocationMetadata{
Annotations: map[string]string{
@@ -174,10 +171,7 @@ func Test_encodeDecodeFileMetadata(t *testing.T) {
}
p.SetID()
c := file.Coordinates{
RealPath: "some-file",
FileSystemID: "some-fs-id",
}
c := file.NewCoordinates("some-file", "some-fs-id")
s := sbom.SBOM{
Artifacts: sbom.Artifacts{
syft/format/syftjson/encoder_test.go
+3 -7
View File
@@ -98,7 +98,7 @@ func TestEncode_sameFileInManyArchivesIsReproducible(t *testing.T) {
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
p := pkg.Package{Name: "shared", Version: "1.0"}
for i := range 20 {
c := file.Coordinates{RealPath: "META-INF/MANIFEST.MF", ArchivePath: fmt.Sprintf("/app.war:WEB-INF/lib/%02d.jar", i)}
c := file.Coordinates{RealPath: "/app.war", FullPath: fmt.Sprintf("/app.war!/WEB-INF/lib/%02d.jar!/META-INF/MANIFEST.MF", i)}
s.Artifacts.FileMetadata[c] = file.Metadata{}
p.Locations.Add(file.NewLocationFromCoordinates(c))
}
@@ -168,9 +168,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
Name: "package-1",
Version: "1.0.1",
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a/place/a",
}),
file.NewLocationFromCoordinates(file.NewCoordinates("/a/place/a", "")),
),
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
@@ -191,9 +189,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
Name: "package-2",
Version: "2.0.1",
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b/place/b",
}),
file.NewLocationFromCoordinates(file.NewCoordinates("/b/place/b", "")),
),
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
syft/format/syftjson/model/package.go
+4 -4
View File
@@ -172,17 +172,17 @@ func unpackPkgMetadata(p *Package, unpacker packageMetadataUnpacker) error {
switch unpacker.MetadataType {
case "HackageMetadataType":
for _, l := range p.Locations {
if strings.HasSuffix(l.RealPath, ".yaml.lock") {
if strings.HasSuffix(l.FullNestedPath(), ".yaml.lock") {
ty = "haskell-hackage-stack-lock-entry"
break
} else if strings.HasSuffix(l.RealPath, ".yaml") {
} else if strings.HasSuffix(l.FullNestedPath(), ".yaml") {
ty = "haskell-hackage-stack-entry"
break
}
}
case "RpmMetadata":
for _, l := range p.Locations {
if strings.HasSuffix(l.RealPath, ".rpm") {
if strings.HasSuffix(l.FullNestedPath(), ".rpm") {
ty = "rpm-archive"
break
}
@@ -190,7 +190,7 @@ func unpackPkgMetadata(p *Package, unpacker packageMetadataUnpacker) error {
case "RustCargoPackageMetadata":
var found bool
for _, l := range p.Locations {
if strings.HasSuffix(strings.ToLower(l.RealPath), "cargo.lock") {
if strings.HasSuffix(strings.ToLower(l.FullNestedPath()), "cargo.lock") {
ty = "rust-cargo-lock-entry"
found = true
break
syft/format/syftjson/to_format_model_test.go
+113 -314
View File
@@ -352,10 +352,7 @@ func Test_toFileType(t *testing.T) {
}
func Test_toFileMetadataEntry(t *testing.T) {
coords := file.Coordinates{
RealPath: "/path",
FileSystemID: "x",
}
coords := file.NewCoordinates("/path", "x")
tests := []struct {
name string
metadata *file.Metadata
@@ -453,41 +450,17 @@ func Test_toPackageModel_layerOrdering(t *testing.T) {
Licenses: pkg.NewLicenseSet(pkg.License{
Value: "MIT",
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-3",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/lic-b",
FileSystemID: "fsid-0",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-2",
}),
file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-3")),
file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-1")),
file.NewLocationFromCoordinates(file.NewCoordinates("/lic-b", "fsid-0")),
file.NewLocationFromCoordinates(file.NewCoordinates("/lic-a", "fsid-2")),
),
}),
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-0",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
}),
file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-3")),
file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")),
file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-0")),
file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-2")),
),
},
layerOrder: []string{
@@ -505,38 +478,26 @@ func Test_toPackageModel_layerOrdering(t *testing.T) {
Locations: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/lic-b",
FileSystemID: "fsid-0", // important!
},
AccessPath: "/lic-b",
Coordinates: file.NewCoordinates("/lic-b", "fsid-0"), // important!
AccessPath: "/lic-b",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-1", // important!
},
AccessPath: "/lic-a",
Coordinates: file.NewCoordinates("/lic-a", "fsid-1"), // important!
AccessPath: "/lic-a",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-2", // important!
},
AccessPath: "/lic-a",
Coordinates: file.NewCoordinates("/lic-a", "fsid-2"), // important!
AccessPath: "/lic-a",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-3", // important!
},
AccessPath: "/lic-a",
Coordinates: file.NewCoordinates("/lic-a", "fsid-3"), // important!
AccessPath: "/lic-a",
},
},
},
@@ -545,38 +506,26 @@ func Test_toPackageModel_layerOrdering(t *testing.T) {
Locations: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-0", // important!
},
AccessPath: "/b",
Coordinates: file.NewCoordinates("/b", "fsid-0"), // important!
AccessPath: "/b",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1", // important!
},
AccessPath: "/a",
Coordinates: file.NewCoordinates("/a", "fsid-1"), // important!
AccessPath: "/a",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2", // important!
},
AccessPath: "/a",
Coordinates: file.NewCoordinates("/a", "fsid-2"), // important!
AccessPath: "/a",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3", // important!
},
AccessPath: "/a",
Coordinates: file.NewCoordinates("/a", "fsid-3"), // important!
AccessPath: "/a",
},
},
},
@@ -609,34 +558,22 @@ func Test_toLocationModel(t *testing.T) {
{
name: "nil layer order map",
locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-2",
}),
file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")),
file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-2")),
),
layers: nil, // please don't panic!
want: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
AccessPath: "/a",
Coordinates: file.NewCoordinates("/a", "fsid-1"),
AccessPath: "/a",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-2",
},
AccessPath: "/b",
Coordinates: file.NewCoordinates("/b", "fsid-2"),
AccessPath: "/b",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
@@ -645,18 +582,9 @@ func Test_toLocationModel(t *testing.T) {
{
name: "go case",
locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
}),
file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-3")),
file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")),
file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-2")),
),
layers: []string{
"fsid-1",
@@ -666,31 +594,22 @@ func Test_toLocationModel(t *testing.T) {
want: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
AccessPath: "/b",
Coordinates: file.NewCoordinates("/b", "fsid-1"),
AccessPath: "/b",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
},
AccessPath: "/c",
Coordinates: file.NewCoordinates("/c", "fsid-2"),
AccessPath: "/c",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3",
},
AccessPath: "/a",
Coordinates: file.NewCoordinates("/a", "fsid-3"),
AccessPath: "/a",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
@@ -699,18 +618,9 @@ func Test_toLocationModel(t *testing.T) {
{
name: "same layer different paths", // prove we can sort by path irrespective of layer
locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-1")),
file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-1")),
file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")),
),
layers: []string{
"fsid-1",
@@ -718,31 +628,22 @@ func Test_toLocationModel(t *testing.T) {
want: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
AccessPath: "/a",
Coordinates: file.NewCoordinates("/a", "fsid-1"),
AccessPath: "/a",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
AccessPath: "/b",
Coordinates: file.NewCoordinates("/b", "fsid-1"),
AccessPath: "/b",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-1",
},
AccessPath: "/c",
Coordinates: file.NewCoordinates("/c", "fsid-1"),
AccessPath: "/c",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
@@ -751,22 +652,10 @@ func Test_toLocationModel(t *testing.T) {
{
name: "mixed layers and paths",
locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/z",
FileSystemID: "fsid-3",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
}),
file.NewLocationFromCoordinates(file.NewCoordinates("/z", "fsid-3")),
file.NewLocationFromCoordinates(file.NewCoordinates("/a", "fsid-2")),
file.NewLocationFromCoordinates(file.NewCoordinates("/b", "fsid-1")),
file.NewLocationFromCoordinates(file.NewCoordinates("/c", "fsid-2")),
),
layers: []string{
"fsid-1",
@@ -776,41 +665,29 @@ func Test_toLocationModel(t *testing.T) {
want: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
AccessPath: "/b",
Coordinates: file.NewCoordinates("/b", "fsid-1"),
AccessPath: "/b",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
},
AccessPath: "/a",
Coordinates: file.NewCoordinates("/a", "fsid-2"),
AccessPath: "/a",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
},
AccessPath: "/c",
Coordinates: file.NewCoordinates("/c", "fsid-2"),
AccessPath: "/c",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/z",
FileSystemID: "fsid-3",
},
AccessPath: "/z",
Coordinates: file.NewCoordinates("/z", "fsid-3"),
AccessPath: "/z",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
@@ -845,35 +722,23 @@ func Test_sortFiles(t *testing.T) {
name: "nil layer order map",
files: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
ID: "file-1",
Location: file.NewCoordinates("/a", "fsid-1"),
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-2",
},
ID: "file-2",
Location: file.NewCoordinates("/b", "fsid-2"),
},
},
layers: nil,
want: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
ID: "file-1",
Location: file.NewCoordinates("/a", "fsid-1"),
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-2",
},
ID: "file-2",
Location: file.NewCoordinates("/b", "fsid-2"),
},
},
},
@@ -881,25 +746,16 @@ func Test_sortFiles(t *testing.T) {
name: "layer ordering",
files: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3",
},
ID: "file-1",
Location: file.NewCoordinates("/a", "fsid-3"),
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
ID: "file-2",
Location: file.NewCoordinates("/b", "fsid-1"),
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
},
ID: "file-3",
Location: file.NewCoordinates("/c", "fsid-2"),
},
},
layers: []string{
@@ -909,25 +765,16 @@ func Test_sortFiles(t *testing.T) {
},
want: []model.File{
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
ID: "file-2",
Location: file.NewCoordinates("/b", "fsid-1"),
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
},
ID: "file-3",
Location: file.NewCoordinates("/c", "fsid-2"),
},
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3",
},
ID: "file-1",
Location: file.NewCoordinates("/a", "fsid-3"),
},
},
},
@@ -935,25 +782,16 @@ func Test_sortFiles(t *testing.T) {
name: "same layer different paths",
files: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-1",
},
ID: "file-1",
Location: file.NewCoordinates("/c", "fsid-1"),
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
ID: "file-2",
Location: file.NewCoordinates("/a", "fsid-1"),
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
ID: "file-3",
Location: file.NewCoordinates("/b", "fsid-1"),
},
},
layers: []string{
@@ -961,25 +799,16 @@ func Test_sortFiles(t *testing.T) {
},
want: []model.File{
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
ID: "file-2",
Location: file.NewCoordinates("/a", "fsid-1"),
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
ID: "file-3",
Location: file.NewCoordinates("/b", "fsid-1"),
},
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-1",
},
ID: "file-1",
Location: file.NewCoordinates("/c", "fsid-1"),
},
},
},
@@ -987,25 +816,16 @@ func Test_sortFiles(t *testing.T) {
name: "stability test - preserve original order for equivalent items",
files: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
ID: "file-1",
Location: file.NewCoordinates("/a", "fsid-1"),
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
ID: "file-2",
Location: file.NewCoordinates("/a", "fsid-1"),
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
ID: "file-3",
Location: file.NewCoordinates("/a", "fsid-1"),
},
},
layers: []string{
@@ -1013,25 +833,16 @@ func Test_sortFiles(t *testing.T) {
},
want: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
ID: "file-1",
Location: file.NewCoordinates("/a", "fsid-1"),
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
ID: "file-2",
Location: file.NewCoordinates("/a", "fsid-1"),
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
ID: "file-3",
Location: file.NewCoordinates("/a", "fsid-1"),
},
},
},
@@ -1039,11 +850,8 @@ func Test_sortFiles(t *testing.T) {
name: "complex file metadata doesn't affect sorting",
files: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
},
ID: "file-1",
Location: file.NewCoordinates("/a", "fsid-2"),
Metadata: &model.FileMetadataEntry{
Mode: 0644,
Type: "file",
@@ -1061,11 +869,8 @@ func Test_sortFiles(t *testing.T) {
},
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
ID: "file-2",
Location: file.NewCoordinates("/b", "fsid-1"),
Metadata: &model.FileMetadataEntry{
Mode: 0755,
Type: "directory",
@@ -1082,11 +887,8 @@ func Test_sortFiles(t *testing.T) {
},
want: []model.File{
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
ID: "file-2",
Location: file.NewCoordinates("/b", "fsid-1"),
Metadata: &model.FileMetadataEntry{
Mode: 0755,
Type: "directory",
@@ -1097,11 +899,8 @@ func Test_sortFiles(t *testing.T) {
},
},
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
},
ID: "file-1",
Location: file.NewCoordinates("/a", "fsid-2"),
Metadata: &model.FileMetadataEntry{
Mode: 0644,
Type: "file",
syft/format/syftjson/to_syft_model.go
+2 -2
View File
@@ -90,11 +90,11 @@ func toSyftFiles(files []model.File) sbom.Artifacts {
ret.FileMetadata[coord] = file.Metadata{
FileInfo: stereoscopeFile.ManualInfo{
NameValue: path.Base(coord.RealPath),
NameValue: path.Base(coord.FullNestedPath()),
SizeValue: f.Metadata.Size,
ModeValue: fm,
},
Path: coord.RealPath,
Path: coord.FullNestedPath(),
LinkDestination: f.Metadata.LinkDestination,
UserID: f.Metadata.UserID,
GroupID: f.Metadata.GroupID,
syft/format/syftjson/to_syft_model_test.go
+1 -4
View File
@@ -295,10 +295,7 @@ func Test_idsHaveChanged(t *testing.T) {
}
func Test_toSyftFiles(t *testing.T) {
coord := file.Coordinates{
RealPath: "/somerwhere/place",
FileSystemID: "abc",
}
coord := file.NewCoordinates("/somerwhere/place", "abc")
tests := []struct {
name string
syft/internal/fileresolver/container_image_all_layers.go
+1 -1
View File
@@ -197,7 +197,7 @@ func (r *ContainerImageAllLayers) RelativeFileByPath(location file.Location, pat
func (r *ContainerImageAllLayers) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
entry, err := r.img.FileCatalog.Get(location.Reference())
if err != nil {
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.FullNestedPath(), err)
}
switch entry.Type {
syft/internal/fileresolver/container_image_deep_squash_test.go
+2 -8
View File
@@ -1104,15 +1104,9 @@ func TestContainerImageDeepSquash_MergeLocationStreams_FunCases(t *testing.T) {
}
func makeLocation(path string, layer int) file.Location {
return file.NewLocationFromCoordinates(file.Coordinates{
RealPath: path,
FileSystemID: fmt.Sprintf("layer-%d", layer),
})
return file.NewLocationFromCoordinates(file.NewCoordinates(path, fmt.Sprintf("layer-%d", layer)))
}
func makeVirtualLocation(path, access string, layer int) file.Location {
return file.NewVirtualLocationFromCoordinates(file.Coordinates{
RealPath: path,
FileSystemID: fmt.Sprintf("layer-%d", layer),
}, access)
return file.NewVirtualLocationFromCoordinates(file.NewCoordinates(path, fmt.Sprintf("layer-%d", layer)), access)
}
syft/internal/fileresolver/container_image_squash.go
+1 -1
View File
@@ -148,7 +148,7 @@ func (r *ContainerImageSquash) RelativeFileByPath(_ file.Location, path string)
func (r *ContainerImageSquash) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
entry, err := r.img.FileCatalog.Get(location.Reference())
if err != nil {
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.FullNestedPath(), err)
}
switch entry.Type {
syft/internal/fileresolver/excluding_file.go
+3 -3
View File
@@ -28,14 +28,14 @@ func NewExcludingDecorator(delegate file.Resolver, excludeFn excludeFn) file.Res
func (r *excluding) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
if locationMatches(&location, r.excludeFn) {
return nil, fmt.Errorf("no such location: %+v", location.RealPath)
return nil, fmt.Errorf("no such location: %+v", location.FullNestedPath())
}
return r.delegate.FileContentsByLocation(location)
}
func (r *excluding) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
if locationMatches(&location, r.excludeFn) {
return file.Metadata{}, fmt.Errorf("no such location: %+v", location.RealPath)
return file.Metadata{}, fmt.Errorf("no such location: %+v", location.FullNestedPath())
}
return r.delegate.FileMetadataByLocation(location)
}
@@ -89,7 +89,7 @@ func (r *excluding) AllLocations(ctx context.Context) <-chan file.Location {
}
func locationMatches(location *file.Location, exclusionFn excludeFn) bool {
return exclusionFn(location.RealPath) || exclusionFn(location.AccessPath)
return exclusionFn(location.LocalPath()) || exclusionFn(location.AccessPath)
}
func filterLocations(locations []file.Location, err error, exclusionFn excludeFn) ([]file.Location, error) {
syft/internal/fileresolver/unindexed_directory.go
+1 -1
View File
@@ -89,7 +89,7 @@ func (u UnindexedDirectory) FileContentsByLocation(location file.Location) (io.R
return nil, err
}
if fi.IsDir() {
return nil, fmt.Errorf("unable to get contents of directory: %s", location.RealPath)
return nil, fmt.Errorf("unable to get contents of directory: %s", location.FullNestedPath())
}
return f, nil
}
syft/linux/identify_release.go
+3 -3
View File
@@ -87,20 +87,20 @@ func IdentifyRelease(resolver file.Resolver) *Release {
func tryParseReleaseInfo(resolver file.Resolver, location file.Location, logger logger.Logger, entry parseEntry) *Release {
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
logger.WithFields("error", err, "path", location.RealPath).Trace("unable to get contents")
logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to get contents")
return nil
}
defer internal.CloseAndLogError(contentReader, location.AccessPath)
content, err := io.ReadAll(io.LimitReader(contentReader, 5*1024*1024))
if err != nil {
logger.WithFields("error", err, "path", location.RealPath).Trace("unable to read contents")
logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to read contents")
return nil
}
release, err := entry.fn(string(content))
if err != nil {
logger.WithFields("error", err, "path", location.RealPath).Trace("unable to parse contents")
logger.WithFields("error", err, "path", location.FullNestedPath()).Trace("unable to parse contents")
return nil
}
syft/linux/supplement_release.go
+3 -3
View File
@@ -35,13 +35,13 @@ func supplementDebianVersion(resolver file.Resolver, release *Release) {
func readDebianVersionFile(resolver file.Resolver, location file.Location) string {
rdr, err := resolver.FileContentsByLocation(location)
if err != nil {
log.Debugf("error getting contents for %s: %v", location.RealPath, err)
log.Debugf("error getting contents for %s: %v", location.FullNestedPath(), err)
return ""
}
defer internal.CloseAndLogError(rdr, location.RealPath)
defer internal.CloseAndLogError(rdr, location.FullNestedPath())
contents, err := io.ReadAll(io.LimitReader(rdr, 5*1024*1024))
if err != nil {
log.Debugf("error reading %s: %v", location.RealPath, err)
log.Debugf("error reading %s: %v", location.FullNestedPath(), err)
return ""
}
version := strings.TrimSpace(string(contents))
syft/pkg/cataloger/ai/identity_oci.go
+4 -4
View File
@@ -32,8 +32,8 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver,
// seen wins below, so sort for a stable result across runs. For OCI
// artifacts every RealPath is "/", so in practice this is digest order.
sort.Slice(modelFileLocs, func(i, j int) bool {
if modelFileLocs[i].RealPath != modelFileLocs[j].RealPath {
return modelFileLocs[i].RealPath < modelFileLocs[j].RealPath
if modelFileLocs[i].FullNestedPath() != modelFileLocs[j].FullNestedPath() {
return modelFileLocs[i].FullNestedPath() < modelFileLocs[j].FullNestedPath()
}
return modelFileLocs[i].FileSystemID < modelFileLocs[j].FileSystemID
})
@@ -128,7 +128,7 @@ func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfCo
if err != nil {
return nil, nil
}
defer internal.CloseAndLogError(rc, loc.RealPath)
defer internal.CloseAndLogError(rc, loc.FullNestedPath())
buf, err := readBounded(rc, maxModelFileLayerSize)
if err != nil {
@@ -174,7 +174,7 @@ func readLicenseSPDXIDFromFrontmatter(resolver file.Resolver, loc file.Location)
if err != nil {
return ""
}
defer internal.CloseAndLogError(rc, loc.RealPath)
defer internal.CloseAndLogError(rc, loc.FullNestedPath())
buf, err := readPrefix(rc, maxLicensePrefixSize)
if err != nil {
syft/pkg/cataloger/ai/parse_gguf_model.go
+1 -1
View File
@@ -23,7 +23,7 @@ import (
// parseGGUFModel parses a GGUF model file and returns the discovered package.
// This implementation only reads the header portion of the file, not the entire model.
func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
defer internal.CloseAndLogError(reader, reader.Path())
defer internal.CloseAndLogError(reader, reader.FullNestedPath())
td := tmpdir.FromContext(ctx)
if td == nil {
syft/pkg/cataloger/ai/parse_safetensors_model.go
+1 -1
View File
@@ -18,7 +18,7 @@ import (
// resolution, sibling enrichment, and cross-shard rollup are all handled by
// safeTensorsMergeProcessor.
func parseSafeTensorsFile(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
defer internal.CloseAndLogError(reader, reader.Path())
defer internal.CloseAndLogError(reader, reader.FullNestedPath())
header, err := readSafeTensorsHeader(&io.LimitedReader{R: reader, N: maxSafeTensorsHeaderSize + 8})
if err != nil {
syft/pkg/cataloger/ai/parse_safetensors_oci.go
+2 -2
View File
@@ -47,7 +47,7 @@ type dockerAIModelConfig struct {
// parseSafeTensorsOCIConfig decodes the Docker AI model-config blob
func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
defer internal.CloseAndLogError(reader, reader.Path())
defer internal.CloseAndLogError(reader, reader.FullNestedPath())
body, err := readBounded(reader, maxModelConfigBlobSize)
if err != nil {
@@ -86,7 +86,7 @@ func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.En
// parseSafeTensorsOCILayer decodes the JSON header of a SafeTensors weight
// layer fetched from an OCI model artifact
func parseSafeTensorsOCILayer(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
defer internal.CloseAndLogError(reader, reader.Path())
defer internal.CloseAndLogError(reader, reader.FullNestedPath())
header, err := readSafeTensorsHeader(&io.LimitedReader{R: reader, N: maxSafeTensorsHeaderSize + 8})
if err != nil {
syft/pkg/cataloger/ai/processor.go
+2 -2
View File
@@ -56,7 +56,7 @@ func partitionSafeTensorsPackages(pkgs []pkg.Package) (safeTensors, other []pkg.
// single scan is one source, so the first package is representative of the rest.
func fromOCIArtifact(pkgs []pkg.Package) bool {
loc := primaryEvidenceLocation(pkgs[0])
return loc != nil && loc.RealPath == "/"
return loc != nil && loc.LocalPath() == "/"
}
// mergeOCIModel treats the whole OCI artifact as a single model: every layer
@@ -120,7 +120,7 @@ func groupByParentDir(pkgs []pkg.Package) map[string][]pkg.Package {
if loc == nil {
continue
}
dir := path.Dir(loc.RealPath)
dir := path.Dir(loc.LocalPath())
out[dir] = append(out[dir], p)
}
return out
syft/pkg/cataloger/alpine/parse_apk_db.go
+3 -3
View File
@@ -126,7 +126,7 @@ func parseApkDB(ctx context.Context, resolver file.Resolver, env *generic.Enviro
// This should get fixed with https://gitlab.alpinelinux.org/alpine/apk-tools/-/issues/10875
if r == nil {
// find the repositories file from the relative directory of the DB file
releases := findReleases(resolver, reader.RealPath)
releases := findReleases(resolver, reader.LocalPath())
if len(releases) > 0 {
r = &releases[0]
@@ -163,7 +163,7 @@ func findReleases(resolver file.Resolver, dbPath string) []linux.Release {
log.Tracef("unable to fetch contents for APK repositories file %q: %+v", reposLocation, err)
return nil
}
defer internal.CloseAndLogError(reposReader, location.RealPath)
defer internal.CloseAndLogError(reposReader, location.FullNestedPath())
return parseReleasesFromAPKRepository(file.LocationReadCloser{
Location: location,
@@ -176,7 +176,7 @@ func parseReleasesFromAPKRepository(reader file.LocationReadCloser) []linux.Rele
reposB, err := io.ReadAll(reader) //nolint:gocritic // regex matching requires full buffer
if err != nil {
log.Tracef("unable to read APK repositories file %q: %+v", reader.RealPath, err)
log.Tracef("unable to read APK repositories file %q: %+v", reader.FullNestedPath(), err)
return nil
}
syft/pkg/cataloger/arch/parse_alpm_db.go
+3 -3
View File
@@ -55,7 +55,7 @@ func parseAlpmDB(ctx context.Context, resolver file.Resolver, env *generic.Envir
return nil, nil, nil
}
base := path.Dir(reader.RealPath)
base := path.Dir(reader.LocalPath())
var locs []file.Location
@@ -102,7 +102,7 @@ func fetchPkgFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord, f
if err != nil {
return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err))
}
defer internal.CloseAndLogError(reader, loc.RealPath)
defer internal.CloseAndLogError(reader, loc.FullNestedPath())
pkgFiles, err := parseMtree(reader)
if err != nil {
@@ -126,7 +126,7 @@ func fetchBackupFiles(base string, resolver file.Resolver) ([]pkg.AlpmFileRecord
if err != nil {
return []pkg.AlpmFileRecord{}, loc, unknown.New(loc, fmt.Errorf("failed to get contents: %w", err))
}
defer internal.CloseAndLogError(reader, loc.RealPath)
defer internal.CloseAndLogError(reader, loc.FullNestedPath())
filesMetadata, err := parseAlpmDBEntry(reader)
if err != nil {
syft/pkg/cataloger/binary/classifier_cataloger_test.go
+1 -3
View File
@@ -3410,9 +3410,7 @@ func match(classifier string, paths ...string) pkg.ClassifierMatch {
return pkg.ClassifierMatch{
Classifier: classifier,
Location: file.NewVirtualLocationFromCoordinates(
file.Coordinates{
RealPath: realPath,
},
file.NewCoordinates(realPath, ""),
virtualPath,
),
}
syft/pkg/cataloger/binary/elf_package_cataloger.go
+4 -4
View File
@@ -99,7 +99,7 @@ func (c *elfPackageCataloger) Catalog(ctx context.Context, resolver file.Resolve
func parseElfPackageNotes(resolver file.Resolver, location file.Location, c *elfPackageCataloger) (*elfBinaryPackageNotes, elfPackageKey, error) {
reader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, elfPackageKey{}, fmt.Errorf("unable to get binary contents %q: %w", location.Path(), err)
return nil, elfPackageKey{}, fmt.Errorf("unable to get binary contents %q: %w", location.FullNestedPath(), err)
}
defer internal.CloseAndLogError(reader, location.AccessPath)
@@ -109,7 +109,7 @@ func parseElfPackageNotes(resolver file.Resolver, location file.Location, c *elf
})
if err != nil {
log.WithFields("file", location.Path(), "error", err).Trace("unable to parse ELF notes")
log.WithFields("file", location.FullNestedPath(), "error", err).Trace("unable to parse ELF notes")
return nil, elfPackageKey{}, err
}
@@ -148,7 +148,7 @@ func getELFNotes(r file.LocationReadCloser) (*elfBinaryPackageNotes, error) {
f, err := elfutil.NewFile(unionReader)
if f == nil || err != nil {
log.WithFields("file", r.Location.Path(), "error", err).Trace("unable to parse binary as ELF")
log.WithFields("file", r.Location.FullNestedPath(), "error", err).Trace("unable to parse binary as ELF")
return nil, nil
}
@@ -182,7 +182,7 @@ func getELFNotes(r file.LocationReadCloser) (*elfBinaryPackageNotes, error) {
if metadata, err = unmarshalELFPackageNotesPayload(newPayload); err == nil {
return metadata, nil
}
log.WithFields("file", r.Location.Path(), "error", err).Trace("unable to unmarshal ELF package notes as JSON")
log.WithFields("file", r.Location.FullNestedPath(), "error", err).Trace("unable to unmarshal ELF package notes as JSON")
}
}
syft/pkg/cataloger/binary/pe_package.go
+1 -1
View File
@@ -22,7 +22,7 @@ func newPEPackage(versionResources map[string]string, f file.Location) pkg.Packa
if name == "" {
// it's possible that the version resources are empty, so we fall back to the file name
name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.RealPath), ".exe"), ".dll")
name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.LocalPath()), ".exe"), ".dll")
}
p := pkg.Package{
syft/pkg/cataloger/binary/pe_package_cataloger.go
+1 -1
View File
@@ -22,7 +22,7 @@ func NewPEPackageCataloger() pkg.Cataloger {
func parsePE(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
f, err := pe.Read(reader)
if err != nil {
return nil, nil, fmt.Errorf("failed to parse PE file %q: %w", reader.RealPath, err)
return nil, nil, fmt.Errorf("failed to parse PE file %q: %w", reader.FullNestedPath(), err)
}
if f == nil {
syft/pkg/cataloger/bitnami/cataloger.go
+6 -6
View File
@@ -36,13 +36,13 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
}
if s == nil {
log.WithFields("path", reader.RealPath).Trace("file is not an SBOM")
log.WithFields("path", reader.FullNestedPath()).Trace("file is not an SBOM")
return nil, nil, nil
}
// Bitnami exclusively uses SPDX JSON SBOMs
if sFormat != "spdx-json" {
log.WithFields("path", reader.RealPath).Trace("file is not an SPDX JSON SBOM")
log.WithFields("path", reader.FullNestedPath()).Trace("file is not an SPDX JSON SBOM")
return nil, nil, nil
}
@@ -73,7 +73,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
// Bitnami packages reported in a SPDX file are shipped under the same directory
// as the SPDX file itself.
metadata.Path = path.Dir(reader.RealPath)
metadata.Path = path.Dir(reader.LocalPath())
if p.ID() != mainPkgID {
metadata.Files = packageFiles(s.Relationships, p, metadata.Path)
secondaryPkgsFiles = append(secondaryPkgsFiles, metadata.Files...)
@@ -89,12 +89,12 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
}
// Resolve all files owned by the main package in the SBOM and update the metadata
if mainPkgFiles, err := mainPkgFiles(resolver, reader.RealPath, secondaryPkgsFiles); err == nil {
if mainPkgFiles, err := mainPkgFiles(resolver, reader.LocalPath(), secondaryPkgsFiles); err == nil {
for i, p := range pkgs {
if p.ID() == mainPkgID {
metadata, ok := p.Metadata.(*pkg.BitnamiSBOMEntry)
if !ok {
log.WithFields("spdx-filepath", reader.RealPath).Trace("main package in SBOM does not have Bitnami metadata")
log.WithFields("spdx-filepath", reader.FullNestedPath()).Trace("main package in SBOM does not have Bitnami metadata")
continue
}
@@ -103,7 +103,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
}
}
} else {
log.WithFields("spdx-filepath", reader.RealPath, "error", err).Trace("unable to resolve owned files for main package in SBOM")
log.WithFields("spdx-filepath", reader.FullNestedPath(), "error", err).Trace("unable to resolve owned files for main package in SBOM")
}
return pkgs, filterRelationships(s.Relationships, pkgs), nil
syft/pkg/cataloger/bitnami/package.go
+1 -1
View File
@@ -61,7 +61,7 @@ func packageFiles(relationships []artifact.Relationship, p pkg.Package, baseDire
// note: the file.Location is from the SBOM, and all files within the Bitnami SBOM by convention
// are relative to the /opt/bitnami/PRODUCT directory, so we need to prepend the base directory
// so that it's relative to the path found within the container image.
result = append(result, path.Join(baseDirectory, value.RealPath))
result = append(result, path.Join(baseDirectory, value.LocalPath()))
}
}
}
syft/pkg/cataloger/bitnami/parse_components.go
+1 -1
View File
@@ -67,7 +67,7 @@ func parseComponentsJSON(_ context.Context, _ file.Resolver, _ *generic.Environm
Revision: revision,
Architecture: entry.Arch,
Distro: entry.Distro,
Path: path.Join(path.Dir(reader.RealPath), name),
Path: path.Join(path.Dir(reader.LocalPath()), name),
}
p := pkg.Package{
syft/pkg/cataloger/conda/cataloger.go
+1 -1
View File
@@ -24,7 +24,7 @@ func parseCondaMetaJSON(ctx context.Context, _ file.Resolver, _ *generic.Environ
dec := json.NewDecoder(reader)
var meta pkg.CondaMetaPackage
if err := dec.Decode(&meta); err != nil {
return nil, nil, fmt.Errorf("failed to parse conda-meta package file at %s: %w", reader.RealPath, err)
return nil, nil, fmt.Errorf("failed to parse conda-meta package file at %s: %w", reader.FullNestedPath(), err)
}
p := pkg.Package{
syft/pkg/cataloger/cpp/parse_conaninfo.go
+1 -4
View File
@@ -81,10 +81,7 @@ func parseFullRequiresLine(line string, reader file.LocationReadCloser, pkgs *[]
// - recipe revision (recipe_hash)
func parseConaninfo(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
// First set the base package info by checking the relative path
fullFilePath := string(reader.Location.LocationData.Reference().RealPath)
if len(fullFilePath) == 0 {
fullFilePath = reader.RealPath
}
fullFilePath := reader.LocalPath()
mainMetadata, err := parseConanMetadataFromFilePath(fullFilePath)
if err != nil {
syft/pkg/cataloger/cpp/parse_vcpkgmanifest.go
+2 -2
View File
@@ -132,7 +132,7 @@ func findManAtLoc(loc file.Location, resolver file.Resolver) (*vcpkg.Vcpkg, erro
if err != nil {
return nil, err
}
defer internal.CloseAndLogError(manCont, loc.RealPath)
defer internal.CloseAndLogError(manCont, loc.FullNestedPath())
manBytes, err := io.ReadAll(io.LimitReader(manCont, maxVcpkgManifestSize))
if err != nil {
return nil, err
@@ -166,7 +166,7 @@ func findVcpkgConfig(resolver file.Resolver) (*vcpkg.Config, error) {
if err != nil {
return nil, err
}
defer internal.CloseAndLogError(cfgCont, locs[0].RealPath)
defer internal.CloseAndLogError(cfgCont, locs[0].FullNestedPath())
cfgBytes, err := io.ReadAll(io.LimitReader(cfgCont, maxVcpkgManifestSize))
if err != nil {
return nil, err
syft/pkg/cataloger/debian/package.go
+4 -4
View File
@@ -250,7 +250,7 @@ func getAdditionalFileListing(resolver file.Resolver, dbLocation file.Location,
// attach the file list
records, err := parseDpkgMD5Info(md5Reader)
if err != nil {
log.Debugf("dpkg md5sums file %q: %v", md5Location.RealPath, err)
log.Debugf("dpkg md5sums file %q: %v", md5Location.FullNestedPath(), err)
}
files = append(files, records...)
@@ -265,7 +265,7 @@ func getAdditionalFileListing(resolver file.Resolver, dbLocation file.Location,
// attach the file list
records, err := parseDpkgConffileInfo(conffilesReader)
if err != nil {
log.Debugf("dpkg conffiles file %q: %v", conffilesLocation.RealPath, err)
log.Debugf("dpkg conffiles file %q: %v", conffilesLocation.FullNestedPath(), err)
}
files = append(files, records...)
@@ -288,7 +288,7 @@ func fetchMd5Contents(resolver file.Resolver, dbLocation file.Location, m pkg.Dp
// and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed
// package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in
// the same directory
searchPath := filepath.Dir(dbLocation.RealPath)
searchPath := filepath.Dir(dbLocation.LocalPath())
if !strings.HasSuffix(searchPath, "status.d") {
searchPath = path.Join(searchPath, "info")
@@ -327,7 +327,7 @@ func fetchConffileContents(resolver file.Resolver, dbLocation file.Location, m p
return nil, nil
}
parentPath := filepath.Dir(dbLocation.RealPath)
parentPath := filepath.Dir(dbLocation.LocalPath())
// look for /var/lib/dpkg/info/NAME:ARCH.conffiles
name := md5Key(m)
syft/pkg/cataloger/debian/parse_dpkg_db.go
+2 -2
View File
@@ -37,7 +37,7 @@ var (
func parseDpkgDB(ctx context.Context, resolver file.Resolver, env *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
metadata, err := parseDpkgStatus(reader)
if err != nil {
return nil, nil, fmt.Errorf("unable to catalog dpkg DB=%q: %w", reader.RealPath, err)
return nil, nil, fmt.Errorf("unable to catalog dpkg DB=%q: %w", reader.FullNestedPath(), err)
}
dbLoc := reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)
@@ -61,7 +61,7 @@ func findDpkgInfoFiles(name string, resolver file.Resolver, dbLocation file.Loca
// and the md5sum information is under /var/lib/dpkg/info/; however, for distroless the installed
// package info is across multiple files under /var/lib/dpkg/status.d/ and the md5sums are contained in
// the same directory
searchPath := path.Dir(dbLocation.RealPath)
searchPath := path.Dir(dbLocation.LocalPath())
if !strings.HasSuffix(searchPath, "status.d") {
searchPath = path.Join(searchPath, "info")
syft/pkg/cataloger/dotnet/deps_binary_cataloger.go
+10 -10
View File
@@ -150,7 +150,7 @@ var runtimeDLLPathPattern = regexp.MustCompile(`/Microsoft\.NETCore\.App/(?P<ver
func isRuntimePackageLocation(loc file.Location) (string, bool) {
// we should look at the realpath to see if it is a "**/Microsoft.NETCore.App/\d+.\d+.\d+/*.dll"
// and if so treat it as a runtime package
if match := runtimeDLLPathPattern.FindStringSubmatch(loc.RealPath); match != nil {
if match := runtimeDLLPathPattern.FindStringSubmatch(loc.FullNestedPath()); match != nil {
versionIndex := runtimeDLLPathPattern.SubexpIndex("version")
if versionIndex != -1 {
version := match[versionIndex]
@@ -177,12 +177,12 @@ func partitionPEs(depJsons []logicalDepsJSON, peFiles []logicalPE) ([]logicalDep
// sort deps.json paths from longest to shortest. This is so we are processing the most specific match first.
sort.Slice(depJsons, func(i, j int) bool {
return depJsons[i].Location.RealPath > depJsons[j].Location.RealPath
return depJsons[i].Location.FullNestedPath() > depJsons[j].Location.FullNestedPath()
})
// we should be processing PE files in a stable order
sort.Slice(peFiles, func(i, j int) bool {
return peFiles[i].Location.RealPath > peFiles[j].Location.RealPath
return peFiles[i].Location.FullNestedPath() > peFiles[j].Location.FullNestedPath()
})
peFilesByPath := make(map[file.Coordinates][]logicalPE)
@@ -191,7 +191,7 @@ func partitionPEs(depJsons []logicalDepsJSON, peFiles []logicalPE) ([]logicalDep
var found bool
for i := range depJsons {
dep := &depJsons[i]
if isParentOf(dep.Location.RealPath, pe.Location.RealPath) && attachAssociatedExecutables(dep, pe) {
if isParentOf(dep.Location.LocalPath(), pe.Location.LocalPath()) && attachAssociatedExecutables(dep, pe) {
peFilesByPath[dep.Location.Coordinates] = append(peFilesByPath[dep.Location.Coordinates], pe)
found = true
// note: we cannot break from the dep JSON search since the same binary could be associated with multiple packages
@@ -222,8 +222,8 @@ func partitionPEs(depJsons []logicalDepsJSON, peFiles []logicalPE) ([]logicalDep
// attachAssociatedExecutables looks for PE files matching runtime or resource entries
// and attaches them to the appropriate package.
func attachAssociatedExecutables(dep *logicalDepsJSON, pe logicalPE) bool {
appDir := path.Dir(dep.Location.RealPath)
relativeDllPath := strings.TrimPrefix(strings.TrimPrefix(pe.Location.RealPath, appDir), "/")
appDir := path.Dir(dep.Location.LocalPath())
relativeDllPath := strings.TrimPrefix(strings.TrimPrefix(pe.Location.LocalPath(), appDir), "/")
var found bool
for key, p := range dep.PackagesByNameVersion {
@@ -471,7 +471,7 @@ func readDepsJSON(resolver file.Resolver, loc file.Location) (*depsJSON, error)
if err != nil {
return nil, unknown.New(loc, fmt.Errorf("unable to read deps.json file: %w", err))
}
defer internal.CloseAndLogError(reader, loc.RealPath)
defer internal.CloseAndLogError(reader, loc.FullNestedPath())
dj, err := newDepsJSON(file.NewLocationReadCloser(loc, reader))
if err != nil {
@@ -479,7 +479,7 @@ func readDepsJSON(resolver file.Resolver, loc file.Location) (*depsJSON, error)
}
if dj == nil {
return nil, unknown.New(loc, fmt.Errorf("expected to find packages in deps.json but did not: %q", loc.RealPath))
return nil, unknown.New(loc, fmt.Errorf("expected to find packages in deps.json but did not: %q", loc.FullNestedPath()))
}
return dj, nil
@@ -520,7 +520,7 @@ func readPEFile(resolver file.Resolver, loc file.Location) (*logicalPE, error) {
if err != nil {
return nil, unknown.New(loc, fmt.Errorf("unable to read PE file: %w", err))
}
defer internal.CloseAndLogError(reader, loc.RealPath)
defer internal.CloseAndLogError(reader, loc.FullNestedPath())
ldpe, err := readLogicalPE(file.NewLocationReadCloser(loc, reader))
if err != nil {
@@ -566,7 +566,7 @@ func readELFBundledDepsJSON(resolver file.Resolver, loc file.Location) (*logical
if err != nil {
return nil, err
}
defer internal.CloseAndLogError(reader, loc.RealPath)
defer internal.CloseAndLogError(reader, loc.FullNestedPath())
header := make([]byte, 4)
if _, err := io.ReadFull(reader, header); err != nil {
syft/pkg/cataloger/dotnet/deps_json.go
+1 -1
View File
@@ -161,7 +161,7 @@ type logicalDepsJSON struct {
}
func (l logicalDepsJSON) RootPackage() (logicalDepsJSONPackage, bool) {
rootName := getDepsJSONFilePrefix(l.Location.RealPath)
rootName := getDepsJSONFilePrefix(l.Location.FullNestedPath())
if rootName == "" {
return logicalDepsJSONPackage{}, false
}
syft/pkg/cataloger/dotnet/libman_json.go
+2 -2
View File
@@ -87,14 +87,14 @@ func newLibmanJSON(reader file.LocationReadCloser) (*libmanJSON, error) {
}
func findLibmanJSON(resolver file.Resolver, depsJSON file.Location) (*libmanJSON, error) {
parent := path.Dir(depsJSON.RealPath)
parent := path.Dir(depsJSON.LocalPath())
loc := resolver.RelativeFileByPath(depsJSON, path.Join(parent, "libman.json"))
if loc == nil {
return nil, nil
}
reader, err := resolver.FileContentsByLocation(*loc)
defer internal.CloseAndLogError(reader, loc.RealPath)
defer internal.CloseAndLogError(reader, loc.FullNestedPath())
if err != nil {
return nil, err
}
syft/pkg/cataloger/dotnet/package.go
+1 -1
View File
@@ -265,7 +265,7 @@ func newDotnetBinaryPackage(versionResources map[string]string, f file.Location)
if name == "" {
// older .NET runtime dlls may not have any version resources
name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.RealPath), ".exe"), ".dll")
name = strings.TrimSuffix(strings.TrimSuffix(path.Base(f.FullNestedPath()), ".exe"), ".dll")
}
ver := findVersionFromVersionResources(versionResources)
syft/pkg/cataloger/elixir/parse_mix_lock.go
+1 -1
View File
@@ -83,7 +83,7 @@ func parseMixLock(_ context.Context, _ file.Resolver, _ *generic.Environment, re
}
if name == "" {
log.WithFields("path", reader.RealPath).Debug("skipping empty package name from mix.lock file")
log.WithFields("path", reader.FullNestedPath()).Debug("skipping empty package name from mix.lock file")
errs = unknown.Appendf(errs, reader, "skipping empty package name from mix.lock file, for line: %d: %s", lineNum, line)
continue
}
syft/pkg/cataloger/generic/cataloger.go
+4 -4
View File
@@ -203,7 +203,7 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.
errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(c.selectFiles(resolver)), func(req request) (result, error) {
location, parser := req.Location, req.Parser
log.WithFields("path", location.RealPath).Trace("parsing file contents")
log.WithFields("path", location.FullNestedPath()).Trace("parsing file contents")
discoveredPackages, discoveredRelationships, err := invokeParser(ctx, resolver, location, lgr, parser, &env)
if err != nil {
@@ -243,7 +243,7 @@ func (c *Cataloger) processSafely(ctx context.Context, p processExecutor, resolv
func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) (pkgs []pkg.Package, rels []artifact.Relationship, err error) {
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
logger.WithFields("location", location.RealPath, "error", err).Debug("unable to fetch contents")
logger.WithFields("location", location.FullNestedPath(), "error", err).Debug("unable to fetch contents")
return nil, nil, err
}
defer internal.CloseAndLogError(contentReader, location.AccessPath)
@@ -251,7 +251,7 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc
defer func() {
if r := recover(); r != nil {
// a malformed file must not take down the whole SBOM; the caller records this as an unknown
logger.WithFields("location", location.RealPath, "panic", r).Debug("recovered from panic while parsing file")
logger.WithFields("location", location.FullNestedPath(), "panic", r).Debug("recovered from panic while parsing file")
logger.Tracef("parser panic stack:\n%s", debug.Stack())
pkgs, rels, err = nil, nil, fmt.Errorf("recovered from panic while parsing file: %v", r)
}
@@ -260,7 +260,7 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc
discoveredPackages, discoveredRelationships, err := parser(ctx, resolver, env, file.NewLocationReadCloser(location, contentReader))
if err != nil {
// these errors are propagated up, and are likely to be coordinate errors
logger.WithFields("location", location.RealPath, "error", err).Trace("cataloger returned errors")
logger.WithFields("location", location.FullNestedPath(), "error", err).Trace("cataloger returned errors")
}
return discoveredPackages, discoveredRelationships, err
syft/pkg/cataloger/generic/cataloger_test.go
+4 -4
View File
@@ -19,7 +19,7 @@ import (
func Test_Cataloger(t *testing.T) {
allParsedPaths := make(map[string]bool)
parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
allParsedPaths[reader.Path()] = true
allParsedPaths[reader.FullNestedPath()] = true
contents, err := io.ReadAll(reader)
require.NoError(t, err)
@@ -219,7 +219,7 @@ func TestParserPanicBecomesUnknown(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
parser := func(_ context.Context, _ file.Resolver, _ *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
if reader.Path() == "testdata/another-path.txt" {
if reader.FullNestedPath() == "testdata/another-path.txt" {
tt.doPanic()
}
return []pkg.Package{{Name: "good", Locations: file.NewLocationSet(reader.Location)}}, nil, nil
@@ -308,7 +308,7 @@ func TestProcessorPanicKeepsResults(t *testing.T) {
func Test_CatalogerWithParserByMediaType(t *testing.T) {
allParsedPaths := make(map[string]bool)
parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
allParsedPaths[reader.Path()] = true
allParsedPaths[reader.FullNestedPath()] = true
contents, err := io.ReadAll(reader)
require.NoError(t, err)
@@ -358,7 +358,7 @@ func Test_genericCatalogerReturnsErrors(t *testing.T) {
genericErrorReturning := NewCataloger("error returning").WithParserByGlobs(func(ctx context.Context, resolver file.Resolver, environment *Environment, locationReader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
return []pkg.Package{
{
Name: "some-package-" + locationReader.Path(),
Name: "some-package-" + locationReader.FullNestedPath(),
},
}, nil, unknown.Newf(locationReader, "unable to read")
}, "**/*")
syft/pkg/cataloger/gentoo/license.go
+4 -4
View File
@@ -71,7 +71,7 @@ func extractLicenses(resolver file.Resolver, closestLocation *file.Location, rea
if err := scanner.Err(); err != nil {
fields := []any{"error", err}
if closestLocation != nil {
fields = append(fields, "path", closestLocation.RealPath)
fields = append(fields, "path", closestLocation.FullNestedPath())
}
log.WithFields(fields...).Debug("failed to fully read portage LICENSE")
}
@@ -115,9 +115,9 @@ func readLicenseGroups(resolver file.Resolver, closestLocation *file.Location) m
}
groupReader, err := resolver.FileContentsByLocation(*groupLocation)
defer internal.CloseAndLogError(groupReader, groupLocation.RealPath)
defer internal.CloseAndLogError(groupReader, groupLocation.FullNestedPath())
if err != nil {
log.WithFields("path", groupLocation.RealPath, "error", err).Debug("failed to fetch portage LICENSE")
log.WithFields("path", groupLocation.FullNestedPath(), "error", err).Debug("failed to fetch portage LICENSE")
return nil
}
@@ -127,7 +127,7 @@ func readLicenseGroups(resolver file.Resolver, closestLocation *file.Location) m
licenseGroups, err = parseLicenseGroups(groupReader)
if err != nil {
log.WithFields("path", groupLocation.RealPath, "error", err).Debug("failed to parse portage LICENSE")
log.WithFields("path", groupLocation.FullNestedPath(), "error", err).Debug("failed to parse portage LICENSE")
}
return licenseGroups

Some files were not shown because too many files have changed in this diff Show More