fix: file selection within archives, spdx / cyclonedx file representation

Signed-off-by: Keith Zantow <kzantow@gmail.com>
This commit is contained in:
Keith Zantow
2026-10-27 14:52:18 -04:00
parent 6d34eba977
commit 851dc327c5
27 changed files with 865 additions and 340 deletions
+1
View File
@@ -146,6 +146,7 @@ func (cfg Catalog) ToComplianceConfig() cataloging.ComplianceConfig {
func (cfg Catalog) ToUnknownsConfig() cataloging.UnknownsConfig {
return cataloging.UnknownsConfig{
RemoveWhenPackagesDefined: cfg.Unknowns.RemoveWhenPackagesDefined,
IncludeExecutablesWithoutPackages: cfg.Unknowns.ExecutablesWithoutPackages,
IncludeUnexpandedArchives: cfg.Unknowns.UnexpandedArchives,
}
+5 -8
View File
@@ -273,8 +273,8 @@ func (r *Resolver) fullPath(rooted string) string {
}
// rootedPath is the tree path a query names, which is either form a path takes: a full path is trimmed to
// the entry path within this archive, and anything else (a LocalPath) is taken as archive-relative
// already. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root)
// the entry path within this archive, and anything else (a LocalPath, with or without its leading slash)
// is taken as archive-relative already, since the tree looks a relative path up from its root. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root)
// names the root. A full path into another archive in the scan names nothing here and reports false.
func (r *Resolver) rootedPath(userPath string) (string, bool) {
if rest, ok := strings.CutPrefix(userPath, r.archiveCoordinates.FullNestedPath()); ok {
@@ -289,9 +289,6 @@ func (r *Resolver) rootedPath(userPath string) (string, bool) {
if r.outside(userPath) {
return "", false
}
if !strings.HasPrefix(userPath, "/") {
userPath = "/" + userPath
}
return path.Clean(userPath), true
}
@@ -420,10 +417,10 @@ func (r *Resolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
}
// searchByGlob is the index search, except that stereoscope answers "<dir>/*" by looking the directory
// up by name and the root has none, so "/*" walks the tree instead.
// up by name and the root has none, so "/*" (or "*") walks the tree instead.
func (r *Resolver) searchByGlob(pattern string) ([]stereoscopeFile.Resolution, error) {
if pattern == "/*" {
return r.tree.FilesByGlob(pattern, filetree.FollowBasenameLinks)
if strings.TrimPrefix(pattern, "/") == "*" {
return r.tree.FilesByGlob("/*", filetree.FollowBasenameLinks)
}
return r.search.SearchByGlob(pattern, filetree.FollowBasenameLinks)
}
+15 -4
View File
@@ -38,11 +38,8 @@ func FullPathIn(containerPath, entryPath string) string {
if containerPath == "" {
return entryPath
}
if !strings.HasPrefix(entryPath, "/") {
entryPath = "/" + entryPath
}
// cleaning an already clean path allocates nothing
return containerPath + file.ArchivePathSeparator + path.Clean(entryPath)[1:]
return containerPath + file.ArchivePathSeparator + strings.TrimPrefix(path.Clean(entryPath), "/")
}
// OwnerContainers are the containers a package owns files within: the container of each of its
@@ -100,6 +97,20 @@ func ContainerPath(c file.Coordinates) string {
return c.RealPath
}
// ContainingArchive returns the coordinates of the archive directly holding the file at c, reporting false
// for a file on the scanned filesystem.
func ContainingArchive(c file.Coordinates) (file.Coordinates, bool) {
containerPath := ContainerPath(c)
if containerPath == "" {
return file.Coordinates{}, false
}
container := file.Coordinates{RealPath: c.RealPath, FileSystemID: c.FileSystemID}
if containerPath != c.RealPath {
container.FullPath = containerPath
}
return container, true
}
// archiveEntries is the "!/"-delimited chain of entry paths after RealPath, reporting false for a file
// not inside an archive. RealPath is split off first, so a "!/" in it is not taken for a boundary.
func archiveEntries(c file.Coordinates) (string, bool) {
+9 -2
View File
@@ -141,7 +141,7 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) {
scanDir := t.TempDir()
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
s := runNesting(t, scanDir, 3, defaultLimits(), markerTask())
s := withArchiveContains(t, runNesting(t, scanDir, 3, defaultLimits(), markerTask()))
// each archive's coordinates as seen in its parent; the outermost sits in the scanned
// directory, so it carries no filesystem id of its own
@@ -178,7 +178,7 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) {
"expected a CONTAINS edge from %s to the leaf package", nested.paths[len(nested.paths)-1])
// with file cataloging on, files inside an archive get an edge from it too
s = runNesting(t, scanDir, 3, defaultLimits(), markerTask(), fileMetadataTask())
s = withArchiveContains(t, runNesting(t, scanDir, 3, defaultLimits(), markerTask(), fileMetadataTask()))
var leafEdge bool
for _, rel := range s.Relationships {
from, _ := rel.From.(file.Coordinates)
@@ -379,6 +379,13 @@ func runNesting(t *testing.T, scanDir string, depth int, bounds archive.Limits,
}
}
// withArchiveContains runs the task that follows every scan with archive cataloging on.
func withArchiveContains(t *testing.T, s *sbom.SBOM) *sbom.SBOM {
t.Helper()
require.NoError(t, NewArchiveContainsTask().Execute(context.Background(), nil, sbomsync.NewBuilder(s)))
return s
}
// markerTask emits one package per "marker.txt" found, so the matrix asserts behavior without
// depending on any real cataloger's fixture format.
func markerTask() Task {
+37 -58
View File
@@ -32,7 +32,7 @@ const ArchiveCatalogerTaskName = "archive-cataloger"
// NewArchiveCatalogerTask returns a task that extracts every archive it finds, runs the same tasks
// against the archive's contents as a standalone filesystem, and records
// CONTAINS relationships from the archive to what was found inside it. It recurses into the archives
// CONTAINS relationships from the archive to the packages found directly inside it. It recurses into the archives
// it extracts up to cfg.MaxDepth; the task drops itself from the tasks so that is the only recursion.
//
// Each archive is handled as the scan root is: pkgTasks, then the archives nested in it, then fileTasks,
@@ -267,12 +267,14 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi
// as at the scan root, nested archives are cataloged before this archive's files, so the packages
// describing them (located in this archive) select their files here
children := c.discoverArchives(extracted)
errs := c.catalogCandidates(ctx, extracted, children, depth+1, foundBuilder)
errs := c.catalogCandidates(ctx, extracted, c.discoverArchives(extracted), depth+1, foundBuilder)
started = time.Now()
runArchiveTasks(ctx, c.fileTasks, extracted, location.Coordinates, foundBuilder)
addContains(location.Coordinates, archivePath, direct, children, found)
// the files within it are related once the scan has decided which it describes (see NewArchiveContainsTask)
for _, p := range direct {
found.Relationships = append(found.Relationships, artifact.Relationship{From: location.Coordinates, To: p, Type: artifact.ContainsRelationship})
}
mergeArchiveResults(found, builder)
c.progress.AtomicStage.Set(humanize.Comma(c.progress.Current()) + " archives (" + archivePath + ")")
@@ -440,64 +442,41 @@ func mergeArchiveResults(found *sbom.SBOM, builder sbomsync.Builder) {
}
}
// addContains adds a CONTAINS relationship from the archive to each package found directly in it, each
// file directly in it that the SBOM describes, and each nested archive, so the hierarchy is navigable
// whatever file selection is configured.
//
// Files directly in the archive are those whose container is the archive's own full path, which leaves
// out the archive itself (a java package or an unknown recorded at its coordinates would otherwise
// become a self-loop) and the files of nested archives, contained by those.
func addContains(archiveCoordinates file.Coordinates, archivePath string, direct []pkg.Package, children []archiveCandidate, found *sbom.SBOM) {
contained := file.NewCoordinateSet()
for _, child := range children {
contained.Add(child.location.Coordinates)
// NewArchiveContainsTask relates each archive to every file the SBOM describes directly within it. It must
// run after every other task, so the files within archives are those the scan root's configuration
// selects (file selection, unknowns, package evidence) and no others.
func NewArchiveContainsTask() Task {
fn := func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error {
var rels []artifact.Relationship
builder.(sbomsync.Accessor).ReadFromSBOM(func(s *sbom.SBOM) {
rels = archiveContains(s.AllCoordinates())
})
builder.AddRelationships(rels...)
return nil
}
addInside := func(coordinates file.Coordinates) {
if archive.ContainerPath(coordinates) == archivePath {
contained.Add(coordinates)
}
}
eachFileArtifactCoordinates(&found.Artifacts, addInside)
for _, r := range found.Relationships {
if c, ok := r.From.(file.Coordinates); ok {
addInside(c)
}
if c, ok := r.To.(file.Coordinates); ok {
addInside(c)
}
}
var rels []artifact.Relationship
for _, p := range direct {
rels = append(rels, artifact.Relationship{From: archiveCoordinates, To: p, Type: artifact.ContainsRelationship})
}
for _, coordinates := range contained.ToSlice() {
rels = append(rels, artifact.Relationship{From: archiveCoordinates, To: coordinates, Type: artifact.ContainsRelationship})
}
found.Relationships = append(found.Relationships, rels...)
return NewTask("archive-contains", fn)
}
// eachFileArtifactCoordinates calls fn with the coordinates of every file artifact, once per artifact
// kind describing it.
func eachFileArtifactCoordinates(a *sbom.Artifacts, fn func(file.Coordinates)) {
for c := range a.FileMetadata {
fn(c)
}
for c := range a.FileDigests {
fn(c)
}
for c := range a.FileContents {
fn(c)
}
for c := range a.FileLicenses {
fn(c)
}
for c := range a.Executables {
fn(c)
}
for c := range a.Unknowns {
fn(c)
// archiveContains relates the archive holding each file within one to that file, and each such archive to
// the archive holding it, so an archive the SBOM does not otherwise describe still links what is within it.
func archiveContains(coordinates []file.Coordinates) []artifact.Relationship {
related := file.NewCoordinateSet(coordinates...)
var rels []artifact.Relationship
for _, c := range coordinates {
for {
container, ok := archive.ContainingArchive(c)
if !ok {
break
}
rels = append(rels, artifact.Relationship{From: container, To: c, Type: artifact.ContainsRelationship})
if related.Contains(container) {
break
}
related.Add(container)
c = container
}
}
return rels
}
func mergeFileArtifacts(dst, src *sbom.Artifacts) {
+32
View File
@@ -337,6 +337,7 @@ func Test_archiveCataloger_noSelfContainsEdge(t *testing.T) {
s := newTestSBOM()
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
_ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))
withArchiveContains(t, s)
require.Contains(t, s.Artifacts.Unknowns, file.NewCoordinates("/x.jar", ""), "the fixture must reproduce the unknown")
var contains int
@@ -350,6 +351,37 @@ func Test_archiveCataloger_noSelfContainsEdge(t *testing.T) {
assert.Positive(t, contains, "the archive still contains its files")
}
func Test_NewArchiveContainsTask(t *testing.T) {
// a file within an archive is contained only when the SBOM describes it, as a file at the scan root is
// listed only then; an executable no package owns is recorded whatever the file selection
in := func(fullPath string) file.Coordinates {
return file.Coordinates{RealPath: "/outer.tar", FileSystemID: "layer", FullPath: fullPath}
}
outer := file.NewCoordinates("/outer.tar", "layer")
jar := in("/outer.tar!/lib/app.jar")
manifest := in("/outer.tar!/lib/app.jar!/META-INF/MANIFEST.MF")
unknownExe := in("/outer.tar!/bin/setup.exe")
s := newTestSBOM()
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{manifest: {}, file.NewCoordinates("/loose.txt", "layer"): {}}
s.Artifacts.Executables = map[file.Coordinates]file.Executable{in("/outer.tar!/bin/tool"): {}}
s.Artifacts.Unknowns = map[file.Coordinates][]string{unknownExe: {"no package identified in executable file"}}
withArchiveContains(t, s)
type edge struct{ from, to file.Coordinates }
var edges []edge
for _, r := range s.Relationships {
assert.Equal(t, artifact.ContainsRelationship, r.Type)
edges = append(edges, edge{r.From.(file.Coordinates), r.To.(file.Coordinates)})
}
assert.ElementsMatch(t, []edge{
{jar, manifest},
{outer, jar}, // the jar is not described itself, but is on the way to what is
{outer, unknownExe},
}, edges)
}
func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) {
// a copy of itself in the sub-pipeline would process every nesting level twice
cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)
+10 -3
View File
@@ -3,9 +3,11 @@ package task
import (
"context"
"fmt"
"path"
"strings"
"unicode"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/relationship"
@@ -236,10 +238,10 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
locations := map[artifact.ID]file.Location{}
for _, path := range fileOwner.OwnedFiles() {
pathRefs, err := resolver.FilesByPath(path)
for _, ownedPath := range fileOwner.OwnedFiles() {
pathRefs, err := resolver.FilesByPath(ownedPath)
if err != nil {
return nil, fmt.Errorf("unable to find path for path=%q: %w", path, err)
return nil, fmt.Errorf("unable to find path for path=%q: %w", ownedPath, err)
}
if len(pathRefs) == 0 {
@@ -250,6 +252,11 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
}
for _, ref := range pathRefs {
// an archive looks a relative path up from its root, but an owned path may be relative to
// anything (a python RECORD is relative to site-packages), so only an absolute one owns a file in one
if archive.ContainerPath(ref.Coordinates) != "" && !path.IsAbs(ownedPath) {
continue
}
if oldRef, ok := locations[ref.ID()]; ok {
log.Debugf("found path duplicate of %s", oldRef.FullNestedPath())
}
@@ -1,11 +1,15 @@
package task
import (
"bytes"
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/testutils"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cpe"
@@ -281,3 +285,28 @@ func TestApplyComplianceRules_DropAndStub(t *testing.T) {
assert.False(t, isCompliant)
assert.Equal(t, cataloging.UnknownStubValue, p.Version)
}
func Test_packageFileOwnershipRelationships_exactPathsWithinAnArchive(t *testing.T) {
// a wheel's RECORD is relative to site-packages: looked up from the wheel's root it finds the file, but
// only an exact path owns one, as at the scan root
data := testutils.Zip(t, map[string]string{"pip/__init__.py": "", "bin/tool": ""})
whl, err := archive.Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("/pip.whl", ""),
archive.NewLimiter(archive.Limits{MaxMemoryBytes: 1 << 20}), archive.NewExclusions(nil))
require.NoError(t, err)
t.Cleanup(whl.Cleanup)
found, err := whl.FilesByPath("pip/__init__.py")
require.NoError(t, err)
require.Len(t, found, 1, "a relative path still finds the file")
p := pkg.Package{Name: "pip", Metadata: pkg.PythonPackage{Files: []pkg.PythonFileRecord{{Path: "pip/__init__.py"}, {Path: "/bin/tool"}}}}
p.SetID()
rels, err := packageFileOwnershipRelationships(p, whl)
require.NoError(t, err)
var owned []string
for _, r := range rels {
owned = append(owned, r.To.(file.Coordinates).FullNestedPath())
}
assert.Equal(t, []string{"/pip.whl!/bin/tool"}, owned)
}
+7 -3
View File
@@ -2,6 +2,7 @@ package task
import (
"context"
"path"
"slices"
"strings"
@@ -106,9 +107,12 @@ func coordinateReferenceLookup(resolver file.Resolver, s *sbom.SBOM) func(coords
}
}
for _, ownedFilePath := range f.OwnedFiles() {
for _, container := range archives {
// an archive's contents are not in this resolver to follow links in
allPackageCoords.Add(archive.OwnedCoordinates(container, ownedFilePath))
// an archive's contents are not in this resolver to follow links in, and only an absolute owned
// path names a file in one (see packageFileOwnershipRelationships)
if path.IsAbs(ownedFilePath) {
for _, container := range archives {
allPackageCoords.Add(archive.OwnedCoordinates(container, ownedFilePath))
}
}
if !atRoot {
continue
+52
View File
@@ -5,6 +5,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -29,6 +30,57 @@ func Test_unknownsLabeler_archiveReasonsSurvivePackages(t *testing.T) {
"a package describing the archive explains a parse failure, not contents left out")
}
func Test_unknownsLabeler_archiveContentsAreReportedByFullPath(t *testing.T) {
// a package describing an archive does not account for the files within it
war := file.NewCoordinates("/app.war", "layer")
in := func(fullPath string) file.Coordinates {
return file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: fullPath}
}
nativeLib := in("/app.war!/WEB-INF/lib/jna.jar!/linux-x86-64/libjnidispatch.so")
unidentifiedJar := in("/app.war!/WEB-INF/lib/javax.inject-1.jar")
p := pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(war))}
p.SetID()
noPackage := formatUnknown("no package identified in archive", "java-archive-cataloger")
s := &sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: pkg.NewCollection(p),
Executables: map[file.Coordinates]file.Executable{nativeLib: {}},
Unknowns: map[file.Coordinates][]string{unidentifiedJar: {noPackage}},
},
Relationships: []artifact.Relationship{{From: war, To: unidentifiedJar, Type: artifact.ContainsRelationship}},
}
unknownsLabelerTask{cataloging.UnknownsConfig{RemoveWhenPackagesDefined: true, IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths(), s)
assert.Equal(t, map[file.Coordinates][]string{
nativeLib: {formatUnknown("no package identified in executable file", unknownsLabelerTaskName)},
unidentifiedJar: {noPackage},
}, s.Artifacts.Unknowns)
}
func Test_unknownsLabeler_relativeOwnedPathsNameNothingInAnArchive(t *testing.T) {
// a python RECORD lists paths relative to site-packages, which resolve nowhere at the scan root either
whl := func(fullPath string) file.Coordinates {
return file.Coordinates{RealPath: "/pip.whl", FullPath: fullPath}
}
tool := whl("/pip.whl!/pip/tool")
p := pkg.Package{
Name: "pip",
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(whl("/pip.whl!/pip.dist-info/METADATA"))),
Metadata: pkg.PythonPackage{Files: []pkg.PythonFileRecord{{Path: "pip/tool"}}},
}
p.SetID()
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
Packages: pkg.NewCollection(p),
Executables: map[file.Coordinates]file.Executable{tool: {}},
}}
unknownsLabelerTask{cataloging.UnknownsConfig{IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths(), s)
assert.NotEmpty(t, s.Artifacts.Unknowns[tool])
}
func Test_unknownsLabeler_ownedFilesAreWithinTheOwnersContainer(t *testing.T) {
atRoot := file.NewCoordinates("/usr/bin/foo", "")
inArchive := file.Coordinates{RealPath: "/rootfs.tar", FullPath: "/rootfs.tar!/usr/bin/foo"}
+32
View File
@@ -184,6 +184,13 @@ func TestArchiveCataloger(t *testing.T) {
}, containers, "the archive file, named in its own parent, contains the package")
})
t.Run("each file within an archive is contained by that archive", func(t *testing.T) {
containers := fileContainers(s)
assert.Equal(t, "app.zip", containers["app.zip!/nested/keep.txt"])
assert.Equal(t, "outer.zip!/deeper/inner.zip", containers["outer.zip!/deeper/inner.zip!/nested/marker.txt"])
assert.Equal(t, "outer.zip", containers["outer.zip!/deeper/inner.zip"])
})
t.Run("exclusions are the scan's own", func(t *testing.T) {
// there is no archive-specific exclusion setting: a scan pattern's shape decides whether it
// reaches inside an archive, and an excluded entry is absent from the archive's filesystem
@@ -389,6 +396,8 @@ func TestArchiveCataloger_defaultFileSelectionCoversNestedJars(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "outer.zip"), testutils.Zip(t, map[string][]byte{
"lib/inner.jar": testutils.Jar(t, "inner", "1.0", nil),
"readme.txt": []byte("hello"),
"empty.zip": testutils.Zip(t, map[string]string{"a.txt": "hello"}),
}), 0o644))
cfg := DefaultCreateSBOMConfig().
@@ -400,6 +409,16 @@ func TestArchiveCataloger_defaultFileSelectionCoversNestedJars(t *testing.T) {
jar := file.Coordinates{RealPath: "outer.zip", FullPath: "outer.zip!/lib/inner.jar"}
assert.Contains(t, s.Artifacts.FileMetadata, jar)
assert.Contains(t, s.Artifacts.FileDigests, jar)
assert.Equal(t, "outer.zip", fileContainers(s)["outer.zip!/lib/inner.jar"])
// as at the root, a file no package owns is not described, nor is an archive in which nothing was found
var described []string
for _, c := range s.AllCoordinates() {
described = append(described, c.FullNestedPath())
}
assert.NotContains(t, described, "outer.zip!/readme.txt")
assert.NotContains(t, described, "outer.zip!/empty.zip")
assert.NotContains(t, described, "outer.zip!/empty.zip!/a.txt")
}
func TestArchiveCataloger_archiveUnknowns(t *testing.T) {
@@ -629,6 +648,19 @@ func (p *recordingBus) Publish(e partybus.Event) {
p.events = append(p.events, e)
}
// fileContainers maps the full path of each file an archive CONTAINS to the archive's full path.
func fileContainers(s *sbom.SBOM) map[string]string {
out := map[string]string{}
for _, r := range s.Relationships {
from, fromFile := r.From.(file.Coordinates)
to, toFile := r.To.(file.Coordinates)
if r.Type == artifact.ContainsRelationship && fromFile && toFile {
out[to.FullNestedPath()] = from.FullNestedPath()
}
}
return out
}
// scanDirWithExclusions scans with the given exclusion patterns configured on the source, where
// exclusions live; archive cataloging has no exclusion setting of its own.
func scanDirWithExclusions(t *testing.T, dir string, cfg *CreateSBOMConfig, exclude ...string) *sbom.SBOM {
+5
View File
@@ -254,6 +254,11 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description, exclusions []s
taskGroups = append(taskGroups, osFeatureDetectionTasks)
}
// archives are related to the files within them only once every task has decided which files are described
if archiveTask != nil {
taskGroups = append(taskGroups, []task.Task{task.NewArchiveContainsTask()})
}
// identifying the environment (i.e. the linux release) must be done first as this is required for package cataloging
taskGroups = append(
[][]task.Task{
@@ -14,6 +14,7 @@ import (
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/file"
formatInternal "github.com/anchore/syft/syft/format/internal"
"github.com/anchore/syft/syft/format/internal/cyclonedxutil/helpers"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
@@ -42,7 +43,8 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM {
cdxBOM.SerialNumber = uuid.New().URN()
cdxBOM.Metadata = toBomDescriptor(s.Descriptor.Name, s.Descriptor.Version, s.Source)
coordinates, locationSorter := getCoordinates(s)
locationSorter, coordinateSorter := formatInternal.GetLocationSorters(s)
top, entries := formatInternal.ArchiveEntries(s.AllCoordinates(), coordinateSorter)
// Packages
packages := s.Artifacts.Packages.Sorted()
@@ -52,40 +54,7 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM {
}
components = append(components, toOSComponent(s.Artifacts.LinuxDistribution)...)
artifacts := s.Artifacts
for _, coordinate := range coordinates {
var metadata *file.Metadata
// File Info
fileMetadata, exists := artifacts.FileMetadata[coordinate]
// no file metadata then don't include in SBOM
// the syft config allows for sometimes only capturing files owned by packages
// so there can be a map miss here where we have less metadata than all coordinates
if !exists {
continue
}
if fileMetadata.Type == stfile.TypeDirectory ||
fileMetadata.Type == stfile.TypeSocket ||
fileMetadata.Type == stfile.TypeSymLink {
// skip dir, symlinks and sockets for the final bom
continue
}
metadata = &fileMetadata
// Digests
var digests []file.Digest
if digestsForLocation, exists := artifacts.FileDigests[coordinate]; exists {
digests = digestsForLocation
}
cdxHashes := digestsToHashes(digests)
components = append(components, cyclonedx.Component{
BOMRef: string(coordinate.ID()),
Type: cyclonedx.ComponentTypeFile,
Name: metadata.Path,
Hashes: &cdxHashes,
})
}
components = append(components, toFileComponents(s.Artifacts, entries, top)...)
cdxBOM.Components = &components
dependencies := toDependencies(s.Relationships)
@@ -96,19 +65,41 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM {
return cdxBOM
}
func getCoordinates(s sbom.SBOM) ([]file.Coordinates, func(a, b file.Location) int) {
var layers []string
if m, ok := s.Source.Metadata.(source.ImageMetadata); ok {
for _, l := range m.Layers {
layers = append(layers, l.Digest)
// toFileComponents describes each of coordinates that is a regular file with metadata, named by its path
// (within its archive, for a file within one), with the files within it nested in its component.
func toFileComponents(artifacts sbom.Artifacts, entries map[file.Coordinates][]file.Coordinates, coordinates []file.Coordinates) []cyclonedx.Component {
var components []cyclonedx.Component
for _, coordinate := range coordinates {
nested := toFileComponents(artifacts, entries, entries[coordinate])
// no file metadata then don't include in SBOM
// the syft config allows for sometimes only capturing files owned by packages
// so there can be a map miss here where we have less metadata than all coordinates
metadata, exists := artifacts.FileMetadata[coordinate]
if !isFileComponent(metadata, exists) && len(nested) == 0 {
continue
}
cdxHashes := digestsToHashes(artifacts.FileDigests[coordinate])
component := cyclonedx.Component{
BOMRef: string(coordinate.ID()),
Type: cyclonedx.ComponentTypeFile,
Name: coordinate.LocalPath(),
Hashes: &cdxHashes,
}
if len(nested) > 0 {
component.Components = &nested
}
components = append(components, component)
}
return components
}
coordSorter := file.CoordinatesSorter(layers)
coordinates := s.AllCoordinates()
slices.SortFunc(coordinates, coordSorter)
return coordinates, file.LocationSorter(layers)
// isFileComponent skips dirs, symlinks and sockets for the final bom
func isFileComponent(metadata file.Metadata, exists bool) bool {
return exists &&
metadata.Type != stfile.TypeDirectory &&
metadata.Type != stfile.TypeSocket &&
metadata.Type != stfile.TypeSymLink
}
func digestsToHashes(digests []file.Digest) []cyclonedx.Hash {
@@ -608,3 +608,65 @@ func Test_toOSBomRef(t *testing.T) {
})
}
}
func Test_nestedArchiveFiles(t *testing.T) {
top := file.NewCoordinates("/etc/os-release", "")
inA := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"}
inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"}
inWar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/index.html"}
dir := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/c.jar!/META-INF"}
sha := []file.Digest{{Algorithm: "sha256", Value: "abc"}}
s := sbom.SBOM{Artifacts: sbom.Artifacts{
FileMetadata: map[file.Coordinates]file.Metadata{
// a directory resolver's metadata path is the path on the host, not the one scanned
top: {Path: "/host/scan/etc/os-release"},
inA: {},
inB: {},
inWar: {},
dir: {Type: stfile.TypeDirectory},
},
FileDigests: map[file.Coordinates][]file.Digest{inA: sha},
}}
bom := ToFormatModel(s)
type node struct {
Name string
Children []node
}
var tree func(cs *[]cyclonedx.Component) []node
tree = func(cs *[]cyclonedx.Component) []node {
if cs == nil {
return nil
}
var out []node
for _, c := range *cs {
require.Equal(t, cyclonedx.ComponentTypeFile, c.Type)
out = append(out, node{Name: c.Name, Children: tree(c.Components)})
}
return out
}
// each archive holding a file is a component, synthesized when not otherwise described; an archive
// holding only skipped files is left out
assert.Equal(t, []node{
{Name: "/app.war", Children: []node{
{Name: "/WEB-INF/lib/a.jar", Children: []node{{Name: "/META-INF/MANIFEST.MF"}}},
{Name: "/WEB-INF/lib/b.jar", Children: []node{{Name: "/odd:na%me.txt"}}},
{Name: "/index.html"},
}},
{Name: "/etc/os-release"},
}, tree(bom.Components))
decoded, err := helpers.ToSyftModel(bom)
require.NoError(t, err)
war := file.NewCoordinates("/app.war", "")
a := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar"}
b := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar"}
var got []file.Coordinates
for c := range decoded.Artifacts.FileMetadata {
got = append(got, c)
}
assert.ElementsMatch(t, []file.Coordinates{top, war, a, b, inA, inB, inWar}, got)
assert.Equal(t, map[file.Coordinates][]file.Digest{inA: sha}, decoded.Artifacts.FileDigests)
}
@@ -6,7 +6,6 @@ import (
"crypto/sha1"
"fmt"
"path"
"slices"
"sort"
"strings"
"time"
@@ -53,7 +52,8 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document {
rels := relationship.NewIndex(s.Relationships...)
packages, otherLicenses := toPackages(rels, s.Artifacts.Packages, s)
allRelationships := toRelationships(rels.All())
files, archiveRelationships := toFiles(s)
allRelationships := append(toRelationships(rels.All()), archiveRelationships...)
// for valid SPDX we need a document describes relationship
describesID := spdx.ElementID("DOCUMENT")
@@ -62,8 +62,7 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document {
if rootPackage != nil {
describesID = rootPackage.PackageSPDXIdentifier
// add all relationships from the document root to all other packages
allRelationships = append(allRelationships, toRootRelationships(rootPackage, packages)...)
allRelationships = append(allRelationships, toRootRelationships(rootPackage, packages, files, allRelationships)...)
// append the root package
packages = append(packages, rootPackage)
@@ -155,21 +154,40 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document {
CreatorComment: "",
},
Packages: packages,
Files: toFiles(s),
Files: files,
Relationships: allRelationships,
OtherLicenses: convertOtherLicense(otherLicenses),
}
}
func toRootRelationships(rootPackage *spdx.Package, packages []*spdx.Package) (out []*spdx.Relationship) {
// toRootRelationships relates the root package to each package and file nothing else CONTAINS: what is
// within an archive is reached only through the archive's file, and an owned file through its package.
func toRootRelationships(rootPackage *spdx.Package, packages []*spdx.Package, files []*spdx.File, relationships []*spdx.Relationship) (out []*spdx.Relationship) {
contained := map[spdx.ElementID]bool{}
for _, r := range relationships {
if r.Relationship == string(helpers.ContainsRelationship) {
contained[r.RefB.ElementRefID] = true
}
}
var ids []spdx.ElementID
for _, p := range packages {
ids = append(ids, p.PackageSPDXIdentifier)
}
for _, f := range files {
ids = append(ids, f.FileSPDXIdentifier)
}
for _, id := range ids {
if contained[id] {
continue
}
out = append(out, &spdx.Relationship{
RefA: spdx.DocElementID{
ElementRefID: rootPackage.PackageSPDXIdentifier,
},
Relationship: string(helpers.ContainsRelationship),
RefB: spdx.DocElementID{
ElementRefID: p.PackageSPDXIdentifier,
ElementRefID: id,
},
})
}
@@ -666,7 +684,7 @@ func toRelationships(relationships []artifact.Relationship) (result []*spdx.Rela
// FIXME: we are only currently including Package -> * relationships, plus an archive containing a
// package found inside it, which is the only structure nested archive cataloging records for
// non-java packages. Every file-side From is in the document: toFiles covers every coordinate a
// relationship names.
// relationship names, and itself relates each archive to the files within it.
if _, ok := r.From.(pkg.Package); !ok && !isFileContainsPackage(r) {
log.Debugf("skipping non-package relationship: %+v", r)
continue
@@ -706,82 +724,34 @@ func lookupRelationship(ty artifact.RelationshipType) (bool, helpers.Relationshi
return false, "", ""
}
// file comments carry what an SPDX file has no field for, one "key: value" per line
const (
layerIDCommentPrefix = "layerID: "
archivePathCommentPrefix = "archivePath: "
)
// fileComment carries the layer ID and, for a file inside an archive, its archive path. Such a file is
// named by its full path ("/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"), so a consumer reading
// only the name does not mistake it for a file at that path in the scan root; the entry chain after the
// real path is also kept in the comment so decoding need not guess where it starts.
func fileComment(c file.Coordinates) string {
var entries string
if rest, ok := strings.CutPrefix(c.FullPath, c.RealPath); ok {
entries, _ = strings.CutPrefix(rest, file.ArchivePathSeparator)
}
switch {
case c.FileSystemID != "" && entries != "":
return layerIDCommentPrefix + c.FileSystemID + "\n" + archivePathCommentPrefix + entries
case c.FileSystemID != "":
return layerIDCommentPrefix + c.FileSystemID
case entries != "":
return archivePathCommentPrefix + entries
}
return ""
}
func toFiles(s sbom.SBOM) (results []*spdx.File) {
artifacts := s.Artifacts
const layerIDCommentPrefix = "layerID: "
// toFiles describes every file, including each archive holding one. A file within an archive is named by
// its path in that archive, and the archive CONTAINS it.
func toFiles(s sbom.SBOM) (results []*spdx.File, archiveRelationships []*spdx.Relationship) {
_, coordinateSorter := formatInternal.GetLocationSorters(s)
top, entries := formatInternal.ArchiveEntries(s.AllCoordinates(), coordinateSorter)
coordinates := s.AllCoordinates()
slices.SortFunc(coordinates, coordinateSorter)
for _, c := range coordinates {
var metadata *file.Metadata
if metadataForLocation, exists := artifacts.FileMetadata[c]; exists {
metadata = &metadataForLocation
var addEntries func(archive file.Coordinates)
addEntries = func(archive file.Coordinates) {
for _, c := range entries[archive] {
archiveRelationships = append(archiveRelationships, &spdx.Relationship{
RefA: spdx.DocElementID{ElementRefID: toSPDXID(archive)},
Relationship: string(helpers.ContainsRelationship),
RefB: spdx.DocElementID{ElementRefID: toSPDXID(c)},
})
results = append(results, toFile(s.Artifacts, c, strings.TrimPrefix(c.LocalPath(), "/")))
addEntries(c)
}
var digests []file.Digest
if digestsForLocation, exists := artifacts.FileDigests[c]; exists {
digests = digestsForLocation
}
// if we don't have any metadata or digests for this location
// then the file is most likely a symlink or non-regular file
// for now we include a 0 sha1 digest as requested by the spdx spec
// TODO: update location code in core SBOM so that we can map complex links
// back to their real file digest location.
if len(digests) == 0 {
digests = append(digests, file.Digest{Algorithm: "sha1", Value: "0000000000000000000000000000000000000000"})
}
// TODO: add file classifications (?) and content as a snippet
name := c.FullNestedPath()
relativePath, err := convertAbsoluteToRelative(name)
}
for _, c := range top {
name, err := convertAbsoluteToRelative(c.RealPath)
if err != nil {
log.Debugf("unable to convert relative path '%s' to absolute path: %s", name, err)
relativePath = name
log.Debugf("unable to convert relative path '%s' to absolute path: %s", c.RealPath, err)
name = c.RealPath
}
results = append(results, &spdx.File{
FileSPDXIdentifier: toSPDXID(c),
FileComment: fileComment(c),
// required, no attempt made to determine license information
LicenseConcluded: noAssertion,
FileCopyrightText: noAssertion,
Checksums: toFileChecksums(digests),
FileName: relativePath,
FileTypes: toFileTypes(metadata),
LicenseInfoInFiles: []string{ // required in SPDX 2.2
helpers.NOASSERTION,
},
})
results = append(results, toFile(s.Artifacts, c, name))
addEntries(c)
}
// sort by real path then virtual path to ensure the result is stable across multiple runs
@@ -791,7 +761,49 @@ func toFiles(s sbom.SBOM) (results []*spdx.File) {
}
return results[i].FileName < results[j].FileName
})
return results
return results, archiveRelationships
}
func toFile(artifacts sbom.Artifacts, c file.Coordinates, name string) *spdx.File {
var metadata *file.Metadata
if metadataForLocation, exists := artifacts.FileMetadata[c]; exists {
metadata = &metadataForLocation
}
var digests []file.Digest
if digestsForLocation, exists := artifacts.FileDigests[c]; exists {
digests = digestsForLocation
}
// if we don't have any metadata or digests for this location
// then the file is most likely a symlink or non-regular file
// for now we include a 0 sha1 digest as requested by the spdx spec
// TODO: update location code in core SBOM so that we can map complex links
// back to their real file digest location.
if len(digests) == 0 {
digests = append(digests, file.Digest{Algorithm: "sha1", Value: "0000000000000000000000000000000000000000"})
}
// TODO: add file classifications (?) and content as a snippet
var comment string
if c.FileSystemID != "" {
comment = layerIDCommentPrefix + c.FileSystemID
}
return &spdx.File{
FileSPDXIdentifier: toSPDXID(c),
FileComment: comment,
// required, no attempt made to determine license information
LicenseConcluded: noAssertion,
FileCopyrightText: noAssertion,
Checksums: toFileChecksums(digests),
FileName: name,
FileTypes: toFileTypes(metadata),
LicenseInfoInFiles: []string{ // required in SPDX 2.2
helpers.NOASSERTION,
},
}
}
func toFileChecksums(digests []file.Digest) (checksums []spdx.Checksum) {
@@ -922,6 +934,10 @@ func trimPatchVersion(semver string) string {
// spdx requires that the file name field is a relative filename
// with the root of the package archive or directory
func convertAbsoluteToRelative(absPath string) (string, error) {
// the root itself; an empty file name is not valid SPDX (tag-value omits it and cannot read it back)
if absPath == "" || absPath == "/" {
return ".", nil
}
// Ensure the absolute path is absolute (although it should already be)
if !path.IsAbs(absPath) {
// already relative
@@ -14,6 +14,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
stfile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal/relationship"
"github.com/anchore/syft/internal/sourcemetadata"
"github.com/anchore/syft/syft/artifact"
@@ -588,12 +589,28 @@ func Test_toFiles(t *testing.T) {
FileName: "some/path",
},
},
{
name: "the scanned root has a name",
in: sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{
{RealPath: ""}: {Type: stfile.TypeDirectory},
}}},
want: spdx.File{FileName: "."},
},
{
name: "the absolute root has a name",
in: sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{
{RealPath: "/"}: {Type: stfile.TypeDirectory},
}}},
want: spdx.File{FileName: "."},
},
}
for _, test := range tests {
files := toFiles(test.in)
got := files[0]
assert.Equal(t, test.want.FileName, got.FileName)
t.Run(test.name, func(t *testing.T) {
files, _ := toFiles(test.in)
require.Len(t, files, 1)
assert.Equal(t, test.want.FileName, files[0].FileName)
})
}
}
@@ -1156,38 +1173,99 @@ func Test_otherLicenses(t *testing.T) {
}
}
func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) {
// two archives' META-INF/MANIFEST.MF must not come out as two files with the same name and nothing
// saying which archive each is in
root := file.NewCoordinates("/etc/os-release", "sha256:layer")
func Test_nestedArchiveFiles(t *testing.T) {
top := file.NewCoordinates("/etc/os-release", "sha256:layer")
inA := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", FileSystemID: "sha256:layer"}
inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"}
s := sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{root: {}, inA: {}, inB: {}}}}
byName := map[string]*spdx.File{}
for _, f := range toFiles(s) {
byName[f.FileName] = f
inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt", FileSystemID: "sha256:layer"}
owned := file.NewCoordinates("/usr/bin/tool", "sha256:layer")
a := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar", FileSystemID: "sha256:layer"}
topPkg := pkg.Package{Name: "top"}
topPkg.SetID()
nestedPkg := pkg.Package{Name: "nested"}
nestedPkg.SetID()
s := sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: pkg.NewCollection(topPkg, nestedPkg),
FileMetadata: map[file.Coordinates]file.Metadata{top: {}, inA: {}, inB: {}, owned: {}},
},
Relationships: []artifact.Relationship{
{From: topPkg, To: owned, Type: artifact.ContainsRelationship},
{From: a, To: nestedPkg, Type: artifact.ContainsRelationship},
},
}
require.Len(t, byName, 3)
assert.Equal(t, "layerID: sha256:layer", byName["etc/os-release"].FileComment, "files outside archives are unchanged")
assert.Equal(t, file.NewCoordinates("etc/os-release", "sha256:layer"), toSyftCoordinates(byName["etc/os-release"]))
doc := ToFormatModel(s)
// the name is made relative like any other file's, so the decoded top path is too
a := byName["app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"]
require.NotNil(t, a, "names: %v", byName)
assert.Equal(t, "layerID: sha256:layer\narchivePath: WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", a.FileComment)
wantA := inA
wantA.RealPath = "app.war"
wantA.FullPath = "app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"
assert.Equal(t, wantA, toSyftCoordinates(a))
// each archive holding a described file is described too, named by its path in its own container
war := file.NewCoordinates("/app.war", "sha256:layer")
b := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar", FileSystemID: "sha256:layer"}
names := map[spdx.ElementID]string{}
for _, f := range doc.Files {
names[f.FileSPDXIdentifier] = f.FileName
assert.Equal(t, "layerID: sha256:layer", f.FileComment)
}
assert.Equal(t, map[spdx.ElementID]string{
toSPDXID(top): "etc/os-release",
toSPDXID(owned): "usr/bin/tool",
toSPDXID(war): "app.war",
toSPDXID(a): "WEB-INF/lib/a.jar",
toSPDXID(b): "WEB-INF/lib/b.jar",
toSPDXID(inA): "META-INF/MANIFEST.MF",
toSPDXID(inB): "odd:na%me.txt",
}, names)
b := byName["app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"]
require.NotNil(t, b, "names: %v", byName)
wantB := inB
wantB.RealPath = "app.war"
wantB.FullPath = "app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"
assert.Equal(t, wantB, toSyftCoordinates(b))
var root spdx.ElementID
var contains [][2]spdx.ElementID
for _, r := range doc.Relationships {
switch r.Relationship {
case string(helpers.DescribesRelationship):
root = r.RefB.ElementRefID
case string(helpers.ContainsRelationship):
contains = append(contains, [2]spdx.ElementID{r.RefA.ElementRefID, r.RefB.ElementRefID})
}
}
// the root contains only what nothing else does: not what is within an archive, nor an owned file
assert.ElementsMatch(t, [][2]spdx.ElementID{
{root, toSPDXID(topPkg)},
{root, toSPDXID(top)},
{root, toSPDXID(war)},
{toSPDXID(topPkg), toSPDXID(owned)},
{toSPDXID(a), toSPDXID(nestedPkg)},
{toSPDXID(war), toSPDXID(a)},
{toSPDXID(war), toSPDXID(b)},
{toSPDXID(a), toSPDXID(inA)},
{toSPDXID(b), toSPDXID(inB)},
}, contains)
decoded, err := ToSyftModel(doc)
require.NoError(t, err)
// decoded paths are relative, as SPDX names are
relative := func(c file.Coordinates) file.Coordinates {
c.RealPath = strings.TrimPrefix(c.RealPath, "/")
c.FullPath = strings.TrimPrefix(c.FullPath, "/")
return c
}
var got []file.Coordinates
for c := range decoded.Artifacts.FileMetadata {
got = append(got, c)
}
assert.ElementsMatch(t, []file.Coordinates{relative(top), relative(owned), relative(war), relative(a), relative(b), relative(inA), relative(inB)}, got)
var gotContains []artifact.Relationship
for _, r := range decoded.Relationships {
_, fromFile := r.From.(file.Coordinates)
_, toFile := r.To.(file.Coordinates)
if fromFile && toFile {
gotContains = append(gotContains, r)
}
}
assert.ElementsMatch(t, []artifact.Relationship{
{From: relative(war), To: relative(a), Type: artifact.ContainsRelationship},
{From: relative(war), To: relative(b), Type: artifact.ContainsRelationship},
{From: relative(a), To: relative(inA), Type: artifact.ContainsRelationship},
{From: relative(b), To: relative(inB), Type: artifact.ContainsRelationship},
}, gotContains)
}
func Test_toRootPackage_name(t *testing.T) {
+88 -22
View File
@@ -380,25 +380,74 @@ func collectSyftPackages(s *sbom.SBOM, spdxIDMap map[string]any, doc *spdx.Docum
}
func collectSyftFiles(s *sbom.SBOM, spdxIDMap map[string]any, doc *spdx.Document) {
files := map[spdx.ElementID]*spdx.File{}
for _, p := range doc.Packages {
for _, f := range p.Files {
l := toSyftLocation(f)
spdxIDMap[string(f.FileSPDXIdentifier)] = l
files[f.FileSPDXIdentifier] = f
}
}
for _, f := range doc.Files {
files[f.FileSPDXIdentifier] = f
}
s.Artifacts.FileMetadata[l.Coordinates] = toFileMetadata(f)
s.Artifacts.FileDigests[l.Coordinates] = toFileDigests(f)
nested := newNestedFiles[spdx.ElementID]()
for id, f := range files {
nested.own[id] = toSyftCoordinates(f)
}
for _, r := range doc.Relationships {
container, entry := r.RefA.ElementRefID, r.RefB.ElementRefID
switch helpers.RelationshipType(r.Relationship) {
case helpers.ContainsRelationship:
case helpers.ContainedByRelationship:
container, entry = entry, container
default:
continue
}
if files[container] != nil && files[entry] != nil {
nested.container[entry] = container
}
}
for _, f := range doc.Files {
l := toSyftLocation(f)
spdxIDMap[string(f.FileSPDXIdentifier)] = l
for id, f := range files {
l := file.NewLocationFromCoordinates(nested.coordinates(id))
spdxIDMap[string(id)] = l
s.Artifacts.FileMetadata[l.Coordinates] = toFileMetadata(f)
s.Artifacts.FileDigests[l.Coordinates] = toFileDigests(f)
}
}
// nestedFiles resolves the coordinates of files within archives, where an archive file CONTAINS a file
// named by its path within the archive.
type nestedFiles[K comparable] struct {
own map[K]file.Coordinates
container map[K]K
resolved map[K]file.Coordinates
}
func newNestedFiles[K comparable]() nestedFiles[K] {
return nestedFiles[K]{own: map[K]file.Coordinates{}, container: map[K]K{}, resolved: map[K]file.Coordinates{}}
}
func (n nestedFiles[K]) coordinates(id K) file.Coordinates {
if c, ok := n.resolved[id]; ok {
return c
}
c := n.own[id]
if container, ok := n.container[id]; ok {
// a containment cycle ends where it started
delete(n.container, id)
archive := n.coordinates(container)
c = file.Coordinates{
RealPath: archive.RealPath,
FileSystemID: archive.FileSystemID,
FullPath: archive.FullNestedPath() + file.ArchivePathSeparator + strings.TrimPrefix(c.RealPath, "/"),
}
}
n.resolved[id] = c
return c
}
func toFileDigests(f *spdx.File) (digests []file.Digest) {
for _, digest := range f.Checksums {
digests = append(digests, file.Digest{
@@ -452,6 +501,10 @@ func collectDocRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out
}
a := spdxIDMap[string(r.RefA.ElementRefID)]
b := spdxIDMap[string(r.RefB.ElementRefID)]
if rel, ok := fileContainsRelationship(helpers.RelationshipType(r.Relationship), a, b); ok {
out = append(out, rel)
continue
}
from, fromOk := a.(pkg.Package)
toPackage, toPackageOk := b.(pkg.Package)
toLocation, toLocationOk := b.(file.Location)
@@ -505,6 +558,31 @@ func collectDocRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out
return out
}
// fileContainsRelationship decodes an archive file containing a file or package within it.
func fileContainsRelationship(typ helpers.RelationshipType, a, b any) (artifact.Relationship, bool) {
switch typ {
case helpers.ContainsRelationship:
case helpers.ContainedByRelationship:
a, b = b, a
default:
return artifact.Relationship{}, false
}
container, ok := a.(file.Location)
if !ok {
return artifact.Relationship{}, false
}
var entry artifact.Identifiable
switch to := b.(type) {
case file.Location:
entry = to.Coordinates
case pkg.Package:
entry = to
default:
return artifact.Relationship{}, false
}
return artifact.Relationship{From: container.Coordinates, To: entry, Type: artifact.ContainsRelationship}, true
}
// collectPackageFileRelationships add relationships for direct files
func collectPackageFileRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out []artifact.Relationship) {
for _, p := range doc.Packages {
@@ -533,27 +611,15 @@ func toSyftCoordinates(f *spdx.File) file.Coordinates {
return coordinatesFromFile(f.FileName, f.FileComment, string(f.FileSPDXIdentifier))
}
// coordinatesFromFile decodes what fileComment encodes.
func coordinatesFromFile(name, comment, id string) file.Coordinates {
c := file.NewCoordinates(name, "")
for _, line := range strings.Split(comment, "\n") {
if v, ok := strings.CutPrefix(line, layerIDCommentPrefix); ok {
c.FileSystemID = v
}
if v, ok := strings.CutPrefix(line, archivePathCommentPrefix); ok {
// the name is the full path; the real path is what precedes the entry chain
c.RealPath = strings.TrimSuffix(name, file.ArchivePathSeparator+v)
c.FullPath = name
}
if v, ok := strings.CutPrefix(comment, layerIDCommentPrefix); ok {
c.FileSystemID = v
}
if v, ok := strings.CutPrefix(id, layerIDCommentPrefix); ok {
c.FileSystemID = v
}
return internal.NormalizeCoordinates(c)
}
func toSyftLocation(f *spdx.File) file.Location {
return file.NewLocationFromCoordinates(toSyftCoordinates(f))
return c
}
func requireAndTrimPrefix(val any, prefix string) string {
@@ -344,7 +344,10 @@ func Test_toSyftRelationships(t *testing.T) {
}
pkg3.SetID()
loc1 := file.NewLocationFromCoordinates(file.NewCoordinates("/somewhere/real", "abc"))
loc1 := file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
})
tests := []struct {
name string
@@ -691,8 +694,11 @@ func Test_directPackageFiles(t *testing.T) {
p.OverrideID("1") // the same as the spdxID on the package element
f := file.Location{
LocationData: file.LocationData{
Coordinates: file.NewCoordinates("some-file", ""),
AccessPath: "some-file",
Coordinates: file.Coordinates{
RealPath: "some-file",
FileSystemID: "",
},
AccessPath: "some-file",
},
LocationMetadata: file.LocationMetadata{
Annotations: map[string]string{},
@@ -789,12 +795,3 @@ func Test_skipsPackagesWithGeneratedFromRelationship(t *testing.T) {
assert.NotNil(t, s.Artifacts.Packages.Package("1"))
assert.Nil(t, s.Artifacts.Packages.Package("1-src"))
}
func Test_coordinatesFromFile_nonCanonicalChainIsDropped(t *testing.T) {
// a chain the name doesn't end with leaves FullPath equal to RealPath, which must decode as a plain file
got := coordinatesFromFile("/app.jar", "archivePath: other.jar!/x", "")
assert.Equal(t, file.NewCoordinates("/app.jar", ""), got)
nested := coordinatesFromFile("/app.war!/lib/a.jar", "archivePath: lib/a.jar", "")
assert.Equal(t, file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/lib/a.jar"}, nested)
}
@@ -283,20 +283,42 @@ func v3collectSyftPackages(s *sbom.SBOM, spdxMap ptrMap[any], relationships ptrM
}
func v3collectSyftFiles(s *sbom.SBOM, spdxMap ptrMap[any], doc *spdx.Document) {
found := ptrMap[struct{}]{}
nested := newNestedFiles[reflect.Value]()
var files []spdx.AnyFile
for _, elementList := range []spdx.ElementList{doc.Elements, doc.RootElements} {
for _, f := range elementList.Files() {
if found.Has(f) {
if _, found := nested.own[ptrTo(f)]; found {
continue
}
found.Set(f, struct{}{})
l := v3toSyftLocation(f)
spdxMap.Set(f, l)
s.Artifacts.FileMetadata[l.Coordinates] = v3toFileMetadata(f)
s.Artifacts.FileDigests[l.Coordinates] = v3toFileDigests(f)
nested.own[ptrTo(f)] = v3toSyftCoordinates(f)
files = append(files, f)
}
}
for _, r := range doc.Elements.Relationships() {
container, ok := r.GetFrom().(spdx.AnyFile)
if !ok || r.GetType() != spdx.RelationshipType_Contains {
continue
}
for _, to := range r.GetTo() {
entry, ok := to.(spdx.AnyFile)
if !ok {
continue
}
_, knownContainer := nested.own[ptrTo(container)]
_, knownEntry := nested.own[ptrTo(entry)]
if knownContainer && knownEntry {
nested.container[ptrTo(entry)] = ptrTo(container)
}
}
}
for _, f := range files {
l := file.NewLocationFromCoordinates(nested.coordinates(ptrTo(f)))
spdxMap.Set(f, l)
s.Artifacts.FileMetadata[l.Coordinates] = v3toFileMetadata(f)
s.Artifacts.FileDigests[l.Coordinates] = v3toFileDigests(f)
}
}
func v3toFileDigests(f spdx.AnyFile) (digests []file.Digest) {
@@ -341,6 +363,12 @@ func v3collectDocRelationships(spdxMap ptrMap[any], doc *spdx.Document) (out []a
for _, to := range r.GetTo() {
b := spdxMap.Get(to)
if r.GetType() == spdx.RelationshipType_Contains {
if rel, ok := fileContainsRelationship(helpers.ContainsRelationship, a, b); ok {
out = append(out, rel)
continue
}
}
from, fromOk := a.(pkg.Package)
toPackage, toPackageOk := b.(pkg.Package)
toLocation, toLocationOk := b.(file.Location)
@@ -396,11 +424,6 @@ func v3toSyftCoordinates(f spdx.AnyFile) file.Coordinates {
return coordinatesFromFile(f.GetName(), f.GetComment(), f.GetID())
}
func v3toSyftLocation(f spdx.AnyFile) file.Location {
l := file.NewVirtualLocationFromCoordinates(v3toSyftCoordinates(f), f.GetName())
return l
}
func v3findQualifierValue(purl packageurl.PackageURL, qualifier string) string {
for _, q := range purl.Qualifiers {
if q.Key == qualifier {
@@ -434,3 +434,44 @@ func Test_v3parseSPDXLicenses(t *testing.T) {
require.Len(t, declared, 1)
assert.Equal(t, "Apache-2.0 AND BSD-3-Clause", declared[0].Value)
}
func TestToSyftModelV3_nestedArchiveFiles(t *testing.T) {
war := &spdx.File{ID: "war", Name: "/app.war", Comment: "layerID: sha256:layer"}
jar := &spdx.File{ID: "jar", Name: "WEB-INF/lib/a.jar"}
manifest := &spdx.File{ID: "manifest", Name: "META-INF/MANIFEST.MF"}
p := &spdx.Package{ID: "pkg", Name: "a"}
warContainsJar := &spdx.Relationship{From: war, To: spdx.ElementList{jar}, Type: spdx.RelationshipType_Contains}
jarContains := &spdx.Relationship{From: jar, To: spdx.ElementList{manifest, p}, Type: spdx.RelationshipType_Contains}
doc := &spdx.Document{SpdxDocument: spdx.SpdxDocument{
Elements: spdx.ElementList{war, jar, manifest, p, warContainsJar, jarContains},
}}
s, err := ToSyftModelV3(doc)
require.NoError(t, err)
wantWar := file.NewCoordinates("/app.war", "sha256:layer")
wantJar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar", FileSystemID: "sha256:layer"}
wantManifest := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", FileSystemID: "sha256:layer"}
var got []file.Coordinates
for c := range s.Artifacts.FileMetadata {
got = append(got, c)
}
assert.ElementsMatch(t, []file.Coordinates{wantWar, wantJar, wantManifest}, got)
var fileRels []string
for _, r := range s.Relationships {
if from, ok := r.From.(file.Coordinates); ok {
to := "package"
if c, ok := r.To.(file.Coordinates); ok {
to = c.FullNestedPath()
}
fileRels = append(fileRels, from.FullNestedPath()+" "+string(r.Type)+" "+to)
}
}
assert.ElementsMatch(t, []string{
"/app.war contains /app.war!/WEB-INF/lib/a.jar",
"/app.war!/WEB-INF/lib/a.jar contains /app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF",
"/app.war!/WEB-INF/lib/a.jar contains package",
}, fileRels)
}
+41
View File
@@ -0,0 +1,41 @@
package internal
import (
"slices"
"strings"
"github.com/anchore/syft/syft/file"
)
// ArchiveEntries splits coordinates into those not within an archive and, for each archive, the files
// directly within it, each sorted. An archive not among coordinates is added, so every file within an
// archive has its archive to be described under.
func ArchiveEntries(coordinates []file.Coordinates, sorter func(a, b file.Coordinates) int) (top []file.Coordinates, entries map[file.Coordinates][]file.Coordinates) {
present := file.NewCoordinateSet(coordinates...)
entries = map[file.Coordinates][]file.Coordinates{}
for _, c := range coordinates {
for {
// RealPath is cut off first, so a "!/" in it is not taken for an archive boundary
rest, ok := strings.CutPrefix(c.FullPath, c.RealPath)
if !ok || !strings.HasPrefix(rest, file.ArchivePathSeparator) {
top = append(top, c)
break
}
archive := file.Coordinates{RealPath: c.RealPath, FileSystemID: c.FileSystemID}
if i := strings.LastIndex(rest, file.ArchivePathSeparator); i > 0 {
archive.FullPath = c.FullPath[:len(c.RealPath)+i]
}
entries[archive] = append(entries[archive], c)
if present.Contains(archive) {
break
}
present.Add(archive)
c = archive
}
}
slices.SortFunc(top, sorter)
for _, e := range entries {
slices.SortFunc(e, sorter)
}
return top, entries
}
@@ -0,0 +1,29 @@
package internal
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/file"
)
func TestArchiveEntries(t *testing.T) {
top := file.NewCoordinates("/etc/passwd", "layer")
odd := file.NewCoordinates("/odd!/name", "layer") // a separator in a real path is not an archive boundary
war := file.NewCoordinates("/app.war", "layer")
jar := file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/WEB-INF/lib/a.jar"}
manifest := file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"}
index := file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/index.html"}
byPath := func(a, b file.Coordinates) int { return strings.Compare(a.FullNestedPath(), b.FullNestedPath()) }
root := file.Coordinates{} // a directory scan's root
gotTop, gotEntries := ArchiveEntries([]file.Coordinates{top, root, odd, manifest, index}, byPath)
assert.Equal(t, []file.Coordinates{root, war, top, odd}, gotTop)
assert.Equal(t, map[file.Coordinates][]file.Coordinates{
war: {jar, index},
jar: {manifest},
}, gotEntries)
}
-25
View File
@@ -1,25 +0,0 @@
package internal
import (
"strings"
"github.com/anchore/syft/syft/file"
)
// NormalizeCoordinates clears a decoded FullPath that adds no archive chain to RealPath (one equal to
// RealPath, or one not starting with RealPath and "!/"), so decoded coordinates compare equal to the
// ones a resolver builds.
func NormalizeCoordinates(c file.Coordinates) file.Coordinates {
if !strings.HasPrefix(c.FullPath, c.RealPath+file.ArchivePathSeparator) {
c.FullPath = ""
}
return c
}
// NormalizeLocations applies NormalizeCoordinates to each location in place.
func NormalizeLocations(locations []file.Location) []file.Location {
for i := range locations {
locations[i].Coordinates = NormalizeCoordinates(locations[i].Coordinates)
}
return locations
}
@@ -2,10 +2,12 @@ package helpers
import (
"fmt"
"strings"
"github.com/CycloneDX/cyclonedx-go"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
@@ -32,11 +34,62 @@ func ToSyftModel(bom *cyclonedx.BOM) (*sbom.SBOM, error) {
return nil, err
}
if bom.Components != nil {
collectFiles(*bom.Components, nil, s)
}
collectRelationships(bom, s, idMap)
return s, nil
}
// collectFiles decodes file components, named relative to the scan root; a file component nested in
// another is an entry of that archive.
func collectFiles(components []cyclonedx.Component, container *file.Coordinates, s *sbom.SBOM) {
for i := range components {
component := &components[i]
var coordinates *file.Coordinates
if component.Type == cyclonedx.ComponentTypeFile {
c := file.NewCoordinates("/"+strings.TrimPrefix(component.Name, "/"), "")
if container != nil {
c = file.Coordinates{
RealPath: container.RealPath,
FileSystemID: container.FileSystemID,
FullPath: container.FullNestedPath() + file.ArchivePathSeparator + strings.TrimPrefix(component.Name, "/"),
}
}
coordinates = &c
if s.Artifacts.FileMetadata == nil {
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
}
s.Artifacts.FileMetadata[c] = file.Metadata{}
if digests := toDigests(component.Hashes); len(digests) > 0 {
if s.Artifacts.FileDigests == nil {
s.Artifacts.FileDigests = map[file.Coordinates][]file.Digest{}
}
s.Artifacts.FileDigests[c] = digests
}
}
if component.Components != nil {
collectFiles(*component.Components, coordinates, s)
}
}
}
func toDigests(hashes *[]cyclonedx.Hash) []file.Digest {
if hashes == nil {
return nil
}
var digests []file.Digest
for _, h := range *hashes {
digests = append(digests, file.Digest{
Algorithm: strings.ReplaceAll(strings.ToLower(string(h.Algorithm)), "-", ""),
Value: h.Value,
})
}
return digests
}
func collectBomPackages(bom *cyclonedx.BOM, s *sbom.SBOM, idMap map[string]any) error {
componentsPresent := false
if bom.Components != nil {
+29
View File
@@ -9,6 +9,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
stfile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/format/internal/testutil"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
@@ -227,3 +229,30 @@ func TestEncodeDecodeSourceWithoutName(t *testing.T) {
}
}
}
func TestEncodeDecodeRootDirectoryFile(t *testing.T) {
// a directory scan with all file metadata describes the scanned root, whose relative path is empty
root := file.NewCoordinates("", "")
etc := file.NewCoordinates("/etc", "")
for _, enc := range defaultFormatEncoders() {
t.Run(enc.Version(), func(t *testing.T) {
var buf bytes.Buffer
require.NoError(t, enc.Encode(&buf, sbom.SBOM{
Source: source.Description{Metadata: source.DirectoryMetadata{Path: "src"}},
Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{
root: {Type: stfile.TypeDirectory},
etc: {Type: stfile.TypeDirectory},
}},
}))
s, _, _, err := NewFormatDecoder().Decode(bytes.NewReader(buf.Bytes()))
require.NoError(t, err, "encoded document:\n%s", buf.String())
var names []string
for c := range s.Artifacts.FileMetadata {
names = append(names, c.RealPath)
}
assert.ElementsMatch(t, []string{".", "etc"}, names)
})
}
}
+4 -4
View File
@@ -80,7 +80,7 @@ func toSyftFiles(files []model.File) sbom.Artifacts {
}
for _, f := range files {
coord := internal.NormalizeCoordinates(f.Location)
coord := f.Location
if f.Metadata != nil {
fm, err := safeFileModeConvert(f.Metadata.Mode)
if err != nil {
@@ -163,7 +163,7 @@ func toSyftLicenses(m []model.License) (p []pkg.License) {
SPDXExpression: l.SPDXExpression,
Type: l.Type,
URLs: l.URLs,
Locations: file.NewLocationSet(internal.NormalizeLocations(l.Locations)...),
Locations: file.NewLocationSet(l.Locations...),
Contents: l.Contents,
})
}
@@ -239,7 +239,7 @@ func toSyftRelationships(doc *model.Document, catalog *pkg.Collection, relations
idMap[doc.Source.ID] = toSyftSource(doc.Source)
for _, f := range doc.Files {
idMap[f.ID] = internal.NormalizeCoordinates(f.Location)
idMap[f.ID] = f.Location
}
var out []artifact.Relationship
@@ -346,7 +346,7 @@ func toSyftPackage(p model.Package, idAliases map[string]string) pkg.Package {
Name: p.Name,
Version: p.Version,
FoundBy: p.FoundBy,
Locations: file.NewLocationSet(internal.NormalizeLocations(p.Locations)...),
Locations: file.NewLocationSet(p.Locations...),
Licenses: pkg.NewLicenseSet(toSyftLicenses(p.Licenses)...),
Language: p.Language,
Type: p.Type,
@@ -599,35 +599,3 @@ func Test_safeFileModeConvert(t *testing.T) {
})
}
}
func Test_toSyftModel_fullPathIsNormalized(t *testing.T) {
plain := file.NewCoordinates("/app.jar", "")
nested := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/lib/a.jar"}
decoded := []file.Coordinates{
{RealPath: "/app.jar", FullPath: "/app.jar"},
{RealPath: "/app.jar", FullPath: "/elsewhere!/app.jar"},
nested,
}
want := []file.Coordinates{plain, plain, nested}
var files []model.File
for _, c := range decoded {
files = append(files, model.File{ID: string(c.ID()), Location: c, Unknowns: []string{"x"}})
}
got := toSyftFiles(files)
for _, c := range want {
assert.Contains(t, got.Unknowns, c)
}
assert.Len(t, got.Unknowns, 2)
var locations []file.Location
for _, c := range decoded {
locations = append(locations, file.NewLocationFromCoordinates(c))
}
p := toSyftPackage(model.Package{PackageBasicData: model.PackageBasicData{Locations: locations}}, nil)
var gotCoordinates []file.Coordinates
for _, l := range p.Locations.ToSlice() {
gotCoordinates = append(gotCoordinates, l.Coordinates)
}
assert.ElementsMatch(t, []file.Coordinates{plain, plain, nested}, gotCoordinates)
}