From 851dc327c5536f056bcce1111844e9fca69b82f1 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 27 Oct 2026 14:52:18 -0400 Subject: [PATCH] fix: file selection within archives, spdx / cyclonedx file representation Signed-off-by: Keith Zantow --- cmd/syft/internal/options/catalog.go | 1 + internal/archive/resolver.go | 13 +- internal/archive/traversal.go | 19 +- internal/task/archive_nesting_test.go | 11 +- internal/task/archive_tasks.go | 95 ++++------ internal/task/archive_tasks_test.go | 32 ++++ internal/task/package_task_factory.go | 13 +- internal/task/package_task_factory_test.go | 29 +++ internal/task/unknowns_tasks.go | 10 +- internal/task/unknowns_tasks_test.go | 52 ++++++ syft/archive_cataloger_test.go | 32 ++++ syft/create_sbom_config.go | 5 + .../cyclonedxhelpers/to_format_model.go | 81 ++++---- .../cyclonedxhelpers/to_format_model_test.go | 62 +++++++ .../common/spdxhelpers/to_format_model.go | 174 ++++++++++-------- .../spdxhelpers/to_format_model_test.go | 138 +++++++++++--- .../common/spdxhelpers/to_syft_model.go | 110 ++++++++--- .../common/spdxhelpers/to_syft_model_test.go | 21 +-- .../common/spdxhelpers/to_syft_model_v3.go | 49 +++-- .../spdxhelpers/to_syft_model_v3_test.go | 41 +++++ syft/format/internal/archive_entries.go | 41 +++++ syft/format/internal/archive_entries_test.go | 29 +++ syft/format/internal/coordinates.go | 25 --- .../internal/cyclonedxutil/helpers/decoder.go | 53 ++++++ syft/format/spdxtagvalue/encoder_test.go | 29 +++ syft/format/syftjson/to_syft_model.go | 8 +- syft/format/syftjson/to_syft_model_test.go | 32 ---- 27 files changed, 865 insertions(+), 340 deletions(-) create mode 100644 syft/format/internal/archive_entries.go create mode 100644 syft/format/internal/archive_entries_test.go delete mode 100644 syft/format/internal/coordinates.go diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index 49ffd7c89..4fe7cbff0 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -146,6 +146,7 @@ func (cfg Catalog) ToComplianceConfig() cataloging.ComplianceConfig { func (cfg Catalog) ToUnknownsConfig() cataloging.UnknownsConfig { return cataloging.UnknownsConfig{ + RemoveWhenPackagesDefined: cfg.Unknowns.RemoveWhenPackagesDefined, IncludeExecutablesWithoutPackages: cfg.Unknowns.ExecutablesWithoutPackages, IncludeUnexpandedArchives: cfg.Unknowns.UnexpandedArchives, } diff --git a/internal/archive/resolver.go b/internal/archive/resolver.go index 0a6b1a699..c69b16b6b 100644 --- a/internal/archive/resolver.go +++ b/internal/archive/resolver.go @@ -273,8 +273,8 @@ func (r *Resolver) fullPath(rooted string) string { } // rootedPath is the tree path a query names, which is either form a path takes: a full path is trimmed to -// the entry path within this archive, and anything else (a LocalPath) is taken as archive-relative -// already. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root) +// the entry path within this archive, and anything else (a LocalPath, with or without its leading slash) +// is taken as archive-relative already, since the tree looks a relative path up from its root. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root) // names the root. A full path into another archive in the scan names nothing here and reports false. func (r *Resolver) rootedPath(userPath string) (string, bool) { if rest, ok := strings.CutPrefix(userPath, r.archiveCoordinates.FullNestedPath()); ok { @@ -289,9 +289,6 @@ func (r *Resolver) rootedPath(userPath string) (string, bool) { if r.outside(userPath) { return "", false } - if !strings.HasPrefix(userPath, "/") { - userPath = "/" + userPath - } return path.Clean(userPath), true } @@ -420,10 +417,10 @@ func (r *Resolver) FilesByGlob(patterns ...string) ([]file.Location, error) { } // searchByGlob is the index search, except that stereoscope answers "/*" by looking the directory -// up by name and the root has none, so "/*" walks the tree instead. +// up by name and the root has none, so "/*" (or "*") walks the tree instead. func (r *Resolver) searchByGlob(pattern string) ([]stereoscopeFile.Resolution, error) { - if pattern == "/*" { - return r.tree.FilesByGlob(pattern, filetree.FollowBasenameLinks) + if strings.TrimPrefix(pattern, "/") == "*" { + return r.tree.FilesByGlob("/*", filetree.FollowBasenameLinks) } return r.search.SearchByGlob(pattern, filetree.FollowBasenameLinks) } diff --git a/internal/archive/traversal.go b/internal/archive/traversal.go index 10aa85b1e..dd79d126a 100644 --- a/internal/archive/traversal.go +++ b/internal/archive/traversal.go @@ -38,11 +38,8 @@ func FullPathIn(containerPath, entryPath string) string { if containerPath == "" { return entryPath } - if !strings.HasPrefix(entryPath, "/") { - entryPath = "/" + entryPath - } // cleaning an already clean path allocates nothing - return containerPath + file.ArchivePathSeparator + path.Clean(entryPath)[1:] + return containerPath + file.ArchivePathSeparator + strings.TrimPrefix(path.Clean(entryPath), "/") } // OwnerContainers are the containers a package owns files within: the container of each of its @@ -100,6 +97,20 @@ func ContainerPath(c file.Coordinates) string { return c.RealPath } +// ContainingArchive returns the coordinates of the archive directly holding the file at c, reporting false +// for a file on the scanned filesystem. +func ContainingArchive(c file.Coordinates) (file.Coordinates, bool) { + containerPath := ContainerPath(c) + if containerPath == "" { + return file.Coordinates{}, false + } + container := file.Coordinates{RealPath: c.RealPath, FileSystemID: c.FileSystemID} + if containerPath != c.RealPath { + container.FullPath = containerPath + } + return container, true +} + // archiveEntries is the "!/"-delimited chain of entry paths after RealPath, reporting false for a file // not inside an archive. RealPath is split off first, so a "!/" in it is not taken for a boundary. func archiveEntries(c file.Coordinates) (string, bool) { diff --git a/internal/task/archive_nesting_test.go b/internal/task/archive_nesting_test.go index 16952b6dd..314ef1d8c 100644 --- a/internal/task/archive_nesting_test.go +++ b/internal/task/archive_nesting_test.go @@ -141,7 +141,7 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) { scanDir := t.TempDir() nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()}) - s := runNesting(t, scanDir, 3, defaultLimits(), markerTask()) + s := withArchiveContains(t, runNesting(t, scanDir, 3, defaultLimits(), markerTask())) // each archive's coordinates as seen in its parent; the outermost sits in the scanned // directory, so it carries no filesystem id of its own @@ -178,7 +178,7 @@ func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) { "expected a CONTAINS edge from %s to the leaf package", nested.paths[len(nested.paths)-1]) // with file cataloging on, files inside an archive get an edge from it too - s = runNesting(t, scanDir, 3, defaultLimits(), markerTask(), fileMetadataTask()) + s = withArchiveContains(t, runNesting(t, scanDir, 3, defaultLimits(), markerTask(), fileMetadataTask())) var leafEdge bool for _, rel := range s.Relationships { from, _ := rel.From.(file.Coordinates) @@ -379,6 +379,13 @@ func runNesting(t *testing.T, scanDir string, depth int, bounds archive.Limits, } } +// withArchiveContains runs the task that follows every scan with archive cataloging on. +func withArchiveContains(t *testing.T, s *sbom.SBOM) *sbom.SBOM { + t.Helper() + require.NoError(t, NewArchiveContainsTask().Execute(context.Background(), nil, sbomsync.NewBuilder(s))) + return s +} + // markerTask emits one package per "marker.txt" found, so the matrix asserts behavior without // depending on any real cataloger's fixture format. func markerTask() Task { diff --git a/internal/task/archive_tasks.go b/internal/task/archive_tasks.go index c4b87c69e..aebc7f90b 100644 --- a/internal/task/archive_tasks.go +++ b/internal/task/archive_tasks.go @@ -32,7 +32,7 @@ const ArchiveCatalogerTaskName = "archive-cataloger" // NewArchiveCatalogerTask returns a task that extracts every archive it finds, runs the same tasks // against the archive's contents as a standalone filesystem, and records -// CONTAINS relationships from the archive to what was found inside it. It recurses into the archives +// CONTAINS relationships from the archive to the packages found directly inside it. It recurses into the archives // it extracts up to cfg.MaxDepth; the task drops itself from the tasks so that is the only recursion. // // Each archive is handled as the scan root is: pkgTasks, then the archives nested in it, then fileTasks, @@ -267,12 +267,14 @@ func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver fi // as at the scan root, nested archives are cataloged before this archive's files, so the packages // describing them (located in this archive) select their files here - children := c.discoverArchives(extracted) - errs := c.catalogCandidates(ctx, extracted, children, depth+1, foundBuilder) + errs := c.catalogCandidates(ctx, extracted, c.discoverArchives(extracted), depth+1, foundBuilder) started = time.Now() runArchiveTasks(ctx, c.fileTasks, extracted, location.Coordinates, foundBuilder) - addContains(location.Coordinates, archivePath, direct, children, found) + // the files within it are related once the scan has decided which it describes (see NewArchiveContainsTask) + for _, p := range direct { + found.Relationships = append(found.Relationships, artifact.Relationship{From: location.Coordinates, To: p, Type: artifact.ContainsRelationship}) + } mergeArchiveResults(found, builder) c.progress.AtomicStage.Set(humanize.Comma(c.progress.Current()) + " archives (" + archivePath + ")") @@ -440,64 +442,41 @@ func mergeArchiveResults(found *sbom.SBOM, builder sbomsync.Builder) { } } -// addContains adds a CONTAINS relationship from the archive to each package found directly in it, each -// file directly in it that the SBOM describes, and each nested archive, so the hierarchy is navigable -// whatever file selection is configured. -// -// Files directly in the archive are those whose container is the archive's own full path, which leaves -// out the archive itself (a java package or an unknown recorded at its coordinates would otherwise -// become a self-loop) and the files of nested archives, contained by those. -func addContains(archiveCoordinates file.Coordinates, archivePath string, direct []pkg.Package, children []archiveCandidate, found *sbom.SBOM) { - contained := file.NewCoordinateSet() - for _, child := range children { - contained.Add(child.location.Coordinates) +// NewArchiveContainsTask relates each archive to every file the SBOM describes directly within it. It must +// run after every other task, so the files within archives are those the scan root's configuration +// selects (file selection, unknowns, package evidence) and no others. +func NewArchiveContainsTask() Task { + fn := func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error { + var rels []artifact.Relationship + builder.(sbomsync.Accessor).ReadFromSBOM(func(s *sbom.SBOM) { + rels = archiveContains(s.AllCoordinates()) + }) + builder.AddRelationships(rels...) + return nil } - addInside := func(coordinates file.Coordinates) { - if archive.ContainerPath(coordinates) == archivePath { - contained.Add(coordinates) - } - } - eachFileArtifactCoordinates(&found.Artifacts, addInside) - for _, r := range found.Relationships { - if c, ok := r.From.(file.Coordinates); ok { - addInside(c) - } - if c, ok := r.To.(file.Coordinates); ok { - addInside(c) - } - } - - var rels []artifact.Relationship - for _, p := range direct { - rels = append(rels, artifact.Relationship{From: archiveCoordinates, To: p, Type: artifact.ContainsRelationship}) - } - for _, coordinates := range contained.ToSlice() { - rels = append(rels, artifact.Relationship{From: archiveCoordinates, To: coordinates, Type: artifact.ContainsRelationship}) - } - found.Relationships = append(found.Relationships, rels...) + return NewTask("archive-contains", fn) } -// eachFileArtifactCoordinates calls fn with the coordinates of every file artifact, once per artifact -// kind describing it. -func eachFileArtifactCoordinates(a *sbom.Artifacts, fn func(file.Coordinates)) { - for c := range a.FileMetadata { - fn(c) - } - for c := range a.FileDigests { - fn(c) - } - for c := range a.FileContents { - fn(c) - } - for c := range a.FileLicenses { - fn(c) - } - for c := range a.Executables { - fn(c) - } - for c := range a.Unknowns { - fn(c) +// archiveContains relates the archive holding each file within one to that file, and each such archive to +// the archive holding it, so an archive the SBOM does not otherwise describe still links what is within it. +func archiveContains(coordinates []file.Coordinates) []artifact.Relationship { + related := file.NewCoordinateSet(coordinates...) + var rels []artifact.Relationship + for _, c := range coordinates { + for { + container, ok := archive.ContainingArchive(c) + if !ok { + break + } + rels = append(rels, artifact.Relationship{From: container, To: c, Type: artifact.ContainsRelationship}) + if related.Contains(container) { + break + } + related.Add(container) + c = container + } } + return rels } func mergeFileArtifacts(dst, src *sbom.Artifacts) { diff --git a/internal/task/archive_tasks_test.go b/internal/task/archive_tasks_test.go index e4932b835..53cbe905b 100644 --- a/internal/task/archive_tasks_test.go +++ b/internal/task/archive_tasks_test.go @@ -337,6 +337,7 @@ func Test_archiveCataloger_noSelfContainsEdge(t *testing.T) { s := newTestSBOM() s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} _ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)) + withArchiveContains(t, s) require.Contains(t, s.Artifacts.Unknowns, file.NewCoordinates("/x.jar", ""), "the fixture must reproduce the unknown") var contains int @@ -350,6 +351,37 @@ func Test_archiveCataloger_noSelfContainsEdge(t *testing.T) { assert.Positive(t, contains, "the archive still contains its files") } +func Test_NewArchiveContainsTask(t *testing.T) { + // a file within an archive is contained only when the SBOM describes it, as a file at the scan root is + // listed only then; an executable no package owns is recorded whatever the file selection + in := func(fullPath string) file.Coordinates { + return file.Coordinates{RealPath: "/outer.tar", FileSystemID: "layer", FullPath: fullPath} + } + outer := file.NewCoordinates("/outer.tar", "layer") + jar := in("/outer.tar!/lib/app.jar") + manifest := in("/outer.tar!/lib/app.jar!/META-INF/MANIFEST.MF") + unknownExe := in("/outer.tar!/bin/setup.exe") + + s := newTestSBOM() + s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{manifest: {}, file.NewCoordinates("/loose.txt", "layer"): {}} + s.Artifacts.Executables = map[file.Coordinates]file.Executable{in("/outer.tar!/bin/tool"): {}} + s.Artifacts.Unknowns = map[file.Coordinates][]string{unknownExe: {"no package identified in executable file"}} + + withArchiveContains(t, s) + + type edge struct{ from, to file.Coordinates } + var edges []edge + for _, r := range s.Relationships { + assert.Equal(t, artifact.ContainsRelationship, r.Type) + edges = append(edges, edge{r.From.(file.Coordinates), r.To.(file.Coordinates)}) + } + assert.ElementsMatch(t, []edge{ + {jar, manifest}, + {outer, jar}, // the jar is not described itself, but is on the way to what is + {outer, unknownExe}, + }, edges) +} + func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) { // a copy of itself in the sub-pipeline would process every nesting level twice cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2) diff --git a/internal/task/package_task_factory.go b/internal/task/package_task_factory.go index 26e0b3db3..6ba80554a 100644 --- a/internal/task/package_task_factory.go +++ b/internal/task/package_task_factory.go @@ -3,9 +3,11 @@ package task import ( "context" "fmt" + "path" "strings" "unicode" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/relationship" @@ -236,10 +238,10 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver locations := map[artifact.ID]file.Location{} - for _, path := range fileOwner.OwnedFiles() { - pathRefs, err := resolver.FilesByPath(path) + for _, ownedPath := range fileOwner.OwnedFiles() { + pathRefs, err := resolver.FilesByPath(ownedPath) if err != nil { - return nil, fmt.Errorf("unable to find path for path=%q: %w", path, err) + return nil, fmt.Errorf("unable to find path for path=%q: %w", ownedPath, err) } if len(pathRefs) == 0 { @@ -250,6 +252,11 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver } for _, ref := range pathRefs { + // an archive looks a relative path up from its root, but an owned path may be relative to + // anything (a python RECORD is relative to site-packages), so only an absolute one owns a file in one + if archive.ContainerPath(ref.Coordinates) != "" && !path.IsAbs(ownedPath) { + continue + } if oldRef, ok := locations[ref.ID()]; ok { log.Debugf("found path duplicate of %s", oldRef.FullNestedPath()) } diff --git a/internal/task/package_task_factory_test.go b/internal/task/package_task_factory_test.go index a13650782..04f7d7e11 100644 --- a/internal/task/package_task_factory_test.go +++ b/internal/task/package_task_factory_test.go @@ -1,11 +1,15 @@ package task import ( + "bytes" + "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/anchore/syft/internal/archive" + "github.com/anchore/syft/internal/testutils" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/cpe" @@ -281,3 +285,28 @@ func TestApplyComplianceRules_DropAndStub(t *testing.T) { assert.False(t, isCompliant) assert.Equal(t, cataloging.UnknownStubValue, p.Version) } + +func Test_packageFileOwnershipRelationships_exactPathsWithinAnArchive(t *testing.T) { + // a wheel's RECORD is relative to site-packages: looked up from the wheel's root it finds the file, but + // only an exact path owns one, as at the scan root + data := testutils.Zip(t, map[string]string{"pip/__init__.py": "", "bin/tool": ""}) + whl, err := archive.Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("/pip.whl", ""), + archive.NewLimiter(archive.Limits{MaxMemoryBytes: 1 << 20}), archive.NewExclusions(nil)) + require.NoError(t, err) + t.Cleanup(whl.Cleanup) + + found, err := whl.FilesByPath("pip/__init__.py") + require.NoError(t, err) + require.Len(t, found, 1, "a relative path still finds the file") + + p := pkg.Package{Name: "pip", Metadata: pkg.PythonPackage{Files: []pkg.PythonFileRecord{{Path: "pip/__init__.py"}, {Path: "/bin/tool"}}}} + p.SetID() + rels, err := packageFileOwnershipRelationships(p, whl) + require.NoError(t, err) + + var owned []string + for _, r := range rels { + owned = append(owned, r.To.(file.Coordinates).FullNestedPath()) + } + assert.Equal(t, []string{"/pip.whl!/bin/tool"}, owned) +} diff --git a/internal/task/unknowns_tasks.go b/internal/task/unknowns_tasks.go index b2730a312..fd874b73a 100644 --- a/internal/task/unknowns_tasks.go +++ b/internal/task/unknowns_tasks.go @@ -2,6 +2,7 @@ package task import ( "context" + "path" "slices" "strings" @@ -106,9 +107,12 @@ func coordinateReferenceLookup(resolver file.Resolver, s *sbom.SBOM) func(coords } } for _, ownedFilePath := range f.OwnedFiles() { - for _, container := range archives { - // an archive's contents are not in this resolver to follow links in - allPackageCoords.Add(archive.OwnedCoordinates(container, ownedFilePath)) + // an archive's contents are not in this resolver to follow links in, and only an absolute owned + // path names a file in one (see packageFileOwnershipRelationships) + if path.IsAbs(ownedFilePath) { + for _, container := range archives { + allPackageCoords.Add(archive.OwnedCoordinates(container, ownedFilePath)) + } } if !atRoot { continue diff --git a/internal/task/unknowns_tasks_test.go b/internal/task/unknowns_tasks_test.go index 2466ba044..12c0c74f7 100644 --- a/internal/task/unknowns_tasks_test.go +++ b/internal/task/unknowns_tasks_test.go @@ -5,6 +5,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -29,6 +30,57 @@ func Test_unknownsLabeler_archiveReasonsSurvivePackages(t *testing.T) { "a package describing the archive explains a parse failure, not contents left out") } +func Test_unknownsLabeler_archiveContentsAreReportedByFullPath(t *testing.T) { + // a package describing an archive does not account for the files within it + war := file.NewCoordinates("/app.war", "layer") + in := func(fullPath string) file.Coordinates { + return file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: fullPath} + } + nativeLib := in("/app.war!/WEB-INF/lib/jna.jar!/linux-x86-64/libjnidispatch.so") + unidentifiedJar := in("/app.war!/WEB-INF/lib/javax.inject-1.jar") + p := pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(war))} + p.SetID() + noPackage := formatUnknown("no package identified in archive", "java-archive-cataloger") + + s := &sbom.SBOM{ + Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(p), + Executables: map[file.Coordinates]file.Executable{nativeLib: {}}, + Unknowns: map[file.Coordinates][]string{unidentifiedJar: {noPackage}}, + }, + Relationships: []artifact.Relationship{{From: war, To: unidentifiedJar, Type: artifact.ContainsRelationship}}, + } + + unknownsLabelerTask{cataloging.UnknownsConfig{RemoveWhenPackagesDefined: true, IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths(), s) + + assert.Equal(t, map[file.Coordinates][]string{ + nativeLib: {formatUnknown("no package identified in executable file", unknownsLabelerTaskName)}, + unidentifiedJar: {noPackage}, + }, s.Artifacts.Unknowns) +} + +func Test_unknownsLabeler_relativeOwnedPathsNameNothingInAnArchive(t *testing.T) { + // a python RECORD lists paths relative to site-packages, which resolve nowhere at the scan root either + whl := func(fullPath string) file.Coordinates { + return file.Coordinates{RealPath: "/pip.whl", FullPath: fullPath} + } + tool := whl("/pip.whl!/pip/tool") + p := pkg.Package{ + Name: "pip", + Locations: file.NewLocationSet(file.NewLocationFromCoordinates(whl("/pip.whl!/pip.dist-info/METADATA"))), + Metadata: pkg.PythonPackage{Files: []pkg.PythonFileRecord{{Path: "pip/tool"}}}, + } + p.SetID() + s := &sbom.SBOM{Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(p), + Executables: map[file.Coordinates]file.Executable{tool: {}}, + }} + + unknownsLabelerTask{cataloging.UnknownsConfig{IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths(), s) + + assert.NotEmpty(t, s.Artifacts.Unknowns[tool]) +} + func Test_unknownsLabeler_ownedFilesAreWithinTheOwnersContainer(t *testing.T) { atRoot := file.NewCoordinates("/usr/bin/foo", "") inArchive := file.Coordinates{RealPath: "/rootfs.tar", FullPath: "/rootfs.tar!/usr/bin/foo"} diff --git a/syft/archive_cataloger_test.go b/syft/archive_cataloger_test.go index 74616f71e..e23399c97 100644 --- a/syft/archive_cataloger_test.go +++ b/syft/archive_cataloger_test.go @@ -184,6 +184,13 @@ func TestArchiveCataloger(t *testing.T) { }, containers, "the archive file, named in its own parent, contains the package") }) + t.Run("each file within an archive is contained by that archive", func(t *testing.T) { + containers := fileContainers(s) + assert.Equal(t, "app.zip", containers["app.zip!/nested/keep.txt"]) + assert.Equal(t, "outer.zip!/deeper/inner.zip", containers["outer.zip!/deeper/inner.zip!/nested/marker.txt"]) + assert.Equal(t, "outer.zip", containers["outer.zip!/deeper/inner.zip"]) + }) + t.Run("exclusions are the scan's own", func(t *testing.T) { // there is no archive-specific exclusion setting: a scan pattern's shape decides whether it // reaches inside an archive, and an excluded entry is absent from the archive's filesystem @@ -389,6 +396,8 @@ func TestArchiveCataloger_defaultFileSelectionCoversNestedJars(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "outer.zip"), testutils.Zip(t, map[string][]byte{ "lib/inner.jar": testutils.Jar(t, "inner", "1.0", nil), + "readme.txt": []byte("hello"), + "empty.zip": testutils.Zip(t, map[string]string{"a.txt": "hello"}), }), 0o644)) cfg := DefaultCreateSBOMConfig(). @@ -400,6 +409,16 @@ func TestArchiveCataloger_defaultFileSelectionCoversNestedJars(t *testing.T) { jar := file.Coordinates{RealPath: "outer.zip", FullPath: "outer.zip!/lib/inner.jar"} assert.Contains(t, s.Artifacts.FileMetadata, jar) assert.Contains(t, s.Artifacts.FileDigests, jar) + assert.Equal(t, "outer.zip", fileContainers(s)["outer.zip!/lib/inner.jar"]) + + // as at the root, a file no package owns is not described, nor is an archive in which nothing was found + var described []string + for _, c := range s.AllCoordinates() { + described = append(described, c.FullNestedPath()) + } + assert.NotContains(t, described, "outer.zip!/readme.txt") + assert.NotContains(t, described, "outer.zip!/empty.zip") + assert.NotContains(t, described, "outer.zip!/empty.zip!/a.txt") } func TestArchiveCataloger_archiveUnknowns(t *testing.T) { @@ -629,6 +648,19 @@ func (p *recordingBus) Publish(e partybus.Event) { p.events = append(p.events, e) } +// fileContainers maps the full path of each file an archive CONTAINS to the archive's full path. +func fileContainers(s *sbom.SBOM) map[string]string { + out := map[string]string{} + for _, r := range s.Relationships { + from, fromFile := r.From.(file.Coordinates) + to, toFile := r.To.(file.Coordinates) + if r.Type == artifact.ContainsRelationship && fromFile && toFile { + out[to.FullNestedPath()] = from.FullNestedPath() + } + } + return out +} + // scanDirWithExclusions scans with the given exclusion patterns configured on the source, where // exclusions live; archive cataloging has no exclusion setting of its own. func scanDirWithExclusions(t *testing.T, dir string, cfg *CreateSBOMConfig, exclude ...string) *sbom.SBOM { diff --git a/syft/create_sbom_config.go b/syft/create_sbom_config.go index 3df0f89f6..b3137f238 100644 --- a/syft/create_sbom_config.go +++ b/syft/create_sbom_config.go @@ -254,6 +254,11 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description, exclusions []s taskGroups = append(taskGroups, osFeatureDetectionTasks) } + // archives are related to the files within them only once every task has decided which files are described + if archiveTask != nil { + taskGroups = append(taskGroups, []task.Task{task.NewArchiveContainsTask()}) + } + // identifying the environment (i.e. the linux release) must be done first as this is required for package cataloging taskGroups = append( [][]task.Task{ diff --git a/syft/format/common/cyclonedxhelpers/to_format_model.go b/syft/format/common/cyclonedxhelpers/to_format_model.go index 7cae828c6..3c381c1e4 100644 --- a/syft/format/common/cyclonedxhelpers/to_format_model.go +++ b/syft/format/common/cyclonedxhelpers/to_format_model.go @@ -14,6 +14,7 @@ import ( "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/file" + formatInternal "github.com/anchore/syft/syft/format/internal" "github.com/anchore/syft/syft/format/internal/cyclonedxutil/helpers" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" @@ -42,7 +43,8 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM { cdxBOM.SerialNumber = uuid.New().URN() cdxBOM.Metadata = toBomDescriptor(s.Descriptor.Name, s.Descriptor.Version, s.Source) - coordinates, locationSorter := getCoordinates(s) + locationSorter, coordinateSorter := formatInternal.GetLocationSorters(s) + top, entries := formatInternal.ArchiveEntries(s.AllCoordinates(), coordinateSorter) // Packages packages := s.Artifacts.Packages.Sorted() @@ -52,40 +54,7 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM { } components = append(components, toOSComponent(s.Artifacts.LinuxDistribution)...) - artifacts := s.Artifacts - - for _, coordinate := range coordinates { - var metadata *file.Metadata - // File Info - fileMetadata, exists := artifacts.FileMetadata[coordinate] - // no file metadata then don't include in SBOM - // the syft config allows for sometimes only capturing files owned by packages - // so there can be a map miss here where we have less metadata than all coordinates - if !exists { - continue - } - if fileMetadata.Type == stfile.TypeDirectory || - fileMetadata.Type == stfile.TypeSocket || - fileMetadata.Type == stfile.TypeSymLink { - // skip dir, symlinks and sockets for the final bom - continue - } - metadata = &fileMetadata - - // Digests - var digests []file.Digest - if digestsForLocation, exists := artifacts.FileDigests[coordinate]; exists { - digests = digestsForLocation - } - - cdxHashes := digestsToHashes(digests) - components = append(components, cyclonedx.Component{ - BOMRef: string(coordinate.ID()), - Type: cyclonedx.ComponentTypeFile, - Name: metadata.Path, - Hashes: &cdxHashes, - }) - } + components = append(components, toFileComponents(s.Artifacts, entries, top)...) cdxBOM.Components = &components dependencies := toDependencies(s.Relationships) @@ -96,19 +65,41 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM { return cdxBOM } -func getCoordinates(s sbom.SBOM) ([]file.Coordinates, func(a, b file.Location) int) { - var layers []string - if m, ok := s.Source.Metadata.(source.ImageMetadata); ok { - for _, l := range m.Layers { - layers = append(layers, l.Digest) +// toFileComponents describes each of coordinates that is a regular file with metadata, named by its path +// (within its archive, for a file within one), with the files within it nested in its component. +func toFileComponents(artifacts sbom.Artifacts, entries map[file.Coordinates][]file.Coordinates, coordinates []file.Coordinates) []cyclonedx.Component { + var components []cyclonedx.Component + for _, coordinate := range coordinates { + nested := toFileComponents(artifacts, entries, entries[coordinate]) + // no file metadata then don't include in SBOM + // the syft config allows for sometimes only capturing files owned by packages + // so there can be a map miss here where we have less metadata than all coordinates + metadata, exists := artifacts.FileMetadata[coordinate] + if !isFileComponent(metadata, exists) && len(nested) == 0 { + continue } + + cdxHashes := digestsToHashes(artifacts.FileDigests[coordinate]) + component := cyclonedx.Component{ + BOMRef: string(coordinate.ID()), + Type: cyclonedx.ComponentTypeFile, + Name: coordinate.LocalPath(), + Hashes: &cdxHashes, + } + if len(nested) > 0 { + component.Components = &nested + } + components = append(components, component) } + return components +} - coordSorter := file.CoordinatesSorter(layers) - coordinates := s.AllCoordinates() - - slices.SortFunc(coordinates, coordSorter) - return coordinates, file.LocationSorter(layers) +// isFileComponent skips dirs, symlinks and sockets for the final bom +func isFileComponent(metadata file.Metadata, exists bool) bool { + return exists && + metadata.Type != stfile.TypeDirectory && + metadata.Type != stfile.TypeSocket && + metadata.Type != stfile.TypeSymLink } func digestsToHashes(digests []file.Digest) []cyclonedx.Hash { diff --git a/syft/format/common/cyclonedxhelpers/to_format_model_test.go b/syft/format/common/cyclonedxhelpers/to_format_model_test.go index 5f94b9b57..f0014edee 100644 --- a/syft/format/common/cyclonedxhelpers/to_format_model_test.go +++ b/syft/format/common/cyclonedxhelpers/to_format_model_test.go @@ -608,3 +608,65 @@ func Test_toOSBomRef(t *testing.T) { }) } } + +func Test_nestedArchiveFiles(t *testing.T) { + top := file.NewCoordinates("/etc/os-release", "") + inA := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"} + inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"} + inWar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/index.html"} + dir := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/c.jar!/META-INF"} + sha := []file.Digest{{Algorithm: "sha256", Value: "abc"}} + s := sbom.SBOM{Artifacts: sbom.Artifacts{ + FileMetadata: map[file.Coordinates]file.Metadata{ + // a directory resolver's metadata path is the path on the host, not the one scanned + top: {Path: "/host/scan/etc/os-release"}, + inA: {}, + inB: {}, + inWar: {}, + dir: {Type: stfile.TypeDirectory}, + }, + FileDigests: map[file.Coordinates][]file.Digest{inA: sha}, + }} + + bom := ToFormatModel(s) + + type node struct { + Name string + Children []node + } + var tree func(cs *[]cyclonedx.Component) []node + tree = func(cs *[]cyclonedx.Component) []node { + if cs == nil { + return nil + } + var out []node + for _, c := range *cs { + require.Equal(t, cyclonedx.ComponentTypeFile, c.Type) + out = append(out, node{Name: c.Name, Children: tree(c.Components)}) + } + return out + } + // each archive holding a file is a component, synthesized when not otherwise described; an archive + // holding only skipped files is left out + assert.Equal(t, []node{ + {Name: "/app.war", Children: []node{ + {Name: "/WEB-INF/lib/a.jar", Children: []node{{Name: "/META-INF/MANIFEST.MF"}}}, + {Name: "/WEB-INF/lib/b.jar", Children: []node{{Name: "/odd:na%me.txt"}}}, + {Name: "/index.html"}, + }}, + {Name: "/etc/os-release"}, + }, tree(bom.Components)) + + decoded, err := helpers.ToSyftModel(bom) + require.NoError(t, err) + + war := file.NewCoordinates("/app.war", "") + a := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar"} + b := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar"} + var got []file.Coordinates + for c := range decoded.Artifacts.FileMetadata { + got = append(got, c) + } + assert.ElementsMatch(t, []file.Coordinates{top, war, a, b, inA, inB, inWar}, got) + assert.Equal(t, map[file.Coordinates][]file.Digest{inA: sha}, decoded.Artifacts.FileDigests) +} diff --git a/syft/format/common/spdxhelpers/to_format_model.go b/syft/format/common/spdxhelpers/to_format_model.go index 445ca64b8..40963c047 100644 --- a/syft/format/common/spdxhelpers/to_format_model.go +++ b/syft/format/common/spdxhelpers/to_format_model.go @@ -6,7 +6,6 @@ import ( "crypto/sha1" "fmt" "path" - "slices" "sort" "strings" "time" @@ -53,7 +52,8 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document { rels := relationship.NewIndex(s.Relationships...) packages, otherLicenses := toPackages(rels, s.Artifacts.Packages, s) - allRelationships := toRelationships(rels.All()) + files, archiveRelationships := toFiles(s) + allRelationships := append(toRelationships(rels.All()), archiveRelationships...) // for valid SPDX we need a document describes relationship describesID := spdx.ElementID("DOCUMENT") @@ -62,8 +62,7 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document { if rootPackage != nil { describesID = rootPackage.PackageSPDXIdentifier - // add all relationships from the document root to all other packages - allRelationships = append(allRelationships, toRootRelationships(rootPackage, packages)...) + allRelationships = append(allRelationships, toRootRelationships(rootPackage, packages, files, allRelationships)...) // append the root package packages = append(packages, rootPackage) @@ -155,21 +154,40 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document { CreatorComment: "", }, Packages: packages, - Files: toFiles(s), + Files: files, Relationships: allRelationships, OtherLicenses: convertOtherLicense(otherLicenses), } } -func toRootRelationships(rootPackage *spdx.Package, packages []*spdx.Package) (out []*spdx.Relationship) { +// toRootRelationships relates the root package to each package and file nothing else CONTAINS: what is +// within an archive is reached only through the archive's file, and an owned file through its package. +func toRootRelationships(rootPackage *spdx.Package, packages []*spdx.Package, files []*spdx.File, relationships []*spdx.Relationship) (out []*spdx.Relationship) { + contained := map[spdx.ElementID]bool{} + for _, r := range relationships { + if r.Relationship == string(helpers.ContainsRelationship) { + contained[r.RefB.ElementRefID] = true + } + } + + var ids []spdx.ElementID for _, p := range packages { + ids = append(ids, p.PackageSPDXIdentifier) + } + for _, f := range files { + ids = append(ids, f.FileSPDXIdentifier) + } + for _, id := range ids { + if contained[id] { + continue + } out = append(out, &spdx.Relationship{ RefA: spdx.DocElementID{ ElementRefID: rootPackage.PackageSPDXIdentifier, }, Relationship: string(helpers.ContainsRelationship), RefB: spdx.DocElementID{ - ElementRefID: p.PackageSPDXIdentifier, + ElementRefID: id, }, }) } @@ -666,7 +684,7 @@ func toRelationships(relationships []artifact.Relationship) (result []*spdx.Rela // FIXME: we are only currently including Package -> * relationships, plus an archive containing a // package found inside it, which is the only structure nested archive cataloging records for // non-java packages. Every file-side From is in the document: toFiles covers every coordinate a - // relationship names. + // relationship names, and itself relates each archive to the files within it. if _, ok := r.From.(pkg.Package); !ok && !isFileContainsPackage(r) { log.Debugf("skipping non-package relationship: %+v", r) continue @@ -706,82 +724,34 @@ func lookupRelationship(ty artifact.RelationshipType) (bool, helpers.Relationshi return false, "", "" } -// file comments carry what an SPDX file has no field for, one "key: value" per line -const ( - layerIDCommentPrefix = "layerID: " - archivePathCommentPrefix = "archivePath: " -) - -// fileComment carries the layer ID and, for a file inside an archive, its archive path. Such a file is -// named by its full path ("/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"), so a consumer reading -// only the name does not mistake it for a file at that path in the scan root; the entry chain after the -// real path is also kept in the comment so decoding need not guess where it starts. -func fileComment(c file.Coordinates) string { - var entries string - if rest, ok := strings.CutPrefix(c.FullPath, c.RealPath); ok { - entries, _ = strings.CutPrefix(rest, file.ArchivePathSeparator) - } - switch { - case c.FileSystemID != "" && entries != "": - return layerIDCommentPrefix + c.FileSystemID + "\n" + archivePathCommentPrefix + entries - case c.FileSystemID != "": - return layerIDCommentPrefix + c.FileSystemID - case entries != "": - return archivePathCommentPrefix + entries - } - return "" -} - -func toFiles(s sbom.SBOM) (results []*spdx.File) { - artifacts := s.Artifacts +const layerIDCommentPrefix = "layerID: " +// toFiles describes every file, including each archive holding one. A file within an archive is named by +// its path in that archive, and the archive CONTAINS it. +func toFiles(s sbom.SBOM) (results []*spdx.File, archiveRelationships []*spdx.Relationship) { _, coordinateSorter := formatInternal.GetLocationSorters(s) + top, entries := formatInternal.ArchiveEntries(s.AllCoordinates(), coordinateSorter) - coordinates := s.AllCoordinates() - slices.SortFunc(coordinates, coordinateSorter) - - for _, c := range coordinates { - var metadata *file.Metadata - if metadataForLocation, exists := artifacts.FileMetadata[c]; exists { - metadata = &metadataForLocation + var addEntries func(archive file.Coordinates) + addEntries = func(archive file.Coordinates) { + for _, c := range entries[archive] { + archiveRelationships = append(archiveRelationships, &spdx.Relationship{ + RefA: spdx.DocElementID{ElementRefID: toSPDXID(archive)}, + Relationship: string(helpers.ContainsRelationship), + RefB: spdx.DocElementID{ElementRefID: toSPDXID(c)}, + }) + results = append(results, toFile(s.Artifacts, c, strings.TrimPrefix(c.LocalPath(), "/"))) + addEntries(c) } - - var digests []file.Digest - if digestsForLocation, exists := artifacts.FileDigests[c]; exists { - digests = digestsForLocation - } - - // if we don't have any metadata or digests for this location - // then the file is most likely a symlink or non-regular file - // for now we include a 0 sha1 digest as requested by the spdx spec - // TODO: update location code in core SBOM so that we can map complex links - // back to their real file digest location. - if len(digests) == 0 { - digests = append(digests, file.Digest{Algorithm: "sha1", Value: "0000000000000000000000000000000000000000"}) - } - - // TODO: add file classifications (?) and content as a snippet - - name := c.FullNestedPath() - relativePath, err := convertAbsoluteToRelative(name) + } + for _, c := range top { + name, err := convertAbsoluteToRelative(c.RealPath) if err != nil { - log.Debugf("unable to convert relative path '%s' to absolute path: %s", name, err) - relativePath = name + log.Debugf("unable to convert relative path '%s' to absolute path: %s", c.RealPath, err) + name = c.RealPath } - - results = append(results, &spdx.File{ - FileSPDXIdentifier: toSPDXID(c), - FileComment: fileComment(c), - // required, no attempt made to determine license information - LicenseConcluded: noAssertion, - FileCopyrightText: noAssertion, - Checksums: toFileChecksums(digests), - FileName: relativePath, - FileTypes: toFileTypes(metadata), - LicenseInfoInFiles: []string{ // required in SPDX 2.2 - helpers.NOASSERTION, - }, - }) + results = append(results, toFile(s.Artifacts, c, name)) + addEntries(c) } // sort by real path then virtual path to ensure the result is stable across multiple runs @@ -791,7 +761,49 @@ func toFiles(s sbom.SBOM) (results []*spdx.File) { } return results[i].FileName < results[j].FileName }) - return results + return results, archiveRelationships +} + +func toFile(artifacts sbom.Artifacts, c file.Coordinates, name string) *spdx.File { + var metadata *file.Metadata + if metadataForLocation, exists := artifacts.FileMetadata[c]; exists { + metadata = &metadataForLocation + } + + var digests []file.Digest + if digestsForLocation, exists := artifacts.FileDigests[c]; exists { + digests = digestsForLocation + } + + // if we don't have any metadata or digests for this location + // then the file is most likely a symlink or non-regular file + // for now we include a 0 sha1 digest as requested by the spdx spec + // TODO: update location code in core SBOM so that we can map complex links + // back to their real file digest location. + if len(digests) == 0 { + digests = append(digests, file.Digest{Algorithm: "sha1", Value: "0000000000000000000000000000000000000000"}) + } + + // TODO: add file classifications (?) and content as a snippet + + var comment string + if c.FileSystemID != "" { + comment = layerIDCommentPrefix + c.FileSystemID + } + + return &spdx.File{ + FileSPDXIdentifier: toSPDXID(c), + FileComment: comment, + // required, no attempt made to determine license information + LicenseConcluded: noAssertion, + FileCopyrightText: noAssertion, + Checksums: toFileChecksums(digests), + FileName: name, + FileTypes: toFileTypes(metadata), + LicenseInfoInFiles: []string{ // required in SPDX 2.2 + helpers.NOASSERTION, + }, + } } func toFileChecksums(digests []file.Digest) (checksums []spdx.Checksum) { @@ -922,6 +934,10 @@ func trimPatchVersion(semver string) string { // spdx requires that the file name field is a relative filename // with the root of the package archive or directory func convertAbsoluteToRelative(absPath string) (string, error) { + // the root itself; an empty file name is not valid SPDX (tag-value omits it and cannot read it back) + if absPath == "" || absPath == "/" { + return ".", nil + } // Ensure the absolute path is absolute (although it should already be) if !path.IsAbs(absPath) { // already relative diff --git a/syft/format/common/spdxhelpers/to_format_model_test.go b/syft/format/common/spdxhelpers/to_format_model_test.go index 0692cfe51..2ef2544ef 100644 --- a/syft/format/common/spdxhelpers/to_format_model_test.go +++ b/syft/format/common/spdxhelpers/to_format_model_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + stfile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/internal/relationship" "github.com/anchore/syft/internal/sourcemetadata" "github.com/anchore/syft/syft/artifact" @@ -588,12 +589,28 @@ func Test_toFiles(t *testing.T) { FileName: "some/path", }, }, + { + name: "the scanned root has a name", + in: sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{ + {RealPath: ""}: {Type: stfile.TypeDirectory}, + }}}, + want: spdx.File{FileName: "."}, + }, + { + name: "the absolute root has a name", + in: sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{ + {RealPath: "/"}: {Type: stfile.TypeDirectory}, + }}}, + want: spdx.File{FileName: "."}, + }, } for _, test := range tests { - files := toFiles(test.in) - got := files[0] - assert.Equal(t, test.want.FileName, got.FileName) + t.Run(test.name, func(t *testing.T) { + files, _ := toFiles(test.in) + require.Len(t, files, 1) + assert.Equal(t, test.want.FileName, files[0].FileName) + }) } } @@ -1156,38 +1173,99 @@ func Test_otherLicenses(t *testing.T) { } } -func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) { - // two archives' META-INF/MANIFEST.MF must not come out as two files with the same name and nothing - // saying which archive each is in - root := file.NewCoordinates("/etc/os-release", "sha256:layer") +func Test_nestedArchiveFiles(t *testing.T) { + top := file.NewCoordinates("/etc/os-release", "sha256:layer") inA := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", FileSystemID: "sha256:layer"} - inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"} - s := sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{root: {}, inA: {}, inB: {}}}} - - byName := map[string]*spdx.File{} - for _, f := range toFiles(s) { - byName[f.FileName] = f + inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt", FileSystemID: "sha256:layer"} + owned := file.NewCoordinates("/usr/bin/tool", "sha256:layer") + a := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar", FileSystemID: "sha256:layer"} + topPkg := pkg.Package{Name: "top"} + topPkg.SetID() + nestedPkg := pkg.Package{Name: "nested"} + nestedPkg.SetID() + s := sbom.SBOM{ + Artifacts: sbom.Artifacts{ + Packages: pkg.NewCollection(topPkg, nestedPkg), + FileMetadata: map[file.Coordinates]file.Metadata{top: {}, inA: {}, inB: {}, owned: {}}, + }, + Relationships: []artifact.Relationship{ + {From: topPkg, To: owned, Type: artifact.ContainsRelationship}, + {From: a, To: nestedPkg, Type: artifact.ContainsRelationship}, + }, } - require.Len(t, byName, 3) - assert.Equal(t, "layerID: sha256:layer", byName["etc/os-release"].FileComment, "files outside archives are unchanged") - assert.Equal(t, file.NewCoordinates("etc/os-release", "sha256:layer"), toSyftCoordinates(byName["etc/os-release"])) + doc := ToFormatModel(s) - // the name is made relative like any other file's, so the decoded top path is too - a := byName["app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"] - require.NotNil(t, a, "names: %v", byName) - assert.Equal(t, "layerID: sha256:layer\narchivePath: WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", a.FileComment) - wantA := inA - wantA.RealPath = "app.war" - wantA.FullPath = "app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF" - assert.Equal(t, wantA, toSyftCoordinates(a)) + // each archive holding a described file is described too, named by its path in its own container + war := file.NewCoordinates("/app.war", "sha256:layer") + b := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar", FileSystemID: "sha256:layer"} + names := map[spdx.ElementID]string{} + for _, f := range doc.Files { + names[f.FileSPDXIdentifier] = f.FileName + assert.Equal(t, "layerID: sha256:layer", f.FileComment) + } + assert.Equal(t, map[spdx.ElementID]string{ + toSPDXID(top): "etc/os-release", + toSPDXID(owned): "usr/bin/tool", + toSPDXID(war): "app.war", + toSPDXID(a): "WEB-INF/lib/a.jar", + toSPDXID(b): "WEB-INF/lib/b.jar", + toSPDXID(inA): "META-INF/MANIFEST.MF", + toSPDXID(inB): "odd:na%me.txt", + }, names) - b := byName["app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"] - require.NotNil(t, b, "names: %v", byName) - wantB := inB - wantB.RealPath = "app.war" - wantB.FullPath = "app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt" - assert.Equal(t, wantB, toSyftCoordinates(b)) + var root spdx.ElementID + var contains [][2]spdx.ElementID + for _, r := range doc.Relationships { + switch r.Relationship { + case string(helpers.DescribesRelationship): + root = r.RefB.ElementRefID + case string(helpers.ContainsRelationship): + contains = append(contains, [2]spdx.ElementID{r.RefA.ElementRefID, r.RefB.ElementRefID}) + } + } + // the root contains only what nothing else does: not what is within an archive, nor an owned file + assert.ElementsMatch(t, [][2]spdx.ElementID{ + {root, toSPDXID(topPkg)}, + {root, toSPDXID(top)}, + {root, toSPDXID(war)}, + {toSPDXID(topPkg), toSPDXID(owned)}, + {toSPDXID(a), toSPDXID(nestedPkg)}, + {toSPDXID(war), toSPDXID(a)}, + {toSPDXID(war), toSPDXID(b)}, + {toSPDXID(a), toSPDXID(inA)}, + {toSPDXID(b), toSPDXID(inB)}, + }, contains) + + decoded, err := ToSyftModel(doc) + require.NoError(t, err) + + // decoded paths are relative, as SPDX names are + relative := func(c file.Coordinates) file.Coordinates { + c.RealPath = strings.TrimPrefix(c.RealPath, "/") + c.FullPath = strings.TrimPrefix(c.FullPath, "/") + return c + } + var got []file.Coordinates + for c := range decoded.Artifacts.FileMetadata { + got = append(got, c) + } + assert.ElementsMatch(t, []file.Coordinates{relative(top), relative(owned), relative(war), relative(a), relative(b), relative(inA), relative(inB)}, got) + + var gotContains []artifact.Relationship + for _, r := range decoded.Relationships { + _, fromFile := r.From.(file.Coordinates) + _, toFile := r.To.(file.Coordinates) + if fromFile && toFile { + gotContains = append(gotContains, r) + } + } + assert.ElementsMatch(t, []artifact.Relationship{ + {From: relative(war), To: relative(a), Type: artifact.ContainsRelationship}, + {From: relative(war), To: relative(b), Type: artifact.ContainsRelationship}, + {From: relative(a), To: relative(inA), Type: artifact.ContainsRelationship}, + {From: relative(b), To: relative(inB), Type: artifact.ContainsRelationship}, + }, gotContains) } func Test_toRootPackage_name(t *testing.T) { diff --git a/syft/format/common/spdxhelpers/to_syft_model.go b/syft/format/common/spdxhelpers/to_syft_model.go index b3a281570..79513d46a 100644 --- a/syft/format/common/spdxhelpers/to_syft_model.go +++ b/syft/format/common/spdxhelpers/to_syft_model.go @@ -380,25 +380,74 @@ func collectSyftPackages(s *sbom.SBOM, spdxIDMap map[string]any, doc *spdx.Docum } func collectSyftFiles(s *sbom.SBOM, spdxIDMap map[string]any, doc *spdx.Document) { + files := map[spdx.ElementID]*spdx.File{} for _, p := range doc.Packages { for _, f := range p.Files { - l := toSyftLocation(f) - spdxIDMap[string(f.FileSPDXIdentifier)] = l + files[f.FileSPDXIdentifier] = f + } + } + for _, f := range doc.Files { + files[f.FileSPDXIdentifier] = f + } - s.Artifacts.FileMetadata[l.Coordinates] = toFileMetadata(f) - s.Artifacts.FileDigests[l.Coordinates] = toFileDigests(f) + nested := newNestedFiles[spdx.ElementID]() + for id, f := range files { + nested.own[id] = toSyftCoordinates(f) + } + for _, r := range doc.Relationships { + container, entry := r.RefA.ElementRefID, r.RefB.ElementRefID + switch helpers.RelationshipType(r.Relationship) { + case helpers.ContainsRelationship: + case helpers.ContainedByRelationship: + container, entry = entry, container + default: + continue + } + if files[container] != nil && files[entry] != nil { + nested.container[entry] = container } } - for _, f := range doc.Files { - l := toSyftLocation(f) - spdxIDMap[string(f.FileSPDXIdentifier)] = l + for id, f := range files { + l := file.NewLocationFromCoordinates(nested.coordinates(id)) + spdxIDMap[string(id)] = l s.Artifacts.FileMetadata[l.Coordinates] = toFileMetadata(f) s.Artifacts.FileDigests[l.Coordinates] = toFileDigests(f) } } +// nestedFiles resolves the coordinates of files within archives, where an archive file CONTAINS a file +// named by its path within the archive. +type nestedFiles[K comparable] struct { + own map[K]file.Coordinates + container map[K]K + resolved map[K]file.Coordinates +} + +func newNestedFiles[K comparable]() nestedFiles[K] { + return nestedFiles[K]{own: map[K]file.Coordinates{}, container: map[K]K{}, resolved: map[K]file.Coordinates{}} +} + +func (n nestedFiles[K]) coordinates(id K) file.Coordinates { + if c, ok := n.resolved[id]; ok { + return c + } + c := n.own[id] + if container, ok := n.container[id]; ok { + // a containment cycle ends where it started + delete(n.container, id) + archive := n.coordinates(container) + c = file.Coordinates{ + RealPath: archive.RealPath, + FileSystemID: archive.FileSystemID, + FullPath: archive.FullNestedPath() + file.ArchivePathSeparator + strings.TrimPrefix(c.RealPath, "/"), + } + } + n.resolved[id] = c + return c +} + func toFileDigests(f *spdx.File) (digests []file.Digest) { for _, digest := range f.Checksums { digests = append(digests, file.Digest{ @@ -452,6 +501,10 @@ func collectDocRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out } a := spdxIDMap[string(r.RefA.ElementRefID)] b := spdxIDMap[string(r.RefB.ElementRefID)] + if rel, ok := fileContainsRelationship(helpers.RelationshipType(r.Relationship), a, b); ok { + out = append(out, rel) + continue + } from, fromOk := a.(pkg.Package) toPackage, toPackageOk := b.(pkg.Package) toLocation, toLocationOk := b.(file.Location) @@ -505,6 +558,31 @@ func collectDocRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out return out } +// fileContainsRelationship decodes an archive file containing a file or package within it. +func fileContainsRelationship(typ helpers.RelationshipType, a, b any) (artifact.Relationship, bool) { + switch typ { + case helpers.ContainsRelationship: + case helpers.ContainedByRelationship: + a, b = b, a + default: + return artifact.Relationship{}, false + } + container, ok := a.(file.Location) + if !ok { + return artifact.Relationship{}, false + } + var entry artifact.Identifiable + switch to := b.(type) { + case file.Location: + entry = to.Coordinates + case pkg.Package: + entry = to + default: + return artifact.Relationship{}, false + } + return artifact.Relationship{From: container.Coordinates, To: entry, Type: artifact.ContainsRelationship}, true +} + // collectPackageFileRelationships add relationships for direct files func collectPackageFileRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out []artifact.Relationship) { for _, p := range doc.Packages { @@ -533,27 +611,15 @@ func toSyftCoordinates(f *spdx.File) file.Coordinates { return coordinatesFromFile(f.FileName, f.FileComment, string(f.FileSPDXIdentifier)) } -// coordinatesFromFile decodes what fileComment encodes. func coordinatesFromFile(name, comment, id string) file.Coordinates { c := file.NewCoordinates(name, "") - for _, line := range strings.Split(comment, "\n") { - if v, ok := strings.CutPrefix(line, layerIDCommentPrefix); ok { - c.FileSystemID = v - } - if v, ok := strings.CutPrefix(line, archivePathCommentPrefix); ok { - // the name is the full path; the real path is what precedes the entry chain - c.RealPath = strings.TrimSuffix(name, file.ArchivePathSeparator+v) - c.FullPath = name - } + if v, ok := strings.CutPrefix(comment, layerIDCommentPrefix); ok { + c.FileSystemID = v } if v, ok := strings.CutPrefix(id, layerIDCommentPrefix); ok { c.FileSystemID = v } - return internal.NormalizeCoordinates(c) -} - -func toSyftLocation(f *spdx.File) file.Location { - return file.NewLocationFromCoordinates(toSyftCoordinates(f)) + return c } func requireAndTrimPrefix(val any, prefix string) string { diff --git a/syft/format/common/spdxhelpers/to_syft_model_test.go b/syft/format/common/spdxhelpers/to_syft_model_test.go index d77d48b79..23a8177ca 100644 --- a/syft/format/common/spdxhelpers/to_syft_model_test.go +++ b/syft/format/common/spdxhelpers/to_syft_model_test.go @@ -344,7 +344,10 @@ func Test_toSyftRelationships(t *testing.T) { } pkg3.SetID() - loc1 := file.NewLocationFromCoordinates(file.NewCoordinates("/somewhere/real", "abc")) + loc1 := file.NewLocationFromCoordinates(file.Coordinates{ + RealPath: "/somewhere/real", + FileSystemID: "abc", + }) tests := []struct { name string @@ -691,8 +694,11 @@ func Test_directPackageFiles(t *testing.T) { p.OverrideID("1") // the same as the spdxID on the package element f := file.Location{ LocationData: file.LocationData{ - Coordinates: file.NewCoordinates("some-file", ""), - AccessPath: "some-file", + Coordinates: file.Coordinates{ + RealPath: "some-file", + FileSystemID: "", + }, + AccessPath: "some-file", }, LocationMetadata: file.LocationMetadata{ Annotations: map[string]string{}, @@ -789,12 +795,3 @@ func Test_skipsPackagesWithGeneratedFromRelationship(t *testing.T) { assert.NotNil(t, s.Artifacts.Packages.Package("1")) assert.Nil(t, s.Artifacts.Packages.Package("1-src")) } - -func Test_coordinatesFromFile_nonCanonicalChainIsDropped(t *testing.T) { - // a chain the name doesn't end with leaves FullPath equal to RealPath, which must decode as a plain file - got := coordinatesFromFile("/app.jar", "archivePath: other.jar!/x", "") - assert.Equal(t, file.NewCoordinates("/app.jar", ""), got) - - nested := coordinatesFromFile("/app.war!/lib/a.jar", "archivePath: lib/a.jar", "") - assert.Equal(t, file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/lib/a.jar"}, nested) -} diff --git a/syft/format/common/spdxhelpers/to_syft_model_v3.go b/syft/format/common/spdxhelpers/to_syft_model_v3.go index 9ab7a1fb3..5ba755906 100644 --- a/syft/format/common/spdxhelpers/to_syft_model_v3.go +++ b/syft/format/common/spdxhelpers/to_syft_model_v3.go @@ -283,20 +283,42 @@ func v3collectSyftPackages(s *sbom.SBOM, spdxMap ptrMap[any], relationships ptrM } func v3collectSyftFiles(s *sbom.SBOM, spdxMap ptrMap[any], doc *spdx.Document) { - found := ptrMap[struct{}]{} + nested := newNestedFiles[reflect.Value]() + var files []spdx.AnyFile for _, elementList := range []spdx.ElementList{doc.Elements, doc.RootElements} { for _, f := range elementList.Files() { - if found.Has(f) { + if _, found := nested.own[ptrTo(f)]; found { continue } - found.Set(f, struct{}{}) - l := v3toSyftLocation(f) - spdxMap.Set(f, l) - - s.Artifacts.FileMetadata[l.Coordinates] = v3toFileMetadata(f) - s.Artifacts.FileDigests[l.Coordinates] = v3toFileDigests(f) + nested.own[ptrTo(f)] = v3toSyftCoordinates(f) + files = append(files, f) } } + for _, r := range doc.Elements.Relationships() { + container, ok := r.GetFrom().(spdx.AnyFile) + if !ok || r.GetType() != spdx.RelationshipType_Contains { + continue + } + for _, to := range r.GetTo() { + entry, ok := to.(spdx.AnyFile) + if !ok { + continue + } + _, knownContainer := nested.own[ptrTo(container)] + _, knownEntry := nested.own[ptrTo(entry)] + if knownContainer && knownEntry { + nested.container[ptrTo(entry)] = ptrTo(container) + } + } + } + + for _, f := range files { + l := file.NewLocationFromCoordinates(nested.coordinates(ptrTo(f))) + spdxMap.Set(f, l) + + s.Artifacts.FileMetadata[l.Coordinates] = v3toFileMetadata(f) + s.Artifacts.FileDigests[l.Coordinates] = v3toFileDigests(f) + } } func v3toFileDigests(f spdx.AnyFile) (digests []file.Digest) { @@ -341,6 +363,12 @@ func v3collectDocRelationships(spdxMap ptrMap[any], doc *spdx.Document) (out []a for _, to := range r.GetTo() { b := spdxMap.Get(to) + if r.GetType() == spdx.RelationshipType_Contains { + if rel, ok := fileContainsRelationship(helpers.ContainsRelationship, a, b); ok { + out = append(out, rel) + continue + } + } from, fromOk := a.(pkg.Package) toPackage, toPackageOk := b.(pkg.Package) toLocation, toLocationOk := b.(file.Location) @@ -396,11 +424,6 @@ func v3toSyftCoordinates(f spdx.AnyFile) file.Coordinates { return coordinatesFromFile(f.GetName(), f.GetComment(), f.GetID()) } -func v3toSyftLocation(f spdx.AnyFile) file.Location { - l := file.NewVirtualLocationFromCoordinates(v3toSyftCoordinates(f), f.GetName()) - return l -} - func v3findQualifierValue(purl packageurl.PackageURL, qualifier string) string { for _, q := range purl.Qualifiers { if q.Key == qualifier { diff --git a/syft/format/common/spdxhelpers/to_syft_model_v3_test.go b/syft/format/common/spdxhelpers/to_syft_model_v3_test.go index 9c9dec7cd..90f3ab534 100644 --- a/syft/format/common/spdxhelpers/to_syft_model_v3_test.go +++ b/syft/format/common/spdxhelpers/to_syft_model_v3_test.go @@ -434,3 +434,44 @@ func Test_v3parseSPDXLicenses(t *testing.T) { require.Len(t, declared, 1) assert.Equal(t, "Apache-2.0 AND BSD-3-Clause", declared[0].Value) } + +func TestToSyftModelV3_nestedArchiveFiles(t *testing.T) { + war := &spdx.File{ID: "war", Name: "/app.war", Comment: "layerID: sha256:layer"} + jar := &spdx.File{ID: "jar", Name: "WEB-INF/lib/a.jar"} + manifest := &spdx.File{ID: "manifest", Name: "META-INF/MANIFEST.MF"} + p := &spdx.Package{ID: "pkg", Name: "a"} + warContainsJar := &spdx.Relationship{From: war, To: spdx.ElementList{jar}, Type: spdx.RelationshipType_Contains} + jarContains := &spdx.Relationship{From: jar, To: spdx.ElementList{manifest, p}, Type: spdx.RelationshipType_Contains} + + doc := &spdx.Document{SpdxDocument: spdx.SpdxDocument{ + Elements: spdx.ElementList{war, jar, manifest, p, warContainsJar, jarContains}, + }} + + s, err := ToSyftModelV3(doc) + require.NoError(t, err) + + wantWar := file.NewCoordinates("/app.war", "sha256:layer") + wantJar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar", FileSystemID: "sha256:layer"} + wantManifest := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", FileSystemID: "sha256:layer"} + var got []file.Coordinates + for c := range s.Artifacts.FileMetadata { + got = append(got, c) + } + assert.ElementsMatch(t, []file.Coordinates{wantWar, wantJar, wantManifest}, got) + + var fileRels []string + for _, r := range s.Relationships { + if from, ok := r.From.(file.Coordinates); ok { + to := "package" + if c, ok := r.To.(file.Coordinates); ok { + to = c.FullNestedPath() + } + fileRels = append(fileRels, from.FullNestedPath()+" "+string(r.Type)+" "+to) + } + } + assert.ElementsMatch(t, []string{ + "/app.war contains /app.war!/WEB-INF/lib/a.jar", + "/app.war!/WEB-INF/lib/a.jar contains /app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", + "/app.war!/WEB-INF/lib/a.jar contains package", + }, fileRels) +} diff --git a/syft/format/internal/archive_entries.go b/syft/format/internal/archive_entries.go new file mode 100644 index 000000000..30b83b5fc --- /dev/null +++ b/syft/format/internal/archive_entries.go @@ -0,0 +1,41 @@ +package internal + +import ( + "slices" + "strings" + + "github.com/anchore/syft/syft/file" +) + +// ArchiveEntries splits coordinates into those not within an archive and, for each archive, the files +// directly within it, each sorted. An archive not among coordinates is added, so every file within an +// archive has its archive to be described under. +func ArchiveEntries(coordinates []file.Coordinates, sorter func(a, b file.Coordinates) int) (top []file.Coordinates, entries map[file.Coordinates][]file.Coordinates) { + present := file.NewCoordinateSet(coordinates...) + entries = map[file.Coordinates][]file.Coordinates{} + for _, c := range coordinates { + for { + // RealPath is cut off first, so a "!/" in it is not taken for an archive boundary + rest, ok := strings.CutPrefix(c.FullPath, c.RealPath) + if !ok || !strings.HasPrefix(rest, file.ArchivePathSeparator) { + top = append(top, c) + break + } + archive := file.Coordinates{RealPath: c.RealPath, FileSystemID: c.FileSystemID} + if i := strings.LastIndex(rest, file.ArchivePathSeparator); i > 0 { + archive.FullPath = c.FullPath[:len(c.RealPath)+i] + } + entries[archive] = append(entries[archive], c) + if present.Contains(archive) { + break + } + present.Add(archive) + c = archive + } + } + slices.SortFunc(top, sorter) + for _, e := range entries { + slices.SortFunc(e, sorter) + } + return top, entries +} diff --git a/syft/format/internal/archive_entries_test.go b/syft/format/internal/archive_entries_test.go new file mode 100644 index 000000000..20ab42b36 --- /dev/null +++ b/syft/format/internal/archive_entries_test.go @@ -0,0 +1,29 @@ +package internal + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/file" +) + +func TestArchiveEntries(t *testing.T) { + top := file.NewCoordinates("/etc/passwd", "layer") + odd := file.NewCoordinates("/odd!/name", "layer") // a separator in a real path is not an archive boundary + war := file.NewCoordinates("/app.war", "layer") + jar := file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/WEB-INF/lib/a.jar"} + manifest := file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"} + index := file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/index.html"} + + byPath := func(a, b file.Coordinates) int { return strings.Compare(a.FullNestedPath(), b.FullNestedPath()) } + root := file.Coordinates{} // a directory scan's root + gotTop, gotEntries := ArchiveEntries([]file.Coordinates{top, root, odd, manifest, index}, byPath) + + assert.Equal(t, []file.Coordinates{root, war, top, odd}, gotTop) + assert.Equal(t, map[file.Coordinates][]file.Coordinates{ + war: {jar, index}, + jar: {manifest}, + }, gotEntries) +} diff --git a/syft/format/internal/coordinates.go b/syft/format/internal/coordinates.go deleted file mode 100644 index 5114b1cd3..000000000 --- a/syft/format/internal/coordinates.go +++ /dev/null @@ -1,25 +0,0 @@ -package internal - -import ( - "strings" - - "github.com/anchore/syft/syft/file" -) - -// NormalizeCoordinates clears a decoded FullPath that adds no archive chain to RealPath (one equal to -// RealPath, or one not starting with RealPath and "!/"), so decoded coordinates compare equal to the -// ones a resolver builds. -func NormalizeCoordinates(c file.Coordinates) file.Coordinates { - if !strings.HasPrefix(c.FullPath, c.RealPath+file.ArchivePathSeparator) { - c.FullPath = "" - } - return c -} - -// NormalizeLocations applies NormalizeCoordinates to each location in place. -func NormalizeLocations(locations []file.Location) []file.Location { - for i := range locations { - locations[i].Coordinates = NormalizeCoordinates(locations[i].Coordinates) - } - return locations -} diff --git a/syft/format/internal/cyclonedxutil/helpers/decoder.go b/syft/format/internal/cyclonedxutil/helpers/decoder.go index aff76f66a..a64fb9dae 100644 --- a/syft/format/internal/cyclonedxutil/helpers/decoder.go +++ b/syft/format/internal/cyclonedxutil/helpers/decoder.go @@ -2,10 +2,12 @@ package helpers import ( "fmt" + "strings" "github.com/CycloneDX/cyclonedx-go" "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" @@ -32,11 +34,62 @@ func ToSyftModel(bom *cyclonedx.BOM) (*sbom.SBOM, error) { return nil, err } + if bom.Components != nil { + collectFiles(*bom.Components, nil, s) + } + collectRelationships(bom, s, idMap) return s, nil } +// collectFiles decodes file components, named relative to the scan root; a file component nested in +// another is an entry of that archive. +func collectFiles(components []cyclonedx.Component, container *file.Coordinates, s *sbom.SBOM) { + for i := range components { + component := &components[i] + var coordinates *file.Coordinates + if component.Type == cyclonedx.ComponentTypeFile { + c := file.NewCoordinates("/"+strings.TrimPrefix(component.Name, "/"), "") + if container != nil { + c = file.Coordinates{ + RealPath: container.RealPath, + FileSystemID: container.FileSystemID, + FullPath: container.FullNestedPath() + file.ArchivePathSeparator + strings.TrimPrefix(component.Name, "/"), + } + } + coordinates = &c + if s.Artifacts.FileMetadata == nil { + s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{} + } + s.Artifacts.FileMetadata[c] = file.Metadata{} + if digests := toDigests(component.Hashes); len(digests) > 0 { + if s.Artifacts.FileDigests == nil { + s.Artifacts.FileDigests = map[file.Coordinates][]file.Digest{} + } + s.Artifacts.FileDigests[c] = digests + } + } + if component.Components != nil { + collectFiles(*component.Components, coordinates, s) + } + } +} + +func toDigests(hashes *[]cyclonedx.Hash) []file.Digest { + if hashes == nil { + return nil + } + var digests []file.Digest + for _, h := range *hashes { + digests = append(digests, file.Digest{ + Algorithm: strings.ReplaceAll(strings.ToLower(string(h.Algorithm)), "-", ""), + Value: h.Value, + }) + } + return digests +} + func collectBomPackages(bom *cyclonedx.BOM, s *sbom.SBOM, idMap map[string]any) error { componentsPresent := false if bom.Components != nil { diff --git a/syft/format/spdxtagvalue/encoder_test.go b/syft/format/spdxtagvalue/encoder_test.go index 991d1ad1e..cd99110b4 100644 --- a/syft/format/spdxtagvalue/encoder_test.go +++ b/syft/format/spdxtagvalue/encoder_test.go @@ -9,6 +9,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + stfile "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/format/internal/testutil" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" @@ -227,3 +229,30 @@ func TestEncodeDecodeSourceWithoutName(t *testing.T) { } } } + +func TestEncodeDecodeRootDirectoryFile(t *testing.T) { + // a directory scan with all file metadata describes the scanned root, whose relative path is empty + root := file.NewCoordinates("", "") + etc := file.NewCoordinates("/etc", "") + for _, enc := range defaultFormatEncoders() { + t.Run(enc.Version(), func(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, enc.Encode(&buf, sbom.SBOM{ + Source: source.Description{Metadata: source.DirectoryMetadata{Path: "src"}}, + Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{ + root: {Type: stfile.TypeDirectory}, + etc: {Type: stfile.TypeDirectory}, + }}, + })) + + s, _, _, err := NewFormatDecoder().Decode(bytes.NewReader(buf.Bytes())) + require.NoError(t, err, "encoded document:\n%s", buf.String()) + + var names []string + for c := range s.Artifacts.FileMetadata { + names = append(names, c.RealPath) + } + assert.ElementsMatch(t, []string{".", "etc"}, names) + }) + } +} diff --git a/syft/format/syftjson/to_syft_model.go b/syft/format/syftjson/to_syft_model.go index b65ceb60a..a66db17d0 100644 --- a/syft/format/syftjson/to_syft_model.go +++ b/syft/format/syftjson/to_syft_model.go @@ -80,7 +80,7 @@ func toSyftFiles(files []model.File) sbom.Artifacts { } for _, f := range files { - coord := internal.NormalizeCoordinates(f.Location) + coord := f.Location if f.Metadata != nil { fm, err := safeFileModeConvert(f.Metadata.Mode) if err != nil { @@ -163,7 +163,7 @@ func toSyftLicenses(m []model.License) (p []pkg.License) { SPDXExpression: l.SPDXExpression, Type: l.Type, URLs: l.URLs, - Locations: file.NewLocationSet(internal.NormalizeLocations(l.Locations)...), + Locations: file.NewLocationSet(l.Locations...), Contents: l.Contents, }) } @@ -239,7 +239,7 @@ func toSyftRelationships(doc *model.Document, catalog *pkg.Collection, relations idMap[doc.Source.ID] = toSyftSource(doc.Source) for _, f := range doc.Files { - idMap[f.ID] = internal.NormalizeCoordinates(f.Location) + idMap[f.ID] = f.Location } var out []artifact.Relationship @@ -346,7 +346,7 @@ func toSyftPackage(p model.Package, idAliases map[string]string) pkg.Package { Name: p.Name, Version: p.Version, FoundBy: p.FoundBy, - Locations: file.NewLocationSet(internal.NormalizeLocations(p.Locations)...), + Locations: file.NewLocationSet(p.Locations...), Licenses: pkg.NewLicenseSet(toSyftLicenses(p.Licenses)...), Language: p.Language, Type: p.Type, diff --git a/syft/format/syftjson/to_syft_model_test.go b/syft/format/syftjson/to_syft_model_test.go index 884950983..6e4b4cc68 100644 --- a/syft/format/syftjson/to_syft_model_test.go +++ b/syft/format/syftjson/to_syft_model_test.go @@ -599,35 +599,3 @@ func Test_safeFileModeConvert(t *testing.T) { }) } } - -func Test_toSyftModel_fullPathIsNormalized(t *testing.T) { - plain := file.NewCoordinates("/app.jar", "") - nested := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/lib/a.jar"} - decoded := []file.Coordinates{ - {RealPath: "/app.jar", FullPath: "/app.jar"}, - {RealPath: "/app.jar", FullPath: "/elsewhere!/app.jar"}, - nested, - } - want := []file.Coordinates{plain, plain, nested} - - var files []model.File - for _, c := range decoded { - files = append(files, model.File{ID: string(c.ID()), Location: c, Unknowns: []string{"x"}}) - } - got := toSyftFiles(files) - for _, c := range want { - assert.Contains(t, got.Unknowns, c) - } - assert.Len(t, got.Unknowns, 2) - - var locations []file.Location - for _, c := range decoded { - locations = append(locations, file.NewLocationFromCoordinates(c)) - } - p := toSyftPackage(model.Package{PackageBasicData: model.PackageBasicData{Locations: locations}}, nil) - var gotCoordinates []file.Coordinates - for _, l := range p.Locations.ToSlice() { - gotCoordinates = append(gotCoordinates, l.Coordinates) - } - assert.ElementsMatch(t, []file.Coordinates{plain, plain, nested}, gotCoordinates) -}