fix: SPDX output performance with many relationships (#3053)

This commit is contained in:
Keith Zantow 2024-07-24 10:14:20 -04:00 committed by GitHub
parent 9573f557d1
commit 741c8fb9bd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 412 additions and 293 deletions

View File

@ -5,6 +5,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/relationship"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
) )
@ -44,12 +45,13 @@ func TestBinaryElfRelationships(t *testing.T) {
} }
} }
relationshipIndex := relationship.NewIndex(sbom.Relationships...)
for name, expectedDepNames := range expectedGraph { for name, expectedDepNames := range expectedGraph {
pkgId := nameToId[name] pkgId := nameToId[name]
p := sbom.Artifacts.Packages.Package(pkgId) p := sbom.Artifacts.Packages.Package(pkgId)
require.NotNil(t, p, "expected package %q to be present in the SBOM", name) require.NotNil(t, p, "expected package %q to be present in the SBOM", name)
rels := sbom.RelationshipsForPackage(*p, artifact.DependencyOfRelationship) rels := relationshipIndex.References(*p, artifact.DependencyOfRelationship)
require.NotEmpty(t, rels, "expected package %q to have relationships", name) require.NotEmpty(t, rels, "expected package %q to have relationships", name)
toIds := map[artifact.ID]struct{}{} toIds := map[artifact.ID]struct{}{}

View File

@ -21,27 +21,22 @@ func NewDependencyRelationships(resolver file.Resolver, accessor sbomsync.Access
// 3. craft package-to-package relationships for each binary that represent shared library dependencies // 3. craft package-to-package relationships for each binary that represent shared library dependencies
//note: we only care about package-to-package relationships //note: we only care about package-to-package relationships
var relIndex *relationship.Index return generateRelationships(resolver, accessor, index)
accessor.ReadFromSBOM(func(s *sbom.SBOM) {
relIndex = relationship.NewIndex(s.Relationships...)
})
return generateRelationships(resolver, accessor, index, relIndex)
} }
func generateRelationships(resolver file.Resolver, accessor sbomsync.Accessor, index *sharedLibraryIndex, relIndex *relationship.Index) []artifact.Relationship { func generateRelationships(resolver file.Resolver, accessor sbomsync.Accessor, index *sharedLibraryIndex) []artifact.Relationship {
// read all existing dependencyOf relationships newRelationships := relationship.NewIndex()
accessor.ReadFromSBOM(func(s *sbom.SBOM) {
for _, r := range s.Relationships {
if r.Type != artifact.DependencyOfRelationship {
continue
}
relIndex.Track(r)
}
})
// find all package-to-package relationships for shared library dependencies // find all package-to-package relationships for shared library dependencies
accessor.ReadFromSBOM(func(s *sbom.SBOM) { accessor.ReadFromSBOM(func(s *sbom.SBOM) {
relIndex := relationship.NewIndex(s.Relationships...)
addRelationship := func(r artifact.Relationship) {
if !relIndex.Contains(r) {
newRelationships.Add(r)
}
}
for _, parentPkg := range s.Artifacts.Packages.Sorted(pkg.BinaryPkg) { for _, parentPkg := range s.Artifacts.Packages.Sorted(pkg.BinaryPkg) {
for _, evidentLocation := range parentPkg.Locations.ToSlice() { for _, evidentLocation := range parentPkg.Locations.ToSlice() {
if evidentLocation.Annotations[pkg.EvidenceAnnotationKey] != pkg.PrimaryEvidenceAnnotation { if evidentLocation.Annotations[pkg.EvidenceAnnotationKey] != pkg.PrimaryEvidenceAnnotation {
@ -54,12 +49,12 @@ func generateRelationships(resolver file.Resolver, accessor sbomsync.Accessor, i
continue continue
} }
populateRelationships(exec, parentPkg, resolver, relIndex, index) populateRelationships(exec, parentPkg, resolver, addRelationship, index)
} }
} }
}) })
return relIndex.NewRelationships() return newRelationships.All()
} }
// PackagesToRemove returns a list of binary packages (resolved by the ELF cataloger) that should be removed from the SBOM // PackagesToRemove returns a list of binary packages (resolved by the ELF cataloger) that should be removed from the SBOM
@ -147,7 +142,7 @@ func getBinaryPackagesToDelete(resolver file.Resolver, s *sbom.SBOM) []artifact.
return pkgsToDelete return pkgsToDelete
} }
func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver file.Resolver, relIndex *relationship.Index, index *sharedLibraryIndex) { func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver file.Resolver, addRelationship func(artifact.Relationship), index *sharedLibraryIndex) {
for _, libReference := range exec.ImportedLibraries { for _, libReference := range exec.ImportedLibraries {
// for each library reference, check s.Artifacts.Packages.Sorted(pkg.BinaryPkg) for a binary package that represents that library // for each library reference, check s.Artifacts.Packages.Sorted(pkg.BinaryPkg) for a binary package that represents that library
// if found, create a relationship between the parent package and the library package // if found, create a relationship between the parent package and the library package
@ -167,7 +162,7 @@ func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver
realBaseName := path.Base(loc.RealPath) realBaseName := path.Base(loc.RealPath)
pkgCollection := index.owningLibraryPackage(realBaseName) pkgCollection := index.owningLibraryPackage(realBaseName)
if pkgCollection.PackageCount() < 1 { if pkgCollection.PackageCount() < 1 {
relIndex.Add( addRelationship(
artifact.Relationship{ artifact.Relationship{
From: loc.Coordinates, From: loc.Coordinates,
To: parentPkg, To: parentPkg,
@ -176,7 +171,7 @@ func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver
) )
} }
for _, p := range pkgCollection.Sorted() { for _, p := range pkgCollection.Sorted() {
relIndex.Add( addRelationship(
artifact.Relationship{ artifact.Relationship{
From: p, From: p,
To: parentPkg, To: parentPkg,

View File

@ -2,6 +2,7 @@ package binary
import ( import (
"path" "path"
"strings"
"testing" "testing"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
@ -328,7 +329,20 @@ func relationshipComparer(x, y []artifact.Relationship) string {
artifact.Relationship{}, artifact.Relationship{},
file.LocationSet{}, file.LocationSet{},
pkg.LicenseSet{}, pkg.LicenseSet{},
)) ), cmpopts.SortSlices(lessRelationships))
}
func lessRelationships(r1, r2 artifact.Relationship) bool {
c := strings.Compare(string(r1.Type), string(r2.Type))
if c != 0 {
return c < 0
}
c = strings.Compare(string(r1.From.ID()), string(r2.From.ID()))
if c != 0 {
return c < 0
}
c = strings.Compare(string(r1.To.ID()), string(r2.To.ID()))
return c < 0
} }
func newAccessor(pkgs []pkg.Package, coordinateIndex map[file.Coordinates]file.Executable, preexistingRelationships []artifact.Relationship) sbomsync.Accessor { func newAccessor(pkgs []pkg.Package, coordinateIndex map[file.Coordinates]file.Executable, preexistingRelationships []artifact.Relationship) sbomsync.Accessor {

View File

@ -1,88 +1,181 @@
package relationship package relationship
import ( import (
"github.com/scylladb/go-set/strset" "slices"
"strings"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
) )
// Index indexes relationships, preventing duplicates
type Index struct { type Index struct {
typesByFromTo map[artifact.ID]map[artifact.ID]*strset.Set all []*sortableRelationship
existing []artifact.Relationship fromID map[artifact.ID]*mappedRelationships
additional []artifact.Relationship toID map[artifact.ID]*mappedRelationships
} }
func NewIndex(existing ...artifact.Relationship) *Index { // NewIndex returns a new relationship Index
r := &Index{ func NewIndex(relationships ...artifact.Relationship) *Index {
typesByFromTo: make(map[artifact.ID]map[artifact.ID]*strset.Set), out := Index{}
} out.Add(relationships...)
r.TrackAll(existing...) return &out
return r }
// Add adds all the given relationships to the index, without adding duplicates
func (i *Index) Add(relationships ...artifact.Relationship) {
if i.fromID == nil {
i.fromID = map[artifact.ID]*mappedRelationships{}
}
if i.toID == nil {
i.toID = map[artifact.ID]*mappedRelationships{}
}
// store appropriate indexes for stable ordering to minimize ID() calls
for _, r := range relationships {
// prevent duplicates
if i.Contains(r) {
continue
} }
func (i *Index) track(r artifact.Relationship) bool {
fromID := r.From.ID() fromID := r.From.ID()
if _, ok := i.typesByFromTo[fromID]; !ok {
i.typesByFromTo[fromID] = make(map[artifact.ID]*strset.Set)
}
toID := r.To.ID() toID := r.To.ID()
if _, ok := i.typesByFromTo[fromID][toID]; !ok {
i.typesByFromTo[fromID][toID] = strset.New() relationship := &sortableRelationship{
from: fromID,
to: toID,
relationship: r,
} }
var exists bool // add to all relationships
if i.typesByFromTo[fromID][toID].Has(string(r.Type)) { i.all = append(i.all, relationship)
exists = true
// add from -> to mapping
mapped := i.fromID[fromID]
if mapped == nil {
mapped = &mappedRelationships{}
i.fromID[fromID] = mapped
}
mapped.add(toID, relationship)
// add to -> from mapping
mapped = i.toID[toID]
if mapped == nil {
mapped = &mappedRelationships{}
i.toID[toID] = mapped
}
mapped.add(fromID, relationship)
}
} }
i.typesByFromTo[fromID][toID].Add(string(r.Type)) // From returns all relationships from the given identifiable, with specified types
func (i *Index) From(identifiable artifact.Identifiable, types ...artifact.RelationshipType) []artifact.Relationship {
return !exists return toSortedSlice(fromMapped(i.fromID, identifiable), types)
} }
// Track this relationship as "exists" in the index (this is used to prevent duplicate relationships from being added). // To returns all relationships to the given identifiable, with specified types
// returns true if the relationship is new to the index, false otherwise. func (i *Index) To(identifiable artifact.Identifiable, types ...artifact.RelationshipType) []artifact.Relationship {
func (i *Index) Track(r artifact.Relationship) bool { return toSortedSlice(fromMapped(i.toID, identifiable), types)
unique := i.track(r)
if unique {
i.existing = append(i.existing, r)
}
return unique
} }
// Add a new relationship to the index, returning true if the relationship is new to the index, false otherwise (thus is a duplicate). // References returns all relationships that reference to or from the given identifiable
func (i *Index) Add(r artifact.Relationship) bool { func (i *Index) References(identifiable artifact.Identifiable, types ...artifact.RelationshipType) []artifact.Relationship {
if i.track(r) { return toSortedSlice(append(fromMapped(i.fromID, identifiable), fromMapped(i.toID, identifiable)...), types)
i.additional = append(i.additional, r) }
return true
// Coordinates returns all coordinates for the provided identifiable for provided relationship types
// If no types are provided, all relationship types are considered.
func (i *Index) Coordinates(identifiable artifact.Identifiable, types ...artifact.RelationshipType) []file.Coordinates {
var coordinates []file.Coordinates
for _, relationship := range i.References(identifiable, types...) {
cords := extractCoordinates(relationship)
coordinates = append(coordinates, cords...)
}
return coordinates
}
// Contains indicates the relationship is present in this index
func (i *Index) Contains(r artifact.Relationship) bool {
if mapped := i.fromID[r.From.ID()]; mapped != nil {
if ids := mapped.typeMap[r.Type]; ids != nil {
return ids[r.To.ID()] != nil
}
} }
return false return false
} }
func (i *Index) TrackAll(rs ...artifact.Relationship) { // All returns a sorted set of relationships matching all types, or all relationships if no types specified
for _, r := range rs { func (i *Index) All(types ...artifact.RelationshipType) []artifact.Relationship {
i.Track(r) return toSortedSlice(i.all, types)
}
} }
func (i *Index) AddAll(rs ...artifact.Relationship) { func fromMapped(idMap map[artifact.ID]*mappedRelationships, identifiable artifact.Identifiable) []*sortableRelationship {
for _, r := range rs { if identifiable == nil || idMap == nil {
i.Add(r) return nil
} }
mapped := idMap[identifiable.ID()]
if mapped == nil {
return nil
}
return mapped.allRelated
} }
func (i *Index) NewRelationships() []artifact.Relationship { func toSortedSlice(relationships []*sortableRelationship, types []artifact.RelationshipType) []artifact.Relationship {
return i.additional // always return sorted for SBOM stability
slices.SortFunc(relationships, sortFunc)
var out []artifact.Relationship
for _, r := range relationships {
if len(types) == 0 || slices.Contains(types, r.relationship.Type) {
out = append(out, r.relationship)
}
}
return out
} }
func (i *Index) ExistingRelationships() []artifact.Relationship { func extractCoordinates(relationship artifact.Relationship) (results []file.Coordinates) {
return i.existing if coordinates, exists := relationship.From.(file.Coordinates); exists {
results = append(results, coordinates)
} }
func (i *Index) AllUniqueRelationships() []artifact.Relationship { if coordinates, exists := relationship.To.(file.Coordinates); exists {
var all []artifact.Relationship results = append(results, coordinates)
all = append(all, i.existing...) }
all = append(all, i.additional...)
return all return results
}
type mappedRelationships struct {
typeMap map[artifact.RelationshipType]map[artifact.ID]*sortableRelationship
allRelated []*sortableRelationship
}
func (m *mappedRelationships) add(id artifact.ID, newRelationship *sortableRelationship) {
m.allRelated = append(m.allRelated, newRelationship)
if m.typeMap == nil {
m.typeMap = map[artifact.RelationshipType]map[artifact.ID]*sortableRelationship{}
}
typeMap := m.typeMap[newRelationship.relationship.Type]
if typeMap == nil {
typeMap = map[artifact.ID]*sortableRelationship{}
m.typeMap[newRelationship.relationship.Type] = typeMap
}
typeMap[id] = newRelationship
}
type sortableRelationship struct {
from artifact.ID
to artifact.ID
relationship artifact.Relationship
}
func sortFunc(a, b *sortableRelationship) int {
cmp := strings.Compare(string(a.relationship.Type), string(b.relationship.Type))
if cmp != 0 {
return cmp
}
cmp = strings.Compare(string(a.from), string(b.from))
if cmp != 0 {
return cmp
}
return strings.Compare(string(a.to), string(b.to))
} }

View File

@ -3,223 +3,231 @@ package relationship
import ( import (
"testing" "testing"
"github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
) )
func Test_newRelationshipIndex(t *testing.T) { func Test_Index(t *testing.T) {
from := fakeIdentifiable{id: "from"} p1 := pkg.Package{
to := fakeIdentifiable{id: "to"} Name: "pkg-1",
tests := []struct {
name string
given []artifact.Relationship
track []artifact.Relationship
add []artifact.Relationship
wantExisting []string
wantAdditional []string
}{
{
name: "empty",
},
{
name: "tracks existing relationships",
given: []artifact.Relationship{
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
},
wantExisting: []string{"from [evident-by] to"},
},
{
name: "deduplicate tracked relationships",
given: []artifact.Relationship{
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
},
track: []artifact.Relationship{
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
},
wantExisting: []string{"from [evident-by] to"},
},
{
name: "deduplicate any input relationships",
given: []artifact.Relationship{
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
},
track: []artifact.Relationship{
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
},
add: []artifact.Relationship{
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
},
wantExisting: []string{"from [evident-by] to"},
},
{
name: "deduplicate any added relationships",
add: []artifact.Relationship{
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
{
From: from,
To: to,
Type: artifact.EvidentByRelationship,
},
},
wantAdditional: []string{"from [evident-by] to"},
},
} }
for _, tt := range tests { p2 := pkg.Package{
t.Run(tt.name, func(t *testing.T) { Name: "pkg-2",
idx := NewIndex(tt.given...)
idx.TrackAll(tt.track...)
idx.AddAll(tt.add...)
diffRelationships(t, tt.wantExisting, idx.existing)
diffRelationships(t, tt.wantAdditional, idx.additional)
})
} }
p3 := pkg.Package{
Name: "pkg-3",
}
c1 := file.Coordinates{
RealPath: "/coords/1",
}
c2 := file.Coordinates{
RealPath: "/coords/2",
} }
func diffRelationships(t *testing.T, expected []string, actual []artifact.Relationship) { for _, p := range []*pkg.Package{&p1, &p2, &p3} {
if d := cmp.Diff(expected, stringRelationships(actual)); d != "" { p.SetID()
t.Errorf("unexpected relationships (-want, +got): %s", d)
}
} }
func stringRelationships(relationships []artifact.Relationship) []string { r1 := artifact.Relationship{
var result []string From: p1,
for _, r := range relationships { To: p2,
result = append(result, string(r.From.ID())+" ["+string(r.Type)+"] "+string(r.To.ID())) Type: artifact.DependencyOfRelationship,
} }
return result r2 := artifact.Relationship{
From: p1,
To: p3,
Type: artifact.DependencyOfRelationship,
}
r3 := artifact.Relationship{
From: p1,
To: c1,
Type: artifact.ContainsRelationship,
}
r4 := artifact.Relationship{
From: p2,
To: c2,
Type: artifact.ContainsRelationship,
}
r5 := artifact.Relationship{
From: p3,
To: c2,
Type: artifact.ContainsRelationship,
} }
func Test_relationshipIndex_track(t *testing.T) { dup := artifact.Relationship{
from := fakeIdentifiable{id: "from"} From: p3,
to := fakeIdentifiable{id: "to"} To: c2,
relationship := artifact.Relationship{From: from, To: to, Type: artifact.EvidentByRelationship} Type: artifact.ContainsRelationship,
tests := []struct {
name string
existing []artifact.Relationship
given artifact.Relationship
want bool
}{
{
name: "track returns true for a new relationship",
existing: []artifact.Relationship{},
given: relationship,
want: true,
},
{
name: "track returns false for an existing relationship",
existing: []artifact.Relationship{relationship},
given: relationship,
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
i := NewIndex(tt.existing...)
if got := i.Track(tt.given); got != tt.want {
t.Errorf("track() = %v, want %v", got, tt.want)
}
})
}
} }
func Test_relationshipIndex_add(t *testing.T) { idx := NewIndex(r1, r2, r3, r4, r5, dup)
from := fakeIdentifiable{id: "from"} require.ElementsMatch(t, slice(r1, r2, r3, r4, r5), idx.All())
to := fakeIdentifiable{id: "to"}
relationship := artifact.Relationship{From: from, To: to, Type: artifact.EvidentByRelationship} require.ElementsMatch(t, slice(r1, r4), idx.References(p2))
tests := []struct { require.ElementsMatch(t, slice(r4), idx.References(p2, artifact.ContainsRelationship))
name string
existing []artifact.Relationship require.ElementsMatch(t, slice(r1), idx.To(p2))
given artifact.Relationship require.ElementsMatch(t, []artifact.Relationship(nil), idx.To(p2, artifact.ContainsRelationship))
want bool
}{ require.ElementsMatch(t, slice(r4), idx.From(p2))
{ require.ElementsMatch(t, slice(r4), idx.From(p2, artifact.ContainsRelationship))
name: "add returns true for a new relationship",
existing: []artifact.Relationship{},
given: relationship,
want: true,
},
{
name: "add returns false for an existing relationship",
existing: []artifact.Relationship{relationship},
given: relationship,
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
i := NewIndex(tt.existing...)
if got := i.Add(tt.given); got != tt.want {
t.Errorf("add() = %v, want %v", got, tt.want)
}
})
} }
func Test_sortOrder(t *testing.T) {
r1 := artifact.Relationship{
From: id("1"),
To: id("2"),
Type: "1",
}
r2 := artifact.Relationship{
From: id("2"),
To: id("3"),
Type: "1",
}
r3 := artifact.Relationship{
From: id("3"),
To: id("4"),
Type: "1",
}
r4 := artifact.Relationship{
From: id("1"),
To: id("2"),
Type: "2",
}
r5 := artifact.Relationship{
From: id("2"),
To: id("3"),
Type: "2",
}
dup := artifact.Relationship{
From: id("2"),
To: id("3"),
Type: "2",
}
r6 := artifact.Relationship{
From: id("2"),
To: id("3"),
Type: "3",
} }
type fakeIdentifiable struct { idx := NewIndex(r5, r2, r6, r4, r1, r3, dup)
id string require.EqualValues(t, slice(r1, r2, r3, r4, r5, r6), idx.All())
require.EqualValues(t, slice(r1, r4), idx.From(id("1")))
require.EqualValues(t, slice(r2, r5, r6), idx.To(id("3")))
rLast := artifact.Relationship{
From: id("0"),
To: id("3"),
Type: "9999",
} }
func (f fakeIdentifiable) ID() artifact.ID { rFirst := artifact.Relationship{
return artifact.ID(f.id) From: id("0"),
To: id("3"),
Type: "1",
}
rMid := artifact.Relationship{
From: id("0"),
To: id("1"),
Type: "2",
}
idx.Add(rLast, rFirst, rMid)
require.EqualValues(t, slice(rFirst, r1, r2, r3, rMid, r4, r5, r6, rLast), idx.All())
require.EqualValues(t, slice(rFirst, r2, r5, r6, rLast), idx.To(id("3")))
}
func Test_Coordinates(t *testing.T) {
p1 := pkg.Package{
Name: "pkg-1",
}
p2 := pkg.Package{
Name: "pkg-2",
}
p3 := pkg.Package{
Name: "pkg-3",
}
c1 := file.Coordinates{
RealPath: "/coords/1",
}
c2 := file.Coordinates{
RealPath: "/coords/2",
}
c3 := file.Coordinates{
RealPath: "/coords/3",
}
c4 := file.Coordinates{
RealPath: "/coords/4",
}
for _, p := range []*pkg.Package{&p1, &p2, &p3} {
p.SetID()
}
r1 := artifact.Relationship{
From: p1,
To: p2,
Type: artifact.DependencyOfRelationship,
}
r2 := artifact.Relationship{
From: p1,
To: p3,
Type: artifact.DependencyOfRelationship,
}
r3 := artifact.Relationship{
From: p1,
To: c1,
Type: artifact.ContainsRelationship,
}
r4 := artifact.Relationship{
From: p2,
To: c2,
Type: artifact.ContainsRelationship,
}
r5 := artifact.Relationship{
From: p3,
To: c1,
Type: artifact.ContainsRelationship,
}
r6 := artifact.Relationship{
From: p3,
To: c2,
Type: artifact.ContainsRelationship,
}
r7 := artifact.Relationship{
From: c1,
To: c3,
Type: artifact.ContainsRelationship,
}
r8 := artifact.Relationship{
From: c3,
To: c4,
Type: artifact.ContainsRelationship,
}
idx := NewIndex(r1, r2, r3, r4, r5, r6, r7, r8)
got := idx.Coordinates(p1)
require.ElementsMatch(t, slice(c1), got)
got = idx.Coordinates(p3)
require.ElementsMatch(t, slice(c1, c2), got)
}
type id string
func (i id) ID() artifact.ID {
return artifact.ID(i)
}
func slice[T any](values ...T) []T {
return values
} }

View File

@ -16,6 +16,7 @@ import (
"github.com/anchore/packageurl-go" "github.com/anchore/packageurl-go"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/mimetype" "github.com/anchore/syft/internal/mimetype"
"github.com/anchore/syft/internal/relationship"
"github.com/anchore/syft/internal/spdxlicense" "github.com/anchore/syft/internal/spdxlicense"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
@ -45,9 +46,10 @@ const (
func ToFormatModel(s sbom.SBOM) *spdx.Document { func ToFormatModel(s sbom.SBOM) *spdx.Document {
name, namespace := helpers.DocumentNameAndNamespace(s.Source, s.Descriptor) name, namespace := helpers.DocumentNameAndNamespace(s.Source, s.Descriptor)
packages := toPackages(s.Artifacts.Packages, s) rels := relationship.NewIndex(s.Relationships...)
packages := toPackages(rels, s.Artifacts.Packages, s)
relationships := toRelationships(s.RelationshipsSorted()) allRelationships := toRelationships(rels.All())
// for valid SPDX we need a document describes relationship // for valid SPDX we need a document describes relationship
describesID := spdx.ElementID("DOCUMENT") describesID := spdx.ElementID("DOCUMENT")
@ -57,7 +59,7 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document {
describesID = rootPackage.PackageSPDXIdentifier describesID = rootPackage.PackageSPDXIdentifier
// add all relationships from the document root to all other packages // add all relationships from the document root to all other packages
relationships = append(relationships, toRootRelationships(rootPackage, packages)...) allRelationships = append(allRelationships, toRootRelationships(rootPackage, packages)...)
// append the root package // append the root package
packages = append(packages, rootPackage) packages = append(packages, rootPackage)
@ -75,7 +77,7 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document {
} }
// add the root document relationship // add the root document relationship
relationships = append(relationships, documentDescribesRelationship) allRelationships = append(allRelationships, documentDescribesRelationship)
return &spdx.Document{ return &spdx.Document{
// 6.1: SPDX Version; should be in the format "SPDX-x.x" // 6.1: SPDX Version; should be in the format "SPDX-x.x"
@ -150,7 +152,7 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document {
}, },
Packages: packages, Packages: packages,
Files: toFiles(s), Files: toFiles(s),
Relationships: relationships, Relationships: allRelationships,
OtherLicenses: toOtherLicenses(s.Artifacts.Packages), OtherLicenses: toOtherLicenses(s.Artifacts.Packages),
} }
} }
@ -302,7 +304,7 @@ func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID {
// packages populates all Package Information from the package Collection (see https://spdx.github.io/spdx-spec/3-package-information/) // packages populates all Package Information from the package Collection (see https://spdx.github.io/spdx-spec/3-package-information/)
// //
//nolint:funlen //nolint:funlen
func toPackages(catalog *pkg.Collection, sbom sbom.SBOM) (results []*spdx.Package) { func toPackages(rels *relationship.Index, catalog *pkg.Collection, sbom sbom.SBOM) (results []*spdx.Package) {
for _, p := range catalog.Sorted() { for _, p := range catalog.Sorted() {
// name should be guaranteed to be unique, but semantically useful and stable // name should be guaranteed to be unique, but semantically useful and stable
id := toSPDXID(p) id := toSPDXID(p)
@ -318,7 +320,7 @@ func toPackages(catalog *pkg.Collection, sbom sbom.SBOM) (results []*spdx.Packag
// 2. syft has generated a sha1 digest for the package's contents // 2. syft has generated a sha1 digest for the package's contents
packageChecksums, filesAnalyzed := toPackageChecksums(p) packageChecksums, filesAnalyzed := toPackageChecksums(p)
packageVerificationCode := newPackageVerificationCode(p, sbom) packageVerificationCode := newPackageVerificationCode(rels, p, sbom)
if packageVerificationCode != nil { if packageVerificationCode != nil {
filesAnalyzed = true filesAnalyzed = true
} }
@ -744,12 +746,12 @@ func toOtherLicenses(catalog *pkg.Collection) []*spdx.OtherLicense {
// f file is an "excludes" file, skip it /* exclude SPDX analysis file(s) */ // f file is an "excludes" file, skip it /* exclude SPDX analysis file(s) */
// see: https://spdx.github.io/spdx-spec/v2.3/package-information/#79-package-verification-code-field // see: https://spdx.github.io/spdx-spec/v2.3/package-information/#79-package-verification-code-field
// the above link contains the SPDX algorithm for a package verification code // the above link contains the SPDX algorithm for a package verification code
func newPackageVerificationCode(p pkg.Package, sbom sbom.SBOM) *spdx.PackageVerificationCode { func newPackageVerificationCode(rels *relationship.Index, p pkg.Package, sbom sbom.SBOM) *spdx.PackageVerificationCode {
// key off of the contains relationship; // key off of the contains relationship;
// spdx validator will fail if a package claims to contain a file but no sha1 provided // spdx validator will fail if a package claims to contain a file but no sha1 provided
// if a sha1 for a file is provided then the validator will fail if the package does not have // if a sha1 for a file is provided then the validator will fail if the package does not have
// a package verification code // a package verification code
coordinates := sbom.CoordinatesForPackage(p, artifact.ContainsRelationship) coordinates := rels.Coordinates(p, artifact.ContainsRelationship)
var digests []file.Digest var digests []file.Digest
for _, c := range coordinates { for _, c := range coordinates {
digest := sbom.Artifacts.FileDigests[c] digest := sbom.Artifacts.FileDigests[c]

View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/relationship"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/format/internal/spdxutil/helpers" "github.com/anchore/syft/syft/format/internal/spdxutil/helpers"
@ -665,7 +666,7 @@ func Test_H1Digest(t *testing.T) {
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
catalog := pkg.NewCollection(test.pkg) catalog := pkg.NewCollection(test.pkg)
pkgs := toPackages(catalog, s) pkgs := toPackages(relationship.NewIndex(), catalog, s)
require.Len(t, pkgs, 1) require.Len(t, pkgs, 1)
for _, p := range pkgs { for _, p := range pkgs {
if test.expectedDigest == "" { if test.expectedDigest == "" {

View File

@ -167,7 +167,7 @@ func wheelEggRelationships(ctx context.Context, resolver file.Resolver, pkgs []p
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("failed to resolve relationships for global site package %q: %w", globalSitePackage, err) return nil, nil, fmt.Errorf("failed to resolve relationships for global site package %q: %w", globalSitePackage, err)
} }
relationshipIndex.AddAll(siteRels...) relationshipIndex.Add(siteRels...)
} }
// create relationships between packages within each virtual env site package directory (that doesn't link to a global site-packages directory) // create relationships between packages within each virtual env site package directory (that doesn't link to a global site-packages directory)
@ -180,7 +180,7 @@ func wheelEggRelationships(ctx context.Context, resolver file.Resolver, pkgs []p
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("failed to resolve relationships for virtualenv site package %q: %w", venv.SitePackagesPath, err) return nil, nil, fmt.Errorf("failed to resolve relationships for virtualenv site package %q: %w", venv.SitePackagesPath, err)
} }
relationshipIndex.AddAll(siteRels...) relationshipIndex.Add(siteRels...)
} }
// create relationships between packages within each virtual env site package directory (that links to a global site package directory) // create relationships between packages within each virtual env site package directory (that links to a global site package directory)
@ -197,10 +197,10 @@ func wheelEggRelationships(ctx context.Context, resolver file.Resolver, pkgs []p
return nil, nil, fmt.Errorf("failed to resolve relationships for virtualenv + global site package path %q + %q: %w", venv.SitePackagesPath, globalSitePackage, err) return nil, nil, fmt.Errorf("failed to resolve relationships for virtualenv + global site package path %q + %q: %w", venv.SitePackagesPath, globalSitePackage, err)
} }
relationshipIndex.AddAll(siteRels...) relationshipIndex.Add(siteRels...)
} }
return pkgs, relationshipIndex.AllUniqueRelationships(), err return pkgs, relationshipIndex.All(), err
} }
func collectPackages(pkgsBySitePackageAndName map[string]map[string]pkg.Package, sites []string) []pkg.Package { func collectPackages(pkgsBySitePackageAndName map[string]map[string]pkg.Package, sites []string) []pkg.Package {

View File

@ -35,6 +35,8 @@ type Descriptor struct {
Configuration interface{} Configuration interface{}
} }
// RelationshipsSorted returns a sorted slice of all relationships
// Deprecated -- use relationship.Index
func (s SBOM) RelationshipsSorted() []artifact.Relationship { func (s SBOM) RelationshipsSorted() []artifact.Relationship {
relationships := s.Relationships relationships := s.Relationships
sort.SliceStable(relationships, func(i, j int) bool { sort.SliceStable(relationships, func(i, j int) bool {
@ -70,6 +72,7 @@ func (s SBOM) AllCoordinates() []file.Coordinates {
// RelationshipsForPackage returns all relationships for the provided types. // RelationshipsForPackage returns all relationships for the provided types.
// If no types are provided, all relationships for the package are returned. // If no types are provided, all relationships for the package are returned.
// Deprecated -- use relationship.Index
func (s SBOM) RelationshipsForPackage(p pkg.Package, rt ...artifact.RelationshipType) []artifact.Relationship { func (s SBOM) RelationshipsForPackage(p pkg.Package, rt ...artifact.RelationshipType) []artifact.Relationship {
if len(rt) == 0 { if len(rt) == 0 {
rt = artifact.AllRelationshipTypes() rt = artifact.AllRelationshipTypes()
@ -103,6 +106,7 @@ func (s SBOM) RelationshipsForPackage(p pkg.Package, rt ...artifact.Relationship
// CoordinatesForPackage returns all coordinates for the provided package for provided relationship types // CoordinatesForPackage returns all coordinates for the provided package for provided relationship types
// If no types are provided, all relationship types are considered. // If no types are provided, all relationship types are considered.
// Deprecated -- use relationship.Index
func (s SBOM) CoordinatesForPackage(p pkg.Package, rt ...artifact.RelationshipType) []file.Coordinates { func (s SBOM) CoordinatesForPackage(p pkg.Package, rt ...artifact.RelationshipType) []file.Coordinates {
var coordinates []file.Coordinates var coordinates []file.Coordinates
for _, relationship := range s.RelationshipsForPackage(p, rt...) { for _, relationship := range s.RelationshipsForPackage(p, rt...) {