Add python wheel egg relationships (#2903)

* add python package relationships

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* nil for empty relationships collections

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* new json schema for optional python requiremenets

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update format snapshots for python packages

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* decompose python parsers more + add tests around plural fields

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update JSON schema with python dep refs

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

internal/relationship/binary/binary_dependencies.go

@@ -4,6 +4,7 @@ import (
 	"path"
 
 	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/internal/relationship"
 	"github.com/anchore/syft/internal/sbomsync"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
@@ -20,22 +21,22 @@ func NewDependencyRelationships(resolver file.Resolver, accessor sbomsync.Access
 
 	// 3. craft package-to-package relationships for each binary that represent shared library dependencies
 	//note: we only care about package-to-package relationships
-	var relIndex *relationshipIndex
+	var relIndex *relationship.Index
 	accessor.ReadFromSBOM(func(s *sbom.SBOM) {
-		relIndex = newRelationshipIndex(s.Relationships...)
+		relIndex = relationship.NewIndex(s.Relationships...)
 	})
 
 	return generateRelationships(resolver, accessor, index, relIndex)
 }
 
-func generateRelationships(resolver file.Resolver, accessor sbomsync.Accessor, index *sharedLibraryIndex, relIndex *relationshipIndex) []artifact.Relationship {
+func generateRelationships(resolver file.Resolver, accessor sbomsync.Accessor, index *sharedLibraryIndex, relIndex *relationship.Index) []artifact.Relationship {
 	// read all existing dependencyOf relationships
 	accessor.ReadFromSBOM(func(s *sbom.SBOM) {
 		for _, r := range s.Relationships {
 			if r.Type != artifact.DependencyOfRelationship {
 				continue
 			}
-			relIndex.track(r)
+			relIndex.Track(r)
 		}
 	})
 
@@ -58,7 +59,7 @@ func generateRelationships(resolver file.Resolver, accessor sbomsync.Accessor, i
 		}
 	})
 
-	return relIndex.newRelationships()
+	return relIndex.NewRelationships()
 }
 
 // PackagesToRemove returns a list of binary packages (resolved by the ELF cataloger) that should be removed from the SBOM
@@ -146,7 +147,7 @@ func getBinaryPackagesToDelete(resolver file.Resolver, s *sbom.SBOM) []artifact.
 	return pkgsToDelete
 }
 
-func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver file.Resolver, relIndex *relationshipIndex, index *sharedLibraryIndex) {
+func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver file.Resolver, relIndex *relationship.Index, index *sharedLibraryIndex) {
 	for _, libReference := range exec.ImportedLibraries {
 		// for each library reference, check s.Artifacts.Packages.Sorted(pkg.BinaryPkg) for a binary package that represents that library
 		// if found, create a relationship between the parent package and the library package
@@ -166,7 +167,7 @@ func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver
 			realBaseName := path.Base(loc.RealPath)
 			pkgCollection := index.owningLibraryPackage(realBaseName)
 			if pkgCollection.PackageCount() < 1 {
-				relIndex.add(
+				relIndex.Add(
 					artifact.Relationship{
 						From: loc.Coordinates,
 						To:   parentPkg,
@@ -175,7 +176,7 @@ func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver
 				)
 			}
 			for _, p := range pkgCollection.Sorted() {
-				relIndex.add(
+				relIndex.Add(
 					artifact.Relationship{
 						From: p,
 						To:   parentPkg,

internal/relationship/binary/binary_dependencies_test.go

@@ -225,7 +225,6 @@ func TestNewDependencyRelationships(t *testing.T) {
 			resolver:        nil,
 			coordinateIndex: map[file.Coordinates]file.Executable{},
 			packages:        []pkg.Package{},
-			want:            make([]artifact.Relationship, 0),
 		},
 		{
 			name: "given a package that imports glibc, expect a relationship between the two packages when the package is an executable",
@@ -297,7 +296,6 @@ func TestNewDependencyRelationships(t *testing.T) {
 					Type: artifact.DependencyOfRelationship,
 				},
 			},
-			want: []artifact.Relationship{},
 		},
 		{
 			name:     "given a package that imports a library that is not tracked by the resolver, expect no relationships to be created",
@@ -308,7 +306,6 @@ func TestNewDependencyRelationships(t *testing.T) {
 				parallelLibCoordinate: syftTestFixtureExecutable2,
 			},
 			packages: []pkg.Package{glibCPackage, syftTestFixturePackage},
-			want:     []artifact.Relationship{},
 		},
 	}
 	for _, tt := range tests {

internal/relationship/binary/relationship_index.go

@@ -1,59 +0,0 @@
-package binary
-
-import (
-	"github.com/scylladb/go-set/strset"
-
-	"github.com/anchore/syft/syft/artifact"
-)
-
-type relationshipIndex struct {
-	typesByFromTo map[artifact.ID]map[artifact.ID]*strset.Set
-	additional    []artifact.Relationship
-}
-
-func newRelationshipIndex(existing ...artifact.Relationship) *relationshipIndex {
-	r := &relationshipIndex{
-		typesByFromTo: make(map[artifact.ID]map[artifact.ID]*strset.Set),
-		additional:    make([]artifact.Relationship, 0),
-	}
-	for _, rel := range existing {
-		r.track(rel)
-	}
-	return r
-}
-
-// track this relationship as "exists" in the index (this is used to prevent duplicate relationships from being added).
-// returns true if the relationship is new to the index, false otherwise.
-func (i *relationshipIndex) track(r artifact.Relationship) bool {
-	fromID := r.From.ID()
-	if _, ok := i.typesByFromTo[fromID]; !ok {
-		i.typesByFromTo[fromID] = make(map[artifact.ID]*strset.Set)
-	}
-
-	toID := r.To.ID()
-	if _, ok := i.typesByFromTo[fromID][toID]; !ok {
-		i.typesByFromTo[fromID][toID] = strset.New()
-	}
-
-	var exists bool
-	if i.typesByFromTo[fromID][toID].Has(string(r.Type)) {
-		exists = true
-	}
-
-	i.typesByFromTo[fromID][toID].Add(string(r.Type))
-	return !exists
-}
-
-// add a new relationship to the index, returning true if the relationship is new to the index, false otherwise (thus is a duplicate).
-// nolint:unparam
-func (i *relationshipIndex) add(r artifact.Relationship) bool {
-	if i.track(r) {
-		i.additional = append(i.additional, r)
-		return true
-	}
-	return false
-}
-
-func (i *relationshipIndex) newRelationships() []artifact.Relationship {
-	return i.additional
-}

internal/relationship/binary/relationship_index_test.go

@@ -1,128 +0,0 @@
-package binary
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/scylladb/go-set/strset"
-
-	"github.com/anchore/syft/syft/artifact"
-)
-
-func Test_newRelationshipIndex(t *testing.T) {
-	from := fakeIdentifiable{id: "from"}
-	to := fakeIdentifiable{id: "to"}
-	tests := []struct {
-		name  string
-		given []artifact.Relationship
-		want  *relationshipIndex
-	}{
-		{
-			name: "newRelationshipIndex returns an empty index with no existing relationships",
-			want: &relationshipIndex{
-				typesByFromTo: make(map[artifact.ID]map[artifact.ID]*strset.Set),
-				additional:    make([]artifact.Relationship, 0),
-			},
-		},
-		{
-			name: "newRelationshipIndex returns an index which tracks existing relationships",
-			given: []artifact.Relationship{
-				{
-					From: from,
-					To:   to,
-					Type: artifact.EvidentByRelationship,
-				},
-			},
-			want: &relationshipIndex{
-				typesByFromTo: map[artifact.ID]map[artifact.ID]*strset.Set{
-					"from": {
-						"to": strset.New("evident-by"),
-					},
-				},
-				additional: make([]artifact.Relationship, 0),
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := newRelationshipIndex(tt.given...); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("newRelationshipIndex() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_relationshipIndex_track(t *testing.T) {
-	from := fakeIdentifiable{id: "from"}
-	to := fakeIdentifiable{id: "to"}
-	relationship := artifact.Relationship{From: from, To: to, Type: artifact.EvidentByRelationship}
-	tests := []struct {
-		name     string
-		existing []artifact.Relationship
-		given    artifact.Relationship
-		want     bool
-	}{
-		{
-			name:     "track returns true for a new relationship",
-			existing: []artifact.Relationship{},
-			given:    relationship,
-			want:     true,
-		},
-		{
-			name:     "track returns false for an existing relationship",
-			existing: []artifact.Relationship{relationship},
-			given:    relationship,
-			want:     false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			i := newRelationshipIndex(tt.existing...)
-			if got := i.track(tt.given); got != tt.want {
-				t.Errorf("track() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_relationshipIndex_add(t *testing.T) {
-	from := fakeIdentifiable{id: "from"}
-	to := fakeIdentifiable{id: "to"}
-	relationship := artifact.Relationship{From: from, To: to, Type: artifact.EvidentByRelationship}
-	tests := []struct {
-		name     string
-		existing []artifact.Relationship
-		given    artifact.Relationship
-		want     bool
-	}{
-		{
-			name:     "add returns true for a new relationship",
-			existing: []artifact.Relationship{},
-			given:    relationship,
-			want:     true,
-		},
-		{
-			name:     "add returns false for an existing relationship",
-			existing: []artifact.Relationship{relationship},
-			given:    relationship,
-			want:     false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			i := newRelationshipIndex(tt.existing...)
-			if got := i.add(tt.given); got != tt.want {
-				t.Errorf("add() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-
-}
-
-type fakeIdentifiable struct {
-	id string
-}
-
-func (f fakeIdentifiable) ID() artifact.ID {
-	return artifact.ID(f.id)
-}

internal/relationship/by_file_ownership.go

@@ -32,7 +32,7 @@ type ownershipByFilesMetadata struct {
 	Files []string `json:"files"`
 }
 
-func byFileOwnershipOverlapWorker(accessor sbomsync.Accessor) {
+func ByFileOwnershipOverlapWorker(accessor sbomsync.Accessor) {
 	var relationships []artifact.Relationship
 
 	accessor.ReadFromSBOM(func(s *sbom.SBOM) {

internal/relationship/evident_by.go

@@ -5,7 +5,7 @@ import (
 	"github.com/anchore/syft/syft/pkg"
 )
 
-func evidentBy(catalog *pkg.Collection) []artifact.Relationship {
+func EvidentBy(catalog *pkg.Collection) []artifact.Relationship {
 	var edges []artifact.Relationship
 	for _, p := range catalog.Sorted() {
 		for _, l := range p.Locations.ToSlice() {

internal/relationship/evident_by_test.go

@@ -76,7 +76,7 @@ func TestRelationshipsEvidentBy(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			actual := evidentBy(tt.catalog)
+			actual := EvidentBy(tt.catalog)
 			require.Len(t, actual, len(tt.want))
 			for i := range actual {
 				assert.Equal(t, tt.want[i].From.ID(), actual[i].From.ID(), "from mismatch at index %d", i)

internal/relationship/exclude_binaries_by_file_ownership_overlap.go

@@ -23,7 +23,7 @@ var (
 	}
 )
 
-func excludeBinariesByFileOwnershipOverlap(accessor sbomsync.Accessor) {
+func ExcludeBinariesByFileOwnershipOverlap(accessor sbomsync.Accessor) {
 	accessor.WriteToSBOM(func(s *sbom.SBOM) {
 		for _, r := range s.Relationships {
 			if excludeBinaryByFileOwnershipOverlap(r, s.Artifacts.Packages) {

internal/relationship/finalize.go

@@ -1,50 +0,0 @@
-package relationship
-
-import (
-	"github.com/anchore/syft/internal/relationship/binary"
-	"github.com/anchore/syft/internal/sbomsync"
-	"github.com/anchore/syft/syft/artifact"
-	"github.com/anchore/syft/syft/cataloging"
-	"github.com/anchore/syft/syft/file"
-	"github.com/anchore/syft/syft/sbom"
-)
-
-func Finalize(resolver file.Resolver, builder sbomsync.Builder, cfg cataloging.RelationshipsConfig, src artifact.Identifiable) {
-	accessor := builder.(sbomsync.Accessor)
-
-	// remove ELF packages and Binary packages that are already
-	// represented by a source package (e.g. a package that is evident by some package manager)
-	builder.DeletePackages(binary.PackagesToRemove(resolver, accessor)...)
-
-	// add relationships showing packages that are evident by a file which is owned by another package (package-to-package)
-	if cfg.PackageFileOwnershipOverlap {
-		byFileOwnershipOverlapWorker(accessor)
-	}
-
-	// conditionally remove binary packages based on file ownership overlap relationships found
-	// https://github.com/anchore/syft/issues/931
-	if cfg.ExcludeBinaryPackagesWithFileOwnershipOverlap {
-		excludeBinariesByFileOwnershipOverlap(accessor)
-	}
-
-	// add the new relationships for executables to the SBOM
-	newBinaryRelationships := binary.NewDependencyRelationships(resolver, accessor)
-	accessor.WriteToSBOM(func(s *sbom.SBOM) {
-		s.Relationships = append(s.Relationships, newBinaryRelationships...)
-	})
-	builder.AddRelationships(newBinaryRelationships...)
-	// add source "contains package" relationship (source-to-package)
-	var sourceRelationships []artifact.Relationship
-	accessor.ReadFromSBOM(func(s *sbom.SBOM) {
-		sourceRelationships = toSource(src, s.Artifacts.Packages)
-	})
-	builder.AddRelationships(sourceRelationships...)
-
-	// add evident-by relationships (package-to-file)
-	var evidentByRelationships []artifact.Relationship
-	accessor.ReadFromSBOM(func(s *sbom.SBOM) {
-		evidentByRelationships = evidentBy(s.Artifacts.Packages)
-	})
-
-	builder.AddRelationships(evidentByRelationships...)
-}

internal/relationship/index.go

@@ -0,0 +1,88 @@
+package relationship
+
+import (
+	"github.com/scylladb/go-set/strset"
+
+	"github.com/anchore/syft/syft/artifact"
+)
+
+type Index struct {
+	typesByFromTo map[artifact.ID]map[artifact.ID]*strset.Set
+	existing      []artifact.Relationship
+	additional    []artifact.Relationship
+}
+
+func NewIndex(existing ...artifact.Relationship) *Index {
+	r := &Index{
+		typesByFromTo: make(map[artifact.ID]map[artifact.ID]*strset.Set),
+	}
+	r.TrackAll(existing...)
+	return r
+}
+
+func (i *Index) track(r artifact.Relationship) bool {
+	fromID := r.From.ID()
+	if _, ok := i.typesByFromTo[fromID]; !ok {
+		i.typesByFromTo[fromID] = make(map[artifact.ID]*strset.Set)
+	}
+
+	toID := r.To.ID()
+	if _, ok := i.typesByFromTo[fromID][toID]; !ok {
+		i.typesByFromTo[fromID][toID] = strset.New()
+	}
+
+	var exists bool
+	if i.typesByFromTo[fromID][toID].Has(string(r.Type)) {
+		exists = true
+	}
+
+	i.typesByFromTo[fromID][toID].Add(string(r.Type))
+
+	return !exists
+}
+
+// Track this relationship as "exists" in the index (this is used to prevent duplicate relationships from being added).
+// returns true if the relationship is new to the index, false otherwise.
+func (i *Index) Track(r artifact.Relationship) bool {
+	unique := i.track(r)
+	if unique {
+		i.existing = append(i.existing, r)
+	}
+	return unique
+}
+
+// Add a new relationship to the index, returning true if the relationship is new to the index, false otherwise (thus is a duplicate).
+func (i *Index) Add(r artifact.Relationship) bool {
+	if i.track(r) {
+		i.additional = append(i.additional, r)
+		return true
+	}
+	return false
+}
+
+func (i *Index) TrackAll(rs ...artifact.Relationship) {
+	for _, r := range rs {
+		i.Track(r)
+	}
+}
+
+func (i *Index) AddAll(rs ...artifact.Relationship) {
+	for _, r := range rs {
+		i.Add(r)
+	}
+}
+
+func (i *Index) NewRelationships() []artifact.Relationship {
+	return i.additional
+}
+
+func (i *Index) ExistingRelationships() []artifact.Relationship {
+	return i.existing
+}
+
+func (i *Index) AllUniqueRelationships() []artifact.Relationship {
+	var all []artifact.Relationship
+	all = append(all, i.existing...)
+	all = append(all, i.additional...)
+	return all
+}

internal/relationship/index_test.go

@@ -0,0 +1,225 @@
+package relationship
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/anchore/syft/syft/artifact"
+)
+
+func Test_newRelationshipIndex(t *testing.T) {
+	from := fakeIdentifiable{id: "from"}
+	to := fakeIdentifiable{id: "to"}
+	tests := []struct {
+		name           string
+		given          []artifact.Relationship
+		track          []artifact.Relationship
+		add            []artifact.Relationship
+		wantExisting   []string
+		wantAdditional []string
+	}{
+		{
+			name: "empty",
+		},
+		{
+			name: "tracks existing relationships",
+			given: []artifact.Relationship{
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+			},
+			wantExisting: []string{"from [evident-by] to"},
+		},
+		{
+			name: "deduplicate tracked relationships",
+			given: []artifact.Relationship{
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+			},
+			track: []artifact.Relationship{
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+			},
+			wantExisting: []string{"from [evident-by] to"},
+		},
+		{
+			name: "deduplicate any input relationships",
+			given: []artifact.Relationship{
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+			},
+			track: []artifact.Relationship{
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+			},
+			add: []artifact.Relationship{
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+			},
+			wantExisting: []string{"from [evident-by] to"},
+		},
+		{
+			name: "deduplicate any added relationships",
+			add: []artifact.Relationship{
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+				{
+					From: from,
+					To:   to,
+					Type: artifact.EvidentByRelationship,
+				},
+			},
+			wantAdditional: []string{"from [evident-by] to"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			idx := NewIndex(tt.given...)
+			idx.TrackAll(tt.track...)
+			idx.AddAll(tt.add...)
+			diffRelationships(t, tt.wantExisting, idx.existing)
+			diffRelationships(t, tt.wantAdditional, idx.additional)
+		})
+	}
+}
+
+func diffRelationships(t *testing.T, expected []string, actual []artifact.Relationship) {
+	if d := cmp.Diff(expected, stringRelationships(actual)); d != "" {
+		t.Errorf("unexpected relationships (-want, +got): %s", d)
+	}
+}
+
+func stringRelationships(relationships []artifact.Relationship) []string {
+	var result []string
+	for _, r := range relationships {
+		result = append(result, string(r.From.ID())+" ["+string(r.Type)+"] "+string(r.To.ID()))
+	}
+	return result
+
+}
+
+func Test_relationshipIndex_track(t *testing.T) {
+	from := fakeIdentifiable{id: "from"}
+	to := fakeIdentifiable{id: "to"}
+	relationship := artifact.Relationship{From: from, To: to, Type: artifact.EvidentByRelationship}
+	tests := []struct {
+		name     string
+		existing []artifact.Relationship
+		given    artifact.Relationship
+		want     bool
+	}{
+		{
+			name:     "track returns true for a new relationship",
+			existing: []artifact.Relationship{},
+			given:    relationship,
+			want:     true,
+		},
+		{
+			name:     "track returns false for an existing relationship",
+			existing: []artifact.Relationship{relationship},
+			given:    relationship,
+			want:     false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			i := NewIndex(tt.existing...)
+			if got := i.Track(tt.given); got != tt.want {
+				t.Errorf("track() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_relationshipIndex_add(t *testing.T) {
+	from := fakeIdentifiable{id: "from"}
+	to := fakeIdentifiable{id: "to"}
+	relationship := artifact.Relationship{From: from, To: to, Type: artifact.EvidentByRelationship}
+	tests := []struct {
+		name     string
+		existing []artifact.Relationship
+		given    artifact.Relationship
+		want     bool
+	}{
+		{
+			name:     "add returns true for a new relationship",
+			existing: []artifact.Relationship{},
+			given:    relationship,
+			want:     true,
+		},
+		{
+			name:     "add returns false for an existing relationship",
+			existing: []artifact.Relationship{relationship},
+			given:    relationship,
+			want:     false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			i := NewIndex(tt.existing...)
+			if got := i.Add(tt.given); got != tt.want {
+				t.Errorf("add() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+
+}
+
+type fakeIdentifiable struct {
+	id string
+}
+
+func (f fakeIdentifiable) ID() artifact.ID {
+	return artifact.ID(f.id)
+}

internal/relationship/to_source.go

@@ -5,7 +5,7 @@ import (
 	"github.com/anchore/syft/syft/pkg"
 )
 
-func toSource(src artifact.Identifiable, c *pkg.Collection) []artifact.Relationship {
+func ToSource(src artifact.Identifiable, c *pkg.Collection) []artifact.Relationship {
 	relationships := make([]artifact.Relationship, 0) // Should we pre-allocate this by giving catalog a Len() method?
 	for p := range c.Enumerate() {
 		relationships = append(relationships, artifact.Relationship{