931: binary cataloger exclusion defaults for ownership by overlap (#1948)

Fixes #931

PR #1948 introduces a new implicit exclusion for binary packages that overlap by file ownership and have certain characteristics:

1) the relationship between packages is OwnershipByFileOverlap
2) the parent package is an "os" package - see changelog for included catalogers
3) the child is a synthetic package generated by the binary cataloger - see changelog for included catalogers
4) the package names are identical

---------

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

syft/lib.go

@@ -74,11 +74,34 @@ func CatalogPackages(src source.Source, cfg cataloger.Config) (*pkg.Collection,
 
 	catalog, relationships, err := cataloger.Catalog(resolver, release, cfg.Parallelism, catalogers...)
 
-	relationships = append(relationships, newSourceRelationshipsFromCatalog(src, catalog)...)
+	// apply exclusions to the package catalog
+	// default config value for this is true
+	// https://github.com/anchore/syft/issues/931
+	if cfg.ExcludeBinaryOverlapByOwnership {
+		for _, r := range relationships {
+			if cataloger.Exclude(r, catalog) {
+				catalog.Delete(r.To.ID())
+				relationships = removeRelationshipsByID(relationships, r.To.ID())
+			}
+		}
+	}
 
+	// no need to consider source relationships for os -> binary exclusions
+	relationships = append(relationships, newSourceRelationshipsFromCatalog(src, catalog)...)
 	return catalog, relationships, release, err
 }
 
+func removeRelationshipsByID(relationships []artifact.Relationship, id artifact.ID) []artifact.Relationship {
+	// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
+	filtered := relationships[:0]
+	for _, r := range relationships {
+		if r.To.ID() != id && r.From.ID() != id {
+			filtered = append(filtered, r)
+		}
+	}
+	return filtered
+}
+
 func newSourceRelationshipsFromCatalog(src source.Source, c *pkg.Collection) []artifact.Relationship {
 	relationships := make([]artifact.Relationship, 0) // Should we pre-allocate this by giving catalog a Len() method?
 	for p := range c.Enumerate() {

syft/pkg/cataloger/alpm/cataloger.go

@@ -5,9 +5,9 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
-const catalogerName = "alpmdb-cataloger"
+const CatalogerName = "alpmdb-cataloger"
 
 func NewAlpmdbCataloger() *generic.Cataloger {
-	return generic.NewCataloger(catalogerName).
+	return generic.NewCataloger(CatalogerName).
 		WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob)
 }

syft/pkg/cataloger/apkdb/cataloger.go

@@ -8,10 +8,10 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
-const catalogerName = "apkdb-cataloger"
+const CatalogerName = "apkdb-cataloger"
 
 // NewApkdbCataloger returns a new Alpine DB cataloger object.
 func NewApkdbCataloger() *generic.Cataloger {
-	return generic.NewCataloger(catalogerName).
+	return generic.NewCataloger(CatalogerName).
 		WithParserByGlobs(parseApkDB, pkg.ApkDBGlob)
 }

syft/pkg/cataloger/binary/cataloger.go

@@ -7,7 +7,7 @@ import (
 	"github.com/anchore/syft/syft/pkg"
 )
 
-const catalogerName = "binary-cataloger"
+const CatalogerName = "binary-cataloger"
 
 func NewCataloger() *Cataloger {
 	return &Cataloger{}
@@ -22,7 +22,7 @@ type Cataloger struct{}
 
 // Name returns a string that uniquely describes the Cataloger
 func (c Cataloger) Name() string {
-	return catalogerName
+	return CatalogerName
 }
 
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages

syft/pkg/cataloger/binary/package.go

@@ -31,7 +31,7 @@ func newPackage(classifier classifier, location file.Location, matchMetadata map
 		),
 		Type:         pkg.BinaryPkg,
 		CPEs:         cpes,
-		FoundBy:      catalogerName,
+		FoundBy:      CatalogerName,
 		MetadataType: pkg.BinaryMetadataType,
 		Metadata: pkg.BinaryMetadata{
 			Matches: []pkg.ClassifierMatch{

syft/pkg/cataloger/config.go

@@ -8,23 +8,14 @@ import (
 )
 
 // TODO: these field naming vs helper function naming schemes are inconsistent.
-
 type Config struct {
-	Search      SearchConfig
-	Golang      golang.GoCatalogerOpts
-	LinuxKernel kernel.LinuxCatalogerConfig
-	Python      python.CatalogerConfig
-	Catalogers  []string
-	Parallelism int
-}
-
-func DefaultConfig() Config {
-	return Config{
-		Search:      DefaultSearchConfig(),
-		Parallelism: 1,
-		LinuxKernel: kernel.DefaultLinuxCatalogerConfig(),
-		Python:      python.DefaultCatalogerConfig(),
-	}
+	Search                          SearchConfig
+	Golang                          golang.GoCatalogerOpts
+	LinuxKernel                     kernel.LinuxCatalogerConfig
+	Python                          python.CatalogerConfig
+	Catalogers                      []string
+	Parallelism                     int
+	ExcludeBinaryOverlapByOwnership bool
 }
 
 func (c Config) Java() java.Config {

syft/pkg/cataloger/deb/cataloger.go

@@ -7,11 +7,11 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
-const catalogerName = "dpkgdb-cataloger"
+const CatalogerName = "dpkgdb-cataloger"
 
 // NewDpkgdbCataloger returns a new Deb package cataloger capable of parsing DPKG status DB files.
 func NewDpkgdbCataloger() *generic.Cataloger {
-	return generic.NewCataloger(catalogerName).
+	return generic.NewCataloger(CatalogerName).
 		// note: these globs have been intentionally split up in order to improve search performance,
 		// please do NOT combine into: "**/var/lib/dpkg/{status,status.d/*}"
 		WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status")

syft/pkg/cataloger/nix/cataloger.go

@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	catalogerName = "nix-store-cataloger"
+	CatalogerName = "nix-store-cataloger"
 	nixStoreGlob  = "**/nix/store/*"
 )
 
@@ -24,7 +24,7 @@ func NewStoreCataloger() *StoreCataloger {
 }
 
 func (c *StoreCataloger) Name() string {
-	return catalogerName
+	return CatalogerName
 }
 
 func (c *StoreCataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {

syft/pkg/cataloger/nix/cataloger_test.go

@@ -24,7 +24,7 @@ func TestCataloger_Catalog(t *testing.T) {
 					Version:      "2.34-210",
 					PURL:         "pkg:nix/glibc@2.34-210?output=bin&outputhash=h0cnbmfcn93xm5dg2x27ixhag1cwndga",
 					Locations:    file.NewLocationSet(file.NewLocation("nix/store/h0cnbmfcn93xm5dg2x27ixhag1cwndga-glibc-2.34-210-bin")),
-					FoundBy:      catalogerName,
+					FoundBy:      CatalogerName,
 					Type:         pkg.NixPkg,
 					MetadataType: pkg.NixStoreMetadataType,
 					Metadata: pkg.NixStoreMetadata{

syft/pkg/cataloger/nix/package.go

@@ -10,7 +10,7 @@ func newNixStorePackage(storePath nixStorePath, locations ...file.Location) pkg.
 	p := pkg.Package{
 		Name:         storePath.name,
 		Version:      storePath.version,
-		FoundBy:      catalogerName,
+		FoundBy:      CatalogerName,
 		Locations:    file.NewLocationSet(locations...),
 		Type:         pkg.NixPkg,
 		PURL:         packageURL(storePath),

syft/pkg/cataloger/package_exclusions.go

@@ -0,0 +1,55 @@
+package cataloger
+
+import (
+	"golang.org/x/exp/slices"
+
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/alpm"
+	"github.com/anchore/syft/syft/pkg/cataloger/apkdb"
+	"github.com/anchore/syft/syft/pkg/cataloger/binary"
+	"github.com/anchore/syft/syft/pkg/cataloger/deb"
+	"github.com/anchore/syft/syft/pkg/cataloger/nix"
+	"github.com/anchore/syft/syft/pkg/cataloger/rpm"
+)
+
+var (
+	osCatalogerTypes = []string{
+		apkdb.CatalogerName,
+		alpm.CatalogerName,
+		deb.CatalogerName,
+		nix.CatalogerName,
+		rpm.DBCatalogerName,
+		rpm.FileCatalogerName,
+	}
+	binaryCatalogerTypes = []string{binary.CatalogerName}
+)
+
+// Exclude will remove packages from a collection given the following properties are true
+// 1) the relationship between packages is OwnershipByFileOverlap
+// 2) the parent is an "os" package
+// 3) the child is a synthetic package generated by the binary cataloger
+// 4) the package names are identical
+// This exclude was implemented as a way to help resolve: https://github.com/anchore/syft/issues/931
+func Exclude(r artifact.Relationship, c *pkg.Collection) bool {
+	if artifact.OwnershipByFileOverlapRelationship != r.Type {
+		return false
+	}
+
+	parent := c.Package(r.From.ID())
+	if parent == nil {
+		return false
+	}
+
+	parentInExclusion := slices.Contains(osCatalogerTypes, parent.FoundBy)
+	if !parentInExclusion {
+		return false
+	}
+
+	child := c.Package(r.To.ID())
+	if child == nil {
+		return false
+	}
+
+	return slices.Contains(binaryCatalogerTypes, child.FoundBy)
+}

syft/pkg/cataloger/package_exclusions_test.go

@@ -0,0 +1,78 @@
+package cataloger
+
+import (
+	"testing"
+
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/apkdb"
+	"github.com/anchore/syft/syft/pkg/cataloger/binary"
+)
+
+func TestExclude(t *testing.T) {
+	packageA := pkg.Package{Name: "package-a", Type: pkg.ApkPkg, FoundBy: apkdb.CatalogerName}
+	packageB := pkg.Package{Name: "package-a", Type: pkg.PythonPkg, FoundBy: "language-cataloger"}
+	packageC := pkg.Package{Name: "package-a", Type: pkg.BinaryPkg, FoundBy: binary.CatalogerName}
+	packageD := pkg.Package{Name: "package-d", Type: pkg.BinaryPkg, FoundBy: binary.CatalogerName}
+	for _, p := range []*pkg.Package{&packageA, &packageB, &packageC, &packageD} {
+		p := p
+		p.SetID()
+	}
+
+	tests := []struct {
+		name          string
+		relationship  artifact.Relationship
+		packages      *pkg.Collection
+		shouldExclude bool
+	}{
+		{
+			name: "no exclusions from os -> python",
+			relationship: artifact.Relationship{
+				Type: artifact.OwnershipByFileOverlapRelationship,
+				From: packageA,
+				To:   packageB,
+			},
+			packages:      pkg.NewCollection(packageA, packageB),
+			shouldExclude: false,
+		},
+		{
+			name: "exclusions from os -> binary",
+			relationship: artifact.Relationship{
+				Type: artifact.OwnershipByFileOverlapRelationship,
+				From: packageA,
+				To:   packageC,
+			},
+			packages:      pkg.NewCollection(packageA, packageC),
+			shouldExclude: true,
+		},
+		{
+			name: "no exclusions from python -> binary",
+			relationship: artifact.Relationship{
+				Type: artifact.OwnershipByFileOverlapRelationship,
+				From: packageB,
+				To:   packageC,
+			},
+			packages:      pkg.NewCollection(packageB, packageC),
+			shouldExclude: false,
+		},
+		{
+			name: "no exclusions for different package names",
+			relationship: artifact.Relationship{
+				Type: artifact.OwnershipByFileOverlapRelationship,
+				From: packageA,
+				To:   packageD,
+			},
+			packages:      pkg.NewCollection(packageA, packageD),
+			shouldExclude: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if !Exclude(test.relationship, test.packages) && test.shouldExclude {
+				t.Errorf("expected to exclude relationship %+v", test.relationship)
+			}
+		})
+
+	}
+}

syft/pkg/cataloger/rpm/cataloger.go

@@ -11,6 +11,11 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
+const (
+	DBCatalogerName   = "rpm-db-cataloger"
+	FileCatalogerName = "rpm-file-cataloger"
+)
+
 // NewRpmDBCataloger returns a new RPM DB cataloger object.
 func NewRpmDBCataloger() *generic.Cataloger {
 	// check if a sqlite driver is available
@@ -18,14 +23,14 @@ func NewRpmDBCataloger() *generic.Cataloger {
 		log.Warnf("sqlite driver is not available, newer RPM databases might not be cataloged")
 	}
 
-	return generic.NewCataloger("rpm-db-cataloger").
+	return generic.NewCataloger(DBCatalogerName).
 		WithParserByGlobs(parseRpmDB, pkg.RpmDBGlob).
 		WithParserByGlobs(parseRpmManifest, pkg.RpmManifestGlob)
 }
 
 // NewFileCataloger returns a new RPM file cataloger object.
 func NewFileCataloger() *generic.Cataloger {
-	return generic.NewCataloger("rpm-file-cataloger").
+	return generic.NewCataloger(FileCatalogerName).
 		WithParserByGlobs(parseRpm, "**/*.rpm")
 }