931: binary cataloger exclusion defaults for ownership by overlap (#1948)

Fixes #931

PR #1948 introduces a new implicit exclusion for binary packages that overlap by file ownership and have certain characteristics:

1) the relationship between packages is OwnershipByFileOverlap
2) the parent package is an "os" package - see changelog for included catalogers
3) the child is a synthetic package generated by the binary cataloger - see changelog for included catalogers
4) the package names are identical

---------

Signed-off-by: Christopher Phillips <christopher.phillips@anchore.com>

test/integration/catalog_packages_test.go

@@ -23,7 +23,7 @@ func BenchmarkImagePackageCatalogers(b *testing.B) {
 	tarPath := imagetest.GetFixtureImageTarPath(b, fixtureImageName)
 
 	var pc *pkg.Collection
-	for _, c := range cataloger.ImageCatalogers(cataloger.DefaultConfig()) {
+	for _, c := range cataloger.ImageCatalogers(defaultConfig()) {
 		// in case of future alteration where state is persisted, assume no dependency is safe to reuse
 		userInput := "docker-archive:" + tarPath
 		detection, err := source.Detect(userInput, source.DefaultDetectConfig())
@@ -260,7 +260,7 @@ func TestPkgCoverageCatalogerConfiguration(t *testing.T) {
 	assert.Equal(t, definedLanguages, observedLanguages)
 
 	// Verify that rust isn't actually an image cataloger
-	c := cataloger.DefaultConfig()
+	c := defaultConfig()
 	c.Catalogers = []string{"rust"}
 	assert.Len(t, cataloger.ImageCatalogers(c), 0)
 }

test/integration/encode_decode_cycle_test.go

@@ -26,7 +26,10 @@ import (
 // encode-decode-encode loop which will detect lossy behavior in both directions.
 func TestEncodeDecodeEncodeCycleComparison(t *testing.T) {
 	// use second image for relationships
-	images := []string{"image-pkg-coverage", "image-owning-package"}
+	images := []string{
+		"image-pkg-coverage",
+		"image-owning-package",
+	}
 	tests := []struct {
 		formatOption sbom.FormatID
 		redactor     func(in []byte) []byte

test/integration/package_deduplication_test.go

@@ -22,13 +22,13 @@ func TestPackageDeduplication(t *testing.T) {
 	}{
 		{
 			scope:        source.AllLayersScope,
-			packageCount: 174, // without deduplication this would be 618
+			packageCount: 172, // without deduplication this would be 618
 			instanceCount: map[string]int{
 				"basesystem": 1,
 				"wget":       1,
 				"curl":       2, // upgraded in the image
 				"vsftpd":     1,
-				"httpd":      2, // rpm, binary
+				"httpd":      1, // rpm, - we exclude binary
 			},
 			locationCount: map[string]int{
 				"basesystem-10.0-7.el7.centos": 4,
@@ -37,18 +37,18 @@ func TestPackageDeduplication(t *testing.T) {
 				"wget-1.14-18.el7_6.1":         3,
 				"vsftpd-3.0.2-29.el7_9":        2,
 				"httpd-2.4.6-97.el7.centos.5":  1,
-				"httpd-2.4.6":                  1, // binary
+				// "httpd-2.4.6":                  1, // binary
 			},
 		},
 		{
 			scope:        source.SquashedScope,
-			packageCount: 172,
+			packageCount: 170,
 			instanceCount: map[string]int{
 				"basesystem": 1,
 				"wget":       1,
 				"curl":       1, // upgraded, but the most recent
 				"vsftpd":     1,
-				"httpd":      2, // rpm, binary
+				"httpd":      1, // rpm, binary is now excluded by overlap
 			},
 			locationCount: map[string]int{
 				"basesystem-10.0-7.el7.centos": 1,
@@ -56,7 +56,7 @@ func TestPackageDeduplication(t *testing.T) {
 				"wget-1.14-18.el7_6.1":         1,
 				"vsftpd-3.0.2-29.el7_9":        1,
 				"httpd-2.4.6-97.el7.centos.5":  1,
-				"httpd-2.4.6":                  1, // binary
+				// "httpd-2.4.6":                  1, // binary (excluded)
 			},
 		},
 	}
@@ -64,7 +64,6 @@ func TestPackageDeduplication(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(string(tt.scope), func(t *testing.T) {
 			sbom, _ := catalogFixtureImage(t, "image-vertical-package-dups", tt.scope, nil)
-
 			for _, p := range sbom.Artifacts.Packages.Sorted() {
 				if p.Type == pkg.BinaryPkg {
 					assert.NotEmpty(t, p.Name)

test/integration/package_ownership_relationship_test.go

@@ -7,11 +7,11 @@ import (
 
 	"github.com/anchore/syft/syft/formats/syftjson"
 	syftjsonModel "github.com/anchore/syft/syft/formats/syftjson/model"
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
 )
 
 func TestPackageOwnershipRelationships(t *testing.T) {
-
 	// ensure that the json encoder is applying artifact ownership with an image that has expected ownership relationships
 	tests := []struct {
 		fixture string
@@ -45,3 +45,43 @@ func TestPackageOwnershipRelationships(t *testing.T) {
 	}
 
 }
+
+func TestPackageOwnershipExclusions(t *testing.T) {
+	// ensure that the json encoder is excluding packages by artifact ownership with an image that has expected ownership relationships
+	tests := []struct {
+		name    string
+		fixture string
+	}{
+		{
+			name:    "busybox binary is filtered based on ownership relationship",
+			fixture: "image-os-binary-overlap",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.fixture, func(t *testing.T) {
+			sbom, _ := catalogFixtureImage(t, test.fixture, source.SquashedScope, nil)
+			binaryPackages := make([]pkg.Package, 0)
+			apkPackages := make([]pkg.Package, 0)
+			for p := range sbom.Artifacts.Packages.Enumerate() {
+				if p.Type == pkg.BinaryPkg && p.Name == "busybox" {
+					binaryPackages = append(binaryPackages, p)
+				}
+				if p.Type == pkg.ApkPkg && p.Name == "busybox" {
+					apkPackages = append(apkPackages, p)
+				}
+			}
+
+			if len(binaryPackages) != 0 {
+				packageNames := make([]string, 0)
+				for _, p := range binaryPackages {
+					packageNames = append(packageNames, p.Name)
+				}
+				t.Errorf("expected to find no binary packages but found %d packages: %v", len(binaryPackages), packageNames)
+			}
+			if len(apkPackages) == 0 {
+				t.Errorf("expected to find apk packages but found none")
+			}
+		})
+	}
+}

test/integration/test-fixtures/image-os-binary-overlap/Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine:latest
+
+# syft should not longer show the binary package for this image:
+# https://github.com/anchore/syft/issues/931
+# busybox                 1.36.1       binary <-- not reported synthetic package overlap
+# busybox                 1.36.1-r0    apk
+RUN apk update && apk add busybox

test/integration/utils_test.go

@@ -8,6 +8,8 @@ import (
 	"github.com/anchore/stereoscope/pkg/imagetest"
 	"github.com/anchore/syft/syft"
 	"github.com/anchore/syft/syft/pkg/cataloger"
+	"github.com/anchore/syft/syft/pkg/cataloger/kernel"
+	"github.com/anchore/syft/syft/pkg/cataloger/python"
 	"github.com/anchore/syft/syft/sbom"
 	"github.com/anchore/syft/syft/source"
 )
@@ -24,7 +26,7 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string, scope source.Sco
 		theSource.Close()
 	})
 
-	c := cataloger.DefaultConfig()
+	c := defaultConfig()
 	c.Catalogers = catalogerCfg
 
 	c.Search.Scope = scope
@@ -52,6 +54,16 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string, scope source.Sco
 	}, theSource
 }
 
+func defaultConfig() cataloger.Config {
+	return cataloger.Config{
+		Search:                          cataloger.DefaultSearchConfig(),
+		Parallelism:                     1,
+		LinuxKernel:                     kernel.DefaultLinuxCatalogerConfig(),
+		Python:                          python.DefaultCatalogerConfig(),
+		ExcludeBinaryOverlapByOwnership: true,
+	}
+}
+
 func catalogDirectory(t *testing.T, dir string) (sbom.SBOM, source.Source) {
 	userInput := "dir:" + dir
 	detection, err := source.Detect(userInput, source.DefaultDetectConfig())
@@ -63,7 +75,7 @@ func catalogDirectory(t *testing.T, dir string) (sbom.SBOM, source.Source) {
 	})
 
 	// TODO: this would be better with functional options (after/during API refactor)
-	c := cataloger.DefaultConfig()
+	c := defaultConfig()
 	c.Search.Scope = source.AllLayersScope
 	pkgCatalog, relationships, actualDistro, err := syft.CatalogPackages(theSource, c)
 	if err != nil {