Deduplicate packages across multiple container image layers (#930)

internal/anchore/import_package_sbom_test.go

@@ -56,14 +56,14 @@ func sbomFixture() sbom.SBOM {
 				Name:    "name",
 				Version: "version",
 				FoundBy: "foundBy",
-				Locations: []source.Location{
-					{
+				Locations: source.NewLocationSet(
+					source.Location{
 						Coordinates: source.Coordinates{
 							RealPath:     "path",
 							FileSystemID: "layerID",
 						},
 					},
-				},
+				),
 				Licenses: []string{"license"},
 				Language: pkg.Python,
 				Type:     pkg.PythonPkg,

internal/formats/common/cyclonedxhelpers/component.go

@@ -13,8 +13,9 @@ import (
 func encodeComponent(p pkg.Package) cyclonedx.Component {
 	props := encodeProperties(p, "syft:package")
 	props = append(props, encodeCPEs(p)...)
-	if len(p.Locations) > 0 {
-		props = append(props, encodeProperties(p.Locations, "syft:location")...)
+	locations := p.Locations.ToSlice()
+	if len(locations) > 0 {
+		props = append(props, encodeProperties(locations, "syft:location")...)
 	}
 	if hasMetadata(p) {
 		props = append(props, encodeProperties(p.Metadata, "syft:metadata")...)
@@ -73,10 +74,13 @@ func decodeComponent(c *cyclonedx.Component) *pkg.Package {
 	return p
 }
 
-func decodeLocations(vals map[string]string) []source.Location {
+func decodeLocations(vals map[string]string) source.LocationSet {
 	v := common.Decode(reflect.TypeOf([]source.Location{}), vals, "syft:location", CycloneDXFields)
-	out, _ := v.([]source.Location)
-	return out
+	out, ok := v.([]source.Location)
+	if !ok {
+		out = nil
+	}
+	return source.NewLocationSet(out...)
 }
 
 func decodePackageMetadata(vals map[string]string, c *cyclonedx.Component, typ pkg.MetadataType) interface{} {

internal/formats/common/cyclonedxhelpers/component_test.go

@@ -26,9 +26,9 @@ func Test_encodeComponentProperties(t *testing.T) {
 			name: "from apk",
 			input: pkg.Package{
 				FoundBy: "cataloger",
-				Locations: []source.Location{
-					{Coordinates: source.Coordinates{RealPath: "test"}},
-				},
+				Locations: source.NewLocationSet(
+					source.Location{Coordinates: source.Coordinates{RealPath: "test"}},
+				),
 				Metadata: pkg.ApkMetadata{
 					Package:          "libc-utils",
 					OriginPackage:    "libc-dev",

internal/formats/common/spdxhelpers/source_info.go

@@ -35,7 +35,7 @@ func SourceInfo(p pkg.Package) string {
 		answer = "acquired package info from the following paths"
 	}
 	var paths []string
-	for _, l := range p.Locations {
+	for _, l := range p.Locations.ToSlice() {
 		paths = append(paths, l.RealPath)
 	}
 

internal/formats/common/spdxhelpers/source_info_test.go

@@ -18,10 +18,10 @@ func Test_SourceInfo(t *testing.T) {
 			name: "locations are captured",
 			input: pkg.Package{
 				// note: no type given
-				Locations: []source.Location{
+				Locations: source.NewLocationSet(
 					source.NewVirtualLocation("/a-place", "/b-place"),
 					source.NewVirtualLocation("/c-place", "/d-place"),
-				},
+				),
 			},
 			expected: []string{
 				"from the following paths",

internal/formats/common/testutils/utils.go

@@ -157,9 +157,9 @@ func populateImageCatalog(catalog *pkg.Catalog, img *image.Image) {
 	catalog.Add(pkg.Package{
 		Name:    "package-1",
 		Version: "1.0.1",
-		Locations: []source.Location{
+		Locations: source.NewLocationSet(
 			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
-		},
+		),
 		Type:         pkg.PythonPkg,
 		FoundBy:      "the-cataloger-1",
 		Language:     pkg.Python,
@@ -177,9 +177,9 @@ func populateImageCatalog(catalog *pkg.Catalog, img *image.Image) {
 	catalog.Add(pkg.Package{
 		Name:    "package-2",
 		Version: "2.0.1",
-		Locations: []source.Location{
+		Locations: source.NewLocationSet(
 			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
-		},
+		),
 		Type:         pkg.DebPkg,
 		FoundBy:      "the-cataloger-2",
 		MetadataType: pkg.DpkgMetadataType,
@@ -234,9 +234,9 @@ func newDirectoryCatalog() *pkg.Catalog {
 		Version: "1.0.1",
 		Type:    pkg.PythonPkg,
 		FoundBy: "the-cataloger-1",
-		Locations: []source.Location{
+		Locations: source.NewLocationSet(
 			source.NewLocation("/some/path/pkg1"),
-		},
+		),
 		Language:     pkg.Python,
 		MetadataType: pkg.PythonPackageMetadataType,
 		Licenses:     []string{"MIT"},
@@ -259,9 +259,9 @@ func newDirectoryCatalog() *pkg.Catalog {
 		Version: "2.0.1",
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-2",
-		Locations: []source.Location{
+		Locations: source.NewLocationSet(
 			source.NewLocation("/some/path/pkg1"),
-		},
+		),
 		MetadataType: pkg.DpkgMetadataType,
 		Metadata: pkg.DpkgMetadata{
 			Package: "package-2",

internal/formats/github/encoder.go

@@ -58,8 +58,9 @@ func toSnapshotMetadata(s *sbom.SBOM) Metadata {
 }
 
 func filesystem(p pkg.Package) string {
-	if len(p.Locations) > 0 {
-		return p.Locations[0].FileSystemID
+	locations := p.Locations.ToSlice()
+	if len(locations) > 0 {
+		return locations[0].FileSystemID
 	}
 	return ""
 }
@@ -76,8 +77,9 @@ func toPath(s source.Metadata, p pkg.Package) string {
 	if inputPath == "." {
 		inputPath = ""
 	}
-	if len(p.Locations) > 0 {
-		location := p.Locations[0]
+	locations := p.Locations.ToSlice()
+	if len(locations) > 0 {
+		location := locations[0]
 		packagePath := location.RealPath
 		if location.VirtualPath != "" {
 			packagePath = location.VirtualPath

internal/formats/github/encoder_test.go

@@ -35,32 +35,38 @@ func Test_toGithubModel(t *testing.T) {
 		{
 			Name:    "pkg-1",
 			Version: "1.0.1",
-			Locations: []source.Location{{
-				Coordinates: source.Coordinates{
-					RealPath:     "/usr/lib",
-					FileSystemID: "fsid-1",
+			Locations: source.NewLocationSet(
+				source.Location{
+					Coordinates: source.Coordinates{
+						RealPath:     "/usr/lib",
+						FileSystemID: "fsid-1",
+					},
 				},
-			}},
+			),
 		},
 		{
 			Name:    "pkg-2",
 			Version: "2.0.2",
-			Locations: []source.Location{{
-				Coordinates: source.Coordinates{
-					RealPath:     "/usr/lib",
-					FileSystemID: "fsid-1",
+			Locations: source.NewLocationSet(
+				source.Location{
+					Coordinates: source.Coordinates{
+						RealPath:     "/usr/lib",
+						FileSystemID: "fsid-1",
+					},
 				},
-			}},
+			),
 		},
 		{
 			Name:    "pkg-3",
 			Version: "3.0.3",
-			Locations: []source.Location{{
-				Coordinates: source.Coordinates{
-					RealPath:     "/etc",
-					FileSystemID: "fsid-1",
+			Locations: source.NewLocationSet(
+				source.Location{
+					Coordinates: source.Coordinates{
+						RealPath:     "/etc",
+						FileSystemID: "fsid-1",
+					},
 				},
-			}},
+			),
 		},
 	} {
 		p.PURL = packageurl.NewPackageURL(

internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONDirectoryEncoder.golden

@@ -3,18 +3,18 @@
  "name": "/some/path",
  "spdxVersion": "SPDX-2.2",
  "creationInfo": {
-  "created": "2021-12-20T19:12:47.869816Z",
+  "created": "2022-03-30T21:48:28.297464Z",
   "creators": [
    "Organization: Anchore, Inc",
    "Tool: syft-[not provided]"
   ],
-  "licenseListVersion": "3.15"
+  "licenseListVersion": "3.16"
  },
  "dataLicense": "CC0-1.0",
- "documentNamespace": "https://anchore.com/syft/dir/some/path-4b896ded-7852-4e31-b764-136b53bdf346",
+ "documentNamespace": "https://anchore.com/syft/dir/some/path-e188d59b-76f6-4c7f-a9f2-1ae7d0577781",
  "packages": [
   {
-   "SPDXID": "SPDXRef-1d97af55efe9512f",
+   "SPDXID": "SPDXRef-b85dbb4e6ece5082",
    "name": "package-1",
    "licenseConcluded": "MIT",
    "downloadLocation": "NOASSERTION",
@@ -36,7 +36,7 @@
    "versionInfo": "1.0.1"
   },
   {
-   "SPDXID": "SPDXRef-ad3d1c4abd84bf75",
+   "SPDXID": "SPDXRef-ceda99598967ae8d",
    "name": "package-2",
    "licenseConcluded": "NONE",
    "downloadLocation": "NOASSERTION",

internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONImageEncoder.golden

@@ -3,18 +3,18 @@
  "name": "user-image-input",
  "spdxVersion": "SPDX-2.2",
  "creationInfo": {
-  "created": "2021-12-20T19:13:07.647486Z",
+  "created": "2022-03-30T21:48:28.303986Z",
   "creators": [
    "Organization: Anchore, Inc",
    "Tool: syft-[not provided]"
   ],
-  "licenseListVersion": "3.15"
+  "licenseListVersion": "3.16"
  },
  "dataLicense": "CC0-1.0",
- "documentNamespace": "https://anchore.com/syft/image/user-image-input-174da656-1824-4bd3-8604-28919f8a65bc",
+ "documentNamespace": "https://anchore.com/syft/image/user-image-input-9e4f4190-c5ae-4e31-a852-d1ab71357516",
  "packages": [
   {
-   "SPDXID": "SPDXRef-d16127444133b5c1",
+   "SPDXID": "SPDXRef-2a46171f91c8d4bc",
    "name": "package-1",
    "licenseConcluded": "MIT",
    "downloadLocation": "NOASSERTION",
@@ -36,7 +36,7 @@
    "versionInfo": "1.0.1"
   },
   {
-   "SPDXID": "SPDXRef-24907357f3705420",
+   "SPDXID": "SPDXRef-ae77680e9b1d087e",
    "name": "package-2",
    "licenseConcluded": "NONE",
    "downloadLocation": "NOASSERTION",

internal/formats/spdx22tagvalue/test-fixtures/snapshot/TestSPDXTagValueDirectoryEncoder.golden

@@ -2,16 +2,16 @@ SPDXVersion: SPDX-2.2
 DataLicense: CC0-1.0
 SPDXID: SPDXRef-DOCUMENT
 DocumentName: /some/path
-DocumentNamespace: https://anchore.com/syft/dir/some/path-4b90f56d-d596-4ad8-b6a5-17f7d801350d
+DocumentNamespace: https://anchore.com/syft/dir/some/path-71aa3553-1a73-405f-9f1f-6347d6d4593b
 LicenseListVersion: 3.16
 Creator: Organization: Anchore, Inc
 Creator: Tool: syft-[not provided]
-Created: 2022-02-10T21:09:27Z
+Created: 2022-03-30T21:48:22Z
 
 ##### Package: package-2
 
 PackageName: package-2
-SPDXID: SPDXRef-Package-deb-package-2-ad3d1c4abd84bf75
+SPDXID: SPDXRef-Package-deb-package-2-ceda99598967ae8d
 PackageVersion: 2.0.1
 PackageDownloadLocation: NOASSERTION
 FilesAnalyzed: false
@@ -24,7 +24,7 @@ ExternalRef: PACKAGE_MANAGER purl a-purl-2
 ##### Package: package-1
 
 PackageName: package-1
-SPDXID: SPDXRef-Package-python-package-1-1d97af55efe9512f
+SPDXID: SPDXRef-Package-python-package-1-b85dbb4e6ece5082
 PackageVersion: 1.0.1
 PackageDownloadLocation: NOASSERTION
 FilesAnalyzed: false

internal/formats/spdx22tagvalue/test-fixtures/snapshot/TestSPDXTagValueImageEncoder.golden

@@ -2,16 +2,16 @@ SPDXVersion: SPDX-2.2
 DataLicense: CC0-1.0
 SPDXID: SPDXRef-DOCUMENT
 DocumentName: user-image-input
-DocumentNamespace: https://anchore.com/syft/image/user-image-input-26a2def6-53d0-4504-b99a-a046832508ac
+DocumentNamespace: https://anchore.com/syft/image/user-image-input-e46e20f4-43a4-40e7-9f82-fd55b8a89e5f
 LicenseListVersion: 3.16
 Creator: Organization: Anchore, Inc
 Creator: Tool: syft-[not provided]
-Created: 2022-02-10T21:09:27Z
+Created: 2022-03-30T21:48:22Z
 
 ##### Package: package-2
 
 PackageName: package-2
-SPDXID: SPDXRef-Package-deb-package-2-73f796c846875b9e
+SPDXID: SPDXRef-Package-deb-package-2-ae77680e9b1d087e
 PackageVersion: 2.0.1
 PackageDownloadLocation: NOASSERTION
 FilesAnalyzed: false
@@ -24,7 +24,7 @@ ExternalRef: PACKAGE_MANAGER purl a-purl-2
 ##### Package: package-1
 
 PackageName: package-1
-SPDXID: SPDXRef-Package-python-package-1-d9527e708c11f8b9
+SPDXID: SPDXRef-Package-python-package-1-2a46171f91c8d4bc
 PackageVersion: 1.0.1
 PackageDownloadLocation: NOASSERTION
 FilesAnalyzed: false

internal/formats/syftjson/encoder_test.go

@@ -42,13 +42,13 @@ func TestEncodeFullJSONDocument(t *testing.T) {
 	p1 := pkg.Package{
 		Name:    "package-1",
 		Version: "1.0.1",
-		Locations: []source.Location{
-			{
+		Locations: source.NewLocationSet(
+			source.Location{
 				Coordinates: source.Coordinates{
 					RealPath: "/a/place/a",
 				},
 			},
-		},
+		),
 		Type:         pkg.PythonPkg,
 		FoundBy:      "the-cataloger-1",
 		Language:     pkg.Python,
@@ -68,13 +68,13 @@ func TestEncodeFullJSONDocument(t *testing.T) {
 	p2 := pkg.Package{
 		Name:    "package-2",
 		Version: "2.0.1",
-		Locations: []source.Location{
-			{
+		Locations: source.NewLocationSet(
+			source.Location{
 				Coordinates: source.Coordinates{
 					RealPath: "/b/place/b",
 				},
 			},
-		},
+		),
 		Type:         pkg.DebPkg,
 		FoundBy:      "the-cataloger-2",
 		MetadataType: pkg.DpkgMetadataType,

internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden

@@ -1,7 +1,7 @@
 {
  "artifacts": [
   {
-   "id": "1d97af55efe9512f",
+   "id": "b85dbb4e6ece5082",
    "name": "package-1",
    "version": "1.0.1",
    "type": "python",
@@ -36,7 +36,7 @@
    }
   },
   {
-   "id": "ad3d1c4abd84bf75",
+   "id": "ceda99598967ae8d",
    "name": "package-2",
    "version": "2.0.1",
    "type": "deb",

internal/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden

@@ -1,7 +1,7 @@
 {
  "artifacts": [
   {
-   "id": "d9a7c58726ab4bef",
+   "id": "b3fa3ee64756b0c6",
    "name": "package-1",
    "version": "1.0.1",
    "type": "python",
@@ -31,7 +31,7 @@
    }
   },
   {
-   "id": "ac462e450060da2c",
+   "id": "b324f4d9ee5413fe",
    "name": "package-2",
    "version": "2.0.1",
    "type": "deb",

internal/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden

@@ -1,7 +1,7 @@
 {
  "artifacts": [
   {
-   "id": "d9527e708c11f8b9",
+   "id": "2a46171f91c8d4bc",
    "name": "package-1",
    "version": "1.0.1",
    "type": "python",
@@ -32,7 +32,7 @@
    }
   },
   {
-   "id": "73f796c846875b9e",
+   "id": "ae77680e9b1d087e",
    "name": "package-2",
    "version": "2.0.1",
    "type": "deb",

internal/formats/syftjson/to_format_model.go

@@ -175,8 +175,9 @@ func toPackageModel(p pkg.Package) model.Package {
 		licenses = p.Licenses
 	}
 
-	var coordinates = make([]source.Coordinates, len(p.Locations))
-	for i, l := range p.Locations {
+	locations := p.Locations.ToSlice()
+	var coordinates = make([]source.Coordinates, len(locations))
+	for i, l := range locations {
 		coordinates[i] = l.Coordinates
 	}
 

internal/formats/syftjson/to_syft_model.go

@@ -53,7 +53,8 @@ func toSyftRelationships(doc *model.Document, catalog *pkg.Catalog, relationship
 
 	for _, p := range catalog.Sorted() {
 		idMap[string(p.ID())] = p
-		for _, l := range p.Locations {
+		locations := p.Locations.ToSlice()
+		for _, l := range locations {
 			idMap[string(l.Coordinates.ID())] = l.Coordinates
 		}
 	}
@@ -166,7 +167,7 @@ func toSyftPackage(p model.Package, idAliases map[string]string) pkg.Package {
 		Name:         p.Name,
 		Version:      p.Version,
 		FoundBy:      p.FoundBy,
-		Locations:    locations,
+		Locations:    source.NewLocationSet(locations...),
 		Licenses:     p.Licenses,
 		Language:     p.Language,
 		Type:         p.Type,