Remove MetadataType from core package object and normalize JSON metadataType values (#1983)

* [wip] Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * distinct the package metadata functions Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * remove metadata type from package core model Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * incorporate review feedback for names Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add RPM archive metadata and split parser helpers Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * clarify the python package metadata type Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename the KB metadata type Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * break hackage and composer types by use case Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * linting fix Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix encoding and decoding for syft-json and cyclonedx Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * bump json schema to 11 Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update cyclonedx-json snapshots Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update cyclonedx-xml snapshots Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update spdx-json snapshots Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update spdx-tv snapshots Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update syft-json snapshots Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * correct metadata type in stack yaml parser test Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix bom-ref redactor for cyclonedx-xml Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add tests for legacy package metadata names Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * regenerate json schema v11 Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix legacy HackageMetadataType reflect type value check Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * packagemetadata discovery should account for type shadowing Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix cli tests Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * bump json schema version to v12 Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update json schema to incorporate changes from main Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add syft-json legacy config option Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add tests around v11-v12 json decoding Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add docs for SYFT_JSON_LEGACY Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename structs to be compliant with new naming scheme Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
2026-11-03 07:56:36 +01:00 · 2023-10-30 12:12:04 -04:00
parent f442586ec9
commit 1aaa644007
261 changed files with 7010 additions and 4040 deletions
@@ -6,7 +6,7 @@ import (

 // TODO: it seems that the stack.yaml/stack.lock/cabal.project.freeze have different purposes and could have different installation intentions
 // (some describe intent and are meant to be used by a tool to resolve more dependencies while others describe the actual installed state).
-// This hints at splittin these into multiple catalogers, but for now we'll keep them together.
+// This hints at splitting these into multiple catalogers, but for now we'll keep them together.

 // NewHackageCataloger returns a new Haskell cataloger object.
 func NewHackageCataloger() *generic.Cataloger {
@@ -6,19 +6,15 @@ import (
 	"github.com/anchore/syft/syft/pkg"
 )

-func newPackage(name, version string, m *pkg.HackageMetadata, locations ...file.Location) pkg.Package {
+func newPackage(name, version string, m any, location file.Location) pkg.Package {
 	p := pkg.Package{
 		Name:      name,
 		Version:   version,
-		Locations: file.NewLocationSet(locations...),
+		Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
 		PURL:      packageURL(name, version),
 		Language:  pkg.Haskell,
 		Type:      pkg.HackagePkg,
-	}
-
-	if m != nil {
-		p.MetadataType = pkg.HackageMetadataType
-		p.Metadata = *m
+		Metadata:  m,
 	}

 	p.SetID()
@@ -52,7 +52,7 @@ func parseCabalFreeze(_ file.Resolver, _ *generic.Environment, reader file.Locat
 				pkgName,
 				pkgVersion,
 				nil,
-				reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
+				reader.Location,
 			),
 		)
 	}
@@ -7,6 +7,7 @@ import (

 	"gopkg.in/yaml.v3"

+	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
@@ -47,7 +48,8 @@ func parseStackLock(_ file.Resolver, _ *generic.Environment, reader file.Locatio
 	var lockFile stackLock

 	if err := yaml.Unmarshal(bytes, &lockFile); err != nil {
-		return nil, nil, fmt.Errorf("failed to parse stack.yaml.lock file: %w", err)
+		log.WithFields("error", err).Tracef("failed to parse stack.yaml.lock file %q", reader.RealPath)
+		return nil, nil, nil
 	}

 	var (
@@ -67,11 +69,11 @@ func parseStackLock(_ file.Resolver, _ *generic.Environment, reader file.Locatio
 			newPackage(
 				pkgName,
 				pkgVersion,
-				&pkg.HackageMetadata{
+				pkg.HackageStackYamlLockEntry{
 					PkgHash:     pkgHash,
 					SnapshotURL: snapshotURL,
 				},
-				reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
+				reader.Location,
 			),
 		)
 	}
@@ -16,118 +16,109 @@ func TestParseStackLock(t *testing.T) {

 	expectedPkgs := []pkg.Package{
 		{
-			Name:         "HTTP",
-			Version:      "4000.3.16",
-			PURL:         "pkg:hackage/HTTP@4000.3.16",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "HTTP",
+			Version:   "4000.3.16",
+			PURL:      "pkg:hackage/HTTP@4000.3.16",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlLockEntry{
 				PkgHash:     "6042643c15a0b43e522a6693f1e322f05000d519543a84149cb80aeffee34f71",
 				SnapshotURL: url,
 			},
 		},
 		{
-			Name:         "configurator-pg",
-			Version:      "0.2.6",
-			PURL:         "pkg:hackage/configurator-pg@0.2.6",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "configurator-pg",
+			Version:   "0.2.6",
+			PURL:      "pkg:hackage/configurator-pg@0.2.6",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlLockEntry{
 				PkgHash:     "cd9b06a458428e493a4d6def725af7ab1ab0fef678fbd871f9586fc7f9aa70be",
 				SnapshotURL: url,
 			},
 		},
 		{
-			Name:         "hasql-dynamic-statements",
-			Version:      "0.3.1.1",
-			PURL:         "pkg:hackage/hasql-dynamic-statements@0.3.1.1",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "hasql-dynamic-statements",
+			Version:   "0.3.1.1",
+			PURL:      "pkg:hackage/hasql-dynamic-statements@0.3.1.1",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlLockEntry{
 				PkgHash:     "2cfe6e75990e690f595a87cbe553f2e90fcd738610f6c66749c81cc4396b2cc4",
 				SnapshotURL: url,
 			},
 		},
 		{
-			Name:         "hasql-implicits",
-			Version:      "0.1.0.4",
-			PURL:         "pkg:hackage/hasql-implicits@0.1.0.4",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "hasql-implicits",
+			Version:   "0.1.0.4",
+			PURL:      "pkg:hackage/hasql-implicits@0.1.0.4",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlLockEntry{
 				PkgHash:     "0848d3cbc9d94e1e539948fa0be4d0326b26335034161bf8076785293444ca6f",
 				SnapshotURL: url,
 			},
 		},
 		{
-			Name:         "hasql-pool",
-			Version:      "0.5.2.2",
-			PURL:         "pkg:hackage/hasql-pool@0.5.2.2",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "hasql-pool",
+			Version:   "0.5.2.2",
+			PURL:      "pkg:hackage/hasql-pool@0.5.2.2",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlLockEntry{
 				PkgHash:     "b56d4dea112d97a2ef4b2749508c0ca646828cb2d77b827e8dc433d249bb2062",
 				SnapshotURL: url,
 			},
 		},
 		{
-			Name:         "lens-aeson",
-			Version:      "1.1.3",
-			PURL:         "pkg:hackage/lens-aeson@1.1.3",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "lens-aeson",
+			Version:   "1.1.3",
+			PURL:      "pkg:hackage/lens-aeson@1.1.3",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlLockEntry{
 				PkgHash:     "52c8eaecd2d1c2a969c0762277c4a8ee72c339a686727d5785932e72ef9c3050",
 				SnapshotURL: url,
 			},
 		},
 		{
-			Name:         "optparse-applicative",
-			Version:      "0.16.1.0",
-			PURL:         "pkg:hackage/optparse-applicative@0.16.1.0",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "optparse-applicative",
+			Version:   "0.16.1.0",
+			PURL:      "pkg:hackage/optparse-applicative@0.16.1.0",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlLockEntry{
 				PkgHash:     "418c22ed6a19124d457d96bc66bd22c93ac22fad0c7100fe4972bbb4ac989731",
 				SnapshotURL: url,
 			},
 		},
 		{
-			Name:         "protolude",
-			Version:      "0.3.2",
-			PURL:         "pkg:hackage/protolude@0.3.2",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "protolude",
+			Version:   "0.3.2",
+			PURL:      "pkg:hackage/protolude@0.3.2",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlLockEntry{
 				PkgHash:     "2a38b3dad40d238ab644e234b692c8911423f9d3ed0e36b62287c4a698d92cd1",
 				SnapshotURL: url,
 			},
 		},
 		{
-			Name:         "ptr",
-			Version:      "0.16.8.2",
-			PURL:         "pkg:hackage/ptr@0.16.8.2",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "ptr",
+			Version:   "0.16.8.2",
+			PURL:      "pkg:hackage/ptr@0.16.8.2",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlLockEntry{
 				PkgHash:     "708ebb95117f2872d2c5a554eb6804cf1126e86abe793b2673f913f14e5eb1ac",
 				SnapshotURL: url,
 			},
@@ -6,6 +6,7 @@ import (

 	"gopkg.in/yaml.v3"

+	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
@@ -28,7 +29,8 @@ func parseStackYaml(_ file.Resolver, _ *generic.Environment, reader file.Locatio
 	var stackFile stackYaml

 	if err := yaml.Unmarshal(bytes, &stackFile); err != nil {
-		return nil, nil, fmt.Errorf("failed to parse stack.yaml file: %w", err)
+		log.WithFields("error", err).Tracef("failed to parse stack.yaml file %q", reader.RealPath)
+		return nil, nil, nil
 	}

 	var pkgs []pkg.Package
@@ -39,10 +41,10 @@ func parseStackYaml(_ file.Resolver, _ *generic.Environment, reader file.Locatio
 			newPackage(
 				pkgName,
 				pkgVersion,
-				&pkg.HackageMetadata{
+				pkg.HackageStackYamlEntry{
 					PkgHash: pkgHash,
 				},
-				reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
+				reader.Location,
 			),
 		)
 	}
@@ -15,98 +15,90 @@ func TestParseStackYaml(t *testing.T) {

 	expectedPkgs := []pkg.Package{
 		{
-			Name:         "ShellCheck",
-			Version:      "0.8.0",
-			PURL:         "pkg:hackage/ShellCheck@0.8.0",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "ShellCheck",
+			Version:   "0.8.0",
+			PURL:      "pkg:hackage/ShellCheck@0.8.0",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlEntry{
 				PkgHash: "353c9322847b661e4c6f7c83c2acf8e5c08b682fbe516c7d46c29605937543df",
 			},
 		},
 		{
-			Name:         "colourista",
-			Version:      "0.1.0.1",
-			PURL:         "pkg:hackage/colourista@0.1.0.1",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "colourista",
+			Version:   "0.1.0.1",
+			PURL:      "pkg:hackage/colourista@0.1.0.1",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlEntry{
 				PkgHash: "98353ee0e2f5d97d2148513f084c1cd37dfda03e48aa9dd7a017c9d9c0ba710e",
 			},
 		},
 		{
-			Name:         "language-docker",
-			Version:      "11.0.0",
-			PURL:         "pkg:hackage/language-docker@11.0.0",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "language-docker",
+			Version:   "11.0.0",
+			PURL:      "pkg:hackage/language-docker@11.0.0",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlEntry{
 				PkgHash: "3406ff0c1d592490f53ead8cf2cd22bdf3d79fd125ccaf3add683f6d71c24d55",
 			},
 		},
 		{
-			Name:         "spdx",
-			Version:      "1.0.0.2",
-			PURL:         "pkg:hackage/spdx@1.0.0.2",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "spdx",
+			Version:   "1.0.0.2",
+			PURL:      "pkg:hackage/spdx@1.0.0.2",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlEntry{
 				PkgHash: "7dfac9b454ff2da0abb7560f0ffbe00ae442dd5cb76e8be469f77e6988a70fed",
 			},
 		},
 		{
-			Name:         "hspec",
-			Version:      "2.9.4",
-			PURL:         "pkg:hackage/hspec@2.9.4",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "hspec",
+			Version:   "2.9.4",
+			PURL:      "pkg:hackage/hspec@2.9.4",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlEntry{
 				PkgHash: "658a6a74d5a70c040edd6df2a12228c6d9e63082adaad1ed4d0438ad082a0ef3",
 			},
 		},
 		{
-			Name:         "hspec-core",
-			Version:      "2.9.4",
-			PURL:         "pkg:hackage/hspec-core@2.9.4",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "hspec-core",
+			Version:   "2.9.4",
+			PURL:      "pkg:hackage/hspec-core@2.9.4",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlEntry{
 				PkgHash: "a126e9087409fef8dcafcd2f8656456527ac7bb163ed4d9cb3a57589042a5fe8",
 			},
 		},
 		{
-			Name:         "hspec-discover",
-			Version:      "2.9.4",
-			PURL:         "pkg:hackage/hspec-discover@2.9.4",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "hspec-discover",
+			Version:   "2.9.4",
+			PURL:      "pkg:hackage/hspec-discover@2.9.4",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlEntry{
 				PkgHash: "fbcf49ecfc3d4da53e797fd0275264cba776ffa324ee223e2a3f4ec2d2c9c4a6",
 			},
 		},
 		{
-			Name:         "stm",
-			Version:      "2.5.0.2",
-			PURL:         "pkg:hackage/stm@2.5.0.2",
-			Locations:    locationSet,
-			Language:     pkg.Haskell,
-			Type:         pkg.HackagePkg,
-			MetadataType: pkg.HackageMetadataType,
-			Metadata: pkg.HackageMetadata{
+			Name:      "stm",
+			Version:   "2.5.0.2",
+			PURL:      "pkg:hackage/stm@2.5.0.2",
+			Locations: locationSet,
+			Language:  pkg.Haskell,
+			Type:      pkg.HackagePkg,
+			Metadata: pkg.HackageStackYamlEntry{
 				PkgHash: "e4dc6473faaa75fbd7eccab4e3ee1d651d75bb0e49946ef0b8b751ccde771a55",
 			},
 		},