Remove MetadataType from core package object and normalize JSON metadataType values (#1983)

* [wip] Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * distinct the package metadata functions Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * remove metadata type from package core model Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * incorporate review feedback for names Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add RPM archive metadata and split parser helpers Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * clarify the python package metadata type Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename the KB metadata type Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * break hackage and composer types by use case Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * linting fix Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix encoding and decoding for syft-json and cyclonedx Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * bump json schema to 11 Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update cyclonedx-json snapshots Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update cyclonedx-xml snapshots Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update spdx-json snapshots Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update spdx-tv snapshots Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update syft-json snapshots Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * correct metadata type in stack yaml parser test Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix bom-ref redactor for cyclonedx-xml Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add tests for legacy package metadata names Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * regenerate json schema v11 Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix legacy HackageMetadataType reflect type value check Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * packagemetadata discovery should account for type shadowing Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix cli tests Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * bump json schema version to v12 Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update json schema to incorporate changes from main Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add syft-json legacy config option Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add tests around v11-v12 json decoding Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add docs for SYFT_JSON_LEGACY Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename structs to be compliant with new naming scheme Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
2026-11-02 07:26:34 +01:00 · 2023-10-30 12:12:04 -04:00
parent f442586ec9
commit 1aaa644007
261 changed files with 7010 additions and 4040 deletions
@@ -5,9 +5,11 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"strings"

 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/internal/packagemetadata"
 	"github.com/anchore/syft/syft/license"
 	"github.com/anchore/syft/syft/pkg"
 )
@@ -60,28 +62,28 @@ func newModelLicensesFromValues(licenses []string) (ml []License) {
 }

 func (f *licenses) UnmarshalJSON(b []byte) error {
-	var licenses []License
-	if err := json.Unmarshal(b, &licenses); err != nil {
+	var lics []License
+	if err := json.Unmarshal(b, &lics); err != nil {
 		var simpleLicense []string
 		if err := json.Unmarshal(b, &simpleLicense); err != nil {
 			return fmt.Errorf("unable to unmarshal license: %w", err)
 		}
-		licenses = newModelLicensesFromValues(simpleLicense)
+		lics = newModelLicensesFromValues(simpleLicense)
 	}
-	*f = licenses
+	*f = lics
 	return nil
 }

 // PackageCustomData contains ambiguous values (type-wise) from pkg.Package.
 type PackageCustomData struct {
-	MetadataType pkg.MetadataType `json:"metadataType,omitempty"`
-	Metadata     interface{}      `json:"metadata,omitempty"`
+	MetadataType string `json:"metadataType,omitempty"`
+	Metadata     any    `json:"metadata,omitempty"`
 }

 // packageMetadataUnpacker is all values needed from Package to disambiguate ambiguous fields during json unmarshaling.
 type packageMetadataUnpacker struct {
-	MetadataType pkg.MetadataType `json:"metadataType"`
-	Metadata     json.RawMessage  `json:"metadata"`
+	MetadataType string          `json:"metadataType"`
+	Metadata     json.RawMessage `json:"metadata"`
 }

 func (p *packageMetadataUnpacker) String() string {
@@ -112,32 +114,64 @@ func (p *Package) UnmarshalJSON(b []byte) error {
 }

 func unpackPkgMetadata(p *Package, unpacker packageMetadataUnpacker) error {
-	p.MetadataType = pkg.CleanMetadataType(unpacker.MetadataType)
-
-	typ, ok := pkg.MetadataTypeByName[p.MetadataType]
-	if ok {
-		val := reflect.New(typ).Interface()
-		if len(unpacker.Metadata) > 0 {
-			if err := json.Unmarshal(unpacker.Metadata, val); err != nil {
-				return err
-			}
-		}
-		p.Metadata = reflect.ValueOf(val).Elem().Interface()
+	if unpacker.MetadataType == "" {
 		return nil
 	}

-	// capture unknown metadata as a generic struct
-	if len(unpacker.Metadata) > 0 {
-		var val interface{}
-		if err := json.Unmarshal(unpacker.Metadata, &val); err != nil {
-			return err
+	// check for legacy correction cases from schema v11 -> v12
+	ty := unpacker.MetadataType
+	switch unpacker.MetadataType {
+	case "HackageMetadataType":
+		for _, l := range p.Locations {
+			if strings.HasSuffix(l.RealPath, ".yaml.lock") {
+				ty = "haskell-hackage-stack-lock-entry"
+				break
+			} else if strings.HasSuffix(l.RealPath, ".yaml") {
+				ty = "haskell-hackage-stack-entry"
+				break
+			}
+		}
+	case "RpmMetadata":
+		for _, l := range p.Locations {
+			if strings.HasSuffix(l.RealPath, ".rpm") {
+				ty = "rpm-archive"
+				break
+			}
+		}
+	case "RustCargoPackageMetadata":
+		var found bool
+		for _, l := range p.Locations {
+			if strings.HasSuffix(strings.ToLower(l.RealPath), "cargo.lock") {
+				ty = "rust-cargo-lock-entry"
+				found = true
+				break
+			}
+		}
+		if !found {
+			ty = "rust-cargo-audit-entry"
 		}
-		p.Metadata = val
 	}

-	if p.MetadataType != "" {
+	typ := packagemetadata.ReflectTypeFromJSONName(ty)
+	if typ == nil {
+		// capture unknown metadata as a generic struct
+		if len(unpacker.Metadata) > 0 {
+			var val interface{}
+			if err := json.Unmarshal(unpacker.Metadata, &val); err != nil {
+				return err
+			}
+			p.Metadata = val
+		}
+
 		return errUnknownMetadataType
 	}

+	val := reflect.New(typ).Interface()
+	if len(unpacker.Metadata) > 0 {
+		if err := json.Unmarshal(unpacker.Metadata, val); err != nil {
+			return err
+		}
+	}
+	p.Metadata = reflect.ValueOf(val).Elem().Interface()
 	return nil
 }
@@ -12,7 +12,7 @@ import (
 	"github.com/anchore/syft/syft/pkg"
 )

-func TestUnmarshalPackageGolang(t *testing.T) {
+func Test_UnmarshalJSON(t *testing.T) {
 	tests := []struct {
 		name        string
 		packageData []byte
@@ -50,9 +50,9 @@ func TestUnmarshalPackageGolang(t *testing.T) {
 				}
 			}`),
 			assert: func(p *Package) {
-				assert.NotNil(t, p.Metadata)
-				golangMetadata := p.Metadata.(pkg.GolangBinMetadata)
-				assert.NotEmpty(t, golangMetadata)
+				require.NotNil(t, p.Metadata)
+				golangMetadata := p.Metadata.(pkg.GolangBinaryBuildinfoEntry)
+				require.NotEmpty(t, golangMetadata)
 				assert.Equal(t, "go1.18", golangMetadata.GoCompiledVersion)
 			},
 		},
@@ -169,6 +169,223 @@ func TestUnmarshalPackageGolang(t *testing.T) {
 				}, p.Licenses)
 			},
 		},
+		{
+			name: "breaking v11-v12 schema change: rpm db vs archive (select db)",
+			packageData: []byte(`{
+  "id": "739158935bfffc4d",
+  "name": "dbus",
+  "version": "1:1.12.8-12.el8",
+  "type": "rpm",
+  "foundBy": "rpm-db-cataloger",
+  "locations": [
+    {
+      "path": "/var/lib/rpm/Packages",
+      "layerID": "sha256:d871dadfb37b53ef1ca45be04fc527562b91989991a8f545345ae3be0b93f92a",
+      "annotations": {
+        "evidence": "primary"
+      }
+    }
+  ],
+  "licenses": [],
+  "language": "",
+  "cpes": [],
+  "purl": "pkg:rpm/centos/dbus@1.12.8-12.el8?arch=aarch64&epoch=1&upstream=dbus-1.12.8-12.el8.src.rpm&distro=centos-8",
+  "metadataType": "RpmMetadata",
+  "metadata": {
+    "name": "dbus",
+    "version": "1.12.8",
+    "epoch": 1,
+    "architecture": "aarch64",
+    "release": "12.el8",
+    "sourceRpm": "dbus-1.12.8-12.el8.src.rpm",
+    "size": 0,
+    "vendor": "CentOS",
+    "modularityLabel": "",
+    "files": []
+  }
+}
+`),
+			assert: func(p *Package) {
+				assert.Equal(t, pkg.RpmPkg, p.Type)
+				assert.Equal(t, reflect.TypeOf(pkg.RpmDBEntry{}).Name(), reflect.TypeOf(p.Metadata).Name())
+			},
+		},
+		{
+			name: "breaking v11-v12 schema change: rpm db vs archive (select archive)",
+			packageData: []byte(`{
+  "id": "739158935bfffc4d",
+  "name": "dbus",
+  "version": "1:1.12.8-12.el8",
+  "type": "rpm",
+  "foundBy": "rpm-db-cataloger",
+  "locations": [
+    {
+      "path": "/var/cache/dbus-1.12.8-12.el8.rpm",
+      "layerID": "sha256:d871dadfb37b53ef1ca45be04fc527562b91989991a8f545345ae3be0b93f92a",
+      "annotations": {
+        "evidence": "primary"
+      }
+    }
+  ],
+  "licenses": [],
+  "language": "",
+  "cpes": [],
+  "purl": "pkg:rpm/centos/dbus@1.12.8-12.el8?arch=aarch64&epoch=1&upstream=dbus-1.12.8-12.el8.src.rpm&distro=centos-8",
+  "metadataType": "RpmMetadata",
+  "metadata": {
+    "name": "dbus",
+    "version": "1.12.8",
+    "epoch": 1,
+    "architecture": "aarch64",
+    "release": "12.el8",
+    "sourceRpm": "dbus-1.12.8-12.el8.src.rpm",
+    "size": 0,
+    "vendor": "CentOS",
+    "modularityLabel": "",
+    "files": []
+  }
+}
+`),
+			assert: func(p *Package) {
+				assert.Equal(t, pkg.RpmPkg, p.Type)
+				assert.Equal(t, reflect.TypeOf(pkg.RpmArchive{}).Name(), reflect.TypeOf(p.Metadata).Name())
+			},
+		},
+		{
+			name: "breaking v11-v12 schema change: stack.yaml vs stack.yaml.lock (select stack.yaml)",
+			packageData: []byte(`{
+  "id": "46ff1a71f7715f38",
+  "name": "hspec-discover",
+  "version": "2.9.4",
+  "type": "hackage",
+  "foundBy": "haskell-cataloger",
+  "locations": [
+    {
+      "path": "/stack.yaml",
+      "annotations": {
+        "evidence": "primary"
+      }
+    }
+  ],
+  "licenses": [],
+  "language": "haskell",
+  "cpes": [
+    "cpe:2.3:a:hspec-discover:hspec-discover:2.9.4:*:*:*:*:*:*:*",
+    "cpe:2.3:a:hspec-discover:hspec_discover:2.9.4:*:*:*:*:*:*:*",
+    "cpe:2.3:a:hspec_discover:hspec-discover:2.9.4:*:*:*:*:*:*:*",
+    "cpe:2.3:a:hspec_discover:hspec_discover:2.9.4:*:*:*:*:*:*:*",
+    "cpe:2.3:a:hspec:hspec-discover:2.9.4:*:*:*:*:*:*:*",
+    "cpe:2.3:a:hspec:hspec_discover:2.9.4:*:*:*:*:*:*:*"
+  ],
+  "purl": "pkg:hackage/hspec-discover@2.9.4",
+  "metadataType": "HackageMetadataType",
+  "metadata": {
+    "name": "",
+    "version": "",
+    "pkgHash": "fbcf49ecfc3d4da53e797fd0275264cba776ffa324ee223e2a3f4ec2d2c9c4a6"
+  }
+}`),
+			assert: func(p *Package) {
+				assert.Equal(t, pkg.HackagePkg, p.Type)
+				assert.Equal(t, reflect.TypeOf(pkg.HackageStackYamlEntry{}).Name(), reflect.TypeOf(p.Metadata).Name())
+			},
+		},
+		{
+			name: "breaking v11-v12 schema change: stack.yaml vs stack.yaml.lock (select stack.yaml.lock)",
+			packageData: []byte(`{
+  "id": "87939e95124ceb92",
+  "name": "optparse-applicative",
+  "version": "0.16.1.0",
+  "type": "hackage",
+  "foundBy": "haskell-cataloger",
+  "locations": [
+    {
+      "path": "/stack.yaml.lock",
+      "annotations": {
+        "evidence": "primary"
+      }
+    }
+  ],
+  "licenses": [],
+  "language": "haskell",
+  "cpes": [
+    "cpe:2.3:a:optparse-applicative:optparse-applicative:0.16.1.0:*:*:*:*:*:*:*",
+    "cpe:2.3:a:optparse-applicative:optparse_applicative:0.16.1.0:*:*:*:*:*:*:*",
+    "cpe:2.3:a:optparse_applicative:optparse-applicative:0.16.1.0:*:*:*:*:*:*:*",
+    "cpe:2.3:a:optparse_applicative:optparse_applicative:0.16.1.0:*:*:*:*:*:*:*",
+    "cpe:2.3:a:optparse:optparse-applicative:0.16.1.0:*:*:*:*:*:*:*",
+    "cpe:2.3:a:optparse:optparse_applicative:0.16.1.0:*:*:*:*:*:*:*"
+  ],
+  "purl": "pkg:hackage/optparse-applicative@0.16.1.0",
+  "metadataType": "HackageMetadataType",
+  "metadata": {
+    "name": "",
+    "version": "",
+    "pkgHash": "418c22ed6a19124d457d96bc66bd22c93ac22fad0c7100fe4972bbb4ac989731",
+    "snapshotURL": "https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/19/14.yaml"
+  }
+}`),
+			assert: func(p *Package) {
+				assert.Equal(t, pkg.HackagePkg, p.Type)
+				assert.Equal(t, reflect.TypeOf(pkg.HackageStackYamlLockEntry{}).Name(), reflect.TypeOf(p.Metadata).Name())
+			},
+		},
+		{
+			name: "breaking v11-v12 schema change: rust cargo.lock vs audit (select cargo.lock)",
+			packageData: []byte(`{
+  "id": "95124ceb9287939e",
+  "name": "optpkg",
+  "version": "1.16.1",
+  "type": "hackage",
+  "foundBy": "rust-cataloger",
+  "locations": [
+    {
+      "path": "/cargo.lock",
+      "annotations": {
+        "evidence": "primary"
+      }
+    }
+  ],
+  "licenses": [],
+  "language": "rust",
+  "cpes": [],
+  "purl": "pkg:cargo/optpkg@1.16.1",
+  "metadataType": "RustCargoPackageMetadata",
+  "metadata": {}
+}`),
+			assert: func(p *Package) {
+				assert.Equal(t, pkg.HackagePkg, p.Type)
+				assert.Equal(t, reflect.TypeOf(pkg.RustCargoLockEntry{}).Name(), reflect.TypeOf(p.Metadata).Name())
+			},
+		},
+		{
+			name: "breaking v11-v12 schema change: rust cargo.lock vs audit (select audit binary)",
+			packageData: []byte(`{
+  "id": "95124ceb9287939e",
+  "name": "optpkg",
+  "version": "1.16.1",
+  "type": "hackage",
+  "foundBy": "rust-cataloger",
+  "locations": [
+    {
+      "path": "/my-binary",
+      "annotations": {
+        "evidence": "primary"
+      }
+    }
+  ],
+  "licenses": [],
+  "language": "rust",
+  "cpes": [],
+  "purl": "pkg:cargo/optpkg@1.16.1",
+  "metadataType": "RustCargoPackageMetadata",
+  "metadata": {}
+}`),
+			assert: func(p *Package) {
+				assert.Equal(t, pkg.HackagePkg, p.Type)
+				assert.Equal(t, reflect.TypeOf(pkg.RustBinaryAuditEntry{}).Name(), reflect.TypeOf(p.Metadata).Name())
+			},
+		},
 	}

 	for _, test := range tests {
@@ -185,13 +402,12 @@ func Test_unpackMetadata(t *testing.T) {
 	tests := []struct {
 		name         string
 		packageData  []byte
-		metadataType pkg.MetadataType
-		wantMetadata interface{}
+		wantMetadata any
 		wantErr      require.ErrorAssertionFunc
 	}{
 		{
 			name:         "unmarshal package metadata",
-			metadataType: pkg.GolangBinMetadataType,
+			wantMetadata: pkg.GolangBinaryBuildinfoEntry{},
 			packageData: []byte(`{
 				"id": "8b594519bc23da50",
 				"name": "gopkg.in/square/go-jose.v2",
@@ -217,7 +433,7 @@ func Test_unpackMetadata(t *testing.T) {
 		},
 		{
 			name:         "can handle package without metadata",
-			metadataType: "",
+			wantMetadata: nil,
 			packageData: []byte(`{
 				"id": "8b594519bc23da50",
 				"name": "gopkg.in/square/go-jose.v2",
@@ -237,7 +453,7 @@ func Test_unpackMetadata(t *testing.T) {
 		},
 		{
 			name:         "can handle RpmdbMetadata",
-			metadataType: pkg.RpmMetadataType,
+			wantMetadata: pkg.RpmDBEntry{},
 			packageData: []byte(`{
 				"id": "4ac699c3b8fe1835",
 				"name": "acl",
@@ -272,9 +488,8 @@ func Test_unpackMetadata(t *testing.T) {
 			}`),
 		},
 		{
-			name:         "bad metadata type is an error",
-			metadataType: "BOGOSITY",
-			wantErr:      require.Error,
+			name:    "bad metadata type is an error",
+			wantErr: require.Error,
 			packageData: []byte(`{
 				"id": "8b594519bc23da50",
 				"name": "gopkg.in/square/go-jose.v2",
@@ -301,8 +516,7 @@ func Test_unpackMetadata(t *testing.T) {
 					"thing": "thing-1"
 				}
 			}`),
-			wantErr:      require.Error,
-			metadataType: "NewMetadataType",
+			wantErr: require.Error,
 			wantMetadata: map[string]interface{}{
 				"thing": "thing-1",
 			},
@@ -312,39 +526,31 @@ func Test_unpackMetadata(t *testing.T) {
 			packageData: []byte(`{
 				"metadataType": "GolangBinMetadata"
 			}`),
-			metadataType: pkg.GolangBinMetadataType,
-			wantMetadata: pkg.GolangBinMetadata{},
+			wantMetadata: pkg.GolangBinaryBuildinfoEntry{},
 		},
 		{
 			name: "can handle package with golang bin metadata type",
 			packageData: []byte(`{
 				"metadataType": "GolangBinMetadata"
 			}`),
-			metadataType: pkg.GolangBinMetadataType,
-			wantMetadata: pkg.GolangBinMetadata{},
+			wantMetadata: pkg.GolangBinaryBuildinfoEntry{},
 		},
 		{
-			name: "can handle package with unknonwn metadata type and missing metadata",
+			name: "can handle package with unknown metadata type and missing metadata",
 			packageData: []byte(`{
 				"metadataType": "BadMetadata"
 			}`),
-			wantErr:      require.Error,
-			metadataType: "BadMetadata",
-			wantMetadata: nil,
+			wantErr: require.Error,
 		},
 		{
-			name: "can handle package with unknonwn metadata type and metadata",
+			name: "can handle package with unknown metadata type and metadata",
 			packageData: []byte(`{
 				"metadataType": "BadMetadata",
 				"metadata": {
 					"random": "thing"
 				}
 			}`),
-			wantErr:      require.Error,
-			metadataType: "BadMetadata",
-			wantMetadata: map[string]interface{}{
-				"random": "thing",
-			},
+			wantErr: require.Error,
 		},
 	}

@@ -355,19 +561,18 @@ func Test_unpackMetadata(t *testing.T) {
 			}
 			p := &Package{}

-			var basic PackageBasicData
-			require.NoError(t, json.Unmarshal(test.packageData, &basic))
-			p.PackageBasicData = basic
-
 			var unpacker packageMetadataUnpacker
 			require.NoError(t, json.Unmarshal(test.packageData, &unpacker))

 			err := unpackPkgMetadata(p, unpacker)
-			assert.Equal(t, test.metadataType, p.MetadataType)
 			test.wantErr(t, err)

 			if test.wantMetadata != nil {
-				assert.True(t, reflect.DeepEqual(test.wantMetadata, p.Metadata))
+				if p.Metadata == nil {
+					t.Fatalf("expected metadata to be populated")
+					return
+				}
+				assert.Equal(t, reflect.TypeOf(test.wantMetadata).Name(), reflect.TypeOf(p.Metadata).Name())
 			}
 		})
 	}