Remove MetadataType from core package object and normalize JSON metadataType values (#1983)

* [wip]

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* distinct the package metadata functions

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove metadata type from package core model

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* incorporate review feedback for names

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add RPM archive metadata and split parser helpers

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* clarify the python package metadata type

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename the KB metadata type

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* break hackage and composer types by use case

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* linting fix

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix encoding and decoding for syft-json and cyclonedx

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* bump json schema to 11

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update cyclonedx-json snapshots

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update cyclonedx-xml snapshots

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update spdx-json snapshots

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update spdx-tv snapshots

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update syft-json snapshots

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* correct metadata type in stack yaml parser test

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix bom-ref redactor for cyclonedx-xml

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tests for legacy package metadata names

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* regenerate json schema v11

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix legacy HackageMetadataType reflect type value check

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix linting

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* packagemetadata discovery should account for type shadowing

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix linting

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix cli tests

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* bump json schema version to v12

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update json schema to incorporate changes from main

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add syft-json legacy config option

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tests around v11-v12 json decoding

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add docs for SYFT_JSON_LEGACY

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename structs to be compliant with new naming scheme

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2023-10-30 12:12:04 -04:00
committed by GitHub
parent f442586ec9
commit 1aaa644007
261 changed files with 7010 additions and 4040 deletions
+1
View File
@@ -10,6 +10,7 @@ func NewDotnetDepsCataloger() *generic.Cataloger {
WithParserByGlobs(parseDotnetDeps, "**/*.deps.json")
}
// NewDotnetPortableExecutableCataloger returns a new Dotnet cataloger object base on portable executable files.
func NewDotnetPortableExecutableCataloger() *generic.Cataloger {
return generic.NewCataloger("dotnet-portable-executable-cataloger").
WithParserByGlobs(parseDotnetPortableExecutable, "**/*.dll", "**/*.exe")
+9 -10
View File
@@ -13,7 +13,7 @@ import (
func newDotnetDepsPackage(nameVersion string, lib dotnetDepsLibrary, locations ...file.Location) *pkg.Package {
name, version := extractNameAndVersion(nameVersion)
m := pkg.DotnetDepsMetadata{
m := pkg.DotnetDepsEntry{
Name: name,
Version: version,
Path: lib.Path,
@@ -22,14 +22,13 @@ func newDotnetDepsPackage(nameVersion string, lib dotnetDepsLibrary, locations .
}
p := &pkg.Package{
Name: name,
Version: version,
Locations: file.NewLocationSet(locations...),
PURL: packageURL(m),
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: m,
Name: name,
Version: version,
Locations: file.NewLocationSet(locations...),
PURL: packageURL(m),
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: m,
}
p.SetID()
@@ -58,7 +57,7 @@ func createNameAndVersion(name, version string) (nameVersion string) {
return
}
func packageURL(m pkg.DotnetDepsMetadata) string {
func packageURL(m pkg.DotnetDepsEntry) string {
var qualifiers packageurl.Qualifiers
return packageurl.NewPackageURL(
@@ -14,6 +14,12 @@ import (
var _ generic.Parser = parseDotnetDeps
type dotnetDeps struct {
RuntimeTarget dotnetRuntimeTarget `json:"runtimeTarget"`
Targets map[string]map[string]dotnetDepsTarget `json:"targets"`
Libraries map[string]dotnetDepsLibrary `json:"libraries"`
}
type dotnetRuntimeTarget struct {
Name string `json:"name"`
}
@@ -22,11 +28,6 @@ type dotnetDepsTarget struct {
Dependencies map[string]string `json:"dependencies"`
Runtime map[string]struct{} `json:"runtime"`
}
type dotnetDeps struct {
RuntimeTarget dotnetRuntimeTarget `json:"runtimeTarget"`
Targets map[string]map[string]dotnetDepsTarget `json:"targets"`
Libraries map[string]dotnetDepsLibrary `json:"libraries"`
}
type dotnetDepsLibrary struct {
Type string `json:"type"`
@@ -13,40 +13,37 @@ func TestParseDotnetDeps(t *testing.T) {
fixture := "test-fixtures/TestLibrary.deps.json"
fixtureLocationSet := file.NewLocationSet(file.NewLocation(fixture))
rootPkg := pkg.Package{
Name: "TestLibrary",
Version: "1.0.0",
PURL: "pkg:nuget/TestLibrary@1.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "TestLibrary",
Version: "1.0.0",
PURL: "pkg:nuget/TestLibrary@1.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "TestLibrary",
Version: "1.0.0",
},
}
testCommon := pkg.Package{
Name: "TestCommon",
Version: "1.0.0",
PURL: "pkg:nuget/TestCommon@1.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "TestCommon",
Version: "1.0.0",
PURL: "pkg:nuget/TestCommon@1.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "TestCommon",
Version: "1.0.0",
},
}
awssdkcore := pkg.Package{
Name: "AWSSDK.Core",
Version: "3.7.10.6",
PURL: "pkg:nuget/AWSSDK.Core@3.7.10.6",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "AWSSDK.Core",
Version: "3.7.10.6",
PURL: "pkg:nuget/AWSSDK.Core@3.7.10.6",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "AWSSDK.Core",
Version: "3.7.10.6",
Sha512: "sha512-kHBB+QmosVaG6DpngXQ8OlLVVNMzltNITfsRr68Z90qO7dSqJ2EHNd8dtBU1u3AQQLqqFHOY0lfmbpexeH6Pew==",
@@ -55,14 +52,13 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
msftDependencyInjectionAbstractions := pkg.Package{
Name: "Microsoft.Extensions.DependencyInjection.Abstractions",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.DependencyInjection.Abstractions@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Microsoft.Extensions.DependencyInjection.Abstractions",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.DependencyInjection.Abstractions@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "Microsoft.Extensions.DependencyInjection.Abstractions",
Version: "6.0.0",
Sha512: "sha512-xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==",
@@ -71,14 +67,13 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
msftDependencyInjection := pkg.Package{
Name: "Microsoft.Extensions.DependencyInjection",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.DependencyInjection@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Microsoft.Extensions.DependencyInjection",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.DependencyInjection@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "Microsoft.Extensions.DependencyInjection",
Version: "6.0.0",
Sha512: "sha512-k6PWQMuoBDGGHOQTtyois2u4AwyVcIwL2LaSLlTZQm2CYcJ1pxbt6jfAnpWmzENA/wfrYRI/X9DTLoUkE4AsLw==",
@@ -87,14 +82,13 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
msftLoggingAbstractions := pkg.Package{
Name: "Microsoft.Extensions.Logging.Abstractions",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.Logging.Abstractions@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Microsoft.Extensions.Logging.Abstractions",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.Logging.Abstractions@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "Microsoft.Extensions.Logging.Abstractions",
Version: "6.0.0",
Sha512: "sha512-/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==",
@@ -103,14 +97,13 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
msftExtensionsLogging := pkg.Package{
Name: "Microsoft.Extensions.Logging",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.Logging@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Microsoft.Extensions.Logging",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.Logging@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "Microsoft.Extensions.Logging",
Version: "6.0.0",
Sha512: "sha512-eIbyj40QDg1NDz0HBW0S5f3wrLVnKWnDJ/JtZ+yJDFnDj90VoPuoPmFkeaXrtu+0cKm5GRAwoDf+dBWXK0TUdg==",
@@ -119,14 +112,13 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
msftExtensionsOptions := pkg.Package{
Name: "Microsoft.Extensions.Options",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.Options@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Microsoft.Extensions.Options",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.Options@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "Microsoft.Extensions.Options",
Version: "6.0.0",
Sha512: "sha512-dzXN0+V1AyjOe2xcJ86Qbo233KHuLEY0njf/P2Kw8SfJU+d45HNS2ctJdnEnrWbM9Ye2eFgaC5Mj9otRMU6IsQ==",
@@ -135,14 +127,13 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
msftExtensionsPrimitives := pkg.Package{
Name: "Microsoft.Extensions.Primitives",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.Primitives@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Microsoft.Extensions.Primitives",
Version: "6.0.0",
PURL: "pkg:nuget/Microsoft.Extensions.Primitives@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "Microsoft.Extensions.Primitives",
Version: "6.0.0",
Sha512: "sha512-9+PnzmQFfEFNR9J2aDTfJGGupShHjOuGw4VUv+JB044biSHrnmCIMD+mJHmb2H7YryrfBEXDurxQ47gJZdCKNQ==",
@@ -151,14 +142,13 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
newtonsoftJson := pkg.Package{
Name: "Newtonsoft.Json",
Version: "13.0.1",
PURL: "pkg:nuget/Newtonsoft.Json@13.0.1",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Newtonsoft.Json",
Version: "13.0.1",
PURL: "pkg:nuget/Newtonsoft.Json@13.0.1",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "Newtonsoft.Json",
Version: "13.0.1",
Sha512: "sha512-ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==",
@@ -167,14 +157,13 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
serilogSinksConsole := pkg.Package{
Name: "Serilog.Sinks.Console",
Version: "4.0.1",
PURL: "pkg:nuget/Serilog.Sinks.Console@4.0.1",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Serilog.Sinks.Console",
Version: "4.0.1",
PURL: "pkg:nuget/Serilog.Sinks.Console@4.0.1",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "Serilog.Sinks.Console",
Version: "4.0.1",
Sha512: "sha512-apLOvSJQLlIbKlbx+Y2UDHSP05kJsV7mou+fvJoRGs/iR+jC22r8cuFVMjjfVxz/AD4B2UCltFhE1naRLXwKNw==",
@@ -183,14 +172,13 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
serilog := pkg.Package{
Name: "Serilog",
Version: "2.10.0",
PURL: "pkg:nuget/Serilog@2.10.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Serilog",
Version: "2.10.0",
PURL: "pkg:nuget/Serilog@2.10.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "Serilog",
Version: "2.10.0",
Sha512: "sha512-+QX0hmf37a0/OZLxM3wL7V6/ADvC1XihXN4Kq/p6d8lCPfgkRdiuhbWlMaFjR9Av0dy5F0+MBeDmDdRZN/YwQA==",
@@ -199,14 +187,13 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
systemDiagnosticsDiagnosticsource := pkg.Package{
Name: "System.Diagnostics.DiagnosticSource",
Version: "6.0.0",
PURL: "pkg:nuget/System.Diagnostics.DiagnosticSource@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "System.Diagnostics.DiagnosticSource",
Version: "6.0.0",
PURL: "pkg:nuget/System.Diagnostics.DiagnosticSource@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "System.Diagnostics.DiagnosticSource",
Version: "6.0.0",
Sha512: "sha512-frQDfv0rl209cKm1lnwTgFPzNigy2EKk1BS3uAvHvlBVKe5cymGyHO+Sj+NLv5VF/AhHsqPIUUwya5oV4CHMUw==",
@@ -215,21 +202,19 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
systemRuntimeCompilerServicesUnsafe := pkg.Package{
Name: "System.Runtime.CompilerServices.Unsafe",
Version: "6.0.0",
PURL: "pkg:nuget/System.Runtime.CompilerServices.Unsafe@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "System.Runtime.CompilerServices.Unsafe",
Version: "6.0.0",
PURL: "pkg:nuget/System.Runtime.CompilerServices.Unsafe@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
Metadata: pkg.DotnetDepsEntry{
Name: "System.Runtime.CompilerServices.Unsafe",
Version: "6.0.0",
Sha512: "sha512-/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==",
Path: "system.runtime.compilerservices.unsafe/6.0.0",
HashPath: "system.runtime.compilerservices.unsafe.6.0.0.nupkg.sha512",
},
}
}}
expectedPkgs := []pkg.Package{
awssdkcore,
@@ -61,7 +61,7 @@ func parseDotnetPortableExecutable(_ file.Resolver, _ *generic.Environment, f fi
"",
).ToString()
metadata := pkg.DotnetPortableExecutableMetadata{
metadata := pkg.DotnetPortableExecutableEntry{
AssemblyVersion: versionResources["Assembly Version"],
LegalCopyright: versionResources["LegalCopyright"],
Comments: versionResources["Comments"],
@@ -72,13 +72,12 @@ func parseDotnetPortableExecutable(_ file.Resolver, _ *generic.Environment, f fi
}
p := pkg.Package{
Name: name,
Version: version,
Locations: file.NewLocationSet(f.Location),
Type: pkg.DotnetPkg,
PURL: purl,
MetadataType: pkg.DotnetPortableExecutableMetadataType,
Metadata: metadata,
Name: name,
Version: version,
Locations: file.NewLocationSet(f.Location),
Type: pkg.DotnetPkg,
PURL: purl,
Metadata: metadata,
}
p.SetID()
@@ -15,13 +15,12 @@ func TestParseDotnetPortableExecutable(t *testing.T) {
expected := []pkg.Package{
{
Name: "System.Buffers",
Version: "7.0.923.36201",
Locations: fixtureLocationSet,
Type: pkg.DotnetPkg,
PURL: "pkg:nuget/System.Buffers@7.0.923.36201",
MetadataType: pkg.DotnetPortableExecutableMetadataType,
Metadata: pkg.DotnetPortableExecutableMetadata{
Name: "System.Buffers",
Version: "7.0.923.36201",
Locations: fixtureLocationSet,
Type: pkg.DotnetPkg,
PURL: "pkg:nuget/System.Buffers@7.0.923.36201",
Metadata: pkg.DotnetPortableExecutableEntry{
AssemblyVersion: "7.0.0.0",
LegalCopyright: "© Microsoft Corporation. All rights reserved.",
Comments: "System.Buffers",