feat: Add the origin field to the output format of syftjson (#1327)

* moved the relevant fields to the Metadata field

Signed-off-by: Asaf Greenholts <asaf@cidersecurity.io>

* added metadata types

Signed-off-by: Asaf Greenholts <asaf@cidersecurity.io>

* Added hashes to metadata of packge-lock.json and Pipfile.lock

Signed-off-by: Asaf Greenholts <asaf@cidersecurity.io>

* move package metadata types to "pkg" package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* re-generate json schema to include new npm, python, and binary metadatas

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Signed-off-by: Asaf Greenholts <asaf@cidersecurity.io>
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
Co-authored-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Asi Greenholts 2023-01-12 22:03:05 +02:00 committed by GitHub
parent 85bddaa43d
commit 260cb4c72d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 595 additions and 314 deletions

View File

@ -27,26 +27,33 @@ can be extended to include specific package metadata struct shapes in the future
// This should represent all possible metadatas represented in the pkg.Package.Metadata field (an interface{}). // This should represent all possible metadatas represented in the pkg.Package.Metadata field (an interface{}).
// When a new package metadata definition is created it will need to be manually added here. The variable name does // When a new package metadata definition is created it will need to be manually added here. The variable name does
// not matter as long as it is exported. // not matter as long as it is exported.
// TODO: this should be generated from reflection of whats in the pkg package
type artifactMetadataContainer struct { type artifactMetadataContainer struct {
Apk pkg.ApkMetadata
Alpm pkg.AlpmMetadata Alpm pkg.AlpmMetadata
Dpkg pkg.DpkgMetadata Apk pkg.ApkMetadata
Gem pkg.GemMetadata Binary pkg.BinaryMetadata
Java pkg.JavaMetadata Cocopods pkg.CocoapodsMetadata
Npm pkg.NpmPackageJSONMetadata
Python pkg.PythonPackageMetadata
Rpm pkg.RpmMetadata
Cargo pkg.CargoPackageMetadata
Go pkg.GolangMetadata
Php pkg.PhpComposerJSONMetadata
Dart pkg.DartPubMetadata
Dotnet pkg.DotnetDepsMetadata
Portage pkg.PortageMetadata
Conan pkg.ConanMetadata Conan pkg.ConanMetadata
ConanLock pkg.ConanLockMetadata ConanLock pkg.ConanLockMetadata
KbPackage pkg.KbPackageMetadata Dart pkg.DartPubMetadata
Dotnet pkg.DotnetDepsMetadata
Dpkg pkg.DpkgMetadata
Gem pkg.GemMetadata
Go pkg.GolangMetadata
Hackage pkg.HackageMetadata Hackage pkg.HackageMetadata
SwiftCocopods pkg.CocoapodsMetadata Java pkg.JavaMetadata
KbPackage pkg.KbPackageMetadata
NpmPackage pkg.NpmPackageJSONMetadata
NpmPackageLock pkg.NpmPackageLockJSONMetadata
MixLock pkg.MixLockMetadata
Php pkg.PhpComposerJSONMetadata
Portage pkg.PortageMetadata
PythonPackage pkg.PythonPackageMetadata
PythonPipfilelock pkg.PythonPipfileLockMetadata
Rebar pkg.RebarLockMetadata
Rpm pkg.RpmMetadata
RustCargo pkg.CargoPackageMetadata
} }
func main() { func main() {

View File

@ -200,6 +200,25 @@
"files" "files"
] ]
}, },
"BinaryMetadata": {
"properties": {
"classifier": {
"type": "string"
},
"realPath": {
"type": "string"
},
"virtualPath": {
"type": "string"
}
},
"type": "object",
"required": [
"classifier",
"realPath",
"virtualPath"
]
},
"CargoPackageMetadata": { "CargoPackageMetadata": {
"properties": { "properties": {
"name": { "name": {
@ -766,6 +785,29 @@
}, },
"type": "object" "type": "object"
}, },
"MixLockMetadata": {
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"pkgHash": {
"type": "string"
},
"pkgHashExt": {
"type": "string"
}
},
"type": "object",
"required": [
"name",
"version",
"pkgHash",
"pkgHashExt"
]
},
"NpmPackageJSONMetadata": { "NpmPackageJSONMetadata": {
"properties": { "properties": {
"name": { "name": {
@ -814,6 +856,21 @@
"private" "private"
] ]
}, },
"NpmPackageLockJSONMetadata": {
"properties": {
"resolved": {
"type": "string"
},
"integrity": {
"type": "string"
}
},
"type": "object",
"required": [
"resolved",
"integrity"
]
},
"Package": { "Package": {
"properties": { "properties": {
"id": { "id": {
@ -869,6 +926,9 @@
{ {
"$ref": "#/$defs/ApkMetadata" "$ref": "#/$defs/ApkMetadata"
}, },
{
"$ref": "#/$defs/BinaryMetadata"
},
{ {
"$ref": "#/$defs/CargoPackageMetadata" "$ref": "#/$defs/CargoPackageMetadata"
}, },
@ -905,9 +965,15 @@
{ {
"$ref": "#/$defs/KbPackageMetadata" "$ref": "#/$defs/KbPackageMetadata"
}, },
{
"$ref": "#/$defs/MixLockMetadata"
},
{ {
"$ref": "#/$defs/NpmPackageJSONMetadata" "$ref": "#/$defs/NpmPackageJSONMetadata"
}, },
{
"$ref": "#/$defs/NpmPackageLockJSONMetadata"
},
{ {
"$ref": "#/$defs/PhpComposerJSONMetadata" "$ref": "#/$defs/PhpComposerJSONMetadata"
}, },
@ -917,6 +983,12 @@
{ {
"$ref": "#/$defs/PythonPackageMetadata" "$ref": "#/$defs/PythonPackageMetadata"
}, },
{
"$ref": "#/$defs/PythonPipfileLockMetadata"
},
{
"$ref": "#/$defs/RebarLockMetadata"
},
{ {
"$ref": "#/$defs/RpmMetadata" "$ref": "#/$defs/RpmMetadata"
} }
@ -1291,6 +1363,47 @@
"sitePackagesRootPath" "sitePackagesRootPath"
] ]
}, },
"PythonPipfileLockMetadata": {
"properties": {
"hashes": {
"items": {
"type": "string"
},
"type": "array"
},
"index": {
"type": "string"
}
},
"type": "object",
"required": [
"hashes",
"index"
]
},
"RebarLockMetadata": {
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"pkgHash": {
"type": "string"
},
"pkgHashExt": {
"type": "string"
}
},
"type": "object",
"required": [
"name",
"version",
"pkgHash",
"pkgHashExt"
]
},
"Relationship": { "Relationship": {
"properties": { "properties": {
"parent": { "parent": {

View File

@ -20,6 +20,8 @@ func Test_JavascriptCataloger(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
Licenses: []string{"MIT"}, Licenses: []string{"MIT"},
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/@actions/core/-/core-1.6.0.tgz", Integrity: "sha512-NB1UAZomZlCV/LmJqkLhNTqtKfFXJZAUPcfl/zqG7EfsQdeUJtaWO98SGbuQ3pydJ3fHl2CvI/51OKYlCYYcaw=="},
}, },
{ {
Name: "ansi-regex", Name: "ansi-regex",
@ -29,6 +31,8 @@ func Test_JavascriptCataloger(t *testing.T) {
Locations: locationSet, Locations: locationSet,
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", Integrity: "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg="},
}, },
{ {
Name: "cowsay", Name: "cowsay",
@ -39,6 +43,8 @@ func Test_JavascriptCataloger(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
Licenses: []string{"MIT"}, Licenses: []string{"MIT"},
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/cowsay/-/cowsay-1.4.0.tgz", Integrity: "sha512-rdg5k5PsHFVJheO/pmE3aDg2rUDDTfPJau6yYkZYlHFktUz+UxbE+IgnUAEyyCyv4noL5ltxXD0gZzmHPCy/9g=="},
}, },
{ {
Name: "get-stdin", Name: "get-stdin",
@ -48,6 +54,8 @@ func Test_JavascriptCataloger(t *testing.T) {
Locations: locationSet, Locations: locationSet,
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz", Integrity: "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g="},
}, },
{ {
Name: "is-fullwidth-code-point", Name: "is-fullwidth-code-point",
@ -57,6 +65,8 @@ func Test_JavascriptCataloger(t *testing.T) {
Locations: locationSet, Locations: locationSet,
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", Integrity: "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8="},
}, },
{ {
Name: "minimist", Name: "minimist",
@ -66,6 +76,8 @@ func Test_JavascriptCataloger(t *testing.T) {
Locations: locationSet, Locations: locationSet,
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", Integrity: "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8="},
}, },
{ {
Name: "optimist", Name: "optimist",
@ -75,6 +87,8 @@ func Test_JavascriptCataloger(t *testing.T) {
Locations: locationSet, Locations: locationSet,
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", Integrity: "sha1-2j6nRob6IaGaERwybpDrFaAZZoY="},
}, },
{ {
Name: "string-width", Name: "string-width",
@ -84,6 +98,8 @@ func Test_JavascriptCataloger(t *testing.T) {
Locations: locationSet, Locations: locationSet,
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", Integrity: "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw=="},
}, },
{ {
Name: "strip-ansi", Name: "strip-ansi",
@ -93,6 +109,8 @@ func Test_JavascriptCataloger(t *testing.T) {
Locations: locationSet, Locations: locationSet,
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", Integrity: "sha1-qEeQIusaw2iocTibY1JixQXuNo8="},
}, },
{ {
Name: "strip-eof", Name: "strip-eof",
@ -102,6 +120,8 @@ func Test_JavascriptCataloger(t *testing.T) {
Locations: locationSet, Locations: locationSet,
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", Integrity: "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8="},
}, },
{ {
Name: "wordwrap", Name: "wordwrap",
@ -111,6 +131,8 @@ func Test_JavascriptCataloger(t *testing.T) {
Locations: locationSet, Locations: locationSet,
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", Integrity: "sha1-o9XabNXAvAAI03I0u68b7WMFkQc="},
}, },
} }

View File

@ -69,6 +69,8 @@ func newPackageLockV1Package(resolver source.FileResolver, location source.Locat
PURL: packageURL(name, version), PURL: packageURL(name, version),
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: u.Resolved, Integrity: u.Integrity},
}, },
) )
} }
@ -91,6 +93,8 @@ func newPackageLockV2Package(resolver source.FileResolver, location source.Locat
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
Licenses: licenses, Licenses: licenses,
MetadataType: pkg.NpmPackageLockJSONMetadataType,
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: u.Resolved, Integrity: u.Integrity},
}, },
) )
} }

View File

@ -18,6 +18,8 @@ func TestParsePackageLock(t *testing.T) {
PURL: "pkg:npm/%40actions/core@1.6.0", PURL: "pkg:npm/%40actions/core@1.6.0",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/@actions/core/-/core-1.6.0.tgz", Integrity: "sha512-NB1UAZomZlCV/LmJqkLhNTqtKfFXJZAUPcfl/zqG7EfsQdeUJtaWO98SGbuQ3pydJ3fHl2CvI/51OKYlCYYcaw=="},
}, },
{ {
Name: "ansi-regex", Name: "ansi-regex",
@ -25,6 +27,8 @@ func TestParsePackageLock(t *testing.T) {
PURL: "pkg:npm/ansi-regex@3.0.0", PURL: "pkg:npm/ansi-regex@3.0.0",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", Integrity: "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg="},
}, },
{ {
Name: "cowsay", Name: "cowsay",
@ -32,6 +36,8 @@ func TestParsePackageLock(t *testing.T) {
PURL: "pkg:npm/cowsay@1.4.0", PURL: "pkg:npm/cowsay@1.4.0",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/cowsay/-/cowsay-1.4.0.tgz", Integrity: "sha512-rdg5k5PsHFVJheO/pmE3aDg2rUDDTfPJau6yYkZYlHFktUz+UxbE+IgnUAEyyCyv4noL5ltxXD0gZzmHPCy/9g=="},
}, },
{ {
Name: "get-stdin", Name: "get-stdin",
@ -39,6 +45,8 @@ func TestParsePackageLock(t *testing.T) {
PURL: "pkg:npm/get-stdin@5.0.1", PURL: "pkg:npm/get-stdin@5.0.1",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz", Integrity: "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g="},
}, },
{ {
Name: "is-fullwidth-code-point", Name: "is-fullwidth-code-point",
@ -46,6 +54,8 @@ func TestParsePackageLock(t *testing.T) {
PURL: "pkg:npm/is-fullwidth-code-point@2.0.0", PURL: "pkg:npm/is-fullwidth-code-point@2.0.0",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", Integrity: "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8="},
}, },
{ {
Name: "minimist", Name: "minimist",
@ -53,6 +63,8 @@ func TestParsePackageLock(t *testing.T) {
PURL: "pkg:npm/minimist@0.0.10", PURL: "pkg:npm/minimist@0.0.10",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", Integrity: "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8="},
}, },
{ {
Name: "optimist", Name: "optimist",
@ -60,6 +72,8 @@ func TestParsePackageLock(t *testing.T) {
PURL: "pkg:npm/optimist@0.6.1", PURL: "pkg:npm/optimist@0.6.1",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", Integrity: "sha1-2j6nRob6IaGaERwybpDrFaAZZoY="},
}, },
{ {
Name: "string-width", Name: "string-width",
@ -67,6 +81,8 @@ func TestParsePackageLock(t *testing.T) {
PURL: "pkg:npm/string-width@2.1.1", PURL: "pkg:npm/string-width@2.1.1",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", Integrity: "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw=="},
}, },
{ {
Name: "strip-ansi", Name: "strip-ansi",
@ -74,6 +90,8 @@ func TestParsePackageLock(t *testing.T) {
PURL: "pkg:npm/strip-ansi@4.0.0", PURL: "pkg:npm/strip-ansi@4.0.0",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", Integrity: "sha1-qEeQIusaw2iocTibY1JixQXuNo8="},
}, },
{ {
Name: "strip-eof", Name: "strip-eof",
@ -81,6 +99,8 @@ func TestParsePackageLock(t *testing.T) {
PURL: "pkg:npm/strip-eof@1.0.0", PURL: "pkg:npm/strip-eof@1.0.0",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", Integrity: "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8="},
}, },
{ {
Name: "wordwrap", Name: "wordwrap",
@ -88,6 +108,8 @@ func TestParsePackageLock(t *testing.T) {
PURL: "pkg:npm/wordwrap@0.0.3", PURL: "pkg:npm/wordwrap@0.0.3",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", Integrity: "sha1-o9XabNXAvAAI03I0u68b7WMFkQc="},
}, },
} }
fixture := "test-fixtures/pkg-lock/package-lock.json" fixture := "test-fixtures/pkg-lock/package-lock.json"
@ -108,6 +130,8 @@ func TestParsePackageLockV2(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
PURL: "pkg:npm/npm@6.14.6", PURL: "pkg:npm/npm@6.14.6",
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{},
}, },
{ {
Name: "@types/prop-types", Name: "@types/prop-types",
@ -116,6 +140,8 @@ func TestParsePackageLockV2(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
Licenses: []string{"MIT"}, Licenses: []string{"MIT"},
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", Integrity: "sha1-XxnSuFqY6VWANvajysyIGUIPBc8="},
}, },
{ {
Name: "@types/react", Name: "@types/react",
@ -124,6 +150,8 @@ func TestParsePackageLockV2(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
Licenses: []string{"MIT"}, Licenses: []string{"MIT"},
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/@types/react/-/react-18.0.17.tgz", Integrity: "sha1-RYPZwyLWfv5LOak10iPtzHBQzPQ="},
}, },
{ {
Name: "@types/scheduler", Name: "@types/scheduler",
@ -132,6 +160,8 @@ func TestParsePackageLockV2(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
Licenses: []string{"MIT"}, Licenses: []string{"MIT"},
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", Integrity: "sha1-GmL4lSVyPd4kuhsBsJK/XfitTTk="},
}, },
{ {
Name: "csstype", Name: "csstype",
@ -140,6 +170,8 @@ func TestParsePackageLockV2(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
Licenses: []string{"MIT"}, Licenses: []string{"MIT"},
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/csstype/-/csstype-3.1.0.tgz", Integrity: "sha1-TdysNxjXh8+d8NG30VAzklyPKfI="},
}, },
} }
for i := range expectedPkgs { for i := range expectedPkgs {
@ -158,6 +190,8 @@ func TestParsePackageLockV3(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
PURL: "pkg:npm/lock-v3-fixture@1.0.0", PURL: "pkg:npm/lock-v3-fixture@1.0.0",
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{},
}, },
{ {
Name: "@types/prop-types", Name: "@types/prop-types",
@ -165,6 +199,8 @@ func TestParsePackageLockV3(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
PURL: "pkg:npm/%40types/prop-types@15.7.5", PURL: "pkg:npm/%40types/prop-types@15.7.5",
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", Integrity: "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w=="},
}, },
{ {
Name: "@types/react", Name: "@types/react",
@ -172,6 +208,8 @@ func TestParsePackageLockV3(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
PURL: "pkg:npm/%40types/react@18.0.20", PURL: "pkg:npm/%40types/react@18.0.20",
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/@types/react/-/react-18.0.20.tgz", Integrity: "sha512-MWul1teSPxujEHVwZl4a5HxQ9vVNsjTchVA+xRqv/VYGCuKGAU6UhfrTdF5aBefwD1BHUD8i/zq+O/vyCm/FrA=="},
}, },
{ {
Name: "@types/scheduler", Name: "@types/scheduler",
@ -179,6 +217,8 @@ func TestParsePackageLockV3(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
PURL: "pkg:npm/%40types/scheduler@0.16.2", PURL: "pkg:npm/%40types/scheduler@0.16.2",
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", Integrity: "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew=="},
}, },
{ {
Name: "csstype", Name: "csstype",
@ -186,6 +226,8 @@ func TestParsePackageLockV3(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
PURL: "pkg:npm/csstype@3.1.1", PURL: "pkg:npm/csstype@3.1.1",
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/csstype/-/csstype-3.1.1.tgz", Integrity: "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw=="},
}, },
} }
for i := range expectedPkgs { for i := range expectedPkgs {
@ -203,6 +245,8 @@ func TestParsePackageLockAlias(t *testing.T) {
PURL: "pkg:npm/case@1.6.2", PURL: "pkg:npm/case@1.6.2",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/case/-/case-1.6.2.tgz", Integrity: "sha512-ll380ZRoraT7mUK2G92UbH+FJVD5AwdVIAYk9xhV1tauh0carDgYByUD1HhjCWsWgxrfQvCeHvtfj7IYR6TKeg=="},
}, },
{ {
Name: "case", Name: "case",
@ -210,6 +254,8 @@ func TestParsePackageLockAlias(t *testing.T) {
PURL: "pkg:npm/case@1.6.3", PURL: "pkg:npm/case@1.6.3",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/case/-/case-1.6.3.tgz", Integrity: "sha512-mzDSXIPaFwVDvZAHqZ9VlbyF4yyXRuX6IvB06WvPYkqJVO24kX1PPhv9bfpKNFZyxYFmmgo03HUiD8iklmJYRQ=="},
}, },
{ {
Name: "@bundled-es-modules/chai", Name: "@bundled-es-modules/chai",
@ -217,6 +263,8 @@ func TestParsePackageLockAlias(t *testing.T) {
PURL: "pkg:npm/%40bundled-es-modules/chai@4.2.2", PURL: "pkg:npm/%40bundled-es-modules/chai@4.2.2",
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{Resolved: "https://registry.npmjs.org/@bundled-es-modules/chai/-/chai-4.2.2.tgz", Integrity: "sha512-iGmVYw2/zJCoqyKTtWEYCtFmMyi8WmACQKtky0lpNyEKWX0YIOpKWGD7saMXL+tPpllss0otilxV0SLwyi3Ytg=="},
}, },
} }
@ -227,6 +275,8 @@ func TestParsePackageLockAlias(t *testing.T) {
Language: pkg.JavaScript, Language: pkg.JavaScript,
Type: pkg.NpmPkg, Type: pkg.NpmPkg,
Licenses: []string{"ISC"}, Licenses: []string{"ISC"},
MetadataType: "NpmPackageLockJsonMetadata",
Metadata: pkg.NpmPackageLockJSONMetadata{},
} }
packageLockV1 := "test-fixtures/pkg-lock/alias-package-lock-1.json" packageLockV1 := "test-fixtures/pkg-lock/alias-package-lock-1.json"

View File

@ -23,6 +23,23 @@ func newPackageForIndex(name, version string, locations ...source.Location) pkg.
return p return p
} }
func newPackageForIndexWithMetadata(name, version string, metadata pkg.PythonPipfileLockMetadata, locations ...source.Location) pkg.Package {
p := pkg.Package{
Name: name,
Version: version,
Locations: source.NewLocationSet(locations...),
PURL: packageURL(name, version, nil),
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonPipfileLockMetadataType,
Metadata: metadata,
}
p.SetID()
return p
}
func newPackageForPackage(m pkg.PythonPackageMetadata, sources ...source.Location) pkg.Package { func newPackageForPackage(m pkg.PythonPackageMetadata, sources ...source.Location) pkg.Package {
var licenses []string var licenses []string
if m.License != "" { if m.License != "" {

View File

@ -33,7 +33,9 @@ type pipfileLock struct {
} }
type Dependency struct { type Dependency struct {
Hashes []string `json:"hashes"`
Version string `json:"version"` Version string `json:"version"`
Index string `json:"index"`
} }
var _ generic.Parser = parsePipfileLock var _ generic.Parser = parsePipfileLock
@ -50,9 +52,20 @@ func parsePipfileLock(_ source.FileResolver, _ *generic.Environment, reader sour
} else if err != nil { } else if err != nil {
return nil, nil, fmt.Errorf("failed to parse Pipfile.lock file: %w", err) return nil, nil, fmt.Errorf("failed to parse Pipfile.lock file: %w", err)
} }
sourcesMap := map[string]string{}
for _, source := range lock.Meta.Sources {
sourcesMap[source.Name] = source.URL
}
for name, pkgMeta := range lock.Default { for name, pkgMeta := range lock.Default {
var index string
if pkgMeta.Index != "" {
index = sourcesMap[pkgMeta.Index]
} else {
// https://pipenv.pypa.io/en/latest/advanced/#specifying-package-indexes
index = "https://pypi.org/simple"
}
version := strings.TrimPrefix(pkgMeta.Version, "==") version := strings.TrimPrefix(pkgMeta.Version, "==")
pkgs = append(pkgs, newPackageForIndex(name, version, reader.Location)) pkgs = append(pkgs, newPackageForIndexWithMetadata(name, version, pkg.PythonPipfileLockMetadata{Index: index, Hashes: pkgMeta.Hashes}, reader.Location))
} }
} }

View File

@ -21,6 +21,13 @@ func TestParsePipFileLock(t *testing.T) {
Locations: locations, Locations: locations,
Language: pkg.Python, Language: pkg.Python,
Type: pkg.PythonPkg, Type: pkg.PythonPkg,
MetadataType: pkg.PythonPipfileLockMetadataType,
Metadata: pkg.PythonPipfileLockMetadata{
Index: "https://pypi.org/simple",
Hashes: []string{
"sha256:1d4305a5f78af3857310b4fe48348cdcf6c097e0e275ea88c2cd08570531a369",
"sha256:e69afef8695f47c5d107bbdba21bdb845d5c249acb3be53ef5c2d497b02657c0",
}},
}, },
{ {
Name: "aiodns", Name: "aiodns",
@ -29,6 +36,14 @@ func TestParsePipFileLock(t *testing.T) {
Locations: locations, Locations: locations,
Language: pkg.Python, Language: pkg.Python,
Type: pkg.PythonPkg, Type: pkg.PythonPkg,
MetadataType: pkg.PythonPipfileLockMetadataType,
Metadata: pkg.PythonPipfileLockMetadata{
Index: "https://test.pypi.org/simple",
Hashes: []string{
"sha256:815fdef4607474295d68da46978a54481dd1e7be153c7d60f9e72773cd38d77d",
"sha256:aaa5ac584f40fe778013df0aa6544bf157799bd3f608364b451840ed2c8688de",
},
},
}, },
{ {
Name: "aiohttp", Name: "aiohttp",
@ -37,6 +52,14 @@ func TestParsePipFileLock(t *testing.T) {
Locations: locations, Locations: locations,
Language: pkg.Python, Language: pkg.Python,
Type: pkg.PythonPkg, Type: pkg.PythonPkg,
MetadataType: pkg.PythonPipfileLockMetadataType,
Metadata: pkg.PythonPipfileLockMetadata{
Index: "https://pypi.org/simple",
Hashes: []string{
"sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe",
"sha256:0563c1b3826945eecd62186f3f5c7d31abb7391fedc893b7e2b26303b5a9f3fe",
},
},
}, },
{ {
Name: "aiohttp-jinja2", Name: "aiohttp-jinja2",
@ -45,6 +68,14 @@ func TestParsePipFileLock(t *testing.T) {
Locations: locations, Locations: locations,
Language: pkg.Python, Language: pkg.Python,
Type: pkg.PythonPkg, Type: pkg.PythonPkg,
MetadataType: pkg.PythonPipfileLockMetadataType,
Metadata: pkg.PythonPipfileLockMetadata{
Index: "https://pypi.org/simple",
Hashes: []string{
"sha256:860da7582efa866744bad5883947557d0f82e457d69903ea65d666b66f8a69ca",
"sha256:9c22a0e48e3b277fc145c67dd8c3b8f609dab36bce9eb337f70dfe716663c9a0",
},
},
}, },
} }

View File

@ -12,6 +12,11 @@
"name": "pypi", "name": "pypi",
"url": "https://pypi.org/simple", "url": "https://pypi.org/simple",
"verify_ssl": true "verify_ssl": true
},
{
"name": "test",
"url": "https://test.pypi.org/simple",
"verify_ssl": true
} }
] ]
}, },
@ -29,7 +34,7 @@
"sha256:815fdef4607474295d68da46978a54481dd1e7be153c7d60f9e72773cd38d77d", "sha256:815fdef4607474295d68da46978a54481dd1e7be153c7d60f9e72773cd38d77d",
"sha256:aaa5ac584f40fe778013df0aa6544bf157799bd3f608364b451840ed2c8688de" "sha256:aaa5ac584f40fe778013df0aa6544bf157799bd3f608364b451840ed2c8688de"
], ],
"index": "pypi", "index": "test",
"version": "==2.0.0" "version": "==2.0.0"
}, },
"aiohttp": { "aiohttp": {

View File

@ -9,6 +9,7 @@ type MetadataType string
const ( const (
// this is the full set of data shapes that can be represented within the pkg.Package.Metadata field // this is the full set of data shapes that can be represented within the pkg.Package.Metadata field
UnknownMetadataType MetadataType = "UnknownMetadata"
AlpmMetadataType MetadataType = "AlpmMetadata" AlpmMetadataType MetadataType = "AlpmMetadata"
ApkMetadataType MetadataType = "ApkMetadata" ApkMetadataType MetadataType = "ApkMetadata"
BinaryMetadataType MetadataType = "BinaryMetadata" BinaryMetadataType MetadataType = "BinaryMetadata"
@ -25,13 +26,14 @@ const (
KbPackageMetadataType MetadataType = "KbPackageMetadata" KbPackageMetadataType MetadataType = "KbPackageMetadata"
MixLockMetadataType MetadataType = "MixLockMetadataType" MixLockMetadataType MetadataType = "MixLockMetadataType"
NpmPackageJSONMetadataType MetadataType = "NpmPackageJsonMetadata" NpmPackageJSONMetadataType MetadataType = "NpmPackageJsonMetadata"
NpmPackageLockJSONMetadataType MetadataType = "NpmPackageLockJsonMetadata"
PhpComposerJSONMetadataType MetadataType = "PhpComposerJsonMetadata" PhpComposerJSONMetadataType MetadataType = "PhpComposerJsonMetadata"
PortageMetadataType MetadataType = "PortageMetadata" PortageMetadataType MetadataType = "PortageMetadata"
PythonPackageMetadataType MetadataType = "PythonPackageMetadata" PythonPackageMetadataType MetadataType = "PythonPackageMetadata"
PythonPipfileLockMetadataType MetadataType = "PythonPipfileLockMetadata"
RebarLockMetadataType MetadataType = "RebarLockMetadataType" RebarLockMetadataType MetadataType = "RebarLockMetadataType"
RpmMetadataType MetadataType = "RpmMetadata" RpmMetadataType MetadataType = "RpmMetadata"
RustCargoPackageMetadataType MetadataType = "RustCargoPackageMetadata" RustCargoPackageMetadataType MetadataType = "RustCargoPackageMetadata"
UnknownMetadataType MetadataType = "UnknownMetadata"
) )
var AllMetadataTypes = []MetadataType{ var AllMetadataTypes = []MetadataType{
@ -51,9 +53,11 @@ var AllMetadataTypes = []MetadataType{
KbPackageMetadataType, KbPackageMetadataType,
MixLockMetadataType, MixLockMetadataType,
NpmPackageJSONMetadataType, NpmPackageJSONMetadataType,
NpmPackageLockJSONMetadataType,
PhpComposerJSONMetadataType, PhpComposerJSONMetadataType,
PortageMetadataType, PortageMetadataType,
PythonPackageMetadataType, PythonPackageMetadataType,
PythonPipfileLockMetadataType,
RebarLockMetadataType, RebarLockMetadataType,
RpmMetadataType, RpmMetadataType,
RustCargoPackageMetadataType, RustCargoPackageMetadataType,
@ -76,9 +80,11 @@ var MetadataTypeByName = map[MetadataType]reflect.Type{
KbPackageMetadataType: reflect.TypeOf(KbPackageMetadata{}), KbPackageMetadataType: reflect.TypeOf(KbPackageMetadata{}),
MixLockMetadataType: reflect.TypeOf(MixLockMetadata{}), MixLockMetadataType: reflect.TypeOf(MixLockMetadata{}),
NpmPackageJSONMetadataType: reflect.TypeOf(NpmPackageJSONMetadata{}), NpmPackageJSONMetadataType: reflect.TypeOf(NpmPackageJSONMetadata{}),
NpmPackageLockJSONMetadataType: reflect.TypeOf(NpmPackageLockJSONMetadata{}),
PhpComposerJSONMetadataType: reflect.TypeOf(PhpComposerJSONMetadata{}), PhpComposerJSONMetadataType: reflect.TypeOf(PhpComposerJSONMetadata{}),
PortageMetadataType: reflect.TypeOf(PortageMetadata{}), PortageMetadataType: reflect.TypeOf(PortageMetadata{}),
PythonPackageMetadataType: reflect.TypeOf(PythonPackageMetadata{}), PythonPackageMetadataType: reflect.TypeOf(PythonPackageMetadata{}),
PythonPipfileLockMetadataType: reflect.TypeOf(PythonPipfileLockMetadata{}),
RebarLockMetadataType: reflect.TypeOf(RebarLockMetadata{}), RebarLockMetadataType: reflect.TypeOf(RebarLockMetadata{}),
RpmMetadataType: reflect.TypeOf(RpmMetadata{}), RpmMetadataType: reflect.TypeOf(RpmMetadata{}),
RustCargoPackageMetadataType: reflect.TypeOf(CargoPackageMetadata{}), RustCargoPackageMetadataType: reflect.TypeOf(CargoPackageMetadata{}),

View File

@ -1,6 +1,6 @@
package pkg package pkg
// NpmPackageJSONMetadata holds extra information that is used in pkg.Package // NpmPackageJSONMetadata holds parsing information for a javascript package.json file
type NpmPackageJSONMetadata struct { type NpmPackageJSONMetadata struct {
Name string `mapstructure:"name" json:"name"` Name string `mapstructure:"name" json:"name"`
Version string `mapstructure:"version" json:"version"` Version string `mapstructure:"version" json:"version"`

View File

@ -0,0 +1,7 @@
package pkg
// NpmPackageLockJSONMetadata holds parsing information for a javascript package-lock.json file
type NpmPackageLockJSONMetadata struct {
Resolved string `mapstructure:"resolved" json:"resolved"`
Integrity string `mapstructure:"integrity" json:"integrity"`
}

View File

@ -0,0 +1,6 @@
package pkg
type PythonPipfileLockMetadata struct {
Hashes []string `mapstructure:"hashes" json:"hashes"`
Index string `mapstructure:"index" json:"index"`
}