From 7962002f81af08d6dedcf69b76d2b274ad15a321 Mon Sep 17 00:00:00 2001 From: Zach Hill Date: Fri, 18 Dec 2020 11:08:19 -0800 Subject: [PATCH] Split dpk source into name and version (#297) * Split dpk source into name and version Signed-off-by: Zach Hill * update dpkg status source name parsing Signed-off-by: Alex Goodman Co-authored-by: Dan Luhring Co-authored-by: Alex Goodman --- Makefile | 3 +- internal/constants.go | 2 +- schema/json/schema-1.0.1.json | 682 ++++++++++++++++++ syft/cataloger/deb/parse_dpkg_status.go | 24 + syft/cataloger/deb/parse_dpkg_status_test.go | 40 + syft/pkg/dpkg_metadata.go | 1 + .../snapshot/TestJsonDirsPresenter.golden | 5 +- .../snapshot/TestJsonImgsPresenter.golden | 5 +- 8 files changed, 755 insertions(+), 7 deletions(-) create mode 100644 schema/json/schema-1.0.1.json diff --git a/Makefile b/Makefile index c33097b07..af907b623 100644 --- a/Makefile +++ b/Makefile @@ -154,8 +154,7 @@ fixtures: .PHONY: generate-json-schema generate-json-schema: ## Generate a new json schema - cd schema/json - go run generate.go + cd schema/json && go run generate.go .PHONY: clear-test-cache clear-test-cache: ## Delete all test cache (built docker image tars) diff --git a/internal/constants.go b/internal/constants.go index 5be1e282c..675381dad 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -6,5 +6,5 @@ const ( // JSONSchemaVersion is the current schema version output by the JSON presenter // This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment. - JSONSchemaVersion = "1.0.0" + JSONSchemaVersion = "1.0.1" ) diff --git a/schema/json/schema-1.0.1.json b/schema/json/schema-1.0.1.json new file mode 100644 index 000000000..558acfd19 --- /dev/null +++ b/schema/json/schema-1.0.1.json @@ -0,0 +1,682 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Document", + "definitions": { + "ApkFileRecord": { + "required": [ + "path" + ], + "properties": { + "path": { + "type": "string" + }, + "ownerUid": { + "type": "string" + }, + "ownerGid": { + "type": "string" + }, + "permissions": { + "type": "string" + }, + "checksum": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "ApkMetadata": { + "required": [ + "package", + "originPackage", + "maintainer", + "version", + "license", + "architecture", + "url", + "description", + "size", + "installedSize", + "pullDependencies", + "pullChecksum", + "gitCommitOfApkPort", + "files" + ], + "properties": { + "package": { + "type": "string" + }, + "originPackage": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "version": { + "type": "string" + }, + "license": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "url": { + "type": "string" + }, + "description": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "installedSize": { + "type": "integer" + }, + "pullDependencies": { + "type": "string" + }, + "pullChecksum": { + "type": "string" + }, + "gitCommitOfApkPort": { + "type": "string" + }, + "files": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/ApkFileRecord" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object" + }, + "Descriptor": { + "required": [ + "name", + "version" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "Distribution": { + "required": [ + "name", + "version", + "idLike" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "idLike": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "Document": { + "required": [ + "artifacts", + "source", + "distro", + "descriptor", + "schema" + ], + "properties": { + "artifacts": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Package" + }, + "type": "array" + }, + "source": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Source" + }, + "distro": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Distribution" + }, + "descriptor": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Descriptor" + }, + "schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Schema" + } + }, + "additionalProperties": false, + "type": "object" + }, + "DpkgFileRecord": { + "required": [ + "path", + "md5" + ], + "properties": { + "path": { + "type": "string" + }, + "md5": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "DpkgMetadata": { + "required": [ + "package", + "source", + "version", + "sourceVersion", + "architecture", + "maintainer", + "installedSize", + "files" + ], + "properties": { + "package": { + "type": "string" + }, + "source": { + "type": "string" + }, + "version": { + "type": "string" + }, + "sourceVersion": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "installedSize": { + "type": "integer" + }, + "files": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/DpkgFileRecord" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object" + }, + "GemMetadata": { + "required": [ + "name", + "version" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "type": "string" + }, + "type": "array" + }, + "licenses": { + "items": { + "type": "string" + }, + "type": "array" + }, + "homepage": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "JavaManifest": { + "properties": { + "main": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "namedSections": { + "patternProperties": { + ".*": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "additionalProperties": false, + "type": "object" + }, + "JavaMetadata": { + "required": [ + "virtualPath" + ], + "properties": { + "virtualPath": { + "type": "string" + }, + "manifest": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/JavaManifest" + }, + "pomProperties": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/PomProperties" + } + }, + "additionalProperties": false, + "type": "object" + }, + "Location": { + "required": [ + "path" + ], + "properties": { + "path": { + "type": "string" + }, + "layerID": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "NpmPackageJSONMetadata": { + "required": [ + "author", + "licenses", + "homepage", + "description", + "url" + ], + "properties": { + "files": { + "items": { + "type": "string" + }, + "type": "array" + }, + "author": { + "type": "string" + }, + "licenses": { + "items": { + "type": "string" + }, + "type": "array" + }, + "homepage": { + "type": "string" + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "Package": { + "required": [ + "name", + "version", + "type", + "foundBy", + "locations", + "licenses", + "language", + "cpes", + "purl", + "metadataType", + "metadata" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "type": { + "type": "string" + }, + "foundBy": { + "type": "string" + }, + "locations": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Location" + }, + "type": "array" + }, + "licenses": { + "items": { + "type": "string" + }, + "type": "array" + }, + "language": { + "type": "string" + }, + "cpes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "purl": { + "type": "string" + }, + "metadataType": { + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/definitions/ApkMetadata" + }, + { + "$ref": "#/definitions/DpkgMetadata" + }, + { + "$ref": "#/definitions/GemMetadata" + }, + { + "$ref": "#/definitions/JavaMetadata" + }, + { + "$ref": "#/definitions/NpmPackageJSONMetadata" + }, + { + "$ref": "#/definitions/PythonPackageMetadata" + }, + { + "$ref": "#/definitions/RpmdbMetadata" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "PomProperties": { + "required": [ + "path", + "name", + "groupId", + "artifactId", + "version", + "extraFields" + ], + "properties": { + "path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + }, + "extraFields": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "additionalProperties": false, + "type": "object" + }, + "PythonFileDigest": { + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "PythonFileRecord": { + "required": [ + "path" + ], + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/PythonFileDigest" + }, + "size": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "PythonPackageMetadata": { + "required": [ + "name", + "version", + "license", + "author", + "authorEmail", + "platform", + "sitePackagesRootPath" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "license": { + "type": "string" + }, + "author": { + "type": "string" + }, + "authorEmail": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "files": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/PythonFileRecord" + }, + "type": "array" + }, + "sitePackagesRootPath": { + "type": "string" + }, + "topLevelPackages": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object" + }, + "RpmdbFileRecord": { + "required": [ + "path", + "mode", + "size", + "sha256" + ], + "properties": { + "path": { + "type": "string" + }, + "mode": { + "type": "integer" + }, + "size": { + "type": "integer" + }, + "sha256": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "RpmdbMetadata": { + "required": [ + "name", + "version", + "epoch", + "architecture", + "release", + "sourceRpm", + "size", + "license", + "vendor", + "files" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "epoch": { + "type": "integer" + }, + "architecture": { + "type": "string" + }, + "release": { + "type": "string" + }, + "sourceRpm": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "license": { + "type": "string" + }, + "vendor": { + "type": "string" + }, + "files": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/RpmdbFileRecord" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object" + }, + "Schema": { + "required": [ + "version", + "url" + ], + "properties": { + "version": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "Source": { + "required": [ + "type", + "target" + ], + "properties": { + "type": { + "type": "string" + }, + "target": { + "additionalProperties": true + } + }, + "additionalProperties": false, + "type": "object" + } + } +} diff --git a/syft/cataloger/deb/parse_dpkg_status.go b/syft/cataloger/deb/parse_dpkg_status.go index d33cb5e14..2c14ac14a 100644 --- a/syft/cataloger/deb/parse_dpkg_status.go +++ b/syft/cataloger/deb/parse_dpkg_status.go @@ -5,9 +5,12 @@ import ( "errors" "fmt" "io" + "regexp" "strconv" "strings" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/pkg" "github.com/mitchellh/mapstructure" ) @@ -45,6 +48,7 @@ func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) { } // parseDpkgStatusEntry returns an individual Dpkg entry, or returns errEndOfPackages if there are no more packages to parse from the reader. +// nolint:funlen func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) { dpkgFields := make(map[string]interface{}) var retErr error @@ -105,9 +109,29 @@ func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err err return pkg.DpkgMetadata{}, err } + name, version := extractSourceVersion(entry.Source) + if version != "" { + entry.SourceVersion = version + entry.Source = name + } + return entry, retErr } +// match examples: +// "a-thing (1.2.3)" name="a-thing" version="1.2.3" +// "a-thing" name="a-thing" version="" +// "" name="" version="" +var sourceRegexp = regexp.MustCompile(`(?P\S+)( \((?P.*)\))?`) + +// If the source entry string is of the form " ()" then parse and return the components, if +// of the "" form, then return name and nil +func extractSourceVersion(source string) (string, string) { + // special handling for the Source field since it has formatted data + match := internal.MatchCaptureGroups(sourceRegexp, source) + return match["name"], match["version"] +} + // handleNewKeyValue parse a new key-value pair from the given unprocessed line func handleNewKeyValue(line string) (string, interface{}, error) { if i := strings.Index(line, ":"); i > 0 { diff --git a/syft/cataloger/deb/parse_dpkg_status_test.go b/syft/cataloger/deb/parse_dpkg_status_test.go index d7e311e39..0c4b07202 100644 --- a/syft/cataloger/deb/parse_dpkg_status_test.go +++ b/syft/cataloger/deb/parse_dpkg_status_test.go @@ -115,3 +115,43 @@ func TestMultiplePackages(t *testing.T) { }) } } + +func TestSourceVersionExtract(t *testing.T) { + + tests := []struct { + name string + input string + expected []string + }{ + { + name: "name and version", + input: "test (1.2.3)", + expected: []string{"test", "1.2.3"}, + }, + { + name: "only name", + input: "test", + expected: []string{"test", ""}, + }, + { + name: "empty", + input: "", + expected: []string{"", ""}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + name, version := extractSourceVersion(test.input) + + if name != test.expected[0] { + t.Errorf("mismatch name for %q : %q!=%q", test.input, name, test.expected[0]) + } + + if version != test.expected[1] { + t.Errorf("mismatch version for %q : %q!=%q", test.input, version, test.expected[1]) + } + + }) + } +} diff --git a/syft/pkg/dpkg_metadata.go b/syft/pkg/dpkg_metadata.go index 5e25388c8..d82197029 100644 --- a/syft/pkg/dpkg_metadata.go +++ b/syft/pkg/dpkg_metadata.go @@ -11,6 +11,7 @@ type DpkgMetadata struct { Package string `mapstructure:"Package" json:"package"` Source string `mapstructure:"Source" json:"source"` Version string `mapstructure:"Version" json:"version"` + SourceVersion string `mapstructure:"SourceVersion" json:"sourceVersion"` Architecture string `mapstructure:"Architecture" json:"architecture"` Maintainer string `mapstructure:"Maintainer" json:"maintainer"` InstalledSize int `mapstructure:"InstalledSize" json:"installedSize"` diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden index 31bddc7d6..917b53c8a 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden +++ b/syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden @@ -50,6 +50,7 @@ "package": "package-2", "source": "", "version": "2.0.1", + "sourceVersion": "", "architecture": "", "maintainer": "", "installedSize": 0, @@ -71,7 +72,7 @@ "version": "[not provided]" }, "schema": { - "version": "1.0.0", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.0.json" + "version": "1.0.1", + "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.1.json" } } diff --git a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden index 54be8e80d..6983afac0 100644 --- a/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden +++ b/syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden @@ -52,6 +52,7 @@ "package": "package-2", "source": "", "version": "2.0.1", + "sourceVersion": "", "architecture": "", "maintainer": "", "installedSize": 0, @@ -102,7 +103,7 @@ "version": "[not provided]" }, "schema": { - "version": "1.0.0", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.0.json" + "version": "1.0.1", + "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.1.json" } }