From 624a090631ed08aaa68e0b291e0ea114d7fcb527 Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Sat, 14 Nov 2020 09:50:17 -0500 Subject: [PATCH 1/8] Add test for package.json license object Signed-off-by: Dan Luhring --- .../javascript/parse_package_json_test.go | 17 ++++++++++++++ .../pkg-json/package-license-object.json | 22 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 syft/cataloger/javascript/test-fixtures/pkg-json/package-license-object.json diff --git a/syft/cataloger/javascript/parse_package_json_test.go b/syft/cataloger/javascript/parse_package_json_test.go index 869d4293f..1bd604f8d 100644 --- a/syft/cataloger/javascript/parse_package_json_test.go +++ b/syft/cataloger/javascript/parse_package_json_test.go @@ -30,6 +30,23 @@ func TestParsePackageJSON(t *testing.T) { }, }, }, + { + Fixture: "test-fixtures/pkg-json/package-license-object.json", + ExpectedPkg: pkg.Package{ + Name: "npm", + Version: "6.14.6", + Type: pkg.NpmPkg, + Licenses: []string{"ISC"}, + Language: pkg.JavaScript, + MetadataType: pkg.NpmPackageJSONMetadataType, + Metadata: pkg.NpmPackageJSONMetadata{ + Author: "Isaac Z. Schlueter (http://blog.izs.me)", + Homepage: "https://docs.npmjs.com/", + URL: "https://github.com/npm/cli", + Licenses: []string{"ISC"}, + }, + }, + }, { Fixture: "test-fixtures/pkg-json/package-nested-author.json", ExpectedPkg: pkg.Package{ diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-license-object.json b/syft/cataloger/javascript/test-fixtures/pkg-json/package-license-object.json new file mode 100644 index 000000000..9c5d61567 --- /dev/null +++ b/syft/cataloger/javascript/test-fixtures/pkg-json/package-license-object.json @@ -0,0 +1,22 @@ +{ + "version": "6.14.6", + "name": "npm", + "description": "a package manager for JavaScript", + "homepage": "https://docs.npmjs.com/", + "author": "Isaac Z. Schlueter (http://blog.izs.me)", + "repository": { + "type": "git", + "url": "https://github.com/npm/cli" + }, + "bugs": { + "url": "https://npm.community/c/bugs" + }, + "main": "./lib/npm.js", + "license": { + "type" : "ISC", + "url" : "https://opensource.org/licenses/ISC" + }, + "engines": { + "node": "6 >=6.2.0 || 8 || >=9.3.0" + } +} From bd4c784a6424ca3ff6ba4182ca80adff82125abb Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Sat, 14 Nov 2020 10:13:57 -0500 Subject: [PATCH 2/8] Add test for package.json multiple license objects Signed-off-by: Dan Luhring --- .../javascript/parse_package_json_test.go | 17 ++++++++++++ .../pkg-json/package-license-objects.json | 26 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 syft/cataloger/javascript/test-fixtures/pkg-json/package-license-objects.json diff --git a/syft/cataloger/javascript/parse_package_json_test.go b/syft/cataloger/javascript/parse_package_json_test.go index 1bd604f8d..c16a69006 100644 --- a/syft/cataloger/javascript/parse_package_json_test.go +++ b/syft/cataloger/javascript/parse_package_json_test.go @@ -47,6 +47,23 @@ func TestParsePackageJSON(t *testing.T) { }, }, }, + { + Fixture: "test-fixtures/pkg-json/package-license-objects.json", + ExpectedPkg: pkg.Package{ + Name: "npm", + Version: "6.14.6", + Type: pkg.NpmPkg, + Licenses: []string{"MIT", "Apache-2.0"}, + Language: pkg.JavaScript, + MetadataType: pkg.NpmPackageJSONMetadataType, + Metadata: pkg.NpmPackageJSONMetadata{ + Author: "Isaac Z. Schlueter (http://blog.izs.me)", + Homepage: "https://docs.npmjs.com/", + URL: "https://github.com/npm/cli", + Licenses: []string{"MIT", "Apache-2.0"}, + }, + }, + }, { Fixture: "test-fixtures/pkg-json/package-nested-author.json", ExpectedPkg: pkg.Package{ diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-license-objects.json b/syft/cataloger/javascript/test-fixtures/pkg-json/package-license-objects.json new file mode 100644 index 000000000..605bc3ec9 --- /dev/null +++ b/syft/cataloger/javascript/test-fixtures/pkg-json/package-license-objects.json @@ -0,0 +1,26 @@ +{ + "version": "6.14.6", + "name": "npm", + "description": "a package manager for JavaScript", + "homepage": "https://docs.npmjs.com/", + "author": "Isaac Z. Schlueter (http://blog.izs.me)", + "repository": { + "type": "git", + "url": "https://github.com/npm/cli" + }, + "bugs": { + "url": "https://npm.community/c/bugs" + }, + "main": "./lib/npm.js", + "licenses": [ + { "type": "MIT", + "url": "https://www.opensource.org/licenses/mit-license.php" + }, + { "type": "Apache-2.0", + "url": "https://opensource.org/licenses/apache2.0.php" + } + ], + "engines": { + "node": "6 >=6.2.0 || 8 || >=9.3.0" + } +} From 416afa015e17b94bd64b710bdbdfedd3850104e8 Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Sat, 14 Nov 2020 11:00:13 -0500 Subject: [PATCH 3/8] Add test for package.json with no licenses Signed-off-by: Dan Luhring --- .../javascript/parse_package_json_test.go | 17 +++++++++++++++++ .../pkg-json/package-no-license.json | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 syft/cataloger/javascript/test-fixtures/pkg-json/package-no-license.json diff --git a/syft/cataloger/javascript/parse_package_json_test.go b/syft/cataloger/javascript/parse_package_json_test.go index c16a69006..2b1b5f911 100644 --- a/syft/cataloger/javascript/parse_package_json_test.go +++ b/syft/cataloger/javascript/parse_package_json_test.go @@ -64,6 +64,23 @@ func TestParsePackageJSON(t *testing.T) { }, }, }, + { + Fixture: "test-fixtures/pkg-json/package-no-license.json", + ExpectedPkg: pkg.Package{ + Name: "npm", + Version: "6.14.6", + Type: pkg.NpmPkg, + Licenses: []string{}, + Language: pkg.JavaScript, + MetadataType: pkg.NpmPackageJSONMetadataType, + Metadata: pkg.NpmPackageJSONMetadata{ + Author: "Isaac Z. Schlueter (http://blog.izs.me)", + Homepage: "https://docs.npmjs.com/", + URL: "https://github.com/npm/cli", + Licenses: []string{}, + }, + }, + }, { Fixture: "test-fixtures/pkg-json/package-nested-author.json", ExpectedPkg: pkg.Package{ diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-no-license.json b/syft/cataloger/javascript/test-fixtures/pkg-json/package-no-license.json new file mode 100644 index 000000000..9c1b29e24 --- /dev/null +++ b/syft/cataloger/javascript/test-fixtures/pkg-json/package-no-license.json @@ -0,0 +1,18 @@ +{ + "version": "6.14.6", + "name": "npm", + "description": "a package manager for JavaScript", + "homepage": "https://docs.npmjs.com/", + "author": "Isaac Z. Schlueter (http://blog.izs.me)", + "repository": { + "type": "git", + "url": "https://github.com/npm/cli" + }, + "bugs": { + "url": "https://npm.community/c/bugs" + }, + "main": "./lib/npm.js", + "engines": { + "node": "6 >=6.2.0 || 8 || >=9.3.0" + } +} From 4d4e2422512900e1d246779aec4ba99360857837 Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Sat, 14 Nov 2020 10:19:51 -0500 Subject: [PATCH 4/8] Fix package.json parsing for license object Signed-off-by: Dan Luhring --- .../javascript/parse_package_json.go | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/syft/cataloger/javascript/parse_package_json.go b/syft/cataloger/javascript/parse_package_json.go index 654f1b68b..244cb8585 100644 --- a/syft/cataloger/javascript/parse_package_json.go +++ b/syft/cataloger/javascript/parse_package_json.go @@ -2,6 +2,7 @@ package javascript import ( "encoding/json" + "errors" "fmt" "io" "regexp" @@ -22,7 +23,7 @@ type PackageJSON struct { Version string `json:"version"` Latest []string `json:"latest"` Author Author `json:"author"` - License string `json:"license"` + License json.RawMessage `json:"license"` Name string `json:"name"` Homepage string `json:"homepage"` Description string `json:"description"` @@ -107,6 +108,29 @@ func (r *Repository) UnmarshalJSON(b []byte) error { return nil } +type license struct { + Type string `json:"type"` + URL string `json:"url"` +} + +func licenseFromJSON(b []byte) (string, error) { + // first try as string + var licenseString string + err := json.Unmarshal(b, &licenseString) + if err == nil { + return licenseString, nil + } + + // then try as object (this format is deprecated) + var licenseObject license + err = json.Unmarshal(b, &licenseObject) + if err == nil { + return licenseObject.Type, nil + } + + return "", errors.New("unable to unmarshal license field as either string or object") +} + // parsePackageJson parses a package.json and returns the discovered JavaScript packages. func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { packages := make([]pkg.Package, 0) @@ -120,10 +144,16 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { return nil, fmt.Errorf("failed to parse package.json file: %w", err) } + singleLicense, err := licenseFromJSON(p.License) + if err != nil { + return nil, fmt.Errorf("failed to parse package.json file: %w", err) + } + licenses := []string{singleLicense} + packages = append(packages, pkg.Package{ Name: p.Name, Version: p.Version, - Licenses: []string{p.License}, + Licenses: licenses, Language: pkg.JavaScript, Type: pkg.NpmPkg, MetadataType: pkg.NpmPackageJSONMetadataType, @@ -131,7 +161,7 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { Author: p.Author.AuthorString(), Homepage: p.Homepage, URL: p.Repository.URL, - Licenses: []string{p.License}, + Licenses: licenses, }, }) } From fc99538d78cc40cb30755151fb5666c6617f59a3 Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Sat, 14 Nov 2020 10:35:16 -0500 Subject: [PATCH 5/8] Fix package.json parsing for multiple licenses objects Signed-off-by: Dan Luhring --- .../javascript/parse_package_json.go | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/syft/cataloger/javascript/parse_package_json.go b/syft/cataloger/javascript/parse_package_json.go index 244cb8585..ac22bf2a8 100644 --- a/syft/cataloger/javascript/parse_package_json.go +++ b/syft/cataloger/javascript/parse_package_json.go @@ -24,6 +24,7 @@ type PackageJSON struct { Latest []string `json:"latest"` Author Author `json:"author"` License json.RawMessage `json:"license"` + Licenses []license `json:"licenses,omitempty"` Name string `json:"name"` Homepage string `json:"homepage"` Description string `json:"description"` @@ -131,6 +132,28 @@ func licenseFromJSON(b []byte) (string, error) { return "", errors.New("unable to unmarshal license field as either string or object") } +func licensesFromJSON(p PackageJSON) ([]string, error) { + singleLicense, err := licenseFromJSON(p.License) + if err == nil { + return []string{singleLicense}, nil + } + + // The "licenses" field is deprecated. It should be inspected as a last resort. + if p.Licenses != nil { + mapLicenses := func(licenses []license) []string { + mappedLicenses := make([]string, len(licenses)) + for i, l := range licenses { + mappedLicenses[i] = l.Type + } + return mappedLicenses + } + + return mapLicenses(p.Licenses), nil + } + + return nil, fmt.Errorf("unable to parse license field: %w", err) +} + // parsePackageJson parses a package.json and returns the discovered JavaScript packages. func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { packages := make([]pkg.Package, 0) @@ -144,11 +167,10 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { return nil, fmt.Errorf("failed to parse package.json file: %w", err) } - singleLicense, err := licenseFromJSON(p.License) + licenses, err := licensesFromJSON(p) if err != nil { return nil, fmt.Errorf("failed to parse package.json file: %w", err) } - licenses := []string{singleLicense} packages = append(packages, pkg.Package{ Name: p.Name, From f7be73dbccf457c0de8ce3c36765116a7423cbd2 Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Sat, 14 Nov 2020 11:01:14 -0500 Subject: [PATCH 6/8] Fix package.json parsing for no licenses Signed-off-by: Dan Luhring --- syft/cataloger/javascript/parse_package_json.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/syft/cataloger/javascript/parse_package_json.go b/syft/cataloger/javascript/parse_package_json.go index ac22bf2a8..1a8db0a51 100644 --- a/syft/cataloger/javascript/parse_package_json.go +++ b/syft/cataloger/javascript/parse_package_json.go @@ -133,6 +133,11 @@ func licenseFromJSON(b []byte) (string, error) { } func licensesFromJSON(p PackageJSON) ([]string, error) { + if p.License == nil && p.Licenses == nil { + // This package.json doesn't specify any licenses whatsoever + return []string{}, nil + } + singleLicense, err := licenseFromJSON(p.License) if err == nil { return []string{singleLicense}, nil From 6bde075cd3d6a2c0eeb96b455ae4608286c10ce6 Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Sat, 14 Nov 2020 11:20:06 -0500 Subject: [PATCH 7/8] Remove unused argument value Signed-off-by: Dan Luhring --- syft/cataloger/javascript/parse_package_json_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syft/cataloger/javascript/parse_package_json_test.go b/syft/cataloger/javascript/parse_package_json_test.go index 2b1b5f911..c2940a7a2 100644 --- a/syft/cataloger/javascript/parse_package_json_test.go +++ b/syft/cataloger/javascript/parse_package_json_test.go @@ -124,7 +124,7 @@ func TestParsePackageJSON(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parsePackageJSON(fixture.Name(), fixture) + actual, err := parsePackageJSON("", fixture) if err != nil { t.Fatalf("failed to parse package-lock.json: %+v", err) } From 4861f69d7ce65498c96ec5fd6c7023238b6e2107 Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Sat, 14 Nov 2020 11:35:55 -0500 Subject: [PATCH 8/8] Fix spelling in logger setup Signed-off-by: Dan Luhring --- cmd/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 5d93443bb..f59b08689 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -101,7 +101,7 @@ func initLogging() { logWrapper := logger.NewLogrusLogger(cfg) syft.SetLogger(logWrapper) stereoscope.SetLogger(&logger.LogrusNestedLogger{ - Logger: logWrapper.Logger.WithField("from-lib", "steroscope"), + Logger: logWrapper.Logger.WithField("from-lib", "stereoscope"), }) }