From fb4b83cf3e5699064a6cec1f2d2bc05f4245516c Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Wed, 22 Jul 2020 14:20:10 -0400 Subject: [PATCH 1/3] cataloger: define Javascript (npm) package type Signed-off-by: Alfredo Deza --- imgbom/cataloger/controller.go | 2 ++ imgbom/pkg/language.go | 6 +++--- imgbom/pkg/type.go | 3 +++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/imgbom/cataloger/controller.go b/imgbom/cataloger/controller.go index bd3c488ac..90614aa8e 100644 --- a/imgbom/cataloger/controller.go +++ b/imgbom/cataloger/controller.go @@ -5,6 +5,7 @@ import ( "github.com/anchore/imgbom/imgbom/cataloger/dpkg" golang "github.com/anchore/imgbom/imgbom/cataloger/golang" "github.com/anchore/imgbom/imgbom/cataloger/java" + "github.com/anchore/imgbom/imgbom/cataloger/npm" "github.com/anchore/imgbom/imgbom/cataloger/python" "github.com/anchore/imgbom/imgbom/cataloger/rpmdb" "github.com/anchore/imgbom/imgbom/event" @@ -50,6 +51,7 @@ func newController() controller { ctrlr.add(rpmdb.NewCataloger()) ctrlr.add(java.NewCataloger()) ctrlr.add(golang.NewCataloger()) + ctrlr.add(npm.NewCataloger()) return ctrlr } diff --git a/imgbom/pkg/language.go b/imgbom/pkg/language.go index b7c8c61e0..75032937d 100644 --- a/imgbom/pkg/language.go +++ b/imgbom/pkg/language.go @@ -3,7 +3,7 @@ package pkg const ( UnknownLanguage Language = iota Java - //JavaScript + JavaScript Python Ruby Go @@ -14,7 +14,7 @@ type Language uint var languageStr = []string{ "UnknownLanguage", "java", - //"javascript", + "javascript", "python", "ruby", "go", @@ -22,7 +22,7 @@ var languageStr = []string{ var AllLanguages = []Language{ Java, - //JavaScript, + JavaScript, Python, Ruby, Go, diff --git a/imgbom/pkg/type.go b/imgbom/pkg/type.go index 56d7ce8cd..0fa538d7e 100644 --- a/imgbom/pkg/type.go +++ b/imgbom/pkg/type.go @@ -9,6 +9,7 @@ const ( //PacmanPkg RpmPkg WheelPkg + NpmPkg PythonRequirementsPkg JavaPkg JenkinsPluginPkg @@ -26,6 +27,7 @@ var typeStr = []string{ //"pacman", "rpm", "wheel", + "npm", "python-requirements", "java-archive", "jenkins-plugin", @@ -40,6 +42,7 @@ var AllPkgs = []Type{ //PacmanPkg, RpmPkg, WheelPkg, + NpmPkg, PythonRequirementsPkg, JavaPkg, JenkinsPluginPkg, From 9f314eb493fd45d9a806a5ccdf0acd3688f9db72 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Wed, 22 Jul 2020 14:21:44 -0400 Subject: [PATCH 2/3] cataloger: support npm packages from package-lock.json files Signed-off-by: Alfredo Deza --- imgbom/cataloger/npm/cataloger.go | 34 +++++ imgbom/cataloger/npm/parse_package_lock.go | 48 +++++++ .../cataloger/npm/parse_package_lock_test.go | 119 ++++++++++++++++++ .../test-fixtures/pkg-lock/package-lock.json | 73 +++++++++++ 4 files changed, 274 insertions(+) create mode 100644 imgbom/cataloger/npm/cataloger.go create mode 100644 imgbom/cataloger/npm/parse_package_lock.go create mode 100644 imgbom/cataloger/npm/parse_package_lock_test.go create mode 100644 imgbom/cataloger/npm/test-fixtures/pkg-lock/package-lock.json diff --git a/imgbom/cataloger/npm/cataloger.go b/imgbom/cataloger/npm/cataloger.go new file mode 100644 index 000000000..d3ebc7647 --- /dev/null +++ b/imgbom/cataloger/npm/cataloger.go @@ -0,0 +1,34 @@ +package npm + +import ( + "github.com/anchore/imgbom/imgbom/cataloger/common" + "github.com/anchore/imgbom/imgbom/pkg" + "github.com/anchore/imgbom/imgbom/scope" + "github.com/anchore/stereoscope/pkg/file" +) + +type Cataloger struct { + cataloger common.GenericCataloger +} + +func NewCataloger() *Cataloger { + globParsers := map[string]common.ParserFn{ + "**/package-lock.json": parsePackageLock, + } + + return &Cataloger{ + cataloger: common.NewGenericCataloger(nil, globParsers), + } +} + +func (a *Cataloger) Name() string { + return "npm-cataloger" +} + +func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { + return a.cataloger.SelectFiles(resolver) +} + +func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { + return a.cataloger.Catalog(contents, a.Name()) +} diff --git a/imgbom/cataloger/npm/parse_package_lock.go b/imgbom/cataloger/npm/parse_package_lock.go new file mode 100644 index 000000000..4181cfa25 --- /dev/null +++ b/imgbom/cataloger/npm/parse_package_lock.go @@ -0,0 +1,48 @@ +package npm + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/anchore/imgbom/imgbom/pkg" +) + +type PackageLock struct { + Requires bool `json:"requires"` + LockfileVersion int `json:"lockfileVersion"` + Dependencies Dependencies +} + +type Dependency struct { + Version string `json:"version"` + Resolved string `json:"resolved"` + Integrity string `json:"integrity"` + Requires map[string]string +} + +type Dependencies map[string]Dependency + +func parsePackageLock(_ string, reader io.Reader) ([]pkg.Package, error) { + packages := make([]pkg.Package, 0) + dec := json.NewDecoder(reader) + + for { + var lock PackageLock + if err := dec.Decode(&lock); err == io.EOF { + break + } else if err != nil { + return nil, fmt.Errorf("failed to parse package-lock.json file: %w", err) + } + for name, pkgMeta := range lock.Dependencies { + packages = append(packages, pkg.Package{ + Name: name, + Version: pkgMeta.Version, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }) + } + } + + return packages, nil +} diff --git a/imgbom/cataloger/npm/parse_package_lock_test.go b/imgbom/cataloger/npm/parse_package_lock_test.go new file mode 100644 index 000000000..1ad1bb7a2 --- /dev/null +++ b/imgbom/cataloger/npm/parse_package_lock_test.go @@ -0,0 +1,119 @@ +package npm + +import ( + "os" + "testing" + + "github.com/anchore/imgbom/imgbom/pkg" +) + +func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) { + t.Helper() + if len(actual) != len(expected) { + for _, a := range actual { + t.Log(" ", a) + } + t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected)) + } + + for _, a := range actual { + expectedPkg, ok := expected[a.Name] + if !ok { + t.Errorf("unexpected package found: '%s'", a.Name) + } + + if expectedPkg.Version != a.Version { + t.Errorf("%s : unexpected package version: '%s', expected: '%s'", a.Name, a.Version, expectedPkg.Version) + } + + if a.Language != expectedPkg.Language { + t.Errorf("%s : bad language: '%+v', expected: '%+v'", a.Name, a.Language, expectedPkg.Language) + } + + if a.Type != expectedPkg.Type { + t.Errorf("%s : bad package type: %+v, expected: %+v", a.Name, a.Type, expectedPkg.Type) + } + + if len(a.Licenses) < len(expectedPkg.Licenses) { + t.Errorf("%s : bad package licenses count: '%+v'", a.Name, a.Licenses) + } + + } +} + +func TestParsePackageLock(t *testing.T) { + expected := map[string]pkg.Package{ + "wordwrap": { + Name: "wordwrap", + Version: "0.0.3", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "get-stdin": { + Name: "get-stdin", + Version: "5.0.1", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "minimist": { + Name: "minimist", + Version: "0.0.10", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "optimist": { + Name: "optimist", + Version: "0.6.1", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "string-width": { + Name: "string-width", + Version: "2.1.1", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "strip-ansi": { + Name: "strip-ansi", + Version: "4.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "strip-eof": { + Name: "wordwrap", + Version: "1.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "ansi-regex": { + Name: "ansi-regex", + Version: "3.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "is-fullwidth-code-point": { + Name: "is-fullwidth-code-point", + Version: "2.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "cowsay": { + Name: "cowsay", + Version: "1.4.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + } + fixture, err := os.Open("test-fixtures/pkg-lock/package-lock.json") + if err != nil { + t.Fatalf("failed to open fixture: %+v", err) + } + + actual, err := parsePackageLock(fixture.Name(), fixture) + if err != nil { + t.Fatalf("failed to parse package-lock.json: %+v", err) + } + + assertPkgsEqual(t, actual, expected) + +} diff --git a/imgbom/cataloger/npm/test-fixtures/pkg-lock/package-lock.json b/imgbom/cataloger/npm/test-fixtures/pkg-lock/package-lock.json new file mode 100644 index 000000000..4056cac02 --- /dev/null +++ b/imgbom/cataloger/npm/test-fixtures/pkg-lock/package-lock.json @@ -0,0 +1,73 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" + }, + "cowsay": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/cowsay/-/cowsay-1.4.0.tgz", + "integrity": "sha512-rdg5k5PsHFVJheO/pmE3aDg2rUDDTfPJau6yYkZYlHFktUz+UxbE+IgnUAEyyCyv4noL5ltxXD0gZzmHPCy/9g==", + "requires": { + "get-stdin": "^5.0.1", + "optimist": "~0.6.1", + "string-width": "~2.1.1", + "strip-eof": "^1.0.0" + } + }, + "get-stdin": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz", + "integrity": "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g=" + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" + }, + "minimist": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=" + }, + "optimist": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "requires": { + "minimist": "~0.0.1", + "wordwrap": "~0.0.2" + } + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" + }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" + } + } +} From f1c14d97f6ead264fe2bcb9f1f14116f5f4d2987 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Wed, 22 Jul 2020 14:23:00 -0400 Subject: [PATCH 3/3] integration: validate package-lock.json parsing Signed-off-by: Alfredo Deza --- integration/fixture_pkg_coverage_test.go | 10 +++++++++- .../javascript/package-lock/package-lock.json | 11 +++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 integration/test-fixtures/image-pkg-coverage/javascript/package-lock/package-lock.json diff --git a/integration/fixture_pkg_coverage_test.go b/integration/fixture_pkg_coverage_test.go index b81e8b735..d921f9713 100644 --- a/integration/fixture_pkg_coverage_test.go +++ b/integration/fixture_pkg_coverage_test.go @@ -59,6 +59,14 @@ var cases = []struct { "Pygments": "2.6.1", }, }, + { + name: "find javascript npm packages", + pkgType: pkg.NpmPkg, + pkgLanguage: pkg.JavaScript, + pkgInfo: map[string]string{ + "get-stdin": "8.0.0", + }, + }, { name: "find python egg packages", pkgType: pkg.EggPkg, @@ -181,7 +189,7 @@ func TestPkgCoverageImage(t *testing.T) { } if expectedVersion != a.Version { - t.Errorf("unexpected package version (pkg=%s): %s", a.Name, a.Version) + t.Errorf("unexpected package version (pkg=%s): %s, expected: %s", a.Name, a.Version, expectedVersion) } if a.Language != c.pkgLanguage { diff --git a/integration/test-fixtures/image-pkg-coverage/javascript/package-lock/package-lock.json b/integration/test-fixtures/image-pkg-coverage/javascript/package-lock/package-lock.json new file mode 100644 index 000000000..9eb8f4cf8 --- /dev/null +++ b/integration/test-fixtures/image-pkg-coverage/javascript/package-lock/package-lock.json @@ -0,0 +1,11 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "get-stdin": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz", + "integrity": "sha512-sY22aA6xchAzprjyqmSEQv4UbAAzRN0L2dQB0NlN5acTTK9Don6nhoc3eAbUnpZiCANAMfd/+40kVdKfFygohg==" + } + } +}