From 931c7961586bae00785168dbe13310cda5793ce7 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 20 Oct 2020 15:23:04 -0400 Subject: [PATCH 1/6] add dynamic package.json parsing of author field Signed-off-by: Alex Goodman --- internal/parse.go | 15 ++++ .../javascript/parse_package_json.go | 60 +++++++++++++- .../javascript/parse_package_json_test.go | 78 ++++++++++++------- .../pkg-json/package-nested-author.json | 12 +++ syft/cataloger/ruby/parse_gemspec.go | 16 +--- syft/pkg/metadata.go | 11 +++ 6 files changed, 151 insertions(+), 41 deletions(-) create mode 100644 internal/parse.go create mode 100644 syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json create mode 100644 syft/pkg/metadata.go diff --git a/internal/parse.go b/internal/parse.go new file mode 100644 index 000000000..300825c98 --- /dev/null +++ b/internal/parse.go @@ -0,0 +1,15 @@ +package internal + +import "regexp" + +// MatchCaptureGroups takes a regular expression and string and returns all of the named capture group results in a map. +func MatchCaptureGroups(regEx *regexp.Regexp, str string) map[string]string { + match := regEx.FindStringSubmatch(str) + results := make(map[string]string) + for i, name := range regEx.SubexpNames() { + if i > 0 && i <= len(match) { + results[name] = match[i] + } + } + return results +} diff --git a/syft/cataloger/javascript/parse_package_json.go b/syft/cataloger/javascript/parse_package_json.go index 83642e451..0b3a39a11 100644 --- a/syft/cataloger/javascript/parse_package_json.go +++ b/syft/cataloger/javascript/parse_package_json.go @@ -4,6 +4,11 @@ import ( "encoding/json" "fmt" "io" + "regexp" + + "github.com/anchore/syft/internal" + + "github.com/mitchellh/mapstructure" "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" @@ -16,7 +21,7 @@ var _ common.ParserFn = parsePackageLock type PackageJSON struct { Version string `json:"version"` Latest []string `json:"latest"` - Author string `json:"author"` + Author Author `json:"author"` License string `json:"license"` Name string `json:"name"` Homepage string `json:"homepage"` @@ -24,6 +29,57 @@ type PackageJSON struct { Dependencies map[string]string `json:"dependencies"` } +type Author struct { + Name string `json:"name" mapstruct:"name"` + Email string `json:"email" mapstruct:"email"` + URL string `json:"url" mapstruct:"url"` +} + +var authorPattern = regexp.MustCompile(`^\s*(?P[^<(]*)(\s+<(?P.*)>)?(\s\((?P.*)\))?\s*$`) + +func (a *Author) UnmarshalJSON(b []byte) error { + var authorStr string + if err := json.Unmarshal(b, &authorStr); err != nil { + // string parsing did not work, assume a map was given + // for more information: https://docs.npmjs.com/files/package.json#people-fields-author-contributors + var fields map[string]string + var author Author + if err := json.Unmarshal(b, &fields); err != nil { + return fmt.Errorf("unable to parse package.json author: %w", err) + } + // translate the map into a structure + if err := mapstructure.Decode(fields, &author); err != nil { + return fmt.Errorf("unable to decode package.json author: %w", err) + } + *a = author + } else { + // parse out "name (url)" into an Author struct + var fields = internal.MatchCaptureGroups(authorPattern, authorStr) + *a = Author{ + Name: fields["name"], + Email: fields["email"], + URL: fields["url"], + } + } + + if a.Name == "" { + return fmt.Errorf("package.json author name is empty") + } + + return nil +} + +func (a *Author) String() string { + result := a.Name + if a.Email != "" { + result += fmt.Sprintf(" <%s>", a.Email) + } + if a.URL != "" { + result += fmt.Sprintf(" (%s)", a.URL) + } + return result +} + // parsePackageJson parses a package.json and returns the discovered JavaScript packages. func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { packages := make([]pkg.Package, 0) @@ -44,7 +100,7 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { Language: pkg.JavaScript, Type: pkg.NpmPkg, Metadata: pkg.NpmMetadata{ - Author: p.Author, + Author: p.Author.String(), Homepage: p.Homepage, }, }) diff --git a/syft/cataloger/javascript/parse_package_json_test.go b/syft/cataloger/javascript/parse_package_json_test.go index f8e7b92fb..b843f199d 100644 --- a/syft/cataloger/javascript/parse_package_json_test.go +++ b/syft/cataloger/javascript/parse_package_json_test.go @@ -9,35 +9,61 @@ import ( ) func TestParsePackageJSON(t *testing.T) { - expected := pkg.Package{ - Name: "npm", - Version: "6.14.6", - Type: pkg.NpmPkg, - Licenses: []string{"Artistic-2.0"}, - Language: pkg.JavaScript, - Metadata: pkg.NpmMetadata{ - Author: "Isaac Z. Schlueter (http://blog.izs.me)", - Homepage: "https://docs.npmjs.com/", + tests := []struct { + Fixture string + ExpectedPkg pkg.Package + }{ + { + Fixture: "test-fixtures/pkg-json/package.json", + ExpectedPkg: pkg.Package{ + Name: "npm", + Version: "6.14.6", + Type: pkg.NpmPkg, + Licenses: []string{"Artistic-2.0"}, + Language: pkg.JavaScript, + Metadata: pkg.NpmMetadata{ + Author: "Isaac Z. Schlueter (http://blog.izs.me)", + Homepage: "https://docs.npmjs.com/", + }, + }, + }, + { + Fixture: "test-fixtures/pkg-json/package-nested-author.json", + ExpectedPkg: pkg.Package{ + Name: "npm", + Version: "6.14.6", + Type: pkg.NpmPkg, + Licenses: []string{"Artistic-2.0"}, + Language: pkg.JavaScript, + Metadata: pkg.NpmMetadata{ + Author: "Isaac Z. Schlueter (http://blog.izs.me)", + Homepage: "https://docs.npmjs.com/", + }, + }, }, } - fixture, err := os.Open("test-fixtures/pkg-json/package.json") - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) - } - actual, err := parsePackageJSON(fixture.Name(), fixture) - if err != nil { - t.Fatalf("failed to parse package-lock.json: %+v", err) - } - if len(actual) != 1 { - for _, a := range actual { - t.Log(" ", a) - } - t.Fatalf("unexpected package count: %d!=1", len(actual)) - } + for _, test := range tests { + t.Run(test.Fixture, func(t *testing.T) { + fixture, err := os.Open(test.Fixture) + if err != nil { + t.Fatalf("failed to open fixture: %+v", err) + } - for _, d := range deep.Equal(actual[0], expected) { - t.Errorf("diff: %+v", d) - } + actual, err := parsePackageJSON(fixture.Name(), fixture) + if err != nil { + t.Fatalf("failed to parse package-lock.json: %+v", err) + } + if len(actual) != 1 { + for _, a := range actual { + t.Log(" ", a) + } + t.Fatalf("unexpected package count: %d!=1", len(actual)) + } + for _, d := range deep.Equal(actual[0], test.ExpectedPkg) { + t.Errorf("diff: %+v", d) + } + }) + } } diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json b/syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json new file mode 100644 index 000000000..436fca091 --- /dev/null +++ b/syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json @@ -0,0 +1,12 @@ +{ + "version": "6.14.6", + "name": "npm", + "description": "a package manager for JavaScript", + "homepage": "https://docs.npmjs.com/", + "author": { + "name": "Isaac Z. Schlueter", + "email": "i@izs.me", + "url": "http://blog.izs.me" + }, + "license": "Artistic-2.0" +} \ No newline at end of file diff --git a/syft/cataloger/ruby/parse_gemspec.go b/syft/cataloger/ruby/parse_gemspec.go index 7fe310265..3957c5b5c 100644 --- a/syft/cataloger/ruby/parse_gemspec.go +++ b/syft/cataloger/ruby/parse_gemspec.go @@ -8,6 +8,8 @@ import ( "regexp" "strings" + "github.com/anchore/syft/internal" + "github.com/mitchellh/mapstructure" "github.com/anchore/syft/syft/cataloger/common" @@ -76,7 +78,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) { } for field, pattern := range patterns { - matchMap := matchCaptureGroups(pattern, sanitizedLine) + matchMap := internal.MatchCaptureGroups(pattern, sanitizedLine) if value := matchMap[field]; value != "" { if postProcessor := postProcessors[field]; postProcessor != nil { fields[field] = postProcessor(value) @@ -121,15 +123,3 @@ func renderUtf8(s string) string { }) return fullReplacement } - -// matchCaptureGroups takes a regular expression and string and returns all of the named capture group results in a map. -func matchCaptureGroups(regEx *regexp.Regexp, str string) map[string]string { - match := regEx.FindStringSubmatch(str) - results := make(map[string]string) - for i, name := range regEx.SubexpNames() { - if i > 0 && i <= len(match) { - results[name] = match[i] - } - } - return results -} diff --git a/syft/pkg/metadata.go b/syft/pkg/metadata.go new file mode 100644 index 000000000..2d5eb2def --- /dev/null +++ b/syft/pkg/metadata.go @@ -0,0 +1,11 @@ +package pkg + +type MetadataType string + +const ( + UnknownMetadataType MetadataType = "UnknownMetadata" + ApkMetadataType MetadataType = "apk-metadata" + DpkgMetadataType MetadataType = "dpkg-metadata" + GemgMetadataType MetadataType = "gem-metadata" + RpmdbMetadataType MetadataType = "rpmdb-metadata" +) From 5b08616e475aee486365ce06bae053405a7b4e16 Mon Sep 17 00:00:00 2001 From: Toure Dunnon Date: Wed, 21 Oct 2020 10:44:05 -0400 Subject: [PATCH 2/6] Fix for errors+failures parsing `package.json` closes: #230 Signed-off-by: Toure Dunnon --- go.mod | 1 + .../javascript/parse_package_json.go | 26 ++++++++----------- .../javascript/parse_package_json_test.go | 1 + syft/pkg/metadata.go | 11 -------- 4 files changed, 13 insertions(+), 26 deletions(-) delete mode 100644 syft/pkg/metadata.go diff --git a/go.mod b/go.mod index af71131ea..f7af429f9 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe + github.com/apex/log v1.3.0 github.com/bmatcuk/doublestar v1.3.1 github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible github.com/dustin/go-humanize v1.0.0 diff --git a/syft/cataloger/javascript/parse_package_json.go b/syft/cataloger/javascript/parse_package_json.go index 0b3a39a11..9d6ec9060 100644 --- a/syft/cataloger/javascript/parse_package_json.go +++ b/syft/cataloger/javascript/parse_package_json.go @@ -35,37 +35,33 @@ type Author struct { URL string `json:"url" mapstruct:"url"` } +// match example: "author": "Isaac Z. Schlueter (http://blog.izs.me)" +// ---> name: "Isaac Z. Schlueter" email: "i@izs.me" url: "http://blog.izs.me" var authorPattern = regexp.MustCompile(`^\s*(?P[^<(]*)(\s+<(?P.*)>)?(\s\((?P.*)\))?\s*$`) func (a *Author) UnmarshalJSON(b []byte) error { var authorStr string + var fields map[string]string + var author Author + if err := json.Unmarshal(b, &authorStr); err != nil { // string parsing did not work, assume a map was given // for more information: https://docs.npmjs.com/files/package.json#people-fields-author-contributors - var fields map[string]string - var author Author if err := json.Unmarshal(b, &fields); err != nil { return fmt.Errorf("unable to parse package.json author: %w", err) } - // translate the map into a structure - if err := mapstructure.Decode(fields, &author); err != nil { - return fmt.Errorf("unable to decode package.json author: %w", err) - } - *a = author } else { // parse out "name (url)" into an Author struct - var fields = internal.MatchCaptureGroups(authorPattern, authorStr) - *a = Author{ - Name: fields["name"], - Email: fields["email"], - URL: fields["url"], - } + fields = internal.MatchCaptureGroups(authorPattern, authorStr) } - if a.Name == "" { - return fmt.Errorf("package.json author name is empty") + // translate the map into a structure + if err := mapstructure.Decode(fields, &author); err != nil { + return fmt.Errorf("unable to decode package.json author: %w", err) } + *a = author + return nil } diff --git a/syft/cataloger/javascript/parse_package_json_test.go b/syft/cataloger/javascript/parse_package_json_test.go index b843f199d..ae38d37be 100644 --- a/syft/cataloger/javascript/parse_package_json_test.go +++ b/syft/cataloger/javascript/parse_package_json_test.go @@ -62,6 +62,7 @@ func TestParsePackageJSON(t *testing.T) { } for _, d := range deep.Equal(actual[0], test.ExpectedPkg) { + t.Errorf("diff: %+v", d) } }) diff --git a/syft/pkg/metadata.go b/syft/pkg/metadata.go deleted file mode 100644 index 2d5eb2def..000000000 --- a/syft/pkg/metadata.go +++ /dev/null @@ -1,11 +0,0 @@ -package pkg - -type MetadataType string - -const ( - UnknownMetadataType MetadataType = "UnknownMetadata" - ApkMetadataType MetadataType = "apk-metadata" - DpkgMetadataType MetadataType = "dpkg-metadata" - GemgMetadataType MetadataType = "gem-metadata" - RpmdbMetadataType MetadataType = "rpmdb-metadata" -) From 15379d1075127b7af46f2ec7be937b7fcd2d3555 Mon Sep 17 00:00:00 2001 From: Toure Dunnon Date: Wed, 21 Oct 2020 10:44:05 -0400 Subject: [PATCH 3/6] Fix for errors+failures parsing `package.json` closes: #230 Signed-off-by: Toure Dunnon --- go.mod | 1 + .../javascript/parse_package_json.go | 35 +++++++++++-------- .../javascript/parse_package_json_test.go | 3 ++ .../pkg-json/package-nested-author.json | 4 +++ syft/pkg/metadata.go | 11 ------ syft/pkg/npm_metadata.go | 1 + 6 files changed, 29 insertions(+), 26 deletions(-) delete mode 100644 syft/pkg/metadata.go diff --git a/go.mod b/go.mod index af71131ea..f7af429f9 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe + github.com/apex/log v1.3.0 github.com/bmatcuk/doublestar v1.3.1 github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible github.com/dustin/go-humanize v1.0.0 diff --git a/syft/cataloger/javascript/parse_package_json.go b/syft/cataloger/javascript/parse_package_json.go index 0b3a39a11..a0ebc899f 100644 --- a/syft/cataloger/javascript/parse_package_json.go +++ b/syft/cataloger/javascript/parse_package_json.go @@ -27,6 +27,7 @@ type PackageJSON struct { Homepage string `json:"homepage"` Description string `json:"description"` Dependencies map[string]string `json:"dependencies"` + Repository Repository `json:"repository"` } type Author struct { @@ -35,37 +36,40 @@ type Author struct { URL string `json:"url" mapstruct:"url"` } +type Repository struct { + Type string `json:"type"` + URL string `json:"url"` +} + +// match example: "author": "Isaac Z. Schlueter (http://blog.izs.me)" +// ---> name: "Isaac Z. Schlueter" email: "i@izs.me" url: "http://blog.izs.me" var authorPattern = regexp.MustCompile(`^\s*(?P[^<(]*)(\s+<(?P.*)>)?(\s\((?P.*)\))?\s*$`) +// This method implements the UnmarshalJSON interface to help normalize +// the json structure. func (a *Author) UnmarshalJSON(b []byte) error { var authorStr string + var fields map[string]string + var author Author + if err := json.Unmarshal(b, &authorStr); err != nil { // string parsing did not work, assume a map was given // for more information: https://docs.npmjs.com/files/package.json#people-fields-author-contributors - var fields map[string]string - var author Author if err := json.Unmarshal(b, &fields); err != nil { return fmt.Errorf("unable to parse package.json author: %w", err) } - // translate the map into a structure - if err := mapstructure.Decode(fields, &author); err != nil { - return fmt.Errorf("unable to decode package.json author: %w", err) - } - *a = author } else { // parse out "name (url)" into an Author struct - var fields = internal.MatchCaptureGroups(authorPattern, authorStr) - *a = Author{ - Name: fields["name"], - Email: fields["email"], - URL: fields["url"], - } + fields = internal.MatchCaptureGroups(authorPattern, authorStr) } - if a.Name == "" { - return fmt.Errorf("package.json author name is empty") + // translate the map into a structure + if err := mapstructure.Decode(fields, &author); err != nil { + return fmt.Errorf("unable to decode package.json author: %w", err) } + *a = author + return nil } @@ -102,6 +106,7 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { Metadata: pkg.NpmMetadata{ Author: p.Author.String(), Homepage: p.Homepage, + URL: p.Repository.URL, }, }) } diff --git a/syft/cataloger/javascript/parse_package_json_test.go b/syft/cataloger/javascript/parse_package_json_test.go index b843f199d..b135a8e27 100644 --- a/syft/cataloger/javascript/parse_package_json_test.go +++ b/syft/cataloger/javascript/parse_package_json_test.go @@ -24,6 +24,7 @@ func TestParsePackageJSON(t *testing.T) { Metadata: pkg.NpmMetadata{ Author: "Isaac Z. Schlueter (http://blog.izs.me)", Homepage: "https://docs.npmjs.com/", + URL: "https://github.com/npm/cli", }, }, }, @@ -38,6 +39,7 @@ func TestParsePackageJSON(t *testing.T) { Metadata: pkg.NpmMetadata{ Author: "Isaac Z. Schlueter (http://blog.izs.me)", Homepage: "https://docs.npmjs.com/", + URL: "https://github.com/npm/cli", }, }, }, @@ -62,6 +64,7 @@ func TestParsePackageJSON(t *testing.T) { } for _, d := range deep.Equal(actual[0], test.ExpectedPkg) { + t.Errorf("diff: %+v", d) } }) diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json b/syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json index 436fca091..ac2438a11 100644 --- a/syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json +++ b/syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json @@ -8,5 +8,9 @@ "email": "i@izs.me", "url": "http://blog.izs.me" }, + "repository": { + "type": "git", + "url": "https://github.com/npm/cli" + }, "license": "Artistic-2.0" } \ No newline at end of file diff --git a/syft/pkg/metadata.go b/syft/pkg/metadata.go deleted file mode 100644 index 2d5eb2def..000000000 --- a/syft/pkg/metadata.go +++ /dev/null @@ -1,11 +0,0 @@ -package pkg - -type MetadataType string - -const ( - UnknownMetadataType MetadataType = "UnknownMetadata" - ApkMetadataType MetadataType = "apk-metadata" - DpkgMetadataType MetadataType = "dpkg-metadata" - GemgMetadataType MetadataType = "gem-metadata" - RpmdbMetadataType MetadataType = "rpmdb-metadata" -) diff --git a/syft/pkg/npm_metadata.go b/syft/pkg/npm_metadata.go index af683120b..5b12e5cbc 100644 --- a/syft/pkg/npm_metadata.go +++ b/syft/pkg/npm_metadata.go @@ -9,4 +9,5 @@ type NpmMetadata struct { License string `mapstructure:"license" json:"license"` Homepage string `mapstructure:"homepage" json:"homepage"` Description string `mapstructure:"description" json:"description"` + URL string `mapstructure:"url" json:"url"` } From 48c7dee9dab2a38f56514a65ba321be9c775fbb6 Mon Sep 17 00:00:00 2001 From: Toure Date: Fri, 23 Oct 2020 08:50:38 -0400 Subject: [PATCH 4/6] included additional support for older npm spec. Signed-off-by: Toure --- .../javascript/parse_package_json.go | 35 ++++++++-- .../javascript/parse_package_json_test.go | 15 +++++ .../pkg-json/package-repo-string.json | 66 +++++++++++++++++++ 3 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 syft/cataloger/javascript/test-fixtures/pkg-json/package-repo-string.json diff --git a/syft/cataloger/javascript/parse_package_json.go b/syft/cataloger/javascript/parse_package_json.go index a0ebc899f..b8d5a3dfb 100644 --- a/syft/cataloger/javascript/parse_package_json.go +++ b/syft/cataloger/javascript/parse_package_json.go @@ -37,16 +37,15 @@ type Author struct { } type Repository struct { - Type string `json:"type"` - URL string `json:"url"` + Type string `json:"type" mapstructure:"type"` + URL string `json:"url" mapstructure:"url"` } // match example: "author": "Isaac Z. Schlueter (http://blog.izs.me)" // ---> name: "Isaac Z. Schlueter" email: "i@izs.me" url: "http://blog.izs.me" var authorPattern = regexp.MustCompile(`^\s*(?P[^<(]*)(\s+<(?P.*)>)?(\s\((?P.*)\))?\s*$`) -// This method implements the UnmarshalJSON interface to help normalize -// the json structure. +// Exports Author.UnmarshalJSON interface to help normalize the json structure. func (a *Author) UnmarshalJSON(b []byte) error { var authorStr string var fields map[string]string @@ -73,7 +72,7 @@ func (a *Author) UnmarshalJSON(b []byte) error { return nil } -func (a *Author) String() string { +func (a *Author) AuthorString() string { result := a.Name if a.Email != "" { result += fmt.Sprintf(" <%s>", a.Email) @@ -84,6 +83,30 @@ func (a *Author) String() string { return result } +func (r *Repository) UnmarshalJSON(b []byte) error { + var repositoryStr string + var fields map[string]string + var repository Repository + + if err := json.Unmarshal(b, &repositoryStr); err != nil { + // string parsing did not work, assume a map was given + // for more information: https://docs.npmjs.com/files/package.json#people-fields-author-contributors + if err := json.Unmarshal(b, &fields); err != nil { + return fmt.Errorf("unable to parse package.json author: %w", err) + } + // translate the map into a structure + if err := mapstructure.Decode(fields, &repository); err != nil { + return fmt.Errorf("unable to decode package.json author: %w", err) + } + + *r = repository + } else { + r.URL = repositoryStr + } + + return nil +} + // parsePackageJson parses a package.json and returns the discovered JavaScript packages. func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { packages := make([]pkg.Package, 0) @@ -104,7 +127,7 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { Language: pkg.JavaScript, Type: pkg.NpmPkg, Metadata: pkg.NpmMetadata{ - Author: p.Author.String(), + Author: p.Author.AuthorString(), Homepage: p.Homepage, URL: p.Repository.URL, }, diff --git a/syft/cataloger/javascript/parse_package_json_test.go b/syft/cataloger/javascript/parse_package_json_test.go index b135a8e27..bb5d26d08 100644 --- a/syft/cataloger/javascript/parse_package_json_test.go +++ b/syft/cataloger/javascript/parse_package_json_test.go @@ -43,6 +43,21 @@ func TestParsePackageJSON(t *testing.T) { }, }, }, + { + Fixture: "test-fixtures/pkg-json/package-repo-string.json", + ExpectedPkg: pkg.Package{ + Name: "function-bind", + Version: "1.1.1", + Type: pkg.NpmPkg, + Licenses: []string{"MIT"}, + Language: pkg.JavaScript, + Metadata: pkg.NpmMetadata{ + Author: "Raynos ", + Homepage: "https://github.com/Raynos/function-bind", + URL: "git://github.com/Raynos/function-bind.git", + }, + }, + }, } for _, test := range tests { diff --git a/syft/cataloger/javascript/test-fixtures/pkg-json/package-repo-string.json b/syft/cataloger/javascript/test-fixtures/pkg-json/package-repo-string.json new file mode 100644 index 000000000..d42a73b5c --- /dev/null +++ b/syft/cataloger/javascript/test-fixtures/pkg-json/package-repo-string.json @@ -0,0 +1,66 @@ +{ + "name": "function-bind", + "version": "1.1.1", + "description": "Implementation of Function.prototype.bind", + "keywords": [ + "function", + "bind", + "shim", + "es5" + ], + "author": "Raynos ", + "repository": "git://github.com/Raynos/function-bind.git", + "main": "index", + "homepage": "https://github.com/Raynos/function-bind", + "contributors": [ + { + "name": "Raynos" + }, + { + "name": "Jordan Harband", + "url": "https://github.com/ljharb" + } + ], + "bugs": { + "url": "https://github.com/Raynos/function-bind/issues", + "email": "raynos2@gmail.com" + }, + "dependencies": {}, + "devDependencies": { + "@ljharb/eslint-config": "^12.2.1", + "covert": "^1.1.0", + "eslint": "^4.5.0", + "jscs": "^3.0.7", + "tape": "^4.8.0" + }, + "license": "MIT", + "scripts": { + "pretest": "npm run lint", + "test": "npm run tests-only", + "posttest": "npm run coverage -- --quiet", + "tests-only": "node test", + "coverage": "covert test/*.js", + "lint": "npm run jscs && npm run eslint", + "jscs": "jscs *.js */*.js", + "eslint": "eslint *.js */*.js" + }, + "testling": { + "files": "test/index.js", + "browsers": [ + "ie/8..latest", + "firefox/16..latest", + "firefox/nightly", + "chrome/22..latest", + "chrome/canary", + "opera/12..latest", + "opera/next", + "safari/5.1..latest", + "ipad/6.0..latest", + "iphone/6.0..latest", + "android-browser/4.2..latest" + ] + } +,"_resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz" +,"_integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" +,"_from": "function-bind@1.1.1" +} \ No newline at end of file From 6eb36bc3c8575feb55f4b2fc378be07f5ecacfdc Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 23 Oct 2020 15:00:55 -0400 Subject: [PATCH 5/6] allow for optional fields in npm metadata Signed-off-by: Alex Goodman --- schema/json/schema.json | 81 ++++++++++++++++++---------------------- syft/pkg/npm_metadata.go | 4 +- 2 files changed, 38 insertions(+), 47 deletions(-) diff --git a/schema/json/schema.json b/schema/json/schema.json index 440b271fc..0b32b6a65 100644 --- a/schema/json/schema.json +++ b/schema/json/schema.json @@ -53,62 +53,55 @@ "type": "integer" }, "files": { - "anyOf": [ - { - "type": "null" - }, - { - "items": { - "anyOf": [ - { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "properties": { + "checksum": { "type": "string" }, - { + "digest": { "properties": { - "checksum": { + "algorithm": { "type": "string" }, - "digest": { - "properties": { - "algorithm": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "required": [ - "algorithm", - "value" - ], - "type": "object" - }, - "ownerGid": { - "type": "string" - }, - "ownerUid": { - "type": "string" - }, - "path": { - "type": "string" - }, - "permissions": { - "type": "string" - }, - "size": { + "value": { "type": "string" } }, "required": [ - "path" + "algorithm", + "value" ], "type": "object" + }, + "ownerGid": { + "type": "string" + }, + "ownerUid": { + "type": "string" + }, + "path": { + "type": "string" + }, + "permissions": { + "type": "string" + }, + "size": { + "type": "string" } - ] - }, - "type": "array" - } - ] + }, + "required": [ + "path" + ], + "type": "object" + } + ] + }, + "type": "array" }, "gitCommitOfApkPort": { "type": "string" diff --git a/syft/pkg/npm_metadata.go b/syft/pkg/npm_metadata.go index bf53085a6..316404765 100644 --- a/syft/pkg/npm_metadata.go +++ b/syft/pkg/npm_metadata.go @@ -2,9 +2,7 @@ package pkg // NpmPackageJSONMetadata holds extra information that is used in pkg.Package type NpmPackageJSONMetadata struct { - Name string `mapstructure:"name" json:"name"` - Version string `mapstructure:"version" json:"version"` - Files []string `mapstructure:"files" json:"files"` + Files []string `mapstructure:"files" json:"files,omitempty"` Author string `mapstructure:"author" json:"author"` Licenses []string `mapstructure:"licenses" json:"licenses"` Homepage string `mapstructure:"homepage" json:"homepage"` From cbb7e00a52e7a88b06beca953caade02b1d5d652 Mon Sep 17 00:00:00 2001 From: Toure Dunnon Date: Fri, 23 Oct 2020 16:44:07 -0400 Subject: [PATCH 6/6] Cleaned up go.mod import. Signed-off-by: Toure Dunnon --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index f7af429f9..af71131ea 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe - github.com/apex/log v1.3.0 github.com/bmatcuk/doublestar v1.3.1 github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible github.com/dustin/go-humanize v1.0.0