From f2617285d04993dfe3bca787e02bc8363b95e9d6 Mon Sep 17 00:00:00 2001 From: cipher-ardvark <99636221+cipher-ardvark@users.noreply.github.com> Date: Tue, 8 Mar 2022 09:07:54 -0800 Subject: [PATCH] Update yarn.lock parser to support latest (berry v3) format (#868) * add test cases for yarn parser regex Signed-off-by: Patrick Glass * update yarn.lock parser to support yarn berry Add support for Yarn v3 (berry) which changes the output Collapse regex for parsing scoped and non-scoped packages Add tests for the regex to ensure backwards compatability and to catch issues with future changes. Signed-off-by: Patrick Glass * simplify yarn test expressions Signed-off-by: Alex Goodman Co-authored-by: Patrick Glass Co-authored-by: Alex Goodman --- .../cataloger/javascript/parse_yarn_lock.go | 21 +- .../javascript/parse_yarn_lock_test.go | 186 +++++++++++++++++- .../test-fixtures/yarn-berry/yarn.lock | 110 +++++++++++ 3 files changed, 294 insertions(+), 23 deletions(-) create mode 100644 syft/pkg/cataloger/javascript/test-fixtures/yarn-berry/yarn.lock diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index 7fad405c9..c53e9fc0d 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -16,18 +16,15 @@ import ( var _ common.ParserFn = parseYarnLock var ( - // composedNameExp matches the "composed" variant of yarn.lock entry names, - // where the name appears in quotes and is prefixed with @. - // For example: "@babel/code-frame@^7.0.0" - composedNameExp = regexp.MustCompile(`^"(@[^@]+)`) - - // simpleNameExp matches the "simple" variant of yarn.lock entry names, for packages with no namespace prefix. - // For example: aws-sdk@2.706.0 - simpleNameExp = regexp.MustCompile(`^(\w[\w-_.]*)@`) + // packageNameExp matches the name of the dependency in yarn.lock + // including scope/namespace prefix if found. + // For example: "aws-sdk@2.706.0" returns "aws-sdk" + // "@babel/code-frame@^7.0.0" returns "@babel/code-frame" + packageNameExp = regexp.MustCompile(`^"?((?:@\w[\w-_.]*\/)?\w[\w-_.]*)@`) // versionExp matches the "version" line of a yarn.lock entry and captures the version value. // For example: version "4.10.1" (...and the value "4.10.1" is captured) - versionExp = regexp.MustCompile(`^\W+version\W+"([\w-_.]+)"`) + versionExp = regexp.MustCompile(`^\W+version(?:\W+"|:\W+)([\w-_.]+)"?`) ) const ( @@ -87,11 +84,7 @@ func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re } func findPackageName(line string) string { - if matches := composedNameExp.FindStringSubmatch(line); len(matches) >= 2 { - return matches[1] - } - - if matches := simpleNameExp.FindStringSubmatch(line); len(matches) >= 2 { + if matches := packageNameExp.FindStringSubmatch(line); len(matches) >= 2 { return matches[1] } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index 5ef0242d9..9d43c7c4f 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/anchore/syft/syft/pkg" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestParseYarnLock(t *testing.T) { @@ -65,16 +67,182 @@ func TestParseYarnLock(t *testing.T) { }, } - fixture, err := os.Open("test-fixtures/yarn/yarn.lock") - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) + testFixtures := []string{ + "test-fixtures/yarn/yarn.lock", + "test-fixtures/yarn-berry/yarn.lock", } - // TODO: no relationships are under test yet - actual, _, err := parseYarnLock(fixture.Name(), fixture) - if err != nil { - t.Fatalf("failed to parse yarn.lock: %+v", err) - } + for _, file := range testFixtures { + file := file + t.Run(file, func(t *testing.T) { + t.Parallel() - assertPkgsEqual(t, actual, expected) + fixture, err := os.Open(file) + require.NoError(t, err) + + // TODO: no relationships are under test yet + actual, _, err := parseYarnLock(fixture.Name(), fixture) + require.NoError(t, err) + + assertPkgsEqual(t, actual, expected) + }) + } +} + +func TestParseYarnFindPackageNames(t *testing.T) { + tests := []struct { + line string + expected string + }{ + { + line: `"@babel/code-frame@npm:7.10.4":`, + expected: "@babel/code-frame", + }, + { + line: `"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4":`, + expected: "@babel/code-frame", + }, + { + line: "ajv@^6.10.2, ajv@^6.5.5:", + expected: "ajv", + }, + { + line: "aws-sdk@2.706.0:", + expected: "aws-sdk", + }, + { + line: "asn1.js@^4.0.0:", + expected: "asn1.js", + }, + { + line: "c0n-fab_u.laTION@^7.0.0", + expected: "c0n-fab_u.laTION", + }, + { + line: `"newtest@workspace:.":`, + expected: "newtest", + }, + { + line: `"color-convert@npm:^1.9.0":`, + expected: "color-convert", + }, + { + line: `"@npmcorp/code-frame@^7.1.0", "@npmcorp/code-frame@^7.10.4":`, + expected: "@npmcorp/code-frame", + }, + { + line: `"@npmcorp/code-frame@^7.2.3":`, + expected: "@npmcorp/code-frame", + }, + { + line: `"@s/odd-name@^7.1.2":`, + expected: "@s/odd-name", + }, + { + line: `"@/code-frame@^7.3.4":`, + expected: "", + }, + { + line: `"code-frame":`, + expected: "", + }, + } + + for _, test := range tests { + test := test + t.Run(test.expected, func(t *testing.T) { + t.Parallel() + actual := findPackageName(test.line) + assert.Equal(t, test.expected, actual) + }) + } +} + +func TestParseYarnFindPackageVersions(t *testing.T) { + tests := []struct { + line string + expected string + }{ + { + line: ` version "7.10.4"`, + expected: "7.10.4", + }, + { + line: ` version "7.11.5"`, + expected: "7.11.5", + }, + { + line: `version "7.12.6"`, + expected: "", + }, + { + line: ` version "0.0.0"`, + expected: "0.0.0", + }, + { + line: ` version "2" `, + expected: "2", + }, + { + line: ` version "9.3"`, + expected: "9.3", + }, + { + line: "ajv@^6.10.2, ajv@^6.5.5", + expected: "", + }, + { + line: "atob@^2.1.2:", + expected: "", + }, + { + line: `"color-convert@npm:^1.9.0":`, + expected: "", + }, + { + line: " version: 1.9.3", + expected: "1.9.3", + }, + { + line: " version: 2", + expected: "2", + }, + { + line: " version: 9.3", + expected: "9.3", + }, + { + line: "ajv@^6.10.2, ajv@^6.5.5", + expected: "", + }, + { + line: "atob@^2.1.2:", + expected: "", + }, + { + line: " version: 1.0.0-alpha+001", + expected: "1.0.0-alpha", + }, + { + line: " version: 1.0.0-beta_test+exp.sha.5114f85", + expected: "1.0.0-beta_test", + }, + { + line: " version: 1.0.0+21AF26D3-117B344092BD", + expected: "1.0.0", + }, + { + line: " version: 0.0.0-use.local", + expected: "0.0.0-use.local", + }, + } + + for _, test := range tests { + test := test + t.Run(test.expected, func(t *testing.T) { + t.Parallel() + actual := findPackageVersion(test.line) + assert.Equal(t, test.expected, actual) + }) + } } diff --git a/syft/pkg/cataloger/javascript/test-fixtures/yarn-berry/yarn.lock b/syft/pkg/cataloger/javascript/test-fixtures/yarn-berry/yarn.lock new file mode 100644 index 000000000..8ceaa5008 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/yarn-berry/yarn.lock @@ -0,0 +1,110 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 6 + cacheKey: 8 + +"@babel/code-frame@npm:7.10.4": + version: 7.10.4 + resolution: "@babel/code-frame@npm:7.10.4" + dependencies: + "@babel/highlight": ^7.10.4 + checksum: feb4543c8a509fe30f0f6e8d7aa84f82b41148b963b826cd330e34986f649a85cb63b2f13dd4effdf434ac555d16f14940b8ea5f4433297c2f5ff85486ded019 + languageName: node + linkType: hard + +"@types/minimatch@npm:3.0.3": + version: 3.0.3 + resolution: "@types/minimatch@npm:3.0.3" + checksum: b80259d55b96ef24cb3bb961b6dc18b943f2bb8838b4d8e7bead204f3173e551a416ffa49f9aaf1dc431277fffe36214118628eacf4aea20119df8835229901b + languageName: node + linkType: hard + +"@types/qs@npm:6.9.4": + version: 6.9.4 + resolution: "@types/qs@npm:6.9.4" + checksum: 77e509ed213f7694ae35f84a58b88da8744aad019e93556af6aeab4289287abbe71836c051d00649dbac0289ea199e408442590cfb1785009de11c3c8d0cbbea + languageName: node + linkType: hard + +"@types/qs@npm:6.9.4": + version: 6.9.4 + resolution: "@types/qs@npm:6.9.4" + checksum: 77e509ed213f7694ae35f84a58b88da8744aad019e93556af6aeab4289287abbe71836c051d00649dbac0289ea199e408442590cfb1785009de11c3c8d0cbbea + languageName: node + linkType: hard + +"ajv@npm:6.12.3": + version: 6.12.3 + resolution: "ajv@npm:6.12.3" + dependencies: + fast-deep-equal: ^3.1.1 + fast-json-stable-stringify: ^2.0.0 + json-schema-traverse: ^0.4.1 + uri-js: ^4.2.2 + checksum: ca559d34710e6969d33bc1316282e1ece4d4d99ff5fdca4bfe31947740f8f90e7824238cdc2954e499cf75b2432e3e6c56b32814ebe04fccf8abcc3fbf36b348 + languageName: node + linkType: hard + +"atob@npm:2.1.2": + version: 2.1.2 + resolution: "atob@npm:2.1.2" + bin: + atob: bin/atob.js + checksum: dfeeeb70090c5ebea7be4b9f787f866686c645d9f39a0d184c817252d0cf08455ed25267d79c03254d3be1f03ac399992a792edcd5ffb9c91e097ab5ef42833a + languageName: node + linkType: hard + +"aws-sdk@npm:2.706.0": + version: 2.706.0 + resolution: "aws-sdk@npm:2.706.0" + dependencies: + buffer: 4.9.2 + events: 1.1.1 + ieee754: 1.1.13 + jmespath: 0.15.0 + querystring: 0.2.0 + sax: 1.2.1 + url: 0.10.3 + uuid: 3.3.2 + xml2js: 0.4.19 + checksum: bf8ca2fc4f758bdebd04051ec15729affad3eb0e18eed4ae41db5b7d6ff2aed2cf3a12ae082c11b955df0125378c57b8406e1f91006e48f0c162fdbe4ee4e330 + languageName: node + linkType: hard + +"jhipster-core@npm:7.3.4": + version: 7.3.4 + resolution: "jhipster-core@npm:7.3.4" + dependencies: + chevrotain: 7.0.1 + fs-extra: 8.1.0 + lodash: 4.17.15 + winston: 3.2.1 + checksum: 6a97741d574a42a138f98596c668370b41ec8870335bcd758b6b890e279ba30d4d2be447f8cecbf416286f2c53636b406a63a773c7b00709c95af0a9a3f9b397 + languageName: node + linkType: hard + +"asn1.js@npm:4.10.1": + version: 4.10.1 + resolution: "asn1.js@npm:4.10.1" + dependencies: + bn.js: ^4.0.0 + inherits: ^2.0.1 + minimalistic-assert: ^1.0.0 + checksum: 9289a1a55401238755e3142511d7b8f6fc32f08c86ff68bd7100da8b6c186179dd6b14234fba2f7f6099afcd6758a816708485efe44bc5b2a6ec87d9ceeddbb5 + languageName: node + linkType: hard + +"c0n-fab_u.laTION@workspace:.": + version: 7.7.7 + resolution: "newtest@workspace:." + dependencies: + "@babel/code-frame": 7.10.4 + "@types/minimatch": 3.0.3 + "@types/qs": 6.9.4 + ajv: 6.12.3 + asn1.js: 4.10.1 + atob: 2.1.2 + languageName: unknown + linkType: soft