Update yarn.lock parser to support latest (berry v3) format (#868)

* add test cases for yarn parser regex

Signed-off-by: Patrick Glass <patrickglass@gmail.com>

* update yarn.lock parser to support yarn berry

Add support for Yarn v3 (berry) which changes the output
Collapse regex for parsing scoped and non-scoped packages
Add tests for the regex to ensure backwards compatability
and to catch issues with future changes.

Signed-off-by: Patrick Glass <patrickglass@gmail.com>

* simplify yarn test expressions

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Co-authored-by: Patrick Glass <patrickglass@gmail.com>
Co-authored-by: Alex Goodman <alex.goodman@anchore.com>

syft/pkg/cataloger/javascript/parse_yarn_lock.go

@@ -16,18 +16,15 @@ import (
 var _ common.ParserFn = parseYarnLock
 
 var (
-	// composedNameExp matches the "composed" variant of yarn.lock entry names,
+	// packageNameExp matches the name of the dependency in yarn.lock
-	// where the name appears in quotes and is prefixed with @<some-namespace>.
+	// including scope/namespace prefix if found.
-	// For example: "@babel/code-frame@^7.0.0"
+	// For example: "aws-sdk@2.706.0" returns "aws-sdk"
-	composedNameExp = regexp.MustCompile(`^"(@[^@]+)`)
+	//              "@babel/code-frame@^7.0.0" returns "@babel/code-frame"
-
+	packageNameExp = regexp.MustCompile(`^"?((?:@\w[\w-_.]*\/)?\w[\w-_.]*)@`)
-	// simpleNameExp matches the "simple" variant of yarn.lock entry names, for packages with no namespace prefix.
-	// For example: aws-sdk@2.706.0
-	simpleNameExp = regexp.MustCompile(`^(\w[\w-_.]*)@`)
 
 	// versionExp matches the "version" line of a yarn.lock entry and captures the version value.
 	// For example: version "4.10.1" (...and the value "4.10.1" is captured)
-	versionExp = regexp.MustCompile(`^\W+version\W+"([\w-_.]+)"`)
+	versionExp = regexp.MustCompile(`^\W+version(?:\W+"|:\W+)([\w-_.]+)"?`)
 )
 
 const (
@@ -87,11 +84,7 @@ func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re
 }
 
 func findPackageName(line string) string {
-	if matches := composedNameExp.FindStringSubmatch(line); len(matches) >= 2 {
+	if matches := packageNameExp.FindStringSubmatch(line); len(matches) >= 2 {
-		return matches[1]
-	}
-
-	if matches := simpleNameExp.FindStringSubmatch(line); len(matches) >= 2 {
 		return matches[1]
 	}
 

syft/pkg/cataloger/javascript/parse_yarn_lock_test.go

@@ -5,6 +5,8 @@ import (
 	"testing"
 
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestParseYarnLock(t *testing.T) {
@@ -65,16 +67,182 @@ func TestParseYarnLock(t *testing.T) {
 		},
 	}
 
-	fixture, err := os.Open("test-fixtures/yarn/yarn.lock")
+	testFixtures := []string{
-	if err != nil {
+		"test-fixtures/yarn/yarn.lock",
-		t.Fatalf("failed to open fixture: %+v", err)
+		"test-fixtures/yarn-berry/yarn.lock",
 	}
 
-	// TODO: no relationships are under test yet
+	for _, file := range testFixtures {
-	actual, _, err := parseYarnLock(fixture.Name(), fixture)
+		file := file
-	if err != nil {
+		t.Run(file, func(t *testing.T) {
-		t.Fatalf("failed to parse yarn.lock: %+v", err)
+			t.Parallel()
-	}
 
-	assertPkgsEqual(t, actual, expected)
+			fixture, err := os.Open(file)
+			require.NoError(t, err)
+
+			// TODO: no relationships are under test yet
+			actual, _, err := parseYarnLock(fixture.Name(), fixture)
+			require.NoError(t, err)
+
+			assertPkgsEqual(t, actual, expected)
+		})
+	}
+}
+
+func TestParseYarnFindPackageNames(t *testing.T) {
+	tests := []struct {
+		line     string
+		expected string
+	}{
+		{
+			line:     `"@babel/code-frame@npm:7.10.4":`,
+			expected: "@babel/code-frame",
+		},
+		{
+			line:     `"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4":`,
+			expected: "@babel/code-frame",
+		},
+		{
+			line:     "ajv@^6.10.2, ajv@^6.5.5:",
+			expected: "ajv",
+		},
+		{
+			line:     "aws-sdk@2.706.0:",
+			expected: "aws-sdk",
+		},
+		{
+			line:     "asn1.js@^4.0.0:",
+			expected: "asn1.js",
+		},
+		{
+			line:     "c0n-fab_u.laTION@^7.0.0",
+			expected: "c0n-fab_u.laTION",
+		},
+		{
+			line:     `"newtest@workspace:.":`,
+			expected: "newtest",
+		},
+		{
+			line:     `"color-convert@npm:^1.9.0":`,
+			expected: "color-convert",
+		},
+		{
+			line:     `"@npmcorp/code-frame@^7.1.0", "@npmcorp/code-frame@^7.10.4":`,
+			expected: "@npmcorp/code-frame",
+		},
+		{
+			line:     `"@npmcorp/code-frame@^7.2.3":`,
+			expected: "@npmcorp/code-frame",
+		},
+		{
+			line:     `"@s/odd-name@^7.1.2":`,
+			expected: "@s/odd-name",
+		},
+		{
+			line:     `"@/code-frame@^7.3.4":`,
+			expected: "",
+		},
+		{
+			line:     `"code-frame":`,
+			expected: "",
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.expected, func(t *testing.T) {
+			t.Parallel()
+			actual := findPackageName(test.line)
+			assert.Equal(t, test.expected, actual)
+		})
+	}
+}
+
+func TestParseYarnFindPackageVersions(t *testing.T) {
+	tests := []struct {
+		line     string
+		expected string
+	}{
+		{
+			line:     `  version "7.10.4"`,
+			expected: "7.10.4",
+		},
+		{
+			line:     ` version "7.11.5"`,
+			expected: "7.11.5",
+		},
+		{
+			line:     `version "7.12.6"`,
+			expected: "",
+		},
+		{
+			line:     `  version "0.0.0"`,
+			expected: "0.0.0",
+		},
+		{
+			line:     `  version "2" `,
+			expected: "2",
+		},
+		{
+			line:     `  version "9.3"`,
+			expected: "9.3",
+		},
+		{
+			line:     "ajv@^6.10.2, ajv@^6.5.5",
+			expected: "",
+		},
+		{
+			line:     "atob@^2.1.2:",
+			expected: "",
+		},
+		{
+			line:     `"color-convert@npm:^1.9.0":`,
+			expected: "",
+		},
+		{
+			line:     "  version: 1.9.3",
+			expected: "1.9.3",
+		},
+		{
+			line:     "  version: 2",
+			expected: "2",
+		},
+		{
+			line:     "  version: 9.3",
+			expected: "9.3",
+		},
+		{
+			line:     "ajv@^6.10.2, ajv@^6.5.5",
+			expected: "",
+		},
+		{
+			line:     "atob@^2.1.2:",
+			expected: "",
+		},
+		{
+			line:     "  version: 1.0.0-alpha+001",
+			expected: "1.0.0-alpha",
+		},
+		{
+			line:     "  version: 1.0.0-beta_test+exp.sha.5114f85",
+			expected: "1.0.0-beta_test",
+		},
+		{
+			line:     "  version: 1.0.0+21AF26D3-117B344092BD",
+			expected: "1.0.0",
+		},
+		{
+			line:     "  version: 0.0.0-use.local",
+			expected: "0.0.0-use.local",
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.expected, func(t *testing.T) {
+			t.Parallel()
+			actual := findPackageVersion(test.line)
+			assert.Equal(t, test.expected, actual)
+		})
+	}
 }

syft/pkg/cataloger/javascript/test-fixtures/yarn-berry/yarn.lock

@@ -0,0 +1,110 @@
+# This file is generated by running "yarn install" inside your project.
+# Manual changes might be lost - proceed with caution!
+
+__metadata:
+  version: 6
+  cacheKey: 8
+
+"@babel/code-frame@npm:7.10.4":
+  version: 7.10.4
+  resolution: "@babel/code-frame@npm:7.10.4"
+  dependencies:
+    "@babel/highlight": ^7.10.4
+  checksum: feb4543c8a509fe30f0f6e8d7aa84f82b41148b963b826cd330e34986f649a85cb63b2f13dd4effdf434ac555d16f14940b8ea5f4433297c2f5ff85486ded019
+  languageName: node
+  linkType: hard
+
+"@types/minimatch@npm:3.0.3":
+  version: 3.0.3
+  resolution: "@types/minimatch@npm:3.0.3"
+  checksum: b80259d55b96ef24cb3bb961b6dc18b943f2bb8838b4d8e7bead204f3173e551a416ffa49f9aaf1dc431277fffe36214118628eacf4aea20119df8835229901b
+  languageName: node
+  linkType: hard
+
+"@types/qs@npm:6.9.4":
+  version: 6.9.4
+  resolution: "@types/qs@npm:6.9.4"
+  checksum: 77e509ed213f7694ae35f84a58b88da8744aad019e93556af6aeab4289287abbe71836c051d00649dbac0289ea199e408442590cfb1785009de11c3c8d0cbbea
+  languageName: node
+  linkType: hard
+
+"@types/qs@npm:6.9.4":
+  version: 6.9.4
+  resolution: "@types/qs@npm:6.9.4"
+  checksum: 77e509ed213f7694ae35f84a58b88da8744aad019e93556af6aeab4289287abbe71836c051d00649dbac0289ea199e408442590cfb1785009de11c3c8d0cbbea
+  languageName: node
+  linkType: hard
+
+"ajv@npm:6.12.3":
+  version: 6.12.3
+  resolution: "ajv@npm:6.12.3"
+  dependencies:
+    fast-deep-equal: ^3.1.1
+    fast-json-stable-stringify: ^2.0.0
+    json-schema-traverse: ^0.4.1
+    uri-js: ^4.2.2
+  checksum: ca559d34710e6969d33bc1316282e1ece4d4d99ff5fdca4bfe31947740f8f90e7824238cdc2954e499cf75b2432e3e6c56b32814ebe04fccf8abcc3fbf36b348
+  languageName: node
+  linkType: hard
+
+"atob@npm:2.1.2":
+  version: 2.1.2
+  resolution: "atob@npm:2.1.2"
+  bin:
+    atob: bin/atob.js
+  checksum: dfeeeb70090c5ebea7be4b9f787f866686c645d9f39a0d184c817252d0cf08455ed25267d79c03254d3be1f03ac399992a792edcd5ffb9c91e097ab5ef42833a
+  languageName: node
+  linkType: hard
+
+"aws-sdk@npm:2.706.0":
+  version: 2.706.0
+  resolution: "aws-sdk@npm:2.706.0"
+  dependencies:
+    buffer: 4.9.2
+    events: 1.1.1
+    ieee754: 1.1.13
+    jmespath: 0.15.0
+    querystring: 0.2.0
+    sax: 1.2.1
+    url: 0.10.3
+    uuid: 3.3.2
+    xml2js: 0.4.19
+  checksum: bf8ca2fc4f758bdebd04051ec15729affad3eb0e18eed4ae41db5b7d6ff2aed2cf3a12ae082c11b955df0125378c57b8406e1f91006e48f0c162fdbe4ee4e330
+  languageName: node
+  linkType: hard
+
+"jhipster-core@npm:7.3.4":
+  version: 7.3.4
+  resolution: "jhipster-core@npm:7.3.4"
+  dependencies:
+    chevrotain: 7.0.1
+    fs-extra: 8.1.0
+    lodash: 4.17.15
+    winston: 3.2.1
+  checksum: 6a97741d574a42a138f98596c668370b41ec8870335bcd758b6b890e279ba30d4d2be447f8cecbf416286f2c53636b406a63a773c7b00709c95af0a9a3f9b397
+  languageName: node
+  linkType: hard
+
+"asn1.js@npm:4.10.1":
+  version: 4.10.1
+  resolution: "asn1.js@npm:4.10.1"
+  dependencies:
+    bn.js: ^4.0.0
+    inherits: ^2.0.1
+    minimalistic-assert: ^1.0.0
+  checksum: 9289a1a55401238755e3142511d7b8f6fc32f08c86ff68bd7100da8b6c186179dd6b14234fba2f7f6099afcd6758a816708485efe44bc5b2a6ec87d9ceeddbb5
+  languageName: node
+  linkType: hard
+
+"c0n-fab_u.laTION@workspace:.":
+  version: 7.7.7
+  resolution: "newtest@workspace:."
+  dependencies:
+    "@babel/code-frame": 7.10.4
+    "@types/minimatch": 3.0.3
+    "@types/qs": 6.9.4
+    ajv: 6.12.3
+    asn1.js: 4.10.1
+    atob: 2.1.2
+  languageName: unknown
+  linkType: soft