Adding metadata fields when parsing yarn.lock and poetry.lock (#2350)

* Adding the resolved and integrity fields of yarn.lock to the parsed metadata. This addition is similar to the metadata added when parsing package-lock.json. Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com> * fix comment Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com> * Adding the Index field to metadeta when parsing poetry.lock similarly to the existing Pipfile metadata Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com> * fixing struct accoding to tests Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com> * remove old schema change Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * remove empty constants Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * re-generate JSON schema Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update document ref Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: asi-cider <88270351+asi-cider@users.noreply.github.com> Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>
2026-11-01 06:56:40 +01:00 · 2024-02-02 19:53:42 +00:00
parent d7b9cc70b0
commit 98de2e2f62
14 changed files with 2398 additions and 36 deletions
@@ -110,7 +110,7 @@ func newPnpmPackage(resolver file.Resolver, location file.Location, name, versio
 	)
 }

-func newYarnLockPackage(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name, version string) pkg.Package {
+func newYarnLockPackage(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name, version string, resolved string, integrity string) pkg.Package {
 	var licenseSet pkg.LicenseSet

 	if cfg.SearchRemoteLicenses {
@@ -123,7 +123,6 @@ func newYarnLockPackage(cfg CatalogerConfig, resolver file.Resolver, location fi
 			log.Warnf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, version, err)
 		}
 	}
-
 	return finalizeLockPkg(
 		resolver,
 		location,
@@ -135,6 +134,7 @@ func newYarnLockPackage(cfg CatalogerConfig, resolver file.Resolver, location fi
 			PURL:      packageURL(name, version),
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata:  pkg.YarnLockEntry{Resolved: resolved, Integrity: integrity},
 		},
 	)
 }
@@ -34,11 +34,18 @@ var (
 	//		`resolved "https://registry.yarnpkg.com/@4lolo/resize-observer-polyfill/-/resize-observer-polyfill-1.5.2.tgz#58868fc7224506236b5550d0c68357f0a874b84b"`
 	//			would return "@4lolo/resize-observer-polyfill" and "1.5.2"
 	packageURLExp = regexp.MustCompile(`^\s+resolved\s+"https://registry\.(?:yarnpkg\.com|npmjs\.org)/(.+?)/-/(?:.+?)-(\d+\..+?)\.tgz`)
-)

-const (
-	noPackage = ""
-	noVersion = ""
+	// resolvedExp matches the resolved of the dependency in yarn.lock
+	// For example:
+	// 		resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
+	// 			would return "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
+	resolvedExp = regexp.MustCompile(`^\s+resolved\s+"(.+?)"`)
+
+	// integrityExp matches the integrity of the dependency in yarn.lock
+	// For example:
+	//		integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
+	// 			would return "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==""
+	integrityExp = regexp.MustCompile(`^\s+integrity\s+([^\s]+)`)
 )

 type genericYarnLockAdapter struct {
@@ -59,37 +66,43 @@ func (a genericYarnLockAdapter) parseYarnLock(_ context.Context, resolver file.R
 	}

 	var pkgs []pkg.Package
+	var currentPackage, currentVersion, currentResolved, currentIntegrity string
+
 	scanner := bufio.NewScanner(reader)
 	parsedPackages := strset.New()
-	currentPackage := noPackage
-	currentVersion := noVersion

 	for scanner.Scan() {
 		line := scanner.Text()

-		if packageName := findPackageName(line); packageName != noPackage {
+		if packageName := findPackageName(line); packageName != "" {
 			// When we find a new package, check if we have unsaved identifiers
-			if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) {
-				pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion))
+			if currentPackage != "" && currentVersion != "" && !parsedPackages.Has(currentPackage+"@"+currentVersion) {
+				pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion, currentResolved, currentIntegrity))
 				parsedPackages.Add(currentPackage + "@" + currentVersion)
 			}

 			currentPackage = packageName
-		} else if version := findPackageVersion(line); version != noVersion {
+		} else if version := findPackageVersion(line); version != "" {
 			currentVersion = version
-		} else if packageName, version := findPackageAndVersion(line); packageName != noPackage && version != noVersion && !parsedPackages.Has(packageName+"@"+version) {
-			pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, packageName, version))
-			parsedPackages.Add(packageName + "@" + version)
+		} else if packageName, version, resolved := findResolvedPackageAndVersion(line); packageName != "" && version != "" && resolved != "" {
+			currentResolved = resolved
+			currentPackage = packageName
+			currentVersion = version
+		} else if integrity := findIntegrity(line); integrity != "" && !parsedPackages.Has(currentPackage+"@"+currentVersion) {
+			pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion, currentResolved, integrity))
+			parsedPackages.Add(currentPackage + "@" + currentVersion)

 			// Cleanup to indicate no unsaved identifiers
-			currentPackage = noPackage
-			currentVersion = noVersion
+			currentPackage = ""
+			currentVersion = ""
+			currentResolved = ""
+			currentIntegrity = ""
 		}
 	}

 	// check if we have valid unsaved data after end-of-file has reached
-	if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) {
-		pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion))
+	if currentPackage != "" && currentVersion != "" && !parsedPackages.Has(currentPackage+"@"+currentVersion) {
+		pkgs = append(pkgs, newYarnLockPackage(a.cfg, resolver, reader.Location, currentPackage, currentVersion, currentResolved, currentIntegrity))
 		parsedPackages.Add(currentPackage + "@" + currentVersion)
 	}

@@ -107,7 +120,7 @@ func findPackageName(line string) string {
 		return matches[1]
 	}

-	return noPackage
+	return ""
 }

 func findPackageVersion(line string) string {
@@ -115,13 +128,25 @@ func findPackageVersion(line string) string {
 		return matches[1]
 	}

-	return noVersion
+	return ""
 }

-func findPackageAndVersion(line string) (string, string) {
+func findResolvedPackageAndVersion(line string) (string, string, string) {
+	var resolved string
+	if matches := resolvedExp.FindStringSubmatch(line); len(matches) >= 2 {
+		resolved = matches[1]
+	}
 	if matches := packageURLExp.FindStringSubmatch(line); len(matches) >= 2 {
-		return matches[1], matches[2]
+		return matches[1], matches[2], resolved
 	}

-	return noPackage, noVersion
+	return "", "", ""
+}
+
+func findIntegrity(line string) string {
+	if matches := integrityExp.FindStringSubmatch(line); len(matches) >= 2 {
+		return matches[1]
+	}
+
+	return ""
 }
@@ -28,6 +28,7 @@ func TestParseYarnBerry(t *testing.T) {
 			PURL:      "pkg:npm/%40babel/code-frame@7.10.4",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata:  pkg.YarnLockEntry{},
 		},
 		{
 			Name:      "@types/minimatch",
@@ -36,6 +37,7 @@ func TestParseYarnBerry(t *testing.T) {
 			PURL:      "pkg:npm/%40types/minimatch@3.0.3",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata:  pkg.YarnLockEntry{},
 		},
 		{
 			Name:      "@types/qs",
@@ -44,6 +46,7 @@ func TestParseYarnBerry(t *testing.T) {
 			PURL:      "pkg:npm/%40types/qs@6.9.4",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata:  pkg.YarnLockEntry{},
 		},
 		{
 			Name:      "ajv",
@@ -52,6 +55,7 @@ func TestParseYarnBerry(t *testing.T) {
 			PURL:      "pkg:npm/ajv@6.12.3",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata:  pkg.YarnLockEntry{},
 		},
 		{
 			Name:      "asn1.js",
@@ -60,6 +64,7 @@ func TestParseYarnBerry(t *testing.T) {
 			PURL:      "pkg:npm/asn1.js@4.10.1",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata:  pkg.YarnLockEntry{},
 		},
 		{
 			Name:      "atob",
@@ -68,6 +73,7 @@ func TestParseYarnBerry(t *testing.T) {
 			PURL:      "pkg:npm/atob@2.1.2",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata:  pkg.YarnLockEntry{},
 		},
 		{
 			Name:      "aws-sdk",
@@ -76,6 +82,7 @@ func TestParseYarnBerry(t *testing.T) {
 			Locations: locations,
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata:  pkg.YarnLockEntry{},
 		},
 		{
 			Name:      "c0n-fab_u.laTION",
@@ -84,6 +91,7 @@ func TestParseYarnBerry(t *testing.T) {
 			PURL:      "pkg:npm/c0n-fab_u.laTION@7.7.7",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata:  pkg.YarnLockEntry{},
 		},
 		{
 			Name:      "jhipster-core",
@@ -92,6 +100,7 @@ func TestParseYarnBerry(t *testing.T) {
 			PURL:      "pkg:npm/jhipster-core@7.3.4",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata:  pkg.YarnLockEntry{},
 		},
 	}

@@ -112,6 +121,10 @@ func TestParseYarnLock(t *testing.T) {
 			PURL:      "pkg:npm/%40babel/code-frame@7.10.4",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata: pkg.YarnLockEntry{
+				Resolved:  "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.4.tgz#168da1a36e90da68ae8d49c0f1b48c7c6249213a",
+				Integrity: "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==",
+			},
 		},
 		{
 			Name:      "@types/minimatch",
@@ -120,6 +133,10 @@ func TestParseYarnLock(t *testing.T) {
 			PURL:      "pkg:npm/%40types/minimatch@3.0.3",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata: pkg.YarnLockEntry{
+				Resolved:  "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d",
+				Integrity: "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==",
+			},
 		},
 		{
 			Name:      "@types/qs",
@@ -128,6 +145,10 @@ func TestParseYarnLock(t *testing.T) {
 			PURL:      "pkg:npm/%40types/qs@6.9.4",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata: pkg.YarnLockEntry{
+				Resolved:  "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.4.tgz#a59e851c1ba16c0513ea123830dd639a0a15cb6a",
+				Integrity: "sha512-+wYo+L6ZF6BMoEjtf8zB2esQsqdV6WsjRK/GP9WOgLPrq87PbNWgIxS76dS5uvl/QXtHGakZmwTznIfcPXcKlQ==",
+			},
 		},
 		{
 			Name:      "ajv",
@@ -136,6 +157,10 @@ func TestParseYarnLock(t *testing.T) {
 			PURL:      "pkg:npm/ajv@6.12.3",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata: pkg.YarnLockEntry{
+				Resolved:  "https://registry.yarnpkg.com/ajv/-/ajv-6.12.3.tgz#18c5af38a111ddeb4f2697bd78d68abc1cabd706",
+				Integrity: "sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==",
+			},
 		},
 		{
 			Name:      "asn1.js",
@@ -144,6 +169,10 @@ func TestParseYarnLock(t *testing.T) {
 			PURL:      "pkg:npm/asn1.js@4.10.1",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata: pkg.YarnLockEntry{
+				Resolved:  "https://registry.yarnpkg.com/asn1.js/-/asn1.js-4.10.1.tgz#b9c2bf5805f1e64aadeed6df3a2bfafb5a73f5a0",
+				Integrity: "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==",
+			},
 		},
 		{
 			Name:      "atob",
@@ -153,6 +182,10 @@ func TestParseYarnLock(t *testing.T) {
 			PURL:     "pkg:npm/atob@2.1.2",
 			Language: pkg.JavaScript,
 			Type:     pkg.NpmPkg,
+			Metadata: pkg.YarnLockEntry{
+				Resolved:  "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9",
+				Integrity: "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==",
+			},
 		},
 		{
 			Name:      "aws-sdk",
@@ -161,6 +194,10 @@ func TestParseYarnLock(t *testing.T) {
 			PURL:      "pkg:npm/aws-sdk@2.706.0",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata: pkg.YarnLockEntry{
+				Resolved:  "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.706.0.tgz#09f65e9a91ecac5a635daf934082abae30eca953",
+				Integrity: "sha512-7GT+yrB5Wb/zOReRdv/Pzkb2Qt+hz6B/8FGMVaoysX3NryHvQUdz7EQWi5yhg9CxOjKxdw5lFwYSs69YlSp1KA==",
+			},
 		},
 		{
 			Name:      "jhipster-core",
@@ -169,8 +206,11 @@ func TestParseYarnLock(t *testing.T) {
 			PURL:      "pkg:npm/jhipster-core@7.3.4",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata: pkg.YarnLockEntry{
+				Resolved:  "https://registry.yarnpkg.com/jhipster-core/-/jhipster-core-7.3.4.tgz#c34b8c97c7f4e8b7518dae015517e2112c73cc80",
+				Integrity: "sha512-AUhT69kNkqppaJZVfan/xnKG4Gs9Ggj7YLtTZFVe+xg+THrbMb5Ng7PL07PDlDw4KAEA33GMCwuAf65E8EpC4g==",
+			},
 		},
-
 		{
 			Name:      "something-i-made-up",
 			Version:   "7.7.7",
@@ -178,6 +218,10 @@ func TestParseYarnLock(t *testing.T) {
 			PURL:      "pkg:npm/something-i-made-up@7.7.7",
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,
+			Metadata: pkg.YarnLockEntry{
+				Resolved:  "https://registry.yarnpkg.com/something-i-made-up/-/c0n-fab_u.laTION-7.7.7.tgz#b9c2bf5805f1e64aadeed6df3a2bfafb5a73f5a0",
+				Integrity: "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==",
+			},
 		},
 	}

@@ -221,6 +265,10 @@ func TestSearchYarnForLicenses(t *testing.T) {
 					Licenses:  pkg.NewLicenseSet(pkg.NewLicense("MIT")),
 					Language:  pkg.JavaScript,
 					Type:      pkg.NpmPkg,
+					Metadata: pkg.YarnLockEntry{
+						Resolved:  "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.4.tgz#168da1a36e90da68ae8d49c0f1b48c7c6249213a",
+						Integrity: "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==",
+					},
 				},
 			},
 		},