feat: add option to fetch remote licenses for pnpm-lock.yaml files (#4286)

Signed-off-by: Tim Olshansky <456103+timols@users.noreply.github.com>

syft/pkg/cataloger/javascript/cataloger.go

@@ -18,8 +18,9 @@ func NewPackageCataloger() pkg.Cataloger {
 func NewLockCataloger(cfg CatalogerConfig) pkg.Cataloger {
 	yarnLockAdapter := newGenericYarnLockAdapter(cfg)
 	packageLockAdapter := newGenericPackageLockAdapter(cfg)
+	pnpmLockAdapter := newGenericPnpmLockAdapter(cfg)
 	return generic.NewCataloger("javascript-lock-cataloger").
 		WithParserByGlobs(packageLockAdapter.parsePackageLock, "**/package-lock.json").
 		WithParserByGlobs(yarnLockAdapter.parseYarnLock, "**/yarn.lock").
-		WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml")
+		WithParserByGlobs(pnpmLockAdapter.parsePnpmLock, "**/pnpm-lock.yaml")
 }

syft/pkg/cataloger/javascript/package.go

@@ -107,7 +107,7 @@ func newPackageLockV1Package(ctx context.Context, cfg CatalogerConfig, resolver
 			licenseSet = pkg.NewLicenseSet(licenses...)
 		}
 		if err != nil {
-			log.Debugf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, version, err)
+			log.Debugf("unable to extract licenses from javascript package-lock.json for package %s:%s: %+v", name, version, err)
 		}
 	}
 
@@ -140,7 +140,7 @@ func newPackageLockV2Package(ctx context.Context, cfg CatalogerConfig, resolver
 			licenseSet = pkg.NewLicenseSet(licenses...)
 		}
 		if err != nil {
-			log.Debugf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, u.Version, err)
+			log.Debugf("unable to extract licenses from javascript package-lock.json for package %s:%s: %+v", name, u.Version, err)
 		}
 	}
 
@@ -161,7 +161,19 @@ func newPackageLockV2Package(ctx context.Context, cfg CatalogerConfig, resolver
 	)
 }
 
-func newPnpmPackage(ctx context.Context, resolver file.Resolver, location file.Location, name, version string) pkg.Package {
+func newPnpmPackage(ctx context.Context, cfg CatalogerConfig, resolver file.Resolver, location file.Location, name, version string) pkg.Package {
+	var licenseSet pkg.LicenseSet
+
+	if cfg.SearchRemoteLicenses {
+		license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, version)
+		if err == nil && license != "" {
+			licenses := pkg.NewLicensesFromValuesWithContext(ctx, license)
+			licenseSet = pkg.NewLicenseSet(licenses...)
+		}
+		if err != nil {
+			log.Debugf("unable to extract licenses from javascript pnpm-lock.yaml for package %s:%s: %+v", name, version, err)
+		}
+	}
 	return finalizeLockPkg(
 		ctx,
 		resolver,
@@ -169,6 +181,7 @@ func newPnpmPackage(ctx context.Context, resolver file.Resolver, location file.L
 		pkg.Package{
 			Name:      name,
 			Version:   version,
+			Licenses:  licenseSet,
 			Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
 			PURL:      packageURL(name, version),
 			Language:  pkg.JavaScript,

syft/pkg/cataloger/javascript/parse_pnpm_lock.go

@@ -18,9 +18,6 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
-// integrity check
-var _ generic.Parser = parsePnpmLock
-
 // pnpmPackage holds the raw name and version extracted from the lockfile.
 type pnpmPackage struct {
 	Name    string
@@ -45,6 +42,16 @@ type pnpmV9LockYaml struct {
 	Packages        map[string]interface{} `yaml:"packages"`
 }
 
+type genericPnpmLockAdapter struct {
+	cfg CatalogerConfig
+}
+
+func newGenericPnpmLockAdapter(cfg CatalogerConfig) genericPnpmLockAdapter {
+	return genericPnpmLockAdapter{
+		cfg: cfg,
+	}
+}
+
 // Parse implements the pnpmLockfileParser interface for v6-v8 lockfiles.
 func (p *pnpmV6LockYaml) Parse(version float64, data []byte) ([]pnpmPackage, error) {
 	if err := yaml.Unmarshal(data, p); err != nil {
@@ -116,7 +123,7 @@ func newPnpmLockfileParser(version float64) pnpmLockfileParser {
 }
 
 // parsePnpmLock is the main parser function for pnpm-lock.yaml files.
-func parsePnpmLock(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+func (a genericPnpmLockAdapter) parsePnpmLock(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	data, err := io.ReadAll(reader)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to load pnpm-lock.yaml file: %w", err)
@@ -142,7 +149,7 @@ func parsePnpmLock(ctx context.Context, resolver file.Resolver, _ *generic.Envir
 
 	packages := make([]pkg.Package, len(pnpmPkgs))
 	for i, p := range pnpmPkgs {
-		packages[i] = newPnpmPackage(ctx, resolver, reader.Location, p.Name, p.Version)
+		packages[i] = newPnpmPackage(ctx, a.cfg, resolver, reader.Location, p.Name, p.Version)
 	}
 
 	return packages, nil, unknown.IfEmptyf(packages, "unable to determine packages")

syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go

@@ -1,6 +1,11 @@
 package javascript
 
 import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
 	"testing"
 
 	"github.com/anchore/syft/syft/artifact"
@@ -50,7 +55,8 @@ func TestParsePnpmLock(t *testing.T) {
 		},
 	}
 
-	pkgtest.TestFileParser(t, fixture, parsePnpmLock, expectedPkgs, expectedRelationships)
+	adapter := newGenericPnpmLockAdapter(CatalogerConfig{})
+	pkgtest.TestFileParser(t, fixture, adapter.parsePnpmLock, expectedPkgs, expectedRelationships)
 }
 
 func TestParsePnpmV6Lock(t *testing.T) {
@@ -142,7 +148,8 @@ func TestParsePnpmV6Lock(t *testing.T) {
 		},
 	}
 
-	pkgtest.TestFileParser(t, fixture, parsePnpmLock, expectedPkgs, expectedRelationships)
+	adapter := newGenericPnpmLockAdapter(CatalogerConfig{})
+	pkgtest.TestFileParser(t, fixture, adapter.parsePnpmLock, expectedPkgs, expectedRelationships)
 }
 
 func TestParsePnpmLockV9(t *testing.T) {
@@ -184,14 +191,101 @@ func TestParsePnpmLockV9(t *testing.T) {
 			Type:      pkg.NpmPkg,
 		},
 	}
-
+	adapter := newGenericPnpmLockAdapter(CatalogerConfig{})
 	// TODO: no relationships are under test
-	pkgtest.TestFileParser(t, fixture, parsePnpmLock, expected, expectedRelationships)
+	pkgtest.TestFileParser(t, fixture, adapter.parsePnpmLock, expected, expectedRelationships)
 }
 
+func TestSearchPnpmForLicenses(t *testing.T) {
+	ctx := context.TODO()
+	fixture := "test-fixtures/pnpm-remote/pnpm-lock.yaml"
+	locations := file.NewLocationSet(file.NewLocation(fixture))
+	mux, url, teardown := setupNpmRegistry()
+	defer teardown()
+	tests := []struct {
+		name             string
+		fixture          string
+		config           CatalogerConfig
+		requestHandlers  []handlerPath
+		expectedPackages []pkg.Package
+	}{
+		{
+			name:   "search remote licenses returns the expected licenses when search is set to true",
+			config: CatalogerConfig{SearchRemoteLicenses: true},
+			requestHandlers: []handlerPath{
+				{
+					// https://registry.npmjs.org/nanoid/3.3.4
+					path:    "/nanoid/3.3.4",
+					handler: generateMockNpmRegistryHandler("test-fixtures/pnpm-remote/registry_response.json"),
+				},
+			},
+			expectedPackages: []pkg.Package{
+				{
+					Name:      "nanoid",
+					Version:   "3.3.4",
+					Locations: locations,
+					PURL:      "pkg:npm/nanoid@3.3.4",
+					Licenses:  pkg.NewLicenseSet(pkg.NewLicenseWithContext(ctx, "MIT")),
+					Language:  pkg.JavaScript,
+					Type:      pkg.NpmPkg,
+				},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			// set up the mock server
+			for _, handler := range tc.requestHandlers {
+				mux.HandleFunc(handler.path, handler.handler)
+			}
+			tc.config.NPMBaseURL = url
+			adapter := newGenericPnpmLockAdapter(tc.config)
+			pkgtest.TestFileParser(t, fixture, adapter.parsePnpmLock, tc.expectedPackages, nil)
+		})
+	}
+}
 func Test_corruptPnpmLock(t *testing.T) {
+	adapter := newGenericPnpmLockAdapter(CatalogerConfig{})
 	pkgtest.NewCatalogTester().
 		FromFile(t, "test-fixtures/corrupt/pnpm-lock.yaml").
 		WithError().
-		TestParser(t, parsePnpmLock)
+		TestParser(t, adapter.parsePnpmLock)
+}
+
+func generateMockNpmRegistryHandler(responseFixture string) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		// Copy the file's content to the response writer
+		file, err := os.Open(responseFixture)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		defer file.Close()
+
+		_, err = io.Copy(w, file)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+// setup sets up a test HTTP server for mocking requests to a particular registry.
+// The returned url is injected into the Config so the client uses the test server.
+// Tests should register handlers on mux to simulate the expected request/response structure
+func setupNpmRegistry() (mux *http.ServeMux, serverURL string, teardown func()) {
+	// mux is the HTTP request multiplexer used with the test server.
+	mux = http.NewServeMux()
+
+	// We want to ensure that tests catch mistakes where the endpoint URL is
+	// specified as absolute rather than relative. It only makes a difference
+	// when there's a non-empty base URL path. So, use that. See issue #752.
+	apiHandler := http.NewServeMux()
+	apiHandler.Handle("/", mux)
+	// server is a test HTTP server used to provide mock API responses.
+	server := httptest.NewServer(apiHandler)
+
+	return mux, server.URL, server.Close
 }

syft/pkg/cataloger/javascript/parse_yarn_lock_test.go

@@ -239,7 +239,7 @@ func TestSearchYarnForLicenses(t *testing.T) {
 	ctx := context.TODO()
 	fixture := "test-fixtures/yarn-remote/yarn.lock"
 	locations := file.NewLocationSet(file.NewLocation(fixture))
-	mux, url, teardown := setup()
+	mux, url, teardown := setupYarnRegistry()
 	defer teardown()
 	tests := []struct {
 		name             string
@@ -255,7 +255,7 @@ func TestSearchYarnForLicenses(t *testing.T) {
 				{
 					// https://registry.yarnpkg.com/@babel/code-frame/7.10.4
 					path:    "/@babel/code-frame/7.10.4",
-					handler: generateMockNPMHandler("test-fixtures/yarn-remote/registry_response.json"),
+					handler: generateMockYarnRegistryHandler("test-fixtures/yarn-remote/registry_response.json"),
 				},
 			},
 			expectedPackages: []pkg.Package{
@@ -445,7 +445,7 @@ func TestParseYarnFindPackageVersions(t *testing.T) {
 	}
 }
 
-func generateMockNPMHandler(responseFixture string) func(w http.ResponseWriter, r *http.Request) {
+func generateMockYarnRegistryHandler(responseFixture string) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 		// Copy the file's content to the response writer
@@ -464,10 +464,10 @@ func generateMockNPMHandler(responseFixture string) func(w http.ResponseWriter,
 	}
 }
 
-// setup sets up a test HTTP server for mocking requests to maven central.
+// setup sets up a test HTTP server for mocking requests to a particular registry.
 // The returned url is injected into the Config so the client uses the test server.
 // Tests should register handlers on mux to simulate the expected request/response structure
-func setup() (mux *http.ServeMux, serverURL string, teardown func()) {
+func setupYarnRegistry() (mux *http.ServeMux, serverURL string, teardown func()) {
 	// mux is the HTTP request multiplexer used with the test server.
 	mux = http.NewServeMux()
 

syft/pkg/cataloger/javascript/test-fixtures/pnpm-remote/pnpm-lock.yaml

@@ -0,0 +1,11 @@
+lockfileVersion: 5.4
+
+specifiers:
+  nanoid: ^3.3.4
+
+dependencies:
+  nanoid: 3.3.4
+
+packages:
+  /nanoid/3.3.4:
+    resolution: {integrity: sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==}

syft/pkg/cataloger/javascript/test-fixtures/pnpm-remote/registry_response.json

@@ -0,0 +1,106 @@
+{
+    "name": "nanoid",
+    "version": "3.3.4",
+    "keywords": [
+        "uuid",
+        "random",
+        "id",
+        "url"
+    ],
+    "author": {
+        "name": "Andrey Sitnik",
+        "email": "andrey@sitnik.ru"
+    },
+    "license": "MIT",
+    "_id": "nanoid@3.3.4",
+    "maintainers": [
+        {
+            "name": "ai",
+            "email": "andrey@sitnik.ru"
+        }
+    ],
+    "homepage": "https://github.com/ai/nanoid#readme",
+    "bugs": {
+        "url": "https://github.com/ai/nanoid/issues"
+    },
+    "bin": {
+        "nanoid": "bin/nanoid.cjs"
+    },
+    "dist": {
+        "shasum": "730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab",
+        "tarball": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz",
+        "fileCount": 24,
+        "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==",
+        "signatures": [
+            {
+                "sig": "MEQCIEXG2ta5bIaT6snvQFKV+m1KjuF4DaCpp186tcPo8vsRAiB2Eg9/6nKRi4lZOfwQC1fgq4EzrFjU8T+uqwGxWEQE8A==",
+                "keyid": "SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"
+            }
+        ],
+        "unpackedSize": 21583,
+        "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v4.10.10\r\nComment: https://openpgpjs.org\r\n\r\nwsFzBAEBCAAGBQJicQqNACEJED1NWxICdlZqFiEECWMYAoorWMhJKdjhPU1b\r\nEgJ2Vmp6rw/+IRvv2zOtwi8goF3h1VctIQVWtTtYrobDIVC2W++jyxdbgZoP\r\n2CDj1YWjrr+eM6O6sI1Bj+bF+yoqQ+z8ojtfW3vtRPpjzUf/7Sgs4F2ANshp\r\ne3rqdaQLjpHPriHf6HmPJy3YNJ+7n5TPPGoTEGXAe4eCZdko3XidCMWZdHlf\r\nYQU9CVYiG6mjjORkWw1sYctt8exdcGFMh0QoQq7BEp04QWm04JwvHjUiAgvf\r\nmEQLrNrf9nwzjpnubAJD+1z6fKOc9vUE44MOj2PkPoOr6a+iBBBgwBf45cnj\r\ng8R2G5xzxsRRB0a8XZdp67y3WA8rIaYaUuBFtEWYp7QFoA/tp6AGmHEAhjLa\r\nQKTquG7ejBu21ZsQaxpGc/3WWLEm+7F78GF8CXpQdtg0Kg1eugRotSNnU0SO\r\nPLiyYV4Mw6kXnbVchS5Y+HmcDVEcSBMTve/f1KpmIhJueJ20RCg4MGYZWgI9\r\nNJ1KgH2h4djX4XuoXpcsKnX3oVfinHEMke8sLWXHsMAtOxDipEWgW9cE9hk0\r\n71Y6LAAPBu34pmaj73B0qZiIY7wXxoGWQOCl2STS/VyDG/K9w1T+WiYROu+8\r\nE9Gd+f4qXmdi7Jw6May86DDfauCwBP3gnrB5aeOktCjWsgrrdClN3Hv2pIAN\r\noJcjS3IURf6oeV4+Yw1B5GoJu1Y/6U75fOU=\r\n=IMnM\r\n-----END PGP SIGNATURE-----\r\n"
+    },
+    "main": "index.cjs",
+    "type": "module",
+    "types": "./index.d.ts",
+    "module": "index.js",
+    "browser": {
+        "./index.js": "./index.browser.js",
+        "./index.cjs": "./index.browser.cjs",
+        "./async/index.js": "./async/index.browser.js",
+        "./async/index.cjs": "./async/index.browser.cjs"
+    },
+    "engines": {
+        "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
+    },
+    "exports": {
+        ".": {
+            "types": "./index.d.ts",
+            "import": "./index.js",
+            "browser": "./index.browser.js",
+            "default": "./index.js",
+            "require": "./index.cjs"
+        },
+        "./async": {
+            "import": "./async/index.js",
+            "browser": "./async/index.browser.js",
+            "default": "./async/index.js",
+            "require": "./async/index.cjs"
+        },
+        "./index.d.ts": "./index.d.ts",
+        "./non-secure": {
+            "import": "./non-secure/index.js",
+            "default": "./non-secure/index.js",
+            "require": "./non-secure/index.cjs"
+        },
+        "./package.json": "./package.json",
+        "./url-alphabet": {
+            "import": "./url-alphabet/index.js",
+            "default": "./url-alphabet/index.js",
+            "require": "./url-alphabet/index.cjs"
+        },
+        "./async/package.json": "./async/package.json",
+        "./non-secure/package.json": "./non-secure/package.json",
+        "./url-alphabet/package.json": "./url-alphabet/package.json"
+    },
+    "gitHead": "fc5bd0dbba830b1e6f3e572da8e2bc9ddc1b4b44",
+    "_npmUser": {
+        "name": "ai",
+        "email": "andrey@sitnik.ru"
+    },
+    "repository": {
+        "url": "git+https://github.com/ai/nanoid.git",
+        "type": "git"
+    },
+    "_npmVersion": "8.6.0",
+    "description": "A tiny (116 bytes), secure URL-friendly unique string ID generator",
+    "directories": {},
+    "sideEffects": false,
+    "_nodeVersion": "18.0.0",
+    "react-native": "index.js",
+    "_hasShrinkwrap": false,
+    "_npmOperationalInternal": {
+        "tmp": "tmp/nanoid_3.3.4_1651575437375_0.2288595018362154",
+        "host": "s3://npm-registry-packages"
+    }
+}