diff --git a/syft/pkg/cataloger/javascript/cataloger.go b/syft/pkg/cataloger/javascript/cataloger.go index 649800c6c..7806829ec 100644 --- a/syft/pkg/cataloger/javascript/cataloger.go +++ b/syft/pkg/cataloger/javascript/cataloger.go @@ -18,8 +18,9 @@ func NewPackageCataloger() pkg.Cataloger { func NewLockCataloger(cfg CatalogerConfig) pkg.Cataloger { yarnLockAdapter := newGenericYarnLockAdapter(cfg) packageLockAdapter := newGenericPackageLockAdapter(cfg) + pnpmLockAdapter := newGenericPnpmLockAdapter(cfg) return generic.NewCataloger("javascript-lock-cataloger"). WithParserByGlobs(packageLockAdapter.parsePackageLock, "**/package-lock.json"). WithParserByGlobs(yarnLockAdapter.parseYarnLock, "**/yarn.lock"). - WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml") + WithParserByGlobs(pnpmLockAdapter.parsePnpmLock, "**/pnpm-lock.yaml") } diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go index f74a5da9a..ca0063b65 100644 --- a/syft/pkg/cataloger/javascript/package.go +++ b/syft/pkg/cataloger/javascript/package.go @@ -107,7 +107,7 @@ func newPackageLockV1Package(ctx context.Context, cfg CatalogerConfig, resolver licenseSet = pkg.NewLicenseSet(licenses...) } if err != nil { - log.Debugf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, version, err) + log.Debugf("unable to extract licenses from javascript package-lock.json for package %s:%s: %+v", name, version, err) } } @@ -140,7 +140,7 @@ func newPackageLockV2Package(ctx context.Context, cfg CatalogerConfig, resolver licenseSet = pkg.NewLicenseSet(licenses...) } if err != nil { - log.Debugf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, u.Version, err) + log.Debugf("unable to extract licenses from javascript package-lock.json for package %s:%s: %+v", name, u.Version, err) } } @@ -161,7 +161,19 @@ func newPackageLockV2Package(ctx context.Context, cfg CatalogerConfig, resolver ) } -func newPnpmPackage(ctx context.Context, resolver file.Resolver, location file.Location, name, version string) pkg.Package { +func newPnpmPackage(ctx context.Context, cfg CatalogerConfig, resolver file.Resolver, location file.Location, name, version string) pkg.Package { + var licenseSet pkg.LicenseSet + + if cfg.SearchRemoteLicenses { + license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, version) + if err == nil && license != "" { + licenses := pkg.NewLicensesFromValuesWithContext(ctx, license) + licenseSet = pkg.NewLicenseSet(licenses...) + } + if err != nil { + log.Debugf("unable to extract licenses from javascript pnpm-lock.yaml for package %s:%s: %+v", name, version, err) + } + } return finalizeLockPkg( ctx, resolver, @@ -169,6 +181,7 @@ func newPnpmPackage(ctx context.Context, resolver file.Resolver, location file.L pkg.Package{ Name: name, Version: version, + Licenses: licenseSet, Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), PURL: packageURL(name, version), Language: pkg.JavaScript, diff --git a/syft/pkg/cataloger/javascript/parse_pnpm_lock.go b/syft/pkg/cataloger/javascript/parse_pnpm_lock.go index 706bcdfe9..536c8e3f2 100644 --- a/syft/pkg/cataloger/javascript/parse_pnpm_lock.go +++ b/syft/pkg/cataloger/javascript/parse_pnpm_lock.go @@ -18,9 +18,6 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/generic" ) -// integrity check -var _ generic.Parser = parsePnpmLock - // pnpmPackage holds the raw name and version extracted from the lockfile. type pnpmPackage struct { Name string @@ -45,6 +42,16 @@ type pnpmV9LockYaml struct { Packages map[string]interface{} `yaml:"packages"` } +type genericPnpmLockAdapter struct { + cfg CatalogerConfig +} + +func newGenericPnpmLockAdapter(cfg CatalogerConfig) genericPnpmLockAdapter { + return genericPnpmLockAdapter{ + cfg: cfg, + } +} + // Parse implements the pnpmLockfileParser interface for v6-v8 lockfiles. func (p *pnpmV6LockYaml) Parse(version float64, data []byte) ([]pnpmPackage, error) { if err := yaml.Unmarshal(data, p); err != nil { @@ -116,7 +123,7 @@ func newPnpmLockfileParser(version float64) pnpmLockfileParser { } // parsePnpmLock is the main parser function for pnpm-lock.yaml files. -func parsePnpmLock(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { +func (a genericPnpmLockAdapter) parsePnpmLock(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { data, err := io.ReadAll(reader) if err != nil { return nil, nil, fmt.Errorf("failed to load pnpm-lock.yaml file: %w", err) @@ -142,7 +149,7 @@ func parsePnpmLock(ctx context.Context, resolver file.Resolver, _ *generic.Envir packages := make([]pkg.Package, len(pnpmPkgs)) for i, p := range pnpmPkgs { - packages[i] = newPnpmPackage(ctx, resolver, reader.Location, p.Name, p.Version) + packages[i] = newPnpmPackage(ctx, a.cfg, resolver, reader.Location, p.Name, p.Version) } return packages, nil, unknown.IfEmptyf(packages, "unable to determine packages") diff --git a/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go b/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go index 7ba17a546..d62b95eb8 100644 --- a/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go @@ -1,6 +1,11 @@ package javascript import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" "testing" "github.com/anchore/syft/syft/artifact" @@ -50,7 +55,8 @@ func TestParsePnpmLock(t *testing.T) { }, } - pkgtest.TestFileParser(t, fixture, parsePnpmLock, expectedPkgs, expectedRelationships) + adapter := newGenericPnpmLockAdapter(CatalogerConfig{}) + pkgtest.TestFileParser(t, fixture, adapter.parsePnpmLock, expectedPkgs, expectedRelationships) } func TestParsePnpmV6Lock(t *testing.T) { @@ -142,7 +148,8 @@ func TestParsePnpmV6Lock(t *testing.T) { }, } - pkgtest.TestFileParser(t, fixture, parsePnpmLock, expectedPkgs, expectedRelationships) + adapter := newGenericPnpmLockAdapter(CatalogerConfig{}) + pkgtest.TestFileParser(t, fixture, adapter.parsePnpmLock, expectedPkgs, expectedRelationships) } func TestParsePnpmLockV9(t *testing.T) { @@ -184,14 +191,101 @@ func TestParsePnpmLockV9(t *testing.T) { Type: pkg.NpmPkg, }, } - + adapter := newGenericPnpmLockAdapter(CatalogerConfig{}) // TODO: no relationships are under test - pkgtest.TestFileParser(t, fixture, parsePnpmLock, expected, expectedRelationships) + pkgtest.TestFileParser(t, fixture, adapter.parsePnpmLock, expected, expectedRelationships) } +func TestSearchPnpmForLicenses(t *testing.T) { + ctx := context.TODO() + fixture := "test-fixtures/pnpm-remote/pnpm-lock.yaml" + locations := file.NewLocationSet(file.NewLocation(fixture)) + mux, url, teardown := setupNpmRegistry() + defer teardown() + tests := []struct { + name string + fixture string + config CatalogerConfig + requestHandlers []handlerPath + expectedPackages []pkg.Package + }{ + { + name: "search remote licenses returns the expected licenses when search is set to true", + config: CatalogerConfig{SearchRemoteLicenses: true}, + requestHandlers: []handlerPath{ + { + // https://registry.npmjs.org/nanoid/3.3.4 + path: "/nanoid/3.3.4", + handler: generateMockNpmRegistryHandler("test-fixtures/pnpm-remote/registry_response.json"), + }, + }, + expectedPackages: []pkg.Package{ + { + Name: "nanoid", + Version: "3.3.4", + Locations: locations, + PURL: "pkg:npm/nanoid@3.3.4", + Licenses: pkg.NewLicenseSet(pkg.NewLicenseWithContext(ctx, "MIT")), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // set up the mock server + for _, handler := range tc.requestHandlers { + mux.HandleFunc(handler.path, handler.handler) + } + tc.config.NPMBaseURL = url + adapter := newGenericPnpmLockAdapter(tc.config) + pkgtest.TestFileParser(t, fixture, adapter.parsePnpmLock, tc.expectedPackages, nil) + }) + } +} func Test_corruptPnpmLock(t *testing.T) { + adapter := newGenericPnpmLockAdapter(CatalogerConfig{}) pkgtest.NewCatalogTester(). FromFile(t, "test-fixtures/corrupt/pnpm-lock.yaml"). WithError(). - TestParser(t, parsePnpmLock) + TestParser(t, adapter.parsePnpmLock) +} + +func generateMockNpmRegistryHandler(responseFixture string) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + // Copy the file's content to the response writer + file, err := os.Open(responseFixture) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer file.Close() + + _, err = io.Copy(w, file) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + +// setup sets up a test HTTP server for mocking requests to a particular registry. +// The returned url is injected into the Config so the client uses the test server. +// Tests should register handlers on mux to simulate the expected request/response structure +func setupNpmRegistry() (mux *http.ServeMux, serverURL string, teardown func()) { + // mux is the HTTP request multiplexer used with the test server. + mux = http.NewServeMux() + + // We want to ensure that tests catch mistakes where the endpoint URL is + // specified as absolute rather than relative. It only makes a difference + // when there's a non-empty base URL path. So, use that. See issue #752. + apiHandler := http.NewServeMux() + apiHandler.Handle("/", mux) + // server is a test HTTP server used to provide mock API responses. + server := httptest.NewServer(apiHandler) + + return mux, server.URL, server.Close } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index f85a2cf3a..f6719db4a 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -239,7 +239,7 @@ func TestSearchYarnForLicenses(t *testing.T) { ctx := context.TODO() fixture := "test-fixtures/yarn-remote/yarn.lock" locations := file.NewLocationSet(file.NewLocation(fixture)) - mux, url, teardown := setup() + mux, url, teardown := setupYarnRegistry() defer teardown() tests := []struct { name string @@ -255,7 +255,7 @@ func TestSearchYarnForLicenses(t *testing.T) { { // https://registry.yarnpkg.com/@babel/code-frame/7.10.4 path: "/@babel/code-frame/7.10.4", - handler: generateMockNPMHandler("test-fixtures/yarn-remote/registry_response.json"), + handler: generateMockYarnRegistryHandler("test-fixtures/yarn-remote/registry_response.json"), }, }, expectedPackages: []pkg.Package{ @@ -445,7 +445,7 @@ func TestParseYarnFindPackageVersions(t *testing.T) { } } -func generateMockNPMHandler(responseFixture string) func(w http.ResponseWriter, r *http.Request) { +func generateMockYarnRegistryHandler(responseFixture string) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) // Copy the file's content to the response writer @@ -464,10 +464,10 @@ func generateMockNPMHandler(responseFixture string) func(w http.ResponseWriter, } } -// setup sets up a test HTTP server for mocking requests to maven central. +// setup sets up a test HTTP server for mocking requests to a particular registry. // The returned url is injected into the Config so the client uses the test server. // Tests should register handlers on mux to simulate the expected request/response structure -func setup() (mux *http.ServeMux, serverURL string, teardown func()) { +func setupYarnRegistry() (mux *http.ServeMux, serverURL string, teardown func()) { // mux is the HTTP request multiplexer used with the test server. mux = http.NewServeMux() diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pnpm-remote/pnpm-lock.yaml b/syft/pkg/cataloger/javascript/test-fixtures/pnpm-remote/pnpm-lock.yaml new file mode 100644 index 000000000..27cc4dcf9 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pnpm-remote/pnpm-lock.yaml @@ -0,0 +1,11 @@ +lockfileVersion: 5.4 + +specifiers: + nanoid: ^3.3.4 + +dependencies: + nanoid: 3.3.4 + +packages: + /nanoid/3.3.4: + resolution: {integrity: sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pnpm-remote/registry_response.json b/syft/pkg/cataloger/javascript/test-fixtures/pnpm-remote/registry_response.json new file mode 100644 index 000000000..d67a6bb76 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pnpm-remote/registry_response.json @@ -0,0 +1,106 @@ +{ + "name": "nanoid", + "version": "3.3.4", + "keywords": [ + "uuid", + "random", + "id", + "url" + ], + "author": { + "name": "Andrey Sitnik", + "email": "andrey@sitnik.ru" + }, + "license": "MIT", + "_id": "nanoid@3.3.4", + "maintainers": [ + { + "name": "ai", + "email": "andrey@sitnik.ru" + } + ], + "homepage": "https://github.com/ai/nanoid#readme", + "bugs": { + "url": "https://github.com/ai/nanoid/issues" + }, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "dist": { + "shasum": "730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab", + "tarball": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", + "fileCount": 24, + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", + "signatures": [ + { + "sig": "MEQCIEXG2ta5bIaT6snvQFKV+m1KjuF4DaCpp186tcPo8vsRAiB2Eg9/6nKRi4lZOfwQC1fgq4EzrFjU8T+uqwGxWEQE8A==", + "keyid": "SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA" + } + ], + "unpackedSize": 21583, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v4.10.10\r\nComment: https://openpgpjs.org\r\n\r\nwsFzBAEBCAAGBQJicQqNACEJED1NWxICdlZqFiEECWMYAoorWMhJKdjhPU1b\r\nEgJ2Vmp6rw/+IRvv2zOtwi8goF3h1VctIQVWtTtYrobDIVC2W++jyxdbgZoP\r\n2CDj1YWjrr+eM6O6sI1Bj+bF+yoqQ+z8ojtfW3vtRPpjzUf/7Sgs4F2ANshp\r\ne3rqdaQLjpHPriHf6HmPJy3YNJ+7n5TPPGoTEGXAe4eCZdko3XidCMWZdHlf\r\nYQU9CVYiG6mjjORkWw1sYctt8exdcGFMh0QoQq7BEp04QWm04JwvHjUiAgvf\r\nmEQLrNrf9nwzjpnubAJD+1z6fKOc9vUE44MOj2PkPoOr6a+iBBBgwBf45cnj\r\ng8R2G5xzxsRRB0a8XZdp67y3WA8rIaYaUuBFtEWYp7QFoA/tp6AGmHEAhjLa\r\nQKTquG7ejBu21ZsQaxpGc/3WWLEm+7F78GF8CXpQdtg0Kg1eugRotSNnU0SO\r\nPLiyYV4Mw6kXnbVchS5Y+HmcDVEcSBMTve/f1KpmIhJueJ20RCg4MGYZWgI9\r\nNJ1KgH2h4djX4XuoXpcsKnX3oVfinHEMke8sLWXHsMAtOxDipEWgW9cE9hk0\r\n71Y6LAAPBu34pmaj73B0qZiIY7wXxoGWQOCl2STS/VyDG/K9w1T+WiYROu+8\r\nE9Gd+f4qXmdi7Jw6May86DDfauCwBP3gnrB5aeOktCjWsgrrdClN3Hv2pIAN\r\noJcjS3IURf6oeV4+Yw1B5GoJu1Y/6U75fOU=\r\n=IMnM\r\n-----END PGP SIGNATURE-----\r\n" + }, + "main": "index.cjs", + "type": "module", + "types": "./index.d.ts", + "module": "index.js", + "browser": { + "./index.js": "./index.browser.js", + "./index.cjs": "./index.browser.cjs", + "./async/index.js": "./async/index.browser.js", + "./async/index.cjs": "./async/index.browser.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + }, + "exports": { + ".": { + "types": "./index.d.ts", + "import": "./index.js", + "browser": "./index.browser.js", + "default": "./index.js", + "require": "./index.cjs" + }, + "./async": { + "import": "./async/index.js", + "browser": "./async/index.browser.js", + "default": "./async/index.js", + "require": "./async/index.cjs" + }, + "./index.d.ts": "./index.d.ts", + "./non-secure": { + "import": "./non-secure/index.js", + "default": "./non-secure/index.js", + "require": "./non-secure/index.cjs" + }, + "./package.json": "./package.json", + "./url-alphabet": { + "import": "./url-alphabet/index.js", + "default": "./url-alphabet/index.js", + "require": "./url-alphabet/index.cjs" + }, + "./async/package.json": "./async/package.json", + "./non-secure/package.json": "./non-secure/package.json", + "./url-alphabet/package.json": "./url-alphabet/package.json" + }, + "gitHead": "fc5bd0dbba830b1e6f3e572da8e2bc9ddc1b4b44", + "_npmUser": { + "name": "ai", + "email": "andrey@sitnik.ru" + }, + "repository": { + "url": "git+https://github.com/ai/nanoid.git", + "type": "git" + }, + "_npmVersion": "8.6.0", + "description": "A tiny (116 bytes), secure URL-friendly unique string ID generator", + "directories": {}, + "sideEffects": false, + "_nodeVersion": "18.0.0", + "react-native": "index.js", + "_hasShrinkwrap": false, + "_npmOperationalInternal": { + "tmp": "tmp/nanoid_3.3.4_1651575437375_0.2288595018362154", + "host": "s3://npm-registry-packages" + } +} \ No newline at end of file