Adding the ability to retrieve remote licenses from package.lock (#2708)

Signed-off-by: Colm O hEigeartaigh <coheigea@apache.org>

syft/pkg/cataloger/javascript/cataloger.go

@@ -17,8 +17,9 @@ func NewPackageCataloger() pkg.Cataloger {
 // NewLockCataloger returns a new cataloger object for NPM (and NPM-adjacent, such as yarn) lock files.
 func NewLockCataloger(cfg CatalogerConfig) pkg.Cataloger {
 	yarnLockAdapter := newGenericYarnLockAdapter(cfg)
+	packageLockAdapter := newGenericPackageLockAdapter(cfg)
 	return generic.NewCataloger("javascript-lock-cataloger").
-		WithParserByGlobs(parsePackageLock, "**/package-lock.json").
+		WithParserByGlobs(packageLockAdapter.parsePackageLock, "**/package-lock.json").
 		WithParserByGlobs(yarnLockAdapter.parseYarnLock, "**/yarn.lock").
 		WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml")
 }

syft/pkg/cataloger/javascript/package.go

@@ -47,7 +47,7 @@ func newPackageJSONPackage(u packageJSON, indexLocation file.Location) pkg.Packa
 	return p
 }
 
-func newPackageLockV1Package(resolver file.Resolver, location file.Location, name string, u lockDependency) pkg.Package {
+func newPackageLockV1Package(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name string, u lockDependency) pkg.Package {
 	version := u.Version
 
 	const aliasPrefixPackageLockV1 = "npm:"
@@ -63,12 +63,26 @@ func newPackageLockV1Package(resolver file.Resolver, location file.Location, nam
 		version = canonicalPackageAndVersion[versionSeparator+1:]
 	}
 
+	var licenseSet pkg.LicenseSet
+
+	if cfg.SearchRemoteLicenses {
+		license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, version)
+		if err == nil && license != "" {
+			licenses := pkg.NewLicensesFromValues(license)
+			licenseSet = pkg.NewLicenseSet(licenses...)
+		}
+		if err != nil {
+			log.Warnf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, version, err)
+		}
+	}
+
 	return finalizeLockPkg(
 		resolver,
 		location,
 		pkg.Package{
 			Name:      name,
 			Version:   version,
+			Licenses:  licenseSet,
 			Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
 			PURL:      packageURL(name, version),
 			Language:  pkg.JavaScript,
@@ -78,7 +92,22 @@ func newPackageLockV1Package(resolver file.Resolver, location file.Location, nam
 	)
 }
 
-func newPackageLockV2Package(resolver file.Resolver, location file.Location, name string, u lockPackage) pkg.Package {
+func newPackageLockV2Package(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name string, u lockPackage) pkg.Package {
+	var licenseSet pkg.LicenseSet
+
+	if u.License != nil {
+		licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(location, u.License...)...)
+	} else if cfg.SearchRemoteLicenses {
+		license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, u.Version)
+		if err == nil && license != "" {
+			licenses := pkg.NewLicensesFromValues(license)
+			licenseSet = pkg.NewLicenseSet(licenses...)
+		}
+		if err != nil {
+			log.Warnf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, u.Version, err)
+		}
+	}
+
 	return finalizeLockPkg(
 		resolver,
 		location,
@@ -86,7 +115,7 @@ func newPackageLockV2Package(resolver file.Resolver, location file.Location, nam
 			Name:      name,
 			Version:   u.Version,
 			Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
-			Licenses:  pkg.NewLicenseSet(pkg.NewLicensesFromLocation(location, u.License...)...),
+			Licenses:  licenseSet,
 			PURL:      packageURL(name, u.Version),
 			Language:  pkg.JavaScript,
 			Type:      pkg.NpmPkg,

syft/pkg/cataloger/javascript/parse_package_lock.go

@@ -15,9 +15,6 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
-// integrity check
-var _ generic.Parser = parsePackageLock
-
 // packageLock represents a JavaScript package.lock json file
 type packageLock struct {
 	Requires        bool `json:"requires"`
@@ -44,8 +41,18 @@ type lockPackage struct {
 // packageLockLicense
 type packageLockLicense []string
 
+type genericPackageLockAdapter struct {
+	cfg CatalogerConfig
+}
+
+func newGenericPackageLockAdapter(cfg CatalogerConfig) genericPackageLockAdapter {
+	return genericPackageLockAdapter{
+		cfg: cfg,
+	}
+}
+
 // parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages.
-func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+func (a genericPackageLockAdapter) parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	// in the case we find package-lock.json files in the node_modules directories, skip those
 	// as the whole purpose of the lock file is for the specific dependencies of the root project
 	if pathContainsNodeModulesDirectory(reader.Path()) {
@@ -66,7 +73,7 @@ func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Envi
 
 	if lock.LockfileVersion == 1 {
 		for name, pkgMeta := range lock.Dependencies {
-			pkgs = append(pkgs, newPackageLockV1Package(resolver, reader.Location, name, pkgMeta))
+			pkgs = append(pkgs, newPackageLockV1Package(a.cfg, resolver, reader.Location, name, pkgMeta))
 		}
 	}
 
@@ -86,7 +93,7 @@ func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Envi
 
 			pkgs = append(
 				pkgs,
-				newPackageLockV2Package(resolver, reader.Location, getNameFromPath(name), pkgMeta),
+				newPackageLockV2Package(a.cfg, resolver, reader.Location, getNameFromPath(name), pkgMeta),
 			)
 		}
 	}

syft/pkg/cataloger/javascript/parse_package_lock_test.go

@@ -106,7 +106,8 @@ func TestParsePackageLock(t *testing.T) {
 		expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
 	}
 
-	pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
+	adapter := newGenericPackageLockAdapter(CatalogerConfig{})
+	pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
 }
 
 func TestParsePackageLockV2(t *testing.T) {
@@ -169,7 +170,8 @@ func TestParsePackageLockV2(t *testing.T) {
 	for i := range expectedPkgs {
 		expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
 	}
-	pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
+	adapter := newGenericPackageLockAdapter(CatalogerConfig{})
+	pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
 }
 
 func TestParsePackageLockV3(t *testing.T) {
@@ -220,7 +222,8 @@ func TestParsePackageLockV3(t *testing.T) {
 	for i := range expectedPkgs {
 		expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
 	}
-	pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
+	adapter := newGenericPackageLockAdapter(CatalogerConfig{})
+	pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
 }
 
 func TestParsePackageLockAlias(t *testing.T) {
@@ -279,7 +282,8 @@ func TestParsePackageLockAlias(t *testing.T) {
 		for i := range expected {
 			expected[i].Locations.Add(file.NewLocation(pl))
 		}
-		pkgtest.TestFileParser(t, pl, parsePackageLock, expected, expectedRelationships)
+		adapter := newGenericPackageLockAdapter(CatalogerConfig{})
+		pkgtest.TestFileParser(t, pl, adapter.parsePackageLock, expected, expectedRelationships)
 	}
 }
 
@@ -326,5 +330,6 @@ func TestParsePackageLockLicenseWithArray(t *testing.T) {
 	for i := range expectedPkgs {
 		expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
 	}
-	pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
+	adapter := newGenericPackageLockAdapter(CatalogerConfig{})
+	pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
 }