Create single license scanner for all catalogers (#3348)

* add single license scanner instance Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename testing license scanner Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
2026-11-05 08:56:41 +01:00 · 2024-10-21 16:17:12 +00:00
parent 14355aac21
commit e4e985b9b0
16 changed files with 277 additions and 148 deletions
@@ -3,6 +3,7 @@ package golang
 import (
 	"archive/zip"
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"io/fs"
@@ -79,9 +80,9 @@ func remotesForModule(proxies []string, noProxy []string, module string) []strin
 	return proxies
 }

-func (c *goLicenseResolver) getLicenses(resolver file.Resolver, moduleName, moduleVersion string) ([]pkg.License, error) {
+func (c *goLicenseResolver) getLicenses(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, moduleName, moduleVersion string) ([]pkg.License, error) {
 	// search the scan target first, ignoring local and remote sources
-	goLicenses, err := c.findLicensesInSource(resolver,
+	goLicenses, err := c.findLicensesInSource(ctx, scanner, resolver,
 		fmt.Sprintf(`**/go/pkg/mod/%s@%s/*`, processCaps(moduleName), moduleVersion),
 	)
 	if err != nil || len(goLicenses) > 0 {
@@ -90,7 +91,7 @@ func (c *goLicenseResolver) getLicenses(resolver file.Resolver, moduleName, modu

 	// look in the local host mod directory...
 	if c.opts.SearchLocalModCacheLicenses {
-		goLicenses, err = c.getLicensesFromLocal(moduleName, moduleVersion)
+		goLicenses, err = c.getLicensesFromLocal(ctx, scanner, moduleName, moduleVersion)
 		if err != nil || len(goLicenses) > 0 {
 			return toPkgLicenses(goLicenses), err
 		}
@@ -98,13 +99,13 @@ func (c *goLicenseResolver) getLicenses(resolver file.Resolver, moduleName, modu

 	// download from remote sources
 	if c.opts.SearchRemoteLicenses {
-		goLicenses, err = c.getLicensesFromRemote(moduleName, moduleVersion)
+		goLicenses, err = c.getLicensesFromRemote(ctx, scanner, moduleName, moduleVersion)
 	}

 	return toPkgLicenses(goLicenses), err
 }

-func (c *goLicenseResolver) getLicensesFromLocal(moduleName, moduleVersion string) ([]goLicense, error) {
+func (c *goLicenseResolver) getLicensesFromLocal(ctx context.Context, scanner licenses.Scanner, moduleName, moduleVersion string) ([]goLicense, error) {
 	if c.localModCacheDir == nil {
 		return nil, nil
 	}
@@ -120,10 +121,10 @@ func (c *goLicenseResolver) getLicensesFromLocal(moduleName, moduleVersion strin
 	// if we're running against a directory on the filesystem, it may not include the
 	// user's homedir / GOPATH, so we defer to using the localModCacheResolver
 	// we use $GOPATH/pkg/mod to avoid leaking information about the user's system
-	return c.findLicensesInFS("file://$GOPATH/pkg/mod/"+subdir+"/", dir)
+	return c.findLicensesInFS(ctx, scanner, "file://$GOPATH/pkg/mod/"+subdir+"/", dir)
 }

-func (c *goLicenseResolver) getLicensesFromRemote(moduleName, moduleVersion string) ([]goLicense, error) {
+func (c *goLicenseResolver) getLicensesFromRemote(ctx context.Context, scanner licenses.Scanner, moduleName, moduleVersion string) ([]goLicense, error) {
 	return c.licenseCache.Resolve(fmt.Sprintf("%s/%s", moduleName, moduleVersion), func() ([]goLicense, error) {
 		proxies := remotesForModule(c.opts.Proxies, c.opts.NoProxy, moduleName)

@@ -132,11 +133,11 @@ func (c *goLicenseResolver) getLicensesFromRemote(moduleName, moduleVersion stri
 			return nil, err
 		}

-		return c.findLicensesInFS(urlPrefix, fsys)
+		return c.findLicensesInFS(ctx, scanner, urlPrefix, fsys)
 	})
 }

-func (c *goLicenseResolver) findLicensesInFS(urlPrefix string, fsys fs.FS) ([]goLicense, error) {
+func (c *goLicenseResolver) findLicensesInFS(ctx context.Context, scanner licenses.Scanner, urlPrefix string, fsys fs.FS) ([]goLicense, error) {
 	var out []goLicense
 	err := fs.WalkDir(fsys, ".", func(filePath string, d fs.DirEntry, err error) error {
 		if err != nil {
@@ -156,7 +157,8 @@ func (c *goLicenseResolver) findLicensesInFS(urlPrefix string, fsys fs.FS) ([]go
 			return nil
 		}
 		defer internal.CloseAndLogError(rdr, filePath)
-		parsed, err := licenses.Parse(rdr, file.NewLocation(filePath))
+
+		parsed, err := licenses.Search(ctx, scanner, file.NewLocationReadCloser(file.NewLocation(filePath), rdr))
 		if err != nil {
 			log.Debugf("error parsing license file %s: %v", filePath, err)
 			return nil
@@ -174,7 +176,7 @@ func (c *goLicenseResolver) findLicensesInFS(urlPrefix string, fsys fs.FS) ([]go
 	return out, err
 }

-func (c *goLicenseResolver) findLicensesInSource(resolver file.Resolver, globMatch string) ([]goLicense, error) {
+func (c *goLicenseResolver) findLicensesInSource(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, globMatch string) ([]goLicense, error) {
 	var out []goLicense
 	locations, err := resolver.FilesByGlob(globMatch)
 	if err != nil {
@@ -182,7 +184,7 @@ func (c *goLicenseResolver) findLicensesInSource(resolver file.Resolver, globMat
 	}

 	for _, l := range locations {
-		parsed, err := c.parseLicenseFromLocation(l, resolver)
+		parsed, err := c.parseLicenseFromLocation(ctx, scanner, l, resolver)
 		if err != nil {
 			return nil, err
 		}
@@ -200,7 +202,7 @@ func (c *goLicenseResolver) findLicensesInSource(resolver file.Resolver, globMat
 	return out, nil
 }

-func (c *goLicenseResolver) parseLicenseFromLocation(l file.Location, resolver file.Resolver) ([]goLicense, error) {
+func (c *goLicenseResolver) parseLicenseFromLocation(ctx context.Context, scanner licenses.Scanner, l file.Location, resolver file.Resolver) ([]goLicense, error) {
 	var out []goLicense
 	fileName := path.Base(l.RealPath)
 	if c.lowerLicenseFileNames.Has(strings.ToLower(fileName)) {
@@ -209,7 +211,7 @@ func (c *goLicenseResolver) parseLicenseFromLocation(l file.Location, resolver f
 			return nil, err
 		}
 		defer internal.CloseAndLogError(contents, l.RealPath)
-		parsed, err := licenses.Parse(contents, l)
+		parsed, err := licenses.Search(ctx, scanner, file.NewLocationReadCloser(l, contents))
 		if err != nil {
 			return nil, err
 		}
@@ -3,6 +3,7 @@ package golang
 import (
 	"archive/zip"
 	"bytes"
+	"context"
 	"fmt"
 	"io/fs"
 	"net/http"
@@ -15,6 +16,7 @@ import (

 	"github.com/stretchr/testify/require"

+	"github.com/anchore/syft/internal/licenses"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/internal/fileresolver"
 	"github.com/anchore/syft/syft/license"
@@ -26,6 +28,8 @@ func Test_LocalLicenseSearch(t *testing.T) {
 	loc2 := file.NewLocation("github.com/!cap!o!r!g/!cap!project@v4.111.5/LICENSE.txt")
 	loc3 := file.NewLocation("github.com/someorg/strangelicense@v1.2.3/LiCeNsE.tXt")

+	licenseScanner := licenses.TestingOnlyScanner()
+
 	tests := []struct {
 		name     string
 		version  string
@@ -78,12 +82,12 @@ func Test_LocalLicenseSearch(t *testing.T) {
 					LocalModCacheDir:            filepath.Join(wd, "test-fixtures", "licenses", "pkg", "mod"),
 				},
 			)
-			licenses, err := l.getLicenses(fileresolver.Empty{}, test.name, test.version)
+			lics, err := l.getLicenses(context.Background(), licenseScanner, fileresolver.Empty{}, test.name, test.version)
 			require.NoError(t, err)

-			require.Len(t, licenses, 1)
+			require.Len(t, lics, 1)

-			require.Equal(t, test.expected, licenses[0])
+			require.Equal(t, test.expected, lics[0])
 		})
 	}
 }
@@ -92,6 +96,8 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
 	loc1 := file.NewLocation("github.com/someorg/somename@v0.3.2/LICENSE")
 	loc2 := file.NewLocation("github.com/!cap!o!r!g/!cap!project@v4.111.5/LICENSE.txt")

+	licenseScanner := licenses.TestingOnlyScanner()
+
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		buf := &bytes.Buffer{}
 		uri := strings.TrimPrefix(strings.TrimSuffix(r.RequestURI, ".zip"), "/")
@@ -169,12 +175,12 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
 				},
 			)

-			licenses, err := l.getLicenses(fileresolver.Empty{}, test.name, test.version)
+			lics, err := l.getLicenses(context.Background(), licenseScanner, fileresolver.Empty{}, test.name, test.version)
 			require.NoError(t, err)

-			require.Len(t, licenses, 1)
+			require.Len(t, lics, 1)

-			require.Equal(t, test.expected, licenses[0])
+			require.Equal(t, test.expected, lics[0])
 		})
 	}
 }
@@ -248,7 +254,7 @@ func Test_findVersionPath(t *testing.T) {

 func Test_walkDirErrors(t *testing.T) {
 	resolver := newGoLicenseResolver("", CatalogerConfig{})
-	_, err := resolver.findLicensesInFS("somewhere", badFS{})
+	_, err := resolver.findLicensesInFS(context.Background(), licenses.TestingOnlyScanner(), "somewhere", badFS{})
 	require.Error(t, err)
 }

@@ -266,6 +272,8 @@ func Test_noLocalGoModDir(t *testing.T) {
 	validTmp := t.TempDir()
 	require.NoError(t, os.MkdirAll(filepath.Join(validTmp, "mod@ver"), 0700|os.ModeDir))

+	licenseScanner := licenses.TestingOnlyScanner()
+
 	tests := []struct {
 		name    string
 		dir     string
@@ -299,7 +307,7 @@ func Test_noLocalGoModDir(t *testing.T) {
 				SearchLocalModCacheLicenses: true,
 				LocalModCacheDir:            test.dir,
 			})
-			_, err := resolver.getLicensesFromLocal("mod", "ver")
+			_, err := resolver.getLicensesFromLocal(context.Background(), licenseScanner, "mod", "ver")
 			test.wantErr(t, err)
 		})
 	}
@@ -5,21 +5,15 @@ import (
 	"strings"

 	"github.com/anchore/packageurl-go"
-	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 )

-func (c *goBinaryCataloger) newGoBinaryPackage(resolver file.Resolver, dep *debug.Module, mainModule, goVersion, architecture string, buildSettings pkg.KeyValues, cryptoSettings, experiments []string, locations ...file.Location) pkg.Package {
+func (c *goBinaryCataloger) newGoBinaryPackage(dep *debug.Module, mainModule, goVersion, architecture string, buildSettings pkg.KeyValues, cryptoSettings, experiments []string, licenses []pkg.License, locations ...file.Location) pkg.Package {
 	if dep.Replace != nil {
 		dep = dep.Replace
 	}

-	licenses, err := c.licenseResolver.getLicenses(resolver, dep.Path, dep.Version)
-	if err != nil {
-		log.Tracef("error getting licenses for golang package: %s %v", dep.Path, err)
-	}
-
 	p := pkg.Package{
 		Name:      dep.Path,
 		Version:   dep.Version,
@@ -18,6 +18,7 @@ import (
 	"golang.org/x/mod/module"

 	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/licenses"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
@@ -59,9 +60,11 @@ func newGoBinaryCataloger(opts CatalogerConfig) *goBinaryCataloger {
 }

 // parseGoBinary catalogs packages found in the "buildinfo" section of a binary built by the go compiler.
-func (c *goBinaryCataloger) parseGoBinary(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+func (c *goBinaryCataloger) parseGoBinary(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	var pkgs []pkg.Package

+	licenseScanner := licenses.ContextLicenseScanner(ctx)
+
 	unionReader, err := unionreader.GetUnionReader(reader.ReadCloser)
 	if err != nil {
 		return nil, nil, err
@@ -73,7 +76,7 @@ func (c *goBinaryCataloger) parseGoBinary(_ context.Context, resolver file.Resol
 	var rels []artifact.Relationship
 	for _, mod := range mods {
 		var depPkgs []pkg.Package
-		mainPkg, depPkgs := c.buildGoPkgInfo(resolver, reader.Location, mod, mod.arch, unionReader)
+		mainPkg, depPkgs := c.buildGoPkgInfo(ctx, licenseScanner, resolver, reader.Location, mod, mod.arch, unionReader)
 		if mainPkg != nil {
 			rels = createModuleRelationships(*mainPkg, depPkgs)
 			pkgs = append(pkgs, *mainPkg)
@@ -101,7 +104,7 @@ func createModuleRelationships(main pkg.Package, deps []pkg.Package) []artifact.
 var emptyModule debug.Module
 var moduleFromPartialPackageBuild = debug.Module{Path: "command-line-arguments"}

-func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file.Location, mod *extendedBuildInfo, arch string, reader io.ReadSeekCloser) (*pkg.Package, []pkg.Package) {
+func (c *goBinaryCataloger) buildGoPkgInfo(ctx context.Context, licenseScanner licenses.Scanner, resolver file.Resolver, location file.Location, mod *extendedBuildInfo, arch string, reader io.ReadSeekCloser) (*pkg.Package, []pkg.Package) {
 	if mod == nil {
 		return nil, nil
 	}
@@ -116,9 +119,13 @@ func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file
 			continue
 		}

+		lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, dep.Path, dep.Version)
+		if err != nil {
+			log.Tracef("error getting licenses for golang package: %s %v", dep.Path, err)
+		}
+
 		gover, experiments := getExperimentsFromVersion(mod.GoVersion)
 		p := c.newGoBinaryPackage(
-			resolver,
 			dep,
 			mod.Main.Path,
 			gover,
@@ -126,6 +133,7 @@ func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file
 			nil,
 			mod.cryptoSettings,
 			experiments,
+			lics,
 			location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
 		)
 		if pkg.IsValid(&p) {
@@ -137,7 +145,7 @@ func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file
 		return nil, pkgs
 	}

-	main := c.makeGoMainPackage(resolver, mod, arch, location, reader)
+	main := c.makeGoMainPackage(ctx, licenseScanner, resolver, mod, arch, location, reader)

 	return &main, pkgs
 }
@@ -152,11 +160,16 @@ func missingMainModule(mod *extendedBuildInfo) bool {
 	return mod.Main == moduleFromPartialPackageBuild
 }

-func (c *goBinaryCataloger) makeGoMainPackage(resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location, reader io.ReadSeekCloser) pkg.Package {
+func (c *goBinaryCataloger) makeGoMainPackage(ctx context.Context, licenseScanner licenses.Scanner, resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location, reader io.ReadSeekCloser) pkg.Package {
 	gbs := getBuildSettings(mod.Settings)
+
+	lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, mod.Main.Path, mod.Main.Version)
+	if err != nil {
+		log.Tracef("error getting licenses for golang package: %s %v", mod.Main.Path, err)
+	}
+
 	gover, experiments := getExperimentsFromVersion(mod.GoVersion)
 	main := c.newGoBinaryPackage(
-		resolver,
 		&mod.Main,
 		mod.Main.Path,
 		gover,
@@ -164,6 +177,7 @@ func (c *goBinaryCataloger) makeGoMainPackage(resolver file.Resolver, mod *exten
 		gbs,
 		mod.cryptoSettings,
 		experiments,
+		lics,
 		location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
 	)

@@ -3,6 +3,7 @@ package golang
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"errors"
 	"io"
 	"os"
@@ -17,6 +18,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"

+	"github.com/anchore/syft/internal/licenses"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/internal/fileresolver"
 	"github.com/anchore/syft/syft/internal/unionreader"
@@ -167,6 +169,8 @@ func TestBuildGoPkgInfo(t *testing.T) {
 		},
 	}

+	licenseScanner := licenses.TestingOnlyScanner()
+
 	tests := []struct {
 		name          string
 		mod           *extendedBuildInfo
@@ -1053,7 +1057,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 			c := newGoBinaryCataloger(DefaultCatalogerConfig())
 			reader, err := unionreader.GetUnionReader(io.NopCloser(strings.NewReader(test.binaryContent)))
 			require.NoError(t, err)
-			mainPkg, pkgs := c.buildGoPkgInfo(fileresolver.Empty{}, location, test.mod, test.mod.arch, reader)
+			mainPkg, pkgs := c.buildGoPkgInfo(context.Background(), licenseScanner, fileresolver.Empty{}, location, test.mod, test.mod.arch, reader)
 			if mainPkg != nil {
 				pkgs = append(pkgs, *mainPkg)
 			}
@@ -11,6 +11,7 @@ import (
 	"golang.org/x/mod/modfile"

 	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/licenses"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
@@ -31,9 +32,11 @@ func newGoModCataloger(opts CatalogerConfig) *goModCataloger {
 // parseGoModFile takes a go.mod and lists all packages discovered.
 //
 //nolint:funlen
-func (c *goModCataloger) parseGoModFile(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
 	packages := make(map[string]pkg.Package)

+	licenseScanner := licenses.ContextLicenseScanner(ctx)
+
 	contents, err := io.ReadAll(reader)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to read go module: %w", err)
@@ -50,7 +53,7 @@ func (c *goModCataloger) parseGoModFile(_ context.Context, resolver file.Resolve
 	}

 	for _, m := range f.Require {
-		licenses, err := c.licenseResolver.getLicenses(resolver, m.Mod.Path, m.Mod.Version)
+		lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, m.Mod.Path, m.Mod.Version)
 		if err != nil {
 			log.Tracef("error getting licenses for package: %s %v", m.Mod.Path, err)
 		}
@@ -58,7 +61,7 @@ func (c *goModCataloger) parseGoModFile(_ context.Context, resolver file.Resolve
 		packages[m.Mod.Path] = pkg.Package{
 			Name:      m.Mod.Path,
 			Version:   m.Mod.Version,
-			Licenses:  pkg.NewLicenseSet(licenses...),
+			Licenses:  pkg.NewLicenseSet(lics...),
 			Locations: file.NewLocationSet(reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
 			PURL:      packageURL(m.Mod.Path, m.Mod.Version),
 			Language:  pkg.Go,
@@ -71,7 +74,7 @@ func (c *goModCataloger) parseGoModFile(_ context.Context, resolver file.Resolve

 	// remove any old packages and replace with new ones...
 	for _, m := range f.Replace {
-		licenses, err := c.licenseResolver.getLicenses(resolver, m.New.Path, m.New.Version)
+		lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, m.New.Path, m.New.Version)
 		if err != nil {
 			log.Tracef("error getting licenses for package: %s %v", m.New.Path, err)
 		}
@@ -83,7 +86,7 @@ func (c *goModCataloger) parseGoModFile(_ context.Context, resolver file.Resolve
 		packages[m.New.Path] = pkg.Package{
 			Name:      m.New.Path,
 			Version:   m.New.Version,
-			Licenses:  pkg.NewLicenseSet(licenses...),
+			Licenses:  pkg.NewLicenseSet(lics...),
 			Locations: file.NewLocationSet(reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
 			PURL:      packageURL(m.New.Path, m.New.Version),
 			Language:  pkg.Go,