From a2a56dd3e95e11ae74c58c668bd2d0099c5982e1 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Mon, 6 Jan 2025 11:47:55 -0500 Subject: [PATCH] fix: golang remote license search not executing when error reading local mod dir (#3549) Signed-off-by: Keith Zantow --- syft/pkg/cataloger/golang/licenses.go | 21 ++- syft/pkg/cataloger/golang/licenses_test.go | 174 +++++++++---------- syft/pkg/cataloger/golang/parse_go_binary.go | 13 +- syft/pkg/cataloger/golang/parse_go_mod.go | 11 +- 4 files changed, 103 insertions(+), 116 deletions(-) diff --git a/syft/pkg/cataloger/golang/licenses.go b/syft/pkg/cataloger/golang/licenses.go index 402a01448..da5ef69e0 100644 --- a/syft/pkg/cataloger/golang/licenses.go +++ b/syft/pkg/cataloger/golang/licenses.go @@ -80,29 +80,38 @@ func remotesForModule(proxies []string, noProxy []string, module string) []strin return proxies } -func (c *goLicenseResolver) getLicenses(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, moduleName, moduleVersion string) ([]pkg.License, error) { +func (c *goLicenseResolver) getLicenses(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, moduleName, moduleVersion string) []pkg.License { // search the scan target first, ignoring local and remote sources goLicenses, err := c.findLicensesInSource(ctx, scanner, resolver, fmt.Sprintf(`**/go/pkg/mod/%s@%s/*`, processCaps(moduleName), moduleVersion), ) - if err != nil || len(goLicenses) > 0 { - return toPkgLicenses(goLicenses), err + if err != nil { + log.WithFields("error", err, "module", moduleName, "version", moduleVersion).Trace("unable to read golang licenses from source") + } + if len(goLicenses) > 0 { + return toPkgLicenses(goLicenses) } // look in the local host mod directory... if c.opts.SearchLocalModCacheLicenses { goLicenses, err = c.getLicensesFromLocal(ctx, scanner, moduleName, moduleVersion) - if err != nil || len(goLicenses) > 0 { - return toPkgLicenses(goLicenses), err + if err != nil { + log.WithFields("error", err, "module", moduleName, "version", moduleVersion).Trace("unable to read golang licenses local") + } + if len(goLicenses) > 0 { + return toPkgLicenses(goLicenses) } } // download from remote sources if c.opts.SearchRemoteLicenses { goLicenses, err = c.getLicensesFromRemote(ctx, scanner, moduleName, moduleVersion) + if err != nil { + log.WithFields("error", err, "module", moduleName, "version", moduleVersion).Debug("unable to read golang licenses remote") + } } - return toPkgLicenses(goLicenses), err + return toPkgLicenses(goLicenses) } func (c *goLicenseResolver) getLicensesFromLocal(ctx context.Context, scanner licenses.Scanner, moduleName, moduleVersion string) ([]goLicense, error) { diff --git a/syft/pkg/cataloger/golang/licenses_test.go b/syft/pkg/cataloger/golang/licenses_test.go index 0acde6de9..36e98c86f 100644 --- a/syft/pkg/cataloger/golang/licenses_test.go +++ b/syft/pkg/cataloger/golang/licenses_test.go @@ -23,81 +23,11 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func Test_LocalLicenseSearch(t *testing.T) { +func Test_LicenseSearch(t *testing.T) { loc1 := file.NewLocation("github.com/someorg/somename@v0.3.2/LICENSE") loc2 := file.NewLocation("github.com/!cap!o!r!g/!cap!project@v4.111.5/LICENSE.txt") loc3 := file.NewLocation("github.com/someorg/strangelicense@v1.2.3/LiCeNsE.tXt") - licenseScanner := licenses.TestingOnlyScanner() - - tests := []struct { - name string - version string - expected pkg.License - }{ - { - name: "github.com/someorg/somename", - version: "v0.3.2", - expected: pkg.License{ - Value: "Apache-2.0", - SPDXExpression: "Apache-2.0", - Type: license.Concluded, - URLs: []string{"file://$GOPATH/pkg/mod/" + loc1.RealPath}, - Locations: file.NewLocationSet(), - }, - }, - { - name: "github.com/CapORG/CapProject", - version: "v4.111.5", - expected: pkg.License{ - Value: "MIT", - SPDXExpression: "MIT", - Type: license.Concluded, - URLs: []string{"file://$GOPATH/pkg/mod/" + loc2.RealPath}, - Locations: file.NewLocationSet(), - }, - }, - { - name: "github.com/someorg/strangelicense", - version: "v1.2.3", - expected: pkg.License{ - Value: "Apache-2.0", - SPDXExpression: "Apache-2.0", - Type: license.Concluded, - URLs: []string{"file://$GOPATH/pkg/mod/" + loc3.RealPath}, - Locations: file.NewLocationSet(), - }, - }, - } - - wd, err := os.Getwd() - require.NoError(t, err) - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - l := newGoLicenseResolver( - "", - CatalogerConfig{ - SearchLocalModCacheLicenses: true, - LocalModCacheDir: filepath.Join(wd, "test-fixtures", "licenses", "pkg", "mod"), - }, - ) - lics, err := l.getLicenses(context.Background(), licenseScanner, fileresolver.Empty{}, test.name, test.version) - require.NoError(t, err) - - require.Len(t, lics, 1) - - require.Equal(t, test.expected, lics[0]) - }) - } -} - -func Test_RemoteProxyLicenseSearch(t *testing.T) { - loc1 := file.NewLocation("github.com/someorg/somename@v0.3.2/LICENSE") - loc2 := file.NewLocation("github.com/!cap!o!r!g/!cap!project@v4.111.5/LICENSE.txt") - - licenseScanner := licenses.TestingOnlyScanner() - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { buf := &bytes.Buffer{} uri := strings.TrimPrefix(strings.TrimSuffix(r.RequestURI, ".zip"), "/") @@ -135,52 +65,116 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) { })) defer server.Close() + wd, err := os.Getwd() + require.NoError(t, err) + + licenseScanner := licenses.TestingOnlyScanner() + tests := []struct { name string version string - expected pkg.License + config CatalogerConfig + expected []pkg.License }{ { name: "github.com/someorg/somename", version: "v0.3.2", - expected: pkg.License{ + config: CatalogerConfig{ + SearchLocalModCacheLicenses: true, + LocalModCacheDir: filepath.Join(wd, "test-fixtures", "licenses", "pkg", "mod"), + }, + expected: []pkg.License{{ + Value: "Apache-2.0", + SPDXExpression: "Apache-2.0", + Type: license.Concluded, + URLs: []string{"file://$GOPATH/pkg/mod/" + loc1.RealPath}, + Locations: file.NewLocationSet(), + }}, + }, + { + name: "github.com/CapORG/CapProject", + version: "v4.111.5", + config: CatalogerConfig{ + SearchLocalModCacheLicenses: true, + LocalModCacheDir: filepath.Join(wd, "test-fixtures", "licenses", "pkg", "mod"), + }, + expected: []pkg.License{{ + Value: "MIT", + SPDXExpression: "MIT", + Type: license.Concluded, + URLs: []string{"file://$GOPATH/pkg/mod/" + loc2.RealPath}, + Locations: file.NewLocationSet(), + }}, + }, + { + name: "github.com/someorg/strangelicense", + version: "v1.2.3", + config: CatalogerConfig{ + SearchLocalModCacheLicenses: true, + LocalModCacheDir: filepath.Join(wd, "test-fixtures", "licenses", "pkg", "mod"), + }, + expected: []pkg.License{{ + Value: "Apache-2.0", + SPDXExpression: "Apache-2.0", + Type: license.Concluded, + URLs: []string{"file://$GOPATH/pkg/mod/" + loc3.RealPath}, + Locations: file.NewLocationSet(), + }}, + }, + { + name: "github.com/someorg/somename", + version: "v0.3.2", + config: CatalogerConfig{ + SearchRemoteLicenses: true, + Proxies: []string{server.URL}, + }, + expected: []pkg.License{{ Value: "Apache-2.0", SPDXExpression: "Apache-2.0", Type: license.Concluded, URLs: []string{server.URL + "/github.com/someorg/somename/@v/v0.3.2.zip#" + loc1.RealPath}, Locations: file.NewLocationSet(), - }, + }}, }, { name: "github.com/CapORG/CapProject", version: "v4.111.5", - expected: pkg.License{ + config: CatalogerConfig{ + SearchRemoteLicenses: true, + Proxies: []string{server.URL}, + }, + expected: []pkg.License{{ Value: "MIT", SPDXExpression: "MIT", Type: license.Concluded, URLs: []string{server.URL + "/github.com/CapORG/CapProject/@v/v4.111.5.zip#" + loc2.RealPath}, Locations: file.NewLocationSet(), + }}, + }, + { + name: "github.com/CapORG/CapProject", + version: "v4.111.5", + config: CatalogerConfig{ + SearchLocalModCacheLicenses: true, + LocalModCacheDir: filepath.Join(wd, "test-fixtures"), // valid dir but does not find modules + SearchRemoteLicenses: true, + Proxies: []string{server.URL}, }, + expected: []pkg.License{{ + Value: "MIT", + SPDXExpression: "MIT", + Type: license.Concluded, + URLs: []string{server.URL + "/github.com/CapORG/CapProject/@v/v4.111.5.zip#" + loc2.RealPath}, + Locations: file.NewLocationSet(), + }}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - - l := newGoLicenseResolver( - "", - CatalogerConfig{ - SearchRemoteLicenses: true, - Proxies: []string{server.URL}, - }, - ) - - lics, err := l.getLicenses(context.Background(), licenseScanner, fileresolver.Empty{}, test.name, test.version) - require.NoError(t, err) - - require.Len(t, lics, 1) - - require.Equal(t, test.expected, lics[0]) + l := newGoLicenseResolver("", test.config) + lics := l.getLicenses(context.Background(), licenseScanner, fileresolver.Empty{}, test.name, test.version) + require.EqualValues(t, test.expected, lics) }) } } diff --git a/syft/pkg/cataloger/golang/parse_go_binary.go b/syft/pkg/cataloger/golang/parse_go_binary.go index 187649565..1671886b6 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary.go +++ b/syft/pkg/cataloger/golang/parse_go_binary.go @@ -119,11 +119,7 @@ func (c *goBinaryCataloger) buildGoPkgInfo(ctx context.Context, licenseScanner l continue } - lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, dep.Path, dep.Version) - if err != nil { - log.Tracef("error getting licenses for golang package: %s %v", dep.Path, err) - } - + lics := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, dep.Path, dep.Version) gover, experiments := getExperimentsFromVersion(mod.GoVersion) p := c.newGoBinaryPackage( dep, @@ -162,12 +158,7 @@ func missingMainModule(mod *extendedBuildInfo) bool { func (c *goBinaryCataloger) makeGoMainPackage(ctx context.Context, licenseScanner licenses.Scanner, resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location, reader io.ReadSeekCloser) pkg.Package { gbs := getBuildSettings(mod.Settings) - - lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, mod.Main.Path, mod.Main.Version) - if err != nil { - log.Tracef("error getting licenses for golang package: %s %v", mod.Main.Path, err) - } - + lics := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, mod.Main.Path, mod.Main.Version) gover, experiments := getExperimentsFromVersion(mod.GoVersion) main := c.newGoBinaryPackage( &mod.Main, diff --git a/syft/pkg/cataloger/golang/parse_go_mod.go b/syft/pkg/cataloger/golang/parse_go_mod.go index cfb373e90..4a3f11236 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod.go +++ b/syft/pkg/cataloger/golang/parse_go_mod.go @@ -53,11 +53,7 @@ func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resol } for _, m := range f.Require { - lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, m.Mod.Path, m.Mod.Version) - if err != nil { - log.Tracef("error getting licenses for package: %s %v", m.Mod.Path, err) - } - + lics := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, m.Mod.Path, m.Mod.Version) packages[m.Mod.Path] = pkg.Package{ Name: m.Mod.Path, Version: m.Mod.Version, @@ -74,10 +70,7 @@ func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resol // remove any old packages and replace with new ones... for _, m := range f.Replace { - lics, err := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, m.New.Path, m.New.Version) - if err != nil { - log.Tracef("error getting licenses for package: %s %v", m.New.Path, err) - } + lics := c.licenseResolver.getLicenses(ctx, licenseScanner, resolver, m.New.Path, m.New.Version) // the old path and new path may be the same, in which case this is a noop, // but if they're different we need to remove the old package.