diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index e365b7416..8d96a2f36 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -188,7 +188,8 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config { WithFromContents(cfg.Golang.MainModuleVersion.FromContents). WithFromBuildSettings(cfg.Golang.MainModuleVersion.FromBuildSettings). WithFromLDFlags(cfg.Golang.MainModuleVersion.FromLDFlags), - ), + ). + WithUsePackagesLib(*multiLevelOption(true, enrichmentEnabled(cfg.Enrich, task.Go, task.Golang), cfg.Golang.UsePackagesLib)), JavaScript: javascript.DefaultCatalogerConfig(). WithIncludeDevDependencies(*multiLevelOption(false, cfg.JavaScript.IncludeDevDependencies)). WithSearchRemoteLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.JavaScript, task.Node, task.NPM), cfg.JavaScript.SearchRemoteLicenses)). diff --git a/cmd/syft/internal/options/golang.go b/cmd/syft/internal/options/golang.go index 8eb8e4ba0..10539d79c 100644 --- a/cmd/syft/internal/options/golang.go +++ b/cmd/syft/internal/options/golang.go @@ -16,6 +16,7 @@ type golangConfig struct { Proxy string `json:"proxy" yaml:"proxy" mapstructure:"proxy"` NoProxy string `json:"no-proxy" yaml:"no-proxy" mapstructure:"no-proxy"` MainModuleVersion golangMainModuleVersionConfig `json:"main-module-version" yaml:"main-module-version" mapstructure:"main-module-version"` + UsePackagesLib *bool `json:"use-packages-lib" yaml:"use-packages-lib" mapstructure:"use-packages-lib"` } var _ interface { @@ -37,6 +38,7 @@ if unset this defaults to $GONOPROXY`) descriptions.Add(&o.MainModuleVersion, `the go main module version discovered from binaries built with the go compiler will always show (devel) as the version. Use these options to control heuristics to guess a more accurate version from the binary.`) + descriptions.Add(&o.UsePackagesLib, `use the golang.org/x/tools/go/packages library, which executes golang tooling found on the path in addition to potential network access to get the most accurate results`) descriptions.Add(&o.MainModuleVersion.FromLDFlags, `look for LD flags that appear to be setting a version (e.g. -X main.version=1.0.0)`) descriptions.Add(&o.MainModuleVersion.FromBuildSettings, `use the build settings (e.g. vcs.version & vcs.time) to craft a v0 pseudo version (e.g. v0.0.0-20220308212642-53e6d0aaf6fb) when a more accurate version cannot be found otherwise`) @@ -64,5 +66,6 @@ func defaultGolangConfig() golangConfig { FromContents: def.MainModuleVersion.FromContents, FromBuildSettings: def.MainModuleVersion.FromBuildSettings, }, + UsePackagesLib: nil, // this defaults to true, which is the API default } } diff --git a/syft/pkg/cataloger/golang/config.go b/syft/pkg/cataloger/golang/config.go index d173f4a7c..c90767275 100644 --- a/syft/pkg/cataloger/golang/config.go +++ b/syft/pkg/cataloger/golang/config.go @@ -48,6 +48,9 @@ type CatalogerConfig struct { NoProxy []string `yaml:"no-proxy,omitempty" json:"no-proxy,omitempty" mapstructure:"no-proxy"` MainModuleVersion MainModuleVersionConfig `yaml:"main-module-version" json:"main-module-version" mapstructure:"main-module-version"` + + // Whether to use the golang.org/x/tools/go/packages, which executes golang tooling found on the path in addition to potential network access + UsePackagesLib bool `json:"use-packages-lib" yaml:"use-packages-lib" mapstructure:"use-packages-lib"` } type MainModuleVersionConfig struct { @@ -70,6 +73,7 @@ type MainModuleVersionConfig struct { // - setting the default local module cache dir if none is provided func DefaultCatalogerConfig() CatalogerConfig { g := CatalogerConfig{ + UsePackagesLib: true, MainModuleVersion: DefaultMainModuleVersionConfig(), LocalModCacheDir: defaultGoModDir(), } @@ -180,6 +184,11 @@ func (g CatalogerConfig) WithMainModuleVersion(input MainModuleVersionConfig) Ca return g } +func (g CatalogerConfig) WithUsePackagesLib(useLib bool) CatalogerConfig { + g.UsePackagesLib = useLib + return g +} + func (g MainModuleVersionConfig) WithFromLDFlags(input bool) MainModuleVersionConfig { g.FromLDFlags = input return g diff --git a/syft/pkg/cataloger/golang/config_test.go b/syft/pkg/cataloger/golang/config_test.go index e9e0ef4b0..4b563e656 100644 --- a/syft/pkg/cataloger/golang/config_test.go +++ b/syft/pkg/cataloger/golang/config_test.go @@ -57,6 +57,7 @@ func Test_Config(t *testing.T) { Proxies: []string{"https://my.proxy"}, NoProxy: []string{"my.private", "no.proxy"}, MainModuleVersion: DefaultMainModuleVersionConfig(), + UsePackagesLib: true, }, }, { @@ -84,6 +85,7 @@ func Test_Config(t *testing.T) { Proxies: []string{"https://alt.proxy", "direct"}, NoProxy: []string{"alt.no.proxy"}, MainModuleVersion: DefaultMainModuleVersionConfig(), + UsePackagesLib: true, }, }, } diff --git a/syft/pkg/cataloger/golang/package.go b/syft/pkg/cataloger/golang/package.go index f07867e4c..f689c1a40 100644 --- a/syft/pkg/cataloger/golang/package.go +++ b/syft/pkg/cataloger/golang/package.go @@ -64,27 +64,23 @@ func newBinaryMetadata(dep *debug.Module, mainModule, goVersion, architecture st func packageURL(moduleName, moduleVersion string) string { // source: https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst#golang // note: "The version is often empty when a commit is not specified and should be the commit in most cases when available." - - fields := strings.Split(moduleName, "/") - if len(fields) == 0 { + if moduleName == "" { return "" } namespace := "" - name := "" - // The subpath is used to point to a subpath inside a package (e.g. pkg:golang/google.golang.org/genproto#googleapis/api/annotations) - subpath := "" + name := moduleName - switch len(fields) { - case 1: - name = fields[0] - case 2: - name = fields[1] - namespace = fields[0] - default: - name = fields[2] - namespace = strings.Join(fields[0:2], "/") - subpath = strings.Join(fields[3:], "/") + // golang PURLs from _modules_ are constructed as pkg:golang/@version, where + // the full module name often includes multiple segments including `/v#`-type versions, for example: + // pkg:golang/github.com/cli/cli/v2@2.63.0 + // see: https://github.com/package-url/purl-spec/issues/63 + // and: https://github.com/package-url/purl-spec/blob/main/types-doc/golang-definition.md#subpath-definition + // by setting the namespace this way, it does not escape the slashes: + lastSlash := strings.LastIndex(moduleName, "/") + if lastSlash > 0 && lastSlash < len(moduleName)-1 { + name = moduleName[lastSlash+1:] + namespace = moduleName[0:lastSlash] } return packageurl.NewPackageURL( @@ -93,6 +89,6 @@ func packageURL(moduleName, moduleVersion string) string { name, moduleVersion, nil, - subpath, + "", // subpath is used to reference a specific _package_ within the module ).ToString() } diff --git a/syft/pkg/cataloger/golang/package_test.go b/syft/pkg/cataloger/golang/package_test.go index 199a5d07e..114b64d40 100644 --- a/syft/pkg/cataloger/golang/package_test.go +++ b/syft/pkg/cataloger/golang/package_test.go @@ -38,14 +38,14 @@ func Test_packageURL(t *testing.T) { Name: "github.com/coreos/go-systemd/v22", Version: "v22.1.0", }, - expected: "pkg:golang/github.com/coreos/go-systemd@v22.1.0#v22", + expected: "pkg:golang/github.com/coreos/go-systemd/v22@v22.1.0", }, { name: "golang with subpath deep", pkg: pkg.Package{ Name: "google.golang.org/genproto/googleapis/api/annotations", }, - expected: "pkg:golang/google.golang.org/genproto/googleapis#api/annotations", + expected: "pkg:golang/google.golang.org/genproto/googleapis/api/annotations", }, } diff --git a/syft/pkg/cataloger/golang/parse_go_binary_test.go b/syft/pkg/cataloger/golang/parse_go_binary_test.go index ca124208c..d1b201c72 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary_test.go +++ b/syft/pkg/cataloger/golang/parse_go_binary_test.go @@ -278,7 +278,7 @@ func TestBuildGoPkgInfo(t *testing.T) { { Name: "github.com/a/b/c", Version: "", // this was (devel) but we cleared it explicitly - PURL: "pkg:golang/github.com/a/b#c", + PURL: "pkg:golang/github.com/a/b/c", Language: pkg.Go, Type: pkg.GoModulePkg, Locations: file.NewLocationSet( diff --git a/syft/pkg/cataloger/golang/parse_go_mod.go b/syft/pkg/cataloger/golang/parse_go_mod.go index 23d3721fa..bb495cc40 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod.go +++ b/syft/pkg/cataloger/golang/parse_go_mod.go @@ -8,7 +8,6 @@ import ( "io" "path/filepath" "slices" - "sort" "strings" "github.com/spf13/afero" @@ -26,17 +25,19 @@ import ( ) type goModCataloger struct { + usePackagesLib bool licenseResolver goLicenseResolver } func newGoModCataloger(opts CatalogerConfig) *goModCataloger { return &goModCataloger{ + usePackagesLib: opts.UsePackagesLib, licenseResolver: newGoLicenseResolver(modFileCatalogerName, opts), } } // parseGoModFile takes a go.mod and tries to resolve and lists all packages discovered. -func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { +func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) (pkgs []pkg.Package, relationships []artifact.Relationship, err error) { modDir := filepath.Dir(string(reader.Location.Reference().RealPath)) digests, err := parseGoSumFile(resolver, reader) if err != nil { @@ -48,24 +49,34 @@ func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resol scanRoot = dir.Chroot.Base() } - // source analysis using go toolchain if available - syftSourcePackages, sourceModules, sourceDependencies, unknownErr := c.loadPackages(modDir, reader.Location) - catalogedModules, sourceModuleToPkg := c.catalogModules(ctx, scanRoot, syftSourcePackages, sourceModules, reader, digests) - relationships := buildModuleRelationships(catalogedModules, sourceDependencies, sourceModuleToPkg) - // base case go.mod file parsing modFile, err := c.parseModFileContents(reader) if err != nil { return nil, nil, err } - // only use mod packages NOT found in source analysis + // source analysis using go toolchain if available + var sourceModules map[string]*packages.Module + var catalogedModules []pkg.Package + + if c.usePackagesLib { + var sourcePackages map[string][]pkgInfo + var sourceDependencies map[string][]string + var sourceModuleToPkg map[string]artifact.Identifiable + + sourcePackages, sourceModules, sourceDependencies, err = c.loadPackages(modDir, reader.Location) + catalogedModules, sourceModuleToPkg = c.catalogModules(ctx, scanRoot, sourcePackages, sourceModules, reader, digests) + relationships = buildModuleRelationships(catalogedModules, sourceDependencies, sourceModuleToPkg) + } + + // only use go.mod packages NOT found in source analysis goModPackages := c.createGoModPackages(ctx, resolver, modFile, sourceModules, reader, digests) c.applyReplaceDirectives(ctx, resolver, modFile, goModPackages, reader, digests) c.applyExcludeDirectives(modFile, goModPackages) - finalPkgs := c.assembleResults(catalogedModules, goModPackages) - return finalPkgs, relationships, unknownErr + pkgs = c.assembleResults(catalogedModules, goModPackages) + + return pkgs, relationships, err } // loadPackages uses golang.org/x/tools/go/packages to get dependency information. @@ -327,7 +338,7 @@ func (c *goModCataloger) createGoModPackages(ctx context.Context, resolver file. goModPackages := make(map[string]pkg.Package) for _, m := range modFile.Require { - if _, exists := sourceModules[m.Mod.Path]; !exists { + if sourceModules == nil || sourceModules[m.Mod.Path] == nil { lics := c.licenseResolver.getLicenses(ctx, resolver, m.Mod.Path, m.Mod.Version) goModPkg := pkg.Package{ Name: m.Mod.Path, @@ -392,9 +403,7 @@ func (c *goModCataloger) assembleResults(catalogedPkgs []pkg.Package, goModPacka pkgsSlice = append(pkgsSlice, p) } - sort.SliceStable(pkgsSlice, func(i, j int) bool { - return pkgsSlice[i].Name < pkgsSlice[j].Name - }) + pkg.Sort(pkgsSlice) return pkgsSlice } diff --git a/syft/pkg/cataloger/golang/parse_go_mod_test.go b/syft/pkg/cataloger/golang/parse_go_mod_test.go index 77232133f..45789e441 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod_test.go +++ b/syft/pkg/cataloger/golang/parse_go_mod_test.go @@ -54,7 +54,7 @@ func TestParseGoMod(t *testing.T) { { Name: "github.com/anchore/archiver/v3", Version: "v3.5.2", - PURL: "pkg:golang/github.com/anchore/archiver@v3.5.2#v3", + PURL: "pkg:golang/github.com/anchore/archiver/v3@v3.5.2", Locations: file.NewLocationSet(file.NewLocation("test-fixtures/go-mod-fixtures/many-packages/go.mod")), Language: pkg.Go, Type: pkg.GoModulePkg, @@ -111,7 +111,7 @@ func TestParseGoMod(t *testing.T) { for _, test := range tests { t.Run(test.fixture, func(t *testing.T) { - c := newGoModCataloger(DefaultCatalogerConfig()) + c := newGoModCataloger(DefaultCatalogerConfig().WithUsePackagesLib(false)) pkgtest.NewCatalogTester(). FromFile(t, test.fixture). Expects(test.expected, nil). @@ -172,7 +172,9 @@ func Test_GoSumHashes(t *testing.T) { pkgtest.NewCatalogTester(). FromDirectory(t, test.fixture). Expects(test.expected, nil). - TestCataloger(t, NewGoModuleFileCataloger(CatalogerConfig{})) + TestCataloger(t, NewGoModuleFileCataloger(CatalogerConfig{ + UsePackagesLib: false, + })) }) } } @@ -189,7 +191,6 @@ func Test_parseGoSource_packageResolution(t *testing.T) { tests := []struct { name string fixturePath string - config CatalogerConfig expectedPkgs []string expectedRels []string expectedLicenses map[string][]string @@ -333,7 +334,7 @@ func Test_parseGoSource_packageResolution(t *testing.T) { t.Errorf("mismatch in licenses (-want +got):\n%s", diff) } }). - TestCataloger(t, NewGoModuleFileCataloger(CatalogerConfig{})) + TestCataloger(t, NewGoModuleFileCataloger(DefaultCatalogerConfig().WithUsePackagesLib(true))) }) } }