perf: group golang symbols by package path to reduce the sbom size (#5064)

---------
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>

syft/pkg/cataloger/golang/package.go

@@ -45,7 +45,7 @@ func (c *goBinaryCataloger) newGoBinaryPackage(dep *debug.Module, m pkg.GolangBi
 	return p
 }
 
-func newBinaryMetadata(dep *debug.Module, mainModule, goVersion, architecture string, buildSettings pkg.KeyValues, cryptoSettings, experiments, symbols []string) pkg.GolangBinaryBuildinfoEntry {
+func newBinaryMetadata(dep *debug.Module, mainModule, goVersion, architecture string, buildSettings pkg.KeyValues, cryptoSettings, experiments []string, symbols map[string][]string) pkg.GolangBinaryBuildinfoEntry {
 	if dep.Replace != nil {
 		dep = dep.Replace
 	}

syft/pkg/cataloger/golang/parse_go_binary.go

@@ -54,9 +54,10 @@ type goBinaryCataloger struct {
 	symbolScope       cataloging.SymbolScope
 
 	// stdlibSymbols holds the standard-library function symbols discovered per binary (keyed by the
-	// binary's location), populated during parsing and consumed by stdlibProcessor when it builds the
-	// synthetic "stdlib" package. Guarded by stdlibSymbolsMu because parsers run concurrently.
-	stdlibSymbols   map[file.Coordinates][]string
+	// binary's location), grouped by import path, populated during parsing and consumed by stdlibProcessor
+	// when it builds the synthetic "stdlib" package. Guarded by stdlibSymbolsMu because parsers run
+	// concurrently.
+	stdlibSymbols   map[file.Coordinates]map[string][]string
 	stdlibSymbolsMu sync.Mutex
 }
 
@@ -65,29 +66,49 @@ func newGoBinaryCataloger(opts CatalogerConfig) *goBinaryCataloger {
 		licenseResolver:   newGoLicenseResolver(binaryCatalogerName, opts),
 		mainModuleVersion: opts.MainModuleVersion,
 		symbolScope:       opts.CaptureSymbols,
-		stdlibSymbols:     make(map[file.Coordinates][]string),
+		stdlibSymbols:     make(map[file.Coordinates]map[string][]string),
 	}
 }
 
-// recordStdlibSymbols merges the standard-library symbols discovered for a binary location so the
-// stdlib processor can attach them to the synthetic stdlib package.
-func (c *goBinaryCataloger) recordStdlibSymbols(coord file.Coordinates, symbols []string) {
+// recordStdlibSymbols merges the standard-library symbols discovered for a binary location (grouped by
+// import path) so the stdlib processor can attach them to the synthetic stdlib package.
+func (c *goBinaryCataloger) recordStdlibSymbols(coord file.Coordinates, symbols map[string][]string) {
 	if len(symbols) == 0 {
 		return
 	}
 	c.stdlibSymbolsMu.Lock()
 	defer c.stdlibSymbolsMu.Unlock()
-	merged := slices.Concat(c.stdlibSymbols[coord], symbols)
-	slices.Sort(merged)
-	c.stdlibSymbols[coord] = slices.Compact(merged)
+	existing := c.stdlibSymbols[coord]
+	if existing == nil {
+		existing = make(map[string][]string)
+		c.stdlibSymbols[coord] = existing
+	}
+	for path, names := range symbols {
+		merged := slices.Concat(existing[path], names)
+		slices.Sort(merged)
+		existing[path] = slices.Compact(merged)
+	}
 }
 
-// stdlibSymbolsFor returns the standard-library symbols recorded for a binary location. It returns a copy
-// so callers cannot alias (and later mutate or race on) the map's internal slice.
-func (c *goBinaryCataloger) stdlibSymbolsFor(coord file.Coordinates) []string {
+// stdlibSymbolsFor returns the standard-library symbols recorded for a binary location. It returns a deep
+// copy so callers cannot alias (and later mutate or race on) the map's internal state.
+func (c *goBinaryCataloger) stdlibSymbolsFor(coord file.Coordinates) map[string][]string {
 	c.stdlibSymbolsMu.Lock()
 	defer c.stdlibSymbolsMu.Unlock()
-	return slices.Clone(c.stdlibSymbols[coord])
+	return cloneSymbolGroups(c.stdlibSymbols[coord])
+}
+
+// cloneSymbolGroups returns a deep copy of a symbol group map (import path -> local symbol names), or nil
+// when the input is empty.
+func cloneSymbolGroups(groups map[string][]string) map[string][]string {
+	if len(groups) == 0 {
+		return nil
+	}
+	out := make(map[string][]string, len(groups))
+	for path, names := range groups {
+		out[path] = slices.Clone(names)
+	}
+	return out
 }
 
 // parseGoBinary catalogs packages found in the "buildinfo" section of a binary built by the go compiler.
@@ -221,7 +242,7 @@ func missingMainModule(mod *extendedBuildInfo) bool {
 	return mod.Main == moduleFromPartialPackageBuild
 }
 
-func (c *goBinaryCataloger) makeGoMainPackage(ctx context.Context, resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location, reader io.ReadSeekCloser, symbols []string) pkg.Package {
+func (c *goBinaryCataloger) makeGoMainPackage(ctx context.Context, resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location, reader io.ReadSeekCloser, symbols map[string][]string) pkg.Package {
 	gbs := getBuildSettings(mod.Settings)
 	lics := c.licenseResolver.getLicenses(ctx, resolver, mod.Main.Path, mod.Main.Version)
 	gover, experiments := getExperimentsFromVersion(mod.GoVersion)

syft/pkg/cataloger/golang/parse_go_binary_test.go

@@ -1427,12 +1427,14 @@ func (alwaysErrorReader) Read(_ []byte) (int, error) {
 func Test_buildGoPkgInfo_symbolScope(t *testing.T) {
 	location := file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/a-path", FileSystemID: "layer-id"})
 
-	// the symbols a binary would carry once scanFile has extracted them: one main-package symbol, one
-	// dependency symbol, and one standard-library symbol. For the "none" scope scanFile never runs, so the
-	// build info carries no symbols at all.
+	// the symbols a binary would carry once scanFile has extracted them: one main-package symbol, two
+	// dependency symbols from distinct packages within the same module (so the per-package keying of the
+	// emitted map is visible), and one standard-library symbol. For the "none" scope scanFile never runs,
+	// so the build info carries no symbols at all.
 	populatedSymbols := []binarySymbol{
 		{packagePath: "main", name: "main.main"},
 		{packagePath: "github.com/foo/bar", name: "github.com/foo/bar.Parse"},
+		{packagePath: "github.com/foo/bar/baz", name: "github.com/foo/bar/baz.Helper"},
 		{packagePath: "net/http", name: "net/http.(*Client).Do"},
 	}
 
@@ -1440,9 +1442,9 @@ func Test_buildGoPkgInfo_symbolScope(t *testing.T) {
 		name           string
 		scope          cataloging.SymbolScope
 		symbols        []binarySymbol
-		wantMainSyms   []string
-		wantDepSyms    []string
-		wantStdlibSyms []string
+		wantMainSyms   map[string][]string
+		wantDepSyms    map[string][]string
+		wantStdlibSyms map[string][]string
 	}{
 		{
 			name:    "none captures nothing",
@@ -1453,15 +1455,18 @@ func Test_buildGoPkgInfo_symbolScope(t *testing.T) {
 			name:           "stdlib captures only the stdlib package",
 			scope:          cataloging.SymbolScopeStdlib,
 			symbols:        populatedSymbols,
-			wantStdlibSyms: []string{"net/http.(*Client).Do"},
+			wantStdlibSyms: map[string][]string{"net/http": {"(*Client).Do"}},
 		},
 		{
-			name:           "all captures module and stdlib packages",
-			scope:          cataloging.SymbolScopeAll,
-			symbols:        populatedSymbols,
-			wantMainSyms:   []string{"main.main"},
-			wantDepSyms:    []string{"github.com/foo/bar.Parse"},
-			wantStdlibSyms: []string{"net/http.(*Client).Do"},
+			name:         "all captures module and stdlib packages",
+			scope:        cataloging.SymbolScopeAll,
+			symbols:      populatedSymbols,
+			wantMainSyms: map[string][]string{"main": {"main"}},
+			wantDepSyms: map[string][]string{
+				"github.com/foo/bar":     {"Parse"},
+				"github.com/foo/bar/baz": {"Helper"},
+			},
+			wantStdlibSyms: map[string][]string{"net/http": {"(*Client).Do"}},
 		},
 	}
 
@@ -1491,3 +1496,41 @@ func Test_buildGoPkgInfo_symbolScope(t *testing.T) {
 		})
 	}
 }
+
+// Test_recordStdlibSymbols_merge covers the merge path where the same binary location records stdlib
+// symbols more than once. This happens in production for universal/fat Mach-O binaries: scanFile yields
+// one build info per architecture and each is recorded under the same location coordinates.
+func Test_recordStdlibSymbols_merge(t *testing.T) {
+	coord := file.Coordinates{RealPath: "/a-path", FileSystemID: "layer-id"}
+	c := newGoBinaryCataloger(CatalogerConfig{})
+
+	c.recordStdlibSymbols(coord, map[string][]string{
+		"net/http": {"(*Client).Do", "Get"},
+		"net/url":  {"Parse"},
+	})
+	// a second architecture: overlapping names (must dedup), a new name for an existing package, and a new package
+	c.recordStdlibSymbols(coord, map[string][]string{
+		"net/http": {"Get", "Post"},
+		"os":       {"Open"},
+	})
+
+	assert.Equal(t, map[string][]string{
+		"net/http": {"(*Client).Do", "Get", "Post"},
+		"net/url":  {"Parse"},
+		"os":       {"Open"},
+	}, c.stdlibSymbolsFor(coord))
+}
+
+// Test_stdlibSymbolsFor_isolation asserts the returned map is a deep copy: mutating it (or its slices)
+// must not corrupt the cataloger's guarded internal state.
+func Test_stdlibSymbolsFor_isolation(t *testing.T) {
+	coord := file.Coordinates{RealPath: "/a-path", FileSystemID: "layer-id"}
+	c := newGoBinaryCataloger(CatalogerConfig{})
+	c.recordStdlibSymbols(coord, map[string][]string{"net/http": {"Get"}})
+
+	got := c.stdlibSymbolsFor(coord)
+	got["net/http"] = append(got["net/http"], "MUTATED")
+	got["net/url"] = []string{"Parse"}
+
+	assert.Equal(t, map[string][]string{"net/http": {"Get"}}, c.stdlibSymbolsFor(coord), "internal state must be unaffected by caller mutation")
+}

syft/pkg/cataloger/golang/stdlib_package.go

@@ -50,7 +50,7 @@ func (c *goBinaryCataloger) stdlibPackageAndRelationships(ctx context.Context, p
 	return goCompilerPkgs, relationships
 }
 
-func newGoStdLib(ctx context.Context, version string, location file.LocationSet, symbols []string) *pkg.Package {
+func newGoStdLib(ctx context.Context, version string, location file.LocationSet, symbols map[string][]string) *pkg.Package {
 	stdlibCpe, err := generateStdlibCpe(version)
 	if err != nil {
 		return nil

syft/pkg/cataloger/golang/stdlib_package_test.go

@@ -88,7 +88,7 @@ func Test_stdlibPackageAndRelationships(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			c := &goBinaryCataloger{stdlibSymbols: make(map[file.Coordinates][]string)}
+                        c := &goBinaryCataloger{stdlibSymbols: make(map[file.Coordinates]map[string][]string)}
 			gotPkgs, gotRels := c.stdlibPackageAndRelationships(ctx, tt.pkgs)
 			assert.Len(t, gotPkgs, tt.wantPkgs)
 			assert.Len(t, gotRels, tt.wantRels)
@@ -138,7 +138,7 @@ func Test_stdlibPackageAndRelationships_values(t *testing.T) {
 		Type: artifact.DependencyOfRelationship,
 	}
 
-	c := &goBinaryCataloger{stdlibSymbols: make(map[file.Coordinates][]string)}
+        c := &goBinaryCataloger{stdlibSymbols: make(map[file.Coordinates]map[string][]string)}
 	gotPkgs, gotRels := c.stdlibPackageAndRelationships(ctx, []pkg.Package{p})
 	require.Len(t, gotPkgs, 1)
 

syft/pkg/cataloger/golang/symbols.go

@@ -267,12 +267,15 @@ func readPclntab(r io.ReaderAt) (pclntab []byte, textStart uint64, err error) {
 }
 
 // moduleSymbols attributes each extracted symbol to the module that owns it (by longest module path prefix
-// of the symbol's package path) and returns a sorted, deduplicated list of symbol names per module path.
-// Symbols from the "main" package are attributed to the main module. Standard-library symbols (which belong
-// to no module) are collected separately and returned as the second value so they can be attached to the
+// of the symbol's package path) and returns, per module path, the symbols grouped by the import path of the
+// owning package. Each inner value is a sorted, deduplicated list of symbol names local to that package
+// (the import path prefix stripped, e.g. "github.com/foo/bar.(*T).M" under key "github.com/foo/bar" becomes
+// "(*T).M"). Symbols from the "main" package are attributed to the main module and keyed by the "main"
+// import path the linker assigns. Standard-library symbols (which belong to no module) are collected
+// separately and returned as the second value, grouped by import path, so they can be attached to the
 // synthetic "stdlib" package. Compiler/runtime-internal symbols that are neither module-owned nor a
 // recognizable stdlib import path are dropped.
-func moduleSymbols(symbols []binarySymbol, main *debug.Module, deps []*debug.Module) (byModule map[string][]string, stdlib []string) {
+func moduleSymbols(symbols []binarySymbol, main *debug.Module, deps []*debug.Module) (byModule map[string]map[string][]string, stdlib map[string][]string) {
 	if len(symbols) == 0 {
 		return nil, nil
 	}
@@ -287,41 +290,67 @@ func moduleSymbols(symbols []binarySymbol, main *debug.Module, deps []*debug.Mod
 		}
 	}
 
-	results := make(map[string][]string)
+	results := make(map[string]map[string][]string)
+	stdlib = make(map[string][]string)
 	for _, sym := range symbols {
-		pkgPath := sym.packagePath
-		if pkgPath == mainPackage && main != nil {
-			// the linker renames the main package's import path to "main"
-			pkgPath = main.Path
+		importPath := sym.packagePath
+
+		// the linker renames the main package's import path to "main"; attribute it to the main module,
+		// but keep "main" as the group key since the original import path is not recoverable.
+		attrPath := importPath
+		if importPath == mainPackage && main != nil {
+			attrPath = main.Path
 		}
 
 		var best string
 		for _, modPath := range modulePaths {
-			if len(modPath) > len(best) && (pkgPath == modPath || strings.HasPrefix(pkgPath, modPath+"/")) {
+			if len(modPath) > len(best) && (attrPath == modPath || strings.HasPrefix(attrPath, modPath+"/")) {
 				best = modPath
 			}
 		}
+
+		local := localSymbolName(sym.name, importPath)
 		if best == "" {
-			if pkgPath != mainPackage && isStandardImportPath(pkgPath) {
-				stdlib = append(stdlib, sym.name)
+			if importPath != mainPackage && isStandardImportPath(importPath) {
+				stdlib[importPath] = append(stdlib[importPath], local)
 			}
 			continue
 		}
-		results[best] = append(results[best], sym.name)
+		if results[best] == nil {
+			results[best] = make(map[string][]string)
+		}
+		results[best][importPath] = append(results[best][importPath], local)
 	}
 
-	for modPath, names := range results {
-		slices.Sort(names)
-		results[modPath] = slices.Compact(names)
+	for _, byImport := range results {
+		sortCompactGroups(byImport)
 	}
-	if len(stdlib) > 0 {
-		slices.Sort(stdlib)
-		stdlib = slices.Compact(stdlib)
+	sortCompactGroups(stdlib)
+	if len(stdlib) == 0 {
+		stdlib = nil
 	}
 
 	return results, stdlib
 }
 
+// localSymbolName strips the owning package's import path prefix from a fully qualified symbol name, e.g.
+// "github.com/foo/bar.(*T).M" with import path "github.com/foo/bar" becomes "(*T).M". The name is returned
+// unchanged when it does not carry the expected prefix.
+func localSymbolName(name, importPath string) string {
+	if importPath != "" && strings.HasPrefix(name, importPath+".") {
+		return name[len(importPath)+1:]
+	}
+	return name
+}
+
+// sortCompactGroups sorts and deduplicates each symbol list in a group keyed by import path, in place.
+func sortCompactGroups(groups map[string][]string) {
+	for path, names := range groups {
+		slices.Sort(names)
+		groups[path] = slices.Compact(names)
+	}
+}
+
 // isStandardImportPath reports whether path is a Go standard-library import path. This mirrors the rule
 // the Go toolchain uses: a path is standard if the element before its first slash contains no dot (e.g.
 // "net/http", "runtime", "internal/abi"), which distinguishes it from module paths like

syft/pkg/cataloger/golang/symbols_test.go

@@ -22,8 +22,8 @@ func Test_moduleSymbols(t *testing.T) {
 	tests := []struct {
 		name           string
 		symbols        []binarySymbol
-		expected       map[string][]string
-		expectedStdlib []string
+		expected       map[string]map[string][]string
+		expectedStdlib map[string][]string
 	}{
 		{
 			name:     "no symbols",
@@ -31,47 +31,47 @@ func Test_moduleSymbols(t *testing.T) {
 			expected: nil,
 		},
 		{
-			name: "attribute symbols by longest module path prefix",
+			name: "attribute symbols by longest module path prefix, grouped by import path",
 			symbols: []binarySymbol{
 				{packagePath: "github.com/foo/bar", name: "github.com/foo/bar.Parse"},
 				{packagePath: "github.com/foo/bar/internal/util", name: "github.com/foo/bar/internal/util.(*Helper).Do"},
 				{packagePath: "github.com/foo/bar/v2", name: "github.com/foo/bar/v2.Parse"},
 			},
-			expected: map[string][]string{
+			expected: map[string]map[string][]string{
 				"github.com/foo/bar": {
-					"github.com/foo/bar.Parse",
-					"github.com/foo/bar/internal/util.(*Helper).Do",
+					"github.com/foo/bar":               {"Parse"},
+					"github.com/foo/bar/internal/util": {"(*Helper).Do"},
 				},
 				"github.com/foo/bar/v2": {
-					"github.com/foo/bar/v2.Parse",
+					"github.com/foo/bar/v2": {"Parse"},
 				},
 			},
 		},
 		{
-			name: "main package symbols are attributed to the main module",
+			name: "main package symbols are attributed to the main module and keyed by the main import path",
 			symbols: []binarySymbol{
 				{packagePath: "main", name: "main.main"},
 				{packagePath: "github.com/someorg/somecli/cmd", name: "github.com/someorg/somecli/cmd.Execute"},
 			},
-			expected: map[string][]string{
+			expected: map[string]map[string][]string{
 				"github.com/someorg/somecli": {
-					"github.com/someorg/somecli/cmd.Execute",
-					"main.main",
+					"github.com/someorg/somecli/cmd": {"Execute"},
+					"main":                           {"main"},
 				},
 			},
 		},
 		{
-			name: "stdlib and runtime symbols are collected separately",
+			name: "stdlib and runtime symbols are collected separately, grouped by import path",
 			symbols: []binarySymbol{
 				{packagePath: "runtime", name: "runtime.main"},
 				{packagePath: "net/http", name: "net/http.(*Client).Do"},
 				{packagePath: "internal/abi", name: "internal/abi.(*Type).Kind"},
 			},
-			expected: map[string][]string{},
-			expectedStdlib: []string{
-				"internal/abi.(*Type).Kind",
-				"net/http.(*Client).Do",
-				"runtime.main",
+			expected: map[string]map[string][]string{},
+			expectedStdlib: map[string][]string{
+				"internal/abi": {"(*Type).Kind"},
+				"net/http":     {"(*Client).Do"},
+				"runtime":      {"main"},
 			},
 		},
 		{
@@ -80,9 +80,9 @@ func Test_moduleSymbols(t *testing.T) {
 				{packagePath: "github.com/foo/bar", name: "github.com/foo/bar.Parse"},
 				{packagePath: "github.com/foo/bar", name: "github.com/foo/bar.Parse"},
 			},
-			expected: map[string][]string{
+			expected: map[string]map[string][]string{
 				"github.com/foo/bar": {
-					"github.com/foo/bar.Parse",
+					"github.com/foo/bar": {"Parse"},
 				},
 			},
 		},
@@ -97,6 +97,53 @@ func Test_moduleSymbols(t *testing.T) {
 	}
 }
 
+func Test_localSymbolName(t *testing.T) {
+	tests := []struct {
+		name       string
+		symbol     string
+		importPath string
+		expected   string
+	}{
+		{
+			name:       "function",
+			symbol:     "github.com/foo/bar.Parse",
+			importPath: "github.com/foo/bar",
+			expected:   "Parse",
+		},
+		{
+			name:       "pointer-receiver method",
+			symbol:     "github.com/foo/bar.(*T).M",
+			importPath: "github.com/foo/bar",
+			expected:   "(*T).M",
+		},
+		{
+			// type-argument brackets sit to the right of the stripped prefix and must be preserved
+			name:       "generic instantiation",
+			symbol:     "github.com/foo/bar.Do[net/url.Values]",
+			importPath: "github.com/foo/bar",
+			expected:   "Do[net/url.Values]",
+		},
+		{
+			// a sibling package must not be stripped by a shorter import path (the "." boundary guards this)
+			name:       "prefix mismatch is returned unchanged",
+			symbol:     "github.com/foo/bar/sub.Func",
+			importPath: "github.com/foo/bar",
+			expected:   "github.com/foo/bar/sub.Func",
+		},
+		{
+			name:       "empty import path is returned unchanged",
+			symbol:     "runtime.main",
+			importPath: "",
+			expected:   "runtime.main",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.expected, localSymbolName(tt.symbol, tt.importPath))
+		})
+	}
+}
+
 func Test_getSymbols(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("PE binaries are not supported for symbol extraction")

syft/pkg/golang.go

@@ -23,12 +23,16 @@ type GolangBinaryBuildinfoEntry struct {
 	// GoExperiments lists experimental Go features enabled during compilation (e.g., "arenas", "cgocheck2").
 	GoExperiments []string `json:"goExperiments,omitempty" cyclonedx:"goExperiments"`
 
-	// Symbols are the fully qualified function symbols from this module that are compiled into the binary
-	// (e.g., "github.com/foo/bar.(*Type).Method"), extracted from the binary symbol table (pclntab).
-	// Populated only when the golang cataloger's capture-symbols scope covers this package: the "all" scope
-	// populates every module package plus the synthetic stdlib package, while the "stdlib" scope populates
-	// only the stdlib package.
-	Symbols []string `json:"symbols,omitempty"`
+	// Symbols are the function symbols from this module that are compiled into the binary, extracted from
+	// the binary symbol table (pclntab) and grouped by the import path of the package that owns them. Each
+	// value is the sorted, deduplicated list of symbol names local to that package, i.e. with the import
+	// path prefix stripped (e.g. import path "github.com/foo/bar" -> "(*Type).Method"). The fully qualified
+	// name is the import path, a ".", and the local name. One exception: the binary's main package appears
+	// under the key "main" (the name the linker assigns), not its original source import path, which is not
+	// recoverable from the binary. Populated only when the golang cataloger's capture-symbols scope covers
+	// this package: the "all" scope populates every module package plus the synthetic stdlib package, while
+	// the "stdlib" scope populates only the stdlib package.
+	Symbols map[string][]string `json:"symbols,omitempty"`
 }
 
 // GolangModuleEntry represents all captured data for a Golang source scan with go.mod/go.sum