From 1574fb20ae6b830d6f05c9ecaafba58ea2a71dba Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 8 May 2025 11:28:08 -0400 Subject: [PATCH] merge multiple targets for the same dotnet package (#3869) Signed-off-by: Alex Goodman --- .../cataloger/dotnet/deps_binary_cataloger.go | 13 +- syft/pkg/cataloger/dotnet/deps_json.go | 72 ++++++--- syft/pkg/cataloger/dotnet/deps_json_test.go | 143 ++++++++++++++++++ 3 files changed, 201 insertions(+), 27 deletions(-) diff --git a/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go b/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go index 4829ce7cf..c249261fd 100644 --- a/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go +++ b/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go @@ -149,7 +149,12 @@ func isRuntimePackageLocation(loc file.Location) (string, bool) { func partitionPEs(depJsons []logicalDepsJSON, peFiles []logicalPE) ([]logicalDepsJSON, []logicalPE, []logicalDepsJSON) { // sort deps.json paths from longest to shortest. This is so we are processing the most specific match first. sort.Slice(depJsons, func(i, j int) bool { - return len(depJsons[i].Location.RealPath) > len(depJsons[j].Location.RealPath) + return depJsons[i].Location.RealPath > depJsons[j].Location.RealPath + }) + + // we should be processing PE files in a stable order + sort.Slice(peFiles, func(i, j int) bool { + return peFiles[i].Location.RealPath > peFiles[j].Location.RealPath }) peFilesByPath := make(map[file.Coordinates][]logicalPE) @@ -321,8 +326,7 @@ func relationshipsFromLogicalDepsJSON(doc logicalDepsJSON, pkgMap map[string]pkg if lp.Targets == nil { continue } - for depName, depVersion := range lp.Targets.Dependencies { - depNameVersion := createNameAndVersion(depName, depVersion) + for _, depNameVersion := range lp.dependencyNameVersions() { thisPkg, ok := pkgMap[lp.NameVersion] if !ok { continue @@ -371,8 +375,7 @@ func findNearestDependencyPackages(skippedDep logicalDepsJSONPackage, pkgMap map processed.Add(skippedDep.NameVersion) - for depName, depVersion := range skippedDep.Targets.Dependencies { - depNameVersion := createNameAndVersion(depName, depVersion) + for _, depNameVersion := range skippedDep.dependencyNameVersions() { depPkg, ok := pkgMap[depNameVersion] if !ok { skippedDepPkg, ok := skipped[depNameVersion] diff --git a/syft/pkg/cataloger/dotnet/deps_json.go b/syft/pkg/cataloger/dotnet/deps_json.go index 70d8aa1be..d61f09ce5 100644 --- a/syft/pkg/cataloger/dotnet/deps_json.go +++ b/syft/pkg/cataloger/dotnet/deps_json.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "regexp" + "sort" "strings" "github.com/scylladb/go-set/strset" @@ -66,7 +67,11 @@ func (t depsTarget) resourcePaths() map[string]string { func (t depsTarget) runtimePaths() map[string]string { result := make(map[string]string) for path := range t.Runtime { - result[trimLibPrefix(path)] = path + trimmedPath := trimLibPrefix(path) + if _, exists := result[trimmedPath]; exists { + continue + } + result[trimmedPath] = path } return result } @@ -82,14 +87,14 @@ type depsLibrary struct { // Note: this is not a real construct of the deps.json, just a useful reorganization of the data for downstream processing. type logicalDepsJSONPackage struct { NameVersion string - Targets *depsTarget + Targets []depsTarget Library *depsLibrary - // anyChildClaimsDLLs is a flag that indicates if any of the children of this package claim a DLL associated with them in the deps.json. - anyChildClaimsDLLs bool + // AnyChildClaimsDLLs is a flag that indicates if any of the children of this package claim a DLL associated with them in the deps.json. + AnyChildClaimsDLLs bool - // anyChildHasDLLs is a flag that indicates if any of the children of this package have a DLL associated with them (found on disk). - anyChildHasDLLs bool + // AnyChildHasDLLs is a flag that indicates if any of the children of this package have a DLL associated with them (found on disk). + AnyChildHasDLLs bool // RuntimePathsByRelativeDLLPath is a map of the relative path to the DLL relative to the deps.json file // to the target path as described in the deps.json target entry under "runtime". @@ -103,7 +108,7 @@ type logicalDepsJSONPackage struct { // to the target path as described in the deps.json target entry under "compile". CompilePathsByRelativeDLLPath map[string]string - // NativePathsByRelativeDLLPath is a map of the relative path to the DLL relative to the deps.json file + // NativePaths is a map of the relative path to the DLL relative to the deps.json file // to the target path as described in the deps.json target entry under "native". These should not have // any runtime references to trim from the front of the path. NativePaths *strset.Set @@ -118,11 +123,15 @@ func (l *logicalDepsJSONPackage) dependencyNameVersions() []string { if l.Targets == nil { return nil } - var results []string - for name, version := range l.Targets.Dependencies { - results = append(results, createNameAndVersion(name, version)) + results := strset.New() + for _, t := range l.Targets { + for name, version := range t.Dependencies { + results.Add(createNameAndVersion(name, version)) + } } - return results + r := results.List() + sort.Strings(r) + return r } // ClaimsDLLs indicates if this package has any DLLs associated with it (directly or indirectly with a dependency). @@ -131,7 +140,7 @@ func (l *logicalDepsJSONPackage) ClaimsDLLs(includeChildren bool) bool { if !includeChildren { return selfClaim } - return selfClaim || l.anyChildClaimsDLLs + return selfClaim || l.AnyChildClaimsDLLs } func (l *logicalDepsJSONPackage) FoundDLLs(includeChildren bool) bool { @@ -139,7 +148,7 @@ func (l *logicalDepsJSONPackage) FoundDLLs(includeChildren bool) bool { if !includeChildren { return selfClaim } - return selfClaim || l.anyChildHasDLLs + return selfClaim || l.AnyChildHasDLLs } type logicalDepsJSON struct { @@ -206,6 +215,14 @@ func getLogicalDepsJSON(deps depsJSON, lm *libmanJSON) logicalDepsJSON { for libName, target := range targets { _, exists := packageMap[libName] if exists { + // merge this with existing targets (multiple targets can exist for the same library) + p := packageMap[libName] + p.Targets = append(p.Targets, target) + p.RuntimePathsByRelativeDLLPath = mergeMaps(p.RuntimePathsByRelativeDLLPath, target.runtimePaths()) + p.ResourcePathsByRelativeDLLPath = mergeMaps(p.ResourcePathsByRelativeDLLPath, target.resourcePaths()) + p.CompilePathsByRelativeDLLPath = mergeMaps(p.CompilePathsByRelativeDLLPath, target.compilePaths()) + p.NativePaths = mergeSets(p.NativePaths, target.nativePaths()) + continue } @@ -218,7 +235,7 @@ func getLogicalDepsJSON(deps depsJSON, lm *libmanJSON) logicalDepsJSON { p := &logicalDepsJSONPackage{ NameVersion: libName, Library: lib, - Targets: &target, + Targets: []depsTarget{target}, RuntimePathsByRelativeDLLPath: target.runtimePaths(), ResourcePathsByRelativeDLLPath: target.resourcePaths(), CompilePathsByRelativeDLLPath: target.compilePaths(), @@ -235,8 +252,8 @@ func getLogicalDepsJSON(deps depsJSON, lm *libmanJSON) logicalDepsJSON { if !bundlingDetected && knownBundlers.Has(name) { bundlingDetected = true } - p.anyChildClaimsDLLs = searchForDLLClaims(packageMap, p.dependencyNameVersions()...) - p.anyChildHasDLLs = searchForDLLEvidence(packageMap, p.dependencyNameVersions()...) + p.AnyChildClaimsDLLs = searchForDLLClaims(packageMap, p.dependencyNameVersions()...) + p.AnyChildHasDLLs = searchForDLLEvidence(packageMap, p.dependencyNameVersions()...) packages[p.NameVersion] = *p } @@ -250,6 +267,22 @@ func getLogicalDepsJSON(deps depsJSON, lm *libmanJSON) logicalDepsJSON { } } +func mergeMaps(m1, m2 map[string]string) map[string]string { + if m1 == nil { + m1 = make(map[string]string) + } + for k, v := range m2 { + if _, exists := m1[k]; !exists { + m1[k] = v + } + } + return m1 +} + +func mergeSets(s1, s2 *strset.Set) *strset.Set { + return strset.Union(s1, s2) +} + type visitorFunc func(p *logicalDepsJSONPackage) bool // searchForDLLEvidence recursively searches for executables found for any of the given nameVersions and children recursively. @@ -277,12 +310,7 @@ func traverseDependencies(packageMap map[string]*logicalDepsJSONPackage, visitor return true } - var children []string - for name, version := range p.Targets.Dependencies { - children = append(children, createNameAndVersion(name, version)) - } - - if traverseDependencies(packageMap, visitor, children...) { + if traverseDependencies(packageMap, visitor, p.dependencyNameVersions()...) { return true } } diff --git a/syft/pkg/cataloger/dotnet/deps_json_test.go b/syft/pkg/cataloger/dotnet/deps_json_test.go index 1c8158271..96fd34571 100644 --- a/syft/pkg/cataloger/dotnet/deps_json_test.go +++ b/syft/pkg/cataloger/dotnet/deps_json_test.go @@ -2,6 +2,12 @@ package dotnet import ( "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/file" ) func TestTrimLibPrefix(t *testing.T) { @@ -71,3 +77,140 @@ func TestTrimLibPrefix(t *testing.T) { }) } } + +func TestGetLogicalDepsJSON_MergeTargets(t *testing.T) { + deps := depsJSON{ + Location: file.NewLocation("/path/to/deps.json"), + RuntimeTarget: runtimeTarget{ + Name: ".NETCoreApp,Version=v6.0", + }, + // note: for this test we have two targets with the same name, which will be merged when creating a logical deps + Targets: map[string]map[string]depsTarget{ + ".NETCoreApp,Version=v6.0": { + "Microsoft.CodeAnalysis.CSharp/4.0.0": { + Dependencies: map[string]string{ + "Microsoft.CodeAnalysis.Common": "4.0.0", + }, + Runtime: map[string]map[string]string{ + "lib/netcoreapp3.1/Microsoft.CodeAnalysis.CSharp.dll": { + "assemblyVersion": "4.0.0.0", + "fileVersion": "4.0.21.51404", + }, + }, + Resources: map[string]map[string]string{ + "lib/netcoreapp3.1/cs/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "cs", + }, + "lib/netcoreapp3.1/de/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "de", + }, + "lib/netcoreapp3.1/es/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "es", + }, + "lib/netcoreapp3.1/fr/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "fr", + }, + "lib/netcoreapp3.1/it/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "it", + }, + "lib/netcoreapp3.1/ja/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "ja", + }, + "lib/netcoreapp3.1/ko/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "ko", + }, + "lib/netcoreapp3.1/pl/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "pl", + }, + "lib/netcoreapp3.1/pt-BR/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "pt-BR", + }, + "lib/netcoreapp3.1/ru/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "ru", + }, + "lib/netcoreapp3.1/tr/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "tr", + }, + "lib/netcoreapp3.1/zh-Hans/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "zh-Hans", + }, + "lib/netcoreapp3.1/zh-Hant/Microsoft.CodeAnalysis.CSharp.resources.dll": { + "locale": "zh-Hant", + }, + }, + }, + }, + "net6.0": { + "Microsoft.CodeAnalysis.CSharp/4.0.0": { + Dependencies: map[string]string{ + "Microsoft.CodeAnalysis.Common": "4.0.0", + }, + Compile: map[string]map[string]string{ + "lib/netcoreapp3.1/Microsoft.CodeAnalysis.CSharp.dll": {}, + }, + }, + }, + }, + Libraries: map[string]depsLibrary{ + "Microsoft.CodeAnalysis.CSharp/4.0.0": { + Type: "package", + Path: "microsoft.codeanalysis.csharp/4.0.0", + Sha512: "sha512-example-hash", + HashPath: "microsoft.codeanalysis.csharp.4.0.0.nupkg.sha512", + }, + }, + } + + result := getLogicalDepsJSON(deps, &libmanJSON{}) + + assert.Equal(t, "/path/to/deps.json", result.Location.RealPath) + assert.Equal(t, ".NETCoreApp,Version=v6.0", result.RuntimeTarget.Name) + + libPackage, exists := result.PackagesByNameVersion["Microsoft.CodeAnalysis.CSharp/4.0.0"] + require.True(t, exists, "Expected to find the merged package") + + assert.NotNil(t, libPackage.Library) + assert.Equal(t, "package", libPackage.Library.Type) + assert.Equal(t, "microsoft.codeanalysis.csharp/4.0.0", libPackage.Library.Path) + assert.Equal(t, "sha512-example-hash", libPackage.Library.Sha512) + assert.Equal(t, "microsoft.codeanalysis.csharp.4.0.0.nupkg.sha512", libPackage.Library.HashPath) + + require.Equal(t, 2, len(libPackage.Targets), "Expected 2 targets to be merged") + + expectedRuntimePaths := map[string]string{ + "Microsoft.CodeAnalysis.CSharp.dll": "lib/netcoreapp3.1/Microsoft.CodeAnalysis.CSharp.dll", + } + if diff := cmp.Diff(expectedRuntimePaths, libPackage.RuntimePathsByRelativeDLLPath); diff != "" { + t.Errorf("RuntimePathsByRelativeDLLPath mismatch (-expected +actual):\n%s", diff) + } + + expectedResourcePaths := map[string]string{ + "cs/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/cs/Microsoft.CodeAnalysis.CSharp.resources.dll", + "de/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/de/Microsoft.CodeAnalysis.CSharp.resources.dll", + "es/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/es/Microsoft.CodeAnalysis.CSharp.resources.dll", + "fr/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/fr/Microsoft.CodeAnalysis.CSharp.resources.dll", + "it/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/it/Microsoft.CodeAnalysis.CSharp.resources.dll", + "ja/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/ja/Microsoft.CodeAnalysis.CSharp.resources.dll", + "ko/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/ko/Microsoft.CodeAnalysis.CSharp.resources.dll", + "pl/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/pl/Microsoft.CodeAnalysis.CSharp.resources.dll", + "pt-BR/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/pt-BR/Microsoft.CodeAnalysis.CSharp.resources.dll", + "ru/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/ru/Microsoft.CodeAnalysis.CSharp.resources.dll", + "tr/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/tr/Microsoft.CodeAnalysis.CSharp.resources.dll", + "zh-Hans/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/zh-Hans/Microsoft.CodeAnalysis.CSharp.resources.dll", + "zh-Hant/Microsoft.CodeAnalysis.CSharp.resources.dll": "lib/netcoreapp3.1/zh-Hant/Microsoft.CodeAnalysis.CSharp.resources.dll", + } + if diff := cmp.Diff(expectedResourcePaths, libPackage.ResourcePathsByRelativeDLLPath); diff != "" { + t.Errorf("ResourcePathsByRelativeDLLPath mismatch (-expected +actual):\n%s", diff) + } + + expectedCompilePaths := map[string]string{ + "Microsoft.CodeAnalysis.CSharp.dll": "lib/netcoreapp3.1/Microsoft.CodeAnalysis.CSharp.dll", + } + if diff := cmp.Diff(expectedCompilePaths, libPackage.CompilePathsByRelativeDLLPath); diff != "" { + t.Errorf("CompilePathsByRelativeDLLPath mismatch (-expected +actual):\n%s", diff) + } + + assert.Equal(t, 0, libPackage.NativePaths.Size(), "Expected no native paths") + + assert.True(t, result.PackageNameVersions.Has("Microsoft.CodeAnalysis.CSharp/4.0.0")) +}