From c7a653060d74c7ba03ecbf9484f1066b52800cb8 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 24 Oct 2022 17:17:27 -0400 Subject: [PATCH] port dotnet cataloger to new generic cataloger pattern (#1286) Signed-off-by: Alex Goodman Signed-off-by: Alex Goodman --- syft/pkg/cataloger/dotnet/cataloger.go | 13 +- syft/pkg/cataloger/dotnet/package.go | 55 +++++++ .../pkg/cataloger/dotnet/parse_dotnet_deps.go | 56 +++---- .../dotnet/parse_dotnet_deps_test.go | 138 ++++++++++-------- .../internal/pkgtest/assert_packages_equal.go | 37 +++++ .../internal/pkgtest/test_generic_parser.go | 35 +++++ syft/pkg/dotnet_deps_metadata.go | 18 --- syft/pkg/url_test.go | 14 +- 8 files changed, 230 insertions(+), 136 deletions(-) create mode 100644 syft/pkg/cataloger/dotnet/package.go create mode 100644 syft/pkg/cataloger/internal/pkgtest/assert_packages_equal.go create mode 100644 syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go diff --git a/syft/pkg/cataloger/dotnet/cataloger.go b/syft/pkg/cataloger/dotnet/cataloger.go index e5907e9ec..159edcb27 100644 --- a/syft/pkg/cataloger/dotnet/cataloger.go +++ b/syft/pkg/cataloger/dotnet/cataloger.go @@ -1,14 +1,13 @@ package dotnet import ( - "github.com/anchore/syft/syft/pkg/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/generic" ) -// NewDotnetDepsCataloger returns a new Dotnet cataloger object base on deps json files. -func NewDotnetDepsCataloger() *common.GenericCataloger { - globParsers := map[string]common.ParserFn{ - "**/*.deps.json": parseDotnetDeps, - } +const catalogerName = "dotnet-deps-cataloger" - return common.NewGenericCataloger(nil, globParsers, "dotnet-deps-cataloger") +// NewDotnetDepsCataloger returns a new Dotnet cataloger object base on deps json files. +func NewDotnetDepsCataloger() *generic.Cataloger { + return generic.NewCataloger(catalogerName). + WithParserByGlobs(parseDotnetDeps, "**/*.deps.json") } diff --git a/syft/pkg/cataloger/dotnet/package.go b/syft/pkg/cataloger/dotnet/package.go new file mode 100644 index 000000000..1f5e4255c --- /dev/null +++ b/syft/pkg/cataloger/dotnet/package.go @@ -0,0 +1,55 @@ +package dotnet + +import ( + "strings" + + "github.com/anchore/packageurl-go" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func newDotnetDepsPackage(nameVersion string, lib dotnetDepsLibrary, locations ...source.Location) *pkg.Package { + if lib.Type != "package" { + return nil + } + + fields := strings.Split(nameVersion, "/") + name := fields[0] + version := fields[1] + + m := pkg.DotnetDepsMetadata{ + Name: name, + Version: version, + Path: lib.Path, + Sha512: lib.Sha512, + HashPath: lib.HashPath, + } + + p := &pkg.Package{ + Name: name, + Version: version, + Locations: source.NewLocationSet(locations...), + PURL: packageURL(m), + Language: pkg.Dotnet, + Type: pkg.DotnetPkg, + MetadataType: pkg.DotnetDepsMetadataType, + Metadata: m, + } + + p.SetID() + + return p +} + +func packageURL(m pkg.DotnetDepsMetadata) string { + var qualifiers packageurl.Qualifiers + + return packageurl.NewPackageURL( + packageurl.TypeDotnet, + "", + m.Name, + m.Version, + qualifiers, + "", + ).ToString() +} diff --git a/syft/pkg/cataloger/dotnet/parse_dotnet_deps.go b/syft/pkg/cataloger/dotnet/parse_dotnet_deps.go index c61281975..49a23067c 100644 --- a/syft/pkg/cataloger/dotnet/parse_dotnet_deps.go +++ b/syft/pkg/cataloger/dotnet/parse_dotnet_deps.go @@ -3,16 +3,15 @@ package dotnet import ( "encoding/json" "fmt" - "io" - "strings" + "sort" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/source" ) -// integrity check -var _ common.ParserFn = parseDotnetDeps +var _ generic.Parser = parseDotnetDeps type dotnetDeps struct { Libraries map[string]dotnetDepsLibrary `json:"libraries"` @@ -25,8 +24,8 @@ type dotnetDepsLibrary struct { HashPath string `json:"hashPath"` } -func parseDotnetDeps(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) { - var packages []*pkg.Package +func parseDotnetDeps(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + var pkgs []pkg.Package dec := json.NewDecoder(reader) @@ -35,38 +34,23 @@ func parseDotnetDeps(path string, reader io.Reader) ([]*pkg.Package, []artifact. return nil, nil, fmt.Errorf("failed to parse deps.json file: %w", err) } - for nameVersion, lib := range p.Libraries { - dotnetPkg := newDotnetDepsPackage(nameVersion, lib) + var names []string + + for nameVersion := range p.Libraries { + names = append(names, nameVersion) + } + + // sort the names so that the order of the packages is deterministic + sort.Strings(names) + + for _, nameVersion := range names { + lib := p.Libraries[nameVersion] + dotnetPkg := newDotnetDepsPackage(nameVersion, lib, reader.Location) if dotnetPkg != nil { - packages = append(packages, dotnetPkg) + pkgs = append(pkgs, *dotnetPkg) } } - return packages, nil, nil -} - -func newDotnetDepsPackage(nameVersion string, lib dotnetDepsLibrary) *pkg.Package { - if lib.Type != "package" { - return nil - } - - splitted := strings.Split(nameVersion, "/") - name := splitted[0] - version := splitted[1] - - return &pkg.Package{ - Name: name, - Version: version, - Language: pkg.Dotnet, - Type: pkg.DotnetPkg, - MetadataType: pkg.DotnetDepsMetadataType, - Metadata: &pkg.DotnetDepsMetadata{ - Name: name, - Version: version, - Path: lib.Path, - Sha512: lib.Sha512, - HashPath: lib.HashPath, - }, - } + return pkgs, nil, nil } diff --git a/syft/pkg/cataloger/dotnet/parse_dotnet_deps_test.go b/syft/pkg/cataloger/dotnet/parse_dotnet_deps_test.go index 2907c3628..9f0d2c706 100644 --- a/syft/pkg/cataloger/dotnet/parse_dotnet_deps_test.go +++ b/syft/pkg/cataloger/dotnet/parse_dotnet_deps_test.go @@ -1,23 +1,23 @@ package dotnet import ( - "os" "testing" - "github.com/stretchr/testify/assert" - + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" + "github.com/anchore/syft/syft/source" ) -func assertPackagesEqual(t *testing.T, actual []*pkg.Package, expected map[string]*pkg.Package) { - assert.Len(t, actual, len(expected)) -} - func TestParseDotnetDeps(t *testing.T) { - expected := map[string]*pkg.Package{ - "AWSSDK.Core": { + fixture := "test-fixtures/TestLibrary.deps.json" + fixtureLocationSet := source.NewLocationSet(source.NewLocation(fixture)) + expected := []pkg.Package{ + { Name: "AWSSDK.Core", Version: "3.7.10.6", + PURL: "pkg:dotnet/AWSSDK.Core@3.7.10.6", + Locations: fixtureLocationSet, Language: pkg.Dotnet, Type: pkg.DotnetPkg, MetadataType: pkg.DotnetDepsMetadataType, @@ -29,9 +29,27 @@ func TestParseDotnetDeps(t *testing.T) { HashPath: "awssdk.core.3.7.10.6.nupkg.sha512", }, }, - "Microsoft.Extensions.DependencyInjection": { + { + Name: "Microsoft.Extensions.DependencyInjection.Abstractions", + Version: "6.0.0", + PURL: "pkg:dotnet/Microsoft.Extensions.DependencyInjection.Abstractions@6.0.0", + Locations: fixtureLocationSet, + Language: pkg.Dotnet, + Type: pkg.DotnetPkg, + MetadataType: pkg.DotnetDepsMetadataType, + Metadata: pkg.DotnetDepsMetadata{ + Name: "Microsoft.Extensions.DependencyInjection.Abstractions", + Version: "6.0.0", + Sha512: "sha512-xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==", + Path: "microsoft.extensions.dependencyinjection.abstractions/6.0.0", + HashPath: "microsoft.extensions.dependencyinjection.abstractions.6.0.0.nupkg.sha512", + }, + }, + { Name: "Microsoft.Extensions.DependencyInjection", Version: "6.0.0", + PURL: "pkg:dotnet/Microsoft.Extensions.DependencyInjection@6.0.0", + Locations: fixtureLocationSet, Language: pkg.Dotnet, Type: pkg.DotnetPkg, MetadataType: pkg.DotnetDepsMetadataType, @@ -43,23 +61,27 @@ func TestParseDotnetDeps(t *testing.T) { HashPath: "microsoft.extensions.dependencyinjection.6.0.0.nupkg.sha512", }, }, - "Microsoft.Extensions.DependencyInjection.Abstractions": { - Name: "Microsoft.Extensions.DependencyInjection.Abstractions", + { + Name: "Microsoft.Extensions.Logging.Abstractions", Version: "6.0.0", + PURL: "pkg:dotnet/Microsoft.Extensions.Logging.Abstractions@6.0.0", + Locations: fixtureLocationSet, Language: pkg.Dotnet, Type: pkg.DotnetPkg, MetadataType: pkg.DotnetDepsMetadataType, Metadata: pkg.DotnetDepsMetadata{ - Name: "Microsoft.Extensions.DependencyInjection", + Name: "Microsoft.Extensions.Logging.Abstractions", Version: "6.0.0", - Sha512: "sha512-xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==", - Path: "microsoft.extensions.dependencyinjection.abstractions/6.0.0", - HashPath: "microsoft.extensions.dependencyinjection.abstractions.6.0.0.nupkg.sha512", + Sha512: "sha512-/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==", + Path: "microsoft.extensions.logging.abstractions/6.0.0", + HashPath: "microsoft.extensions.logging.abstractions.6.0.0.nupkg.sha512", }, }, - "Microsoft.Extensions.Logging": { + { Name: "Microsoft.Extensions.Logging", Version: "6.0.0", + PURL: "pkg:dotnet/Microsoft.Extensions.Logging@6.0.0", + Locations: fixtureLocationSet, Language: pkg.Dotnet, Type: pkg.DotnetPkg, MetadataType: pkg.DotnetDepsMetadataType, @@ -71,23 +93,12 @@ func TestParseDotnetDeps(t *testing.T) { HashPath: "microsoft.extensions.logging.6.0.0.nupkg.sha512", }, }, - "Microsoft.Extensions.Logging.Abstractions": { - Name: "Microsoft.Extensions.Logging.Abstractions", - Version: "6.0.0", - Language: pkg.Dotnet, - Type: pkg.DotnetPkg, - MetadataType: pkg.DotnetDepsMetadataType, - Metadata: pkg.DotnetDepsMetadata{ - Name: "Microsoft.Extensions.Logging", - Version: "6.0.0", - Sha512: "sha512-/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==", - Path: "microsoft.extensions.logging.abstractions/6.0.0", - HashPath: "microsoft.extensions.logging.abstractions.6.0.0.nupkg.sha512", - }, - }, - "Microsoft.Extensions.Options": { + + { Name: "Microsoft.Extensions.Options", Version: "6.0.0", + PURL: "pkg:dotnet/Microsoft.Extensions.Options@6.0.0", + Locations: fixtureLocationSet, Language: pkg.Dotnet, Type: pkg.DotnetPkg, MetadataType: pkg.DotnetDepsMetadataType, @@ -99,9 +110,11 @@ func TestParseDotnetDeps(t *testing.T) { HashPath: "microsoft.extensions.options.6.0.0.nupkg.sha512", }, }, - "Microsoft.Extensions.Primitives": { + { Name: "Microsoft.Extensions.Primitives", Version: "6.0.0", + PURL: "pkg:dotnet/Microsoft.Extensions.Primitives@6.0.0", + Locations: fixtureLocationSet, Language: pkg.Dotnet, Type: pkg.DotnetPkg, MetadataType: pkg.DotnetDepsMetadataType, @@ -113,9 +126,11 @@ func TestParseDotnetDeps(t *testing.T) { HashPath: "microsoft.extensions.primitives.6.0.0.nupkg.sha512", }, }, - "Newtonsoft.Json": { + { Name: "Newtonsoft.Json", Version: "13.0.1", + PURL: "pkg:dotnet/Newtonsoft.Json@13.0.1", + Locations: fixtureLocationSet, Language: pkg.Dotnet, Type: pkg.DotnetPkg, MetadataType: pkg.DotnetDepsMetadataType, @@ -127,23 +142,11 @@ func TestParseDotnetDeps(t *testing.T) { HashPath: "newtonsoft.json.13.0.1.nupkg.sha512", }, }, - "Serilog": { - Name: "Serilog", - Version: "2.10.0", - Language: pkg.Dotnet, - Type: pkg.DotnetPkg, - MetadataType: pkg.DotnetDepsMetadataType, - Metadata: pkg.DotnetDepsMetadata{ - Name: "Serilog", - Version: "2.10.0", - Sha512: "sha512-+QX0hmf37a0/OZLxM3wL7V6/ADvC1XihXN4Kq/p6d8lCPfgkRdiuhbWlMaFjR9Av0dy5F0+MBeDmDdRZN/YwQA==", - Path: "serilog/2.10.0", - HashPath: "serilog.2.10.0.nupkg.sha512", - }, - }, - "Serilog.Sinks.Console": { + { Name: "Serilog.Sinks.Console", Version: "4.0.1", + PURL: "pkg:dotnet/Serilog.Sinks.Console@4.0.1", + Locations: fixtureLocationSet, Language: pkg.Dotnet, Type: pkg.DotnetPkg, MetadataType: pkg.DotnetDepsMetadataType, @@ -155,9 +158,27 @@ func TestParseDotnetDeps(t *testing.T) { HashPath: "serilog.sinks.console.4.0.1.nupkg.sha512", }, }, - "System.Diagnostics.DiagnosticSource": { + { + Name: "Serilog", + Version: "2.10.0", + PURL: "pkg:dotnet/Serilog@2.10.0", + Locations: fixtureLocationSet, + Language: pkg.Dotnet, + Type: pkg.DotnetPkg, + MetadataType: pkg.DotnetDepsMetadataType, + Metadata: pkg.DotnetDepsMetadata{ + Name: "Serilog", + Version: "2.10.0", + Sha512: "sha512-+QX0hmf37a0/OZLxM3wL7V6/ADvC1XihXN4Kq/p6d8lCPfgkRdiuhbWlMaFjR9Av0dy5F0+MBeDmDdRZN/YwQA==", + Path: "serilog/2.10.0", + HashPath: "serilog.2.10.0.nupkg.sha512", + }, + }, + { Name: "System.Diagnostics.DiagnosticSource", Version: "6.0.0", + PURL: "pkg:dotnet/System.Diagnostics.DiagnosticSource@6.0.0", + Locations: fixtureLocationSet, Language: pkg.Dotnet, Type: pkg.DotnetPkg, MetadataType: pkg.DotnetDepsMetadataType, @@ -169,9 +190,11 @@ func TestParseDotnetDeps(t *testing.T) { HashPath: "system.diagnostics.diagnosticsource.6.0.0.nupkg.sha512", }, }, - "System.Runtime.CompilerServices.Unsafe": { + { Name: "System.Runtime.CompilerServices.Unsafe", Version: "6.0.0", + PURL: "pkg:dotnet/System.Runtime.CompilerServices.Unsafe@6.0.0", + Locations: fixtureLocationSet, Language: pkg.Dotnet, Type: pkg.DotnetPkg, MetadataType: pkg.DotnetDepsMetadataType, @@ -185,15 +208,6 @@ func TestParseDotnetDeps(t *testing.T) { }, } - fixture, err := os.Open("test-fixtures/TestLibrary.deps.json") - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) - } - - actual, _, err := parseDotnetDeps(fixture.Name(), fixture) - if err != nil { - t.Fatalf("failed to parse deps.json: %+v", err) - } - - assertPackagesEqual(t, actual, expected) + var expectedRelationships []artifact.Relationship + pkgtest.TestGenericParser(t, fixture, parseDotnetDeps, expected, expectedRelationships) } diff --git a/syft/pkg/cataloger/internal/pkgtest/assert_packages_equal.go b/syft/pkg/cataloger/internal/pkgtest/assert_packages_equal.go new file mode 100644 index 000000000..0fbcfd94b --- /dev/null +++ b/syft/pkg/cataloger/internal/pkgtest/assert_packages_equal.go @@ -0,0 +1,37 @@ +package pkgtest + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func AssertPackagesEqual(t testing.TB, expected, actual []pkg.Package) { + if diff := cmp.Diff(expected, actual, + cmpopts.IgnoreFields(pkg.Package{}, "id"), // note: ID is not deterministic for test purposes + cmp.Comparer( + func(x, y source.LocationSet) bool { + xs := x.ToSlice() + ys := y.ToSlice() + + if len(xs) != len(ys) { + return false + } + for i, xe := range xs { + ye := ys[i] + if !(cmp.Equal(xe.Coordinates, ye.Coordinates) && cmp.Equal(xe.VirtualPath, ye.VirtualPath)) { + return false + } + } + + return true + }, + ), + ); diff != "" { + t.Errorf("unexpected packages from parsing (-expected +actual)\n%s", diff) + } +} diff --git a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go new file mode 100644 index 000000000..ddec772a7 --- /dev/null +++ b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go @@ -0,0 +1,35 @@ +package pkgtest + +import ( + "os" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/source" +) + +func TestGenericParser(t *testing.T, fixturePath string, parser generic.Parser, expectedPkgs []pkg.Package, expectedRelationships []artifact.Relationship) { + TestGenericParserWithEnv(t, fixturePath, parser, nil, expectedPkgs, expectedRelationships) +} + +func TestGenericParserWithEnv(t *testing.T, fixturePath string, parser generic.Parser, env *generic.Environment, expectedPkgs []pkg.Package, expectedRelationships []artifact.Relationship) { + fixture, err := os.Open(fixturePath) + require.NoError(t, err) + + actualPkgs, actualRelationships, err := parser(nil, env, source.LocationReadCloser{ + Location: source.NewLocation(fixture.Name()), + ReadCloser: fixture, + }) + require.NoError(t, err) + + AssertPackagesEqual(t, expectedPkgs, actualPkgs) + + if diff := cmp.Diff(expectedRelationships, actualRelationships); diff != "" { + t.Errorf("unexpected relationships from parsing (-expected +actual)\n%s", diff) + } +} diff --git a/syft/pkg/dotnet_deps_metadata.go b/syft/pkg/dotnet_deps_metadata.go index faac92839..2b1413419 100644 --- a/syft/pkg/dotnet_deps_metadata.go +++ b/syft/pkg/dotnet_deps_metadata.go @@ -1,10 +1,5 @@ package pkg -import ( - "github.com/anchore/packageurl-go" - "github.com/anchore/syft/syft/linux" -) - type DotnetDepsMetadata struct { Name string `mapstructure:"name" json:"name"` Version string `mapstructure:"version" json:"version"` @@ -12,16 +7,3 @@ type DotnetDepsMetadata struct { Sha512 string `mapstructure:"sha512" json:"sha512"` HashPath string `mapstructure:"hashPath" json:"hashPath"` } - -func (m DotnetDepsMetadata) PackageURL(_ *linux.Release) string { - var qualifiers packageurl.Qualifiers - - return packageurl.NewPackageURL( - packageurl.TypeDotnet, - "", - m.Name, - m.Version, - qualifiers, - "", - ).ToString() -} diff --git a/syft/pkg/url_test.go b/syft/pkg/url_test.go index 8b13394c7..9afa9c350 100644 --- a/syft/pkg/url_test.go +++ b/syft/pkg/url_test.go @@ -35,19 +35,6 @@ func TestPackageURL(t *testing.T) { }, expected: "pkg:golang/go.opencensus.io@v0.23.0", }, - { - name: "dotnet", - pkg: Package{ - Name: "Microsoft.CodeAnalysis.Razor", - Version: "2.2.0", - Type: DotnetPkg, - Metadata: DotnetDepsMetadata{ - Name: "Microsoft.CodeAnalysis.Razor", - Version: "2.2.0", - }, - }, - expected: "pkg:dotnet/Microsoft.CodeAnalysis.Razor@2.2.0", - }, { name: "python", pkg: Package{ @@ -211,6 +198,7 @@ func TestPackageURL(t *testing.T) { expectedTypes.Remove(string(ApkPkg)) expectedTypes.Remove(string(ConanPkg)) expectedTypes.Remove(string(DartPubPkg)) + expectedTypes.Remove(string(DotnetPkg)) for _, test := range tests { t.Run(test.name, func(t *testing.T) {