port dotnet cataloger to new generic cataloger pattern (#1286)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

syft/pkg/cataloger/dotnet/cataloger.go

@@ -1,14 +1,13 @@
 package dotnet
 
 import (
-	"github.com/anchore/syft/syft/pkg/cataloger/common"
+	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
-// NewDotnetDepsCataloger returns a new Dotnet cataloger object base on deps json files.
+const catalogerName = "dotnet-deps-cataloger"
-func NewDotnetDepsCataloger() *common.GenericCataloger {
-	globParsers := map[string]common.ParserFn{
-		"**/*.deps.json": parseDotnetDeps,
-	}
 
-	return common.NewGenericCataloger(nil, globParsers, "dotnet-deps-cataloger")
+// NewDotnetDepsCataloger returns a new Dotnet cataloger object base on deps json files.
+func NewDotnetDepsCataloger() *generic.Cataloger {
+	return generic.NewCataloger(catalogerName).
+		WithParserByGlobs(parseDotnetDeps, "**/*.deps.json")
 }

syft/pkg/cataloger/dotnet/package.go

@@ -0,0 +1,55 @@
+package dotnet
+
+import (
+	"strings"
+
+	"github.com/anchore/packageurl-go"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+func newDotnetDepsPackage(nameVersion string, lib dotnetDepsLibrary, locations ...source.Location) *pkg.Package {
+	if lib.Type != "package" {
+		return nil
+	}
+
+	fields := strings.Split(nameVersion, "/")
+	name := fields[0]
+	version := fields[1]
+
+	m := pkg.DotnetDepsMetadata{
+		Name:     name,
+		Version:  version,
+		Path:     lib.Path,
+		Sha512:   lib.Sha512,
+		HashPath: lib.HashPath,
+	}
+
+	p := &pkg.Package{
+		Name:         name,
+		Version:      version,
+		Locations:    source.NewLocationSet(locations...),
+		PURL:         packageURL(m),
+		Language:     pkg.Dotnet,
+		Type:         pkg.DotnetPkg,
+		MetadataType: pkg.DotnetDepsMetadataType,
+		Metadata:     m,
+	}
+
+	p.SetID()
+
+	return p
+}
+
+func packageURL(m pkg.DotnetDepsMetadata) string {
+	var qualifiers packageurl.Qualifiers
+
+	return packageurl.NewPackageURL(
+		packageurl.TypeDotnet,
+		"",
+		m.Name,
+		m.Version,
+		qualifiers,
+		"",
+	).ToString()
+}

syft/pkg/cataloger/dotnet/parse_dotnet_deps.go

@@ -3,16 +3,15 @@ package dotnet
 import (
 	"encoding/json"
 	"fmt"
-	"io"
+	"sort"
-	"strings"
 
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/pkg/cataloger/common"
+	"github.com/anchore/syft/syft/pkg/cataloger/generic"
+	"github.com/anchore/syft/syft/source"
 )
 
-// integrity check
+var _ generic.Parser = parseDotnetDeps
-var _ common.ParserFn = parseDotnetDeps
 
 type dotnetDeps struct {
 	Libraries map[string]dotnetDepsLibrary `json:"libraries"`
@@ -25,8 +24,8 @@ type dotnetDepsLibrary struct {
 	HashPath string `json:"hashPath"`
 }
 
-func parseDotnetDeps(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+func parseDotnetDeps(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
-	var packages []*pkg.Package
+	var pkgs []pkg.Package
 
 	dec := json.NewDecoder(reader)
 
@@ -35,38 +34,23 @@ func parseDotnetDeps(path string, reader io.Reader) ([]*pkg.Package, []artifact.
 		return nil, nil, fmt.Errorf("failed to parse deps.json file: %w", err)
 	}
 
-	for nameVersion, lib := range p.Libraries {
+	var names []string
-		dotnetPkg := newDotnetDepsPackage(nameVersion, lib)
+
+	for nameVersion := range p.Libraries {
+		names = append(names, nameVersion)
+	}
+
+	// sort the names so that the order of the packages is deterministic
+	sort.Strings(names)
+
+	for _, nameVersion := range names {
+		lib := p.Libraries[nameVersion]
+		dotnetPkg := newDotnetDepsPackage(nameVersion, lib, reader.Location)
 
 		if dotnetPkg != nil {
-			packages = append(packages, dotnetPkg)
+			pkgs = append(pkgs, *dotnetPkg)
 		}
 	}
 
-	return packages, nil, nil
+	return pkgs, nil, nil
-}
-
-func newDotnetDepsPackage(nameVersion string, lib dotnetDepsLibrary) *pkg.Package {
-	if lib.Type != "package" {
-		return nil
-	}
-
-	splitted := strings.Split(nameVersion, "/")
-	name := splitted[0]
-	version := splitted[1]
-
-	return &pkg.Package{
-		Name:         name,
-		Version:      version,
-		Language:     pkg.Dotnet,
-		Type:         pkg.DotnetPkg,
-		MetadataType: pkg.DotnetDepsMetadataType,
-		Metadata: &pkg.DotnetDepsMetadata{
-			Name:     name,
-			Version:  version,
-			Path:     lib.Path,
-			Sha512:   lib.Sha512,
-			HashPath: lib.HashPath,
-		},
-	}
 }

syft/pkg/cataloger/dotnet/parse_dotnet_deps_test.go

@@ -1,23 +1,23 @@
 package dotnet
 
 import (
-	"os"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/anchore/syft/syft/artifact"
-
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
+	"github.com/anchore/syft/syft/source"
 )
 
-func assertPackagesEqual(t *testing.T, actual []*pkg.Package, expected map[string]*pkg.Package) {
-	assert.Len(t, actual, len(expected))
-}
-
 func TestParseDotnetDeps(t *testing.T) {
-	expected := map[string]*pkg.Package{
+	fixture := "test-fixtures/TestLibrary.deps.json"
-		"AWSSDK.Core": {
+	fixtureLocationSet := source.NewLocationSet(source.NewLocation(fixture))
+	expected := []pkg.Package{
+		{
 			Name:         "AWSSDK.Core",
 			Version:      "3.7.10.6",
+			PURL:         "pkg:dotnet/AWSSDK.Core@3.7.10.6",
+			Locations:    fixtureLocationSet,
 			Language:     pkg.Dotnet,
 			Type:         pkg.DotnetPkg,
 			MetadataType: pkg.DotnetDepsMetadataType,
@@ -29,9 +29,27 @@ func TestParseDotnetDeps(t *testing.T) {
 				HashPath: "awssdk.core.3.7.10.6.nupkg.sha512",
 			},
 		},
-		"Microsoft.Extensions.DependencyInjection": {
+		{
+			Name:         "Microsoft.Extensions.DependencyInjection.Abstractions",
+			Version:      "6.0.0",
+			PURL:         "pkg:dotnet/Microsoft.Extensions.DependencyInjection.Abstractions@6.0.0",
+			Locations:    fixtureLocationSet,
+			Language:     pkg.Dotnet,
+			Type:         pkg.DotnetPkg,
+			MetadataType: pkg.DotnetDepsMetadataType,
+			Metadata: pkg.DotnetDepsMetadata{
+				Name:     "Microsoft.Extensions.DependencyInjection.Abstractions",
+				Version:  "6.0.0",
+				Sha512:   "sha512-xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==",
+				Path:     "microsoft.extensions.dependencyinjection.abstractions/6.0.0",
+				HashPath: "microsoft.extensions.dependencyinjection.abstractions.6.0.0.nupkg.sha512",
+			},
+		},
+		{
 			Name:         "Microsoft.Extensions.DependencyInjection",
 			Version:      "6.0.0",
+			PURL:         "pkg:dotnet/Microsoft.Extensions.DependencyInjection@6.0.0",
+			Locations:    fixtureLocationSet,
 			Language:     pkg.Dotnet,
 			Type:         pkg.DotnetPkg,
 			MetadataType: pkg.DotnetDepsMetadataType,
@@ -43,23 +61,27 @@ func TestParseDotnetDeps(t *testing.T) {
 				HashPath: "microsoft.extensions.dependencyinjection.6.0.0.nupkg.sha512",
 			},
 		},
-		"Microsoft.Extensions.DependencyInjection.Abstractions": {
+		{
-			Name:         "Microsoft.Extensions.DependencyInjection.Abstractions",
+			Name:         "Microsoft.Extensions.Logging.Abstractions",
 			Version:      "6.0.0",
+			PURL:         "pkg:dotnet/Microsoft.Extensions.Logging.Abstractions@6.0.0",
+			Locations:    fixtureLocationSet,
 			Language:     pkg.Dotnet,
 			Type:         pkg.DotnetPkg,
 			MetadataType: pkg.DotnetDepsMetadataType,
 			Metadata: pkg.DotnetDepsMetadata{
-				Name:     "Microsoft.Extensions.DependencyInjection",
+				Name:     "Microsoft.Extensions.Logging.Abstractions",
 				Version:  "6.0.0",
-				Sha512:   "sha512-xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==",
+				Sha512:   "sha512-/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==",
-				Path:     "microsoft.extensions.dependencyinjection.abstractions/6.0.0",
+				Path:     "microsoft.extensions.logging.abstractions/6.0.0",
-				HashPath: "microsoft.extensions.dependencyinjection.abstractions.6.0.0.nupkg.sha512",
+				HashPath: "microsoft.extensions.logging.abstractions.6.0.0.nupkg.sha512",
 			},
 		},
-		"Microsoft.Extensions.Logging": {
+		{
 			Name:         "Microsoft.Extensions.Logging",
 			Version:      "6.0.0",
+			PURL:         "pkg:dotnet/Microsoft.Extensions.Logging@6.0.0",
+			Locations:    fixtureLocationSet,
 			Language:     pkg.Dotnet,
 			Type:         pkg.DotnetPkg,
 			MetadataType: pkg.DotnetDepsMetadataType,
@@ -71,23 +93,12 @@ func TestParseDotnetDeps(t *testing.T) {
 				HashPath: "microsoft.extensions.logging.6.0.0.nupkg.sha512",
 			},
 		},
-		"Microsoft.Extensions.Logging.Abstractions": {
+
-			Name:         "Microsoft.Extensions.Logging.Abstractions",
+		{
-			Version:      "6.0.0",
-			Language:     pkg.Dotnet,
-			Type:         pkg.DotnetPkg,
-			MetadataType: pkg.DotnetDepsMetadataType,
-			Metadata: pkg.DotnetDepsMetadata{
-				Name:     "Microsoft.Extensions.Logging",
-				Version:  "6.0.0",
-				Sha512:   "sha512-/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==",
-				Path:     "microsoft.extensions.logging.abstractions/6.0.0",
-				HashPath: "microsoft.extensions.logging.abstractions.6.0.0.nupkg.sha512",
-			},
-		},
-		"Microsoft.Extensions.Options": {
 			Name:         "Microsoft.Extensions.Options",
 			Version:      "6.0.0",
+			PURL:         "pkg:dotnet/Microsoft.Extensions.Options@6.0.0",
+			Locations:    fixtureLocationSet,
 			Language:     pkg.Dotnet,
 			Type:         pkg.DotnetPkg,
 			MetadataType: pkg.DotnetDepsMetadataType,
@@ -99,9 +110,11 @@ func TestParseDotnetDeps(t *testing.T) {
 				HashPath: "microsoft.extensions.options.6.0.0.nupkg.sha512",
 			},
 		},
-		"Microsoft.Extensions.Primitives": {
+		{
 			Name:         "Microsoft.Extensions.Primitives",
 			Version:      "6.0.0",
+			PURL:         "pkg:dotnet/Microsoft.Extensions.Primitives@6.0.0",
+			Locations:    fixtureLocationSet,
 			Language:     pkg.Dotnet,
 			Type:         pkg.DotnetPkg,
 			MetadataType: pkg.DotnetDepsMetadataType,
@@ -113,9 +126,11 @@ func TestParseDotnetDeps(t *testing.T) {
 				HashPath: "microsoft.extensions.primitives.6.0.0.nupkg.sha512",
 			},
 		},
-		"Newtonsoft.Json": {
+		{
 			Name:         "Newtonsoft.Json",
 			Version:      "13.0.1",
+			PURL:         "pkg:dotnet/Newtonsoft.Json@13.0.1",
+			Locations:    fixtureLocationSet,
 			Language:     pkg.Dotnet,
 			Type:         pkg.DotnetPkg,
 			MetadataType: pkg.DotnetDepsMetadataType,
@@ -127,23 +142,11 @@ func TestParseDotnetDeps(t *testing.T) {
 				HashPath: "newtonsoft.json.13.0.1.nupkg.sha512",
 			},
 		},
-		"Serilog": {
+		{
-			Name:         "Serilog",
-			Version:      "2.10.0",
-			Language:     pkg.Dotnet,
-			Type:         pkg.DotnetPkg,
-			MetadataType: pkg.DotnetDepsMetadataType,
-			Metadata: pkg.DotnetDepsMetadata{
-				Name:     "Serilog",
-				Version:  "2.10.0",
-				Sha512:   "sha512-+QX0hmf37a0/OZLxM3wL7V6/ADvC1XihXN4Kq/p6d8lCPfgkRdiuhbWlMaFjR9Av0dy5F0+MBeDmDdRZN/YwQA==",
-				Path:     "serilog/2.10.0",
-				HashPath: "serilog.2.10.0.nupkg.sha512",
-			},
-		},
-		"Serilog.Sinks.Console": {
 			Name:         "Serilog.Sinks.Console",
 			Version:      "4.0.1",
+			PURL:         "pkg:dotnet/Serilog.Sinks.Console@4.0.1",
+			Locations:    fixtureLocationSet,
 			Language:     pkg.Dotnet,
 			Type:         pkg.DotnetPkg,
 			MetadataType: pkg.DotnetDepsMetadataType,
@@ -155,9 +158,27 @@ func TestParseDotnetDeps(t *testing.T) {
 				HashPath: "serilog.sinks.console.4.0.1.nupkg.sha512",
 			},
 		},
-		"System.Diagnostics.DiagnosticSource": {
+		{
+			Name:         "Serilog",
+			Version:      "2.10.0",
+			PURL:         "pkg:dotnet/Serilog@2.10.0",
+			Locations:    fixtureLocationSet,
+			Language:     pkg.Dotnet,
+			Type:         pkg.DotnetPkg,
+			MetadataType: pkg.DotnetDepsMetadataType,
+			Metadata: pkg.DotnetDepsMetadata{
+				Name:     "Serilog",
+				Version:  "2.10.0",
+				Sha512:   "sha512-+QX0hmf37a0/OZLxM3wL7V6/ADvC1XihXN4Kq/p6d8lCPfgkRdiuhbWlMaFjR9Av0dy5F0+MBeDmDdRZN/YwQA==",
+				Path:     "serilog/2.10.0",
+				HashPath: "serilog.2.10.0.nupkg.sha512",
+			},
+		},
+		{
 			Name:         "System.Diagnostics.DiagnosticSource",
 			Version:      "6.0.0",
+			PURL:         "pkg:dotnet/System.Diagnostics.DiagnosticSource@6.0.0",
+			Locations:    fixtureLocationSet,
 			Language:     pkg.Dotnet,
 			Type:         pkg.DotnetPkg,
 			MetadataType: pkg.DotnetDepsMetadataType,
@@ -169,9 +190,11 @@ func TestParseDotnetDeps(t *testing.T) {
 				HashPath: "system.diagnostics.diagnosticsource.6.0.0.nupkg.sha512",
 			},
 		},
-		"System.Runtime.CompilerServices.Unsafe": {
+		{
 			Name:         "System.Runtime.CompilerServices.Unsafe",
 			Version:      "6.0.0",
+			PURL:         "pkg:dotnet/System.Runtime.CompilerServices.Unsafe@6.0.0",
+			Locations:    fixtureLocationSet,
 			Language:     pkg.Dotnet,
 			Type:         pkg.DotnetPkg,
 			MetadataType: pkg.DotnetDepsMetadataType,
@@ -185,15 +208,6 @@ func TestParseDotnetDeps(t *testing.T) {
 		},
 	}
 
-	fixture, err := os.Open("test-fixtures/TestLibrary.deps.json")
+	var expectedRelationships []artifact.Relationship
-	if err != nil {
+	pkgtest.TestGenericParser(t, fixture, parseDotnetDeps, expected, expectedRelationships)
-		t.Fatalf("failed to open fixture: %+v", err)
-	}
-
-	actual, _, err := parseDotnetDeps(fixture.Name(), fixture)
-	if err != nil {
-		t.Fatalf("failed to parse deps.json: %+v", err)
-	}
-
-	assertPackagesEqual(t, actual, expected)
 }

syft/pkg/cataloger/internal/pkgtest/assert_packages_equal.go

@@ -0,0 +1,37 @@
+package pkgtest
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+func AssertPackagesEqual(t testing.TB, expected, actual []pkg.Package) {
+	if diff := cmp.Diff(expected, actual,
+		cmpopts.IgnoreFields(pkg.Package{}, "id"), // note: ID is not deterministic for test purposes
+		cmp.Comparer(
+			func(x, y source.LocationSet) bool {
+				xs := x.ToSlice()
+				ys := y.ToSlice()
+
+				if len(xs) != len(ys) {
+					return false
+				}
+				for i, xe := range xs {
+					ye := ys[i]
+					if !(cmp.Equal(xe.Coordinates, ye.Coordinates) && cmp.Equal(xe.VirtualPath, ye.VirtualPath)) {
+						return false
+					}
+				}
+
+				return true
+			},
+		),
+	); diff != "" {
+		t.Errorf("unexpected packages from parsing (-expected +actual)\n%s", diff)
+	}
+}

syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go

@@ -0,0 +1,35 @@
+package pkgtest
+
+import (
+	"os"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/generic"
+	"github.com/anchore/syft/syft/source"
+)
+
+func TestGenericParser(t *testing.T, fixturePath string, parser generic.Parser, expectedPkgs []pkg.Package, expectedRelationships []artifact.Relationship) {
+	TestGenericParserWithEnv(t, fixturePath, parser, nil, expectedPkgs, expectedRelationships)
+}
+
+func TestGenericParserWithEnv(t *testing.T, fixturePath string, parser generic.Parser, env *generic.Environment, expectedPkgs []pkg.Package, expectedRelationships []artifact.Relationship) {
+	fixture, err := os.Open(fixturePath)
+	require.NoError(t, err)
+
+	actualPkgs, actualRelationships, err := parser(nil, env, source.LocationReadCloser{
+		Location:   source.NewLocation(fixture.Name()),
+		ReadCloser: fixture,
+	})
+	require.NoError(t, err)
+
+	AssertPackagesEqual(t, expectedPkgs, actualPkgs)
+
+	if diff := cmp.Diff(expectedRelationships, actualRelationships); diff != "" {
+		t.Errorf("unexpected relationships from parsing (-expected +actual)\n%s", diff)
+	}
+}

syft/pkg/dotnet_deps_metadata.go

@@ -1,10 +1,5 @@
 package pkg
 
-import (
-	"github.com/anchore/packageurl-go"
-	"github.com/anchore/syft/syft/linux"
-)
-
 type DotnetDepsMetadata struct {
 	Name     string `mapstructure:"name" json:"name"`
 	Version  string `mapstructure:"version" json:"version"`
@@ -12,16 +7,3 @@ type DotnetDepsMetadata struct {
 	Sha512   string `mapstructure:"sha512" json:"sha512"`
 	HashPath string `mapstructure:"hashPath" json:"hashPath"`
 }
-
-func (m DotnetDepsMetadata) PackageURL(_ *linux.Release) string {
-	var qualifiers packageurl.Qualifiers
-
-	return packageurl.NewPackageURL(
-		packageurl.TypeDotnet,
-		"",
-		m.Name,
-		m.Version,
-		qualifiers,
-		"",
-	).ToString()
-}

syft/pkg/url_test.go

@@ -35,19 +35,6 @@ func TestPackageURL(t *testing.T) {
 			},
 			expected: "pkg:golang/go.opencensus.io@v0.23.0",
 		},
-		{
-			name: "dotnet",
-			pkg: Package{
-				Name:    "Microsoft.CodeAnalysis.Razor",
-				Version: "2.2.0",
-				Type:    DotnetPkg,
-				Metadata: DotnetDepsMetadata{
-					Name:    "Microsoft.CodeAnalysis.Razor",
-					Version: "2.2.0",
-				},
-			},
-			expected: "pkg:dotnet/Microsoft.CodeAnalysis.Razor@2.2.0",
-		},
 		{
 			name: "python",
 			pkg: Package{
@@ -211,6 +198,7 @@ func TestPackageURL(t *testing.T) {
 	expectedTypes.Remove(string(ApkPkg))
 	expectedTypes.Remove(string(ConanPkg))
 	expectedTypes.Remove(string(DartPubPkg))
+	expectedTypes.Remove(string(DotnetPkg))
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {