python-cataloger: normalize package names (#3069)

Signed-off-by: mikcl <mikesmikes400@gmail.com>

cmd/syft/internal/test/integration/catalog_packages_cases_test.go

@@ -35,7 +35,7 @@ var imageOnlyTestCases = []testCase{
 		pkgType:     pkg.PythonPkg,
 		pkgLanguage: pkg.Python,
 		pkgInfo: map[string]string{
-			"Pygments":     "2.6.1",
+			"pygments":     "2.6.1",
 			"requests":     "2.22.0",
 			"somerequests": "3.22.0",
 			"someotherpkg": "3.19.0",
@@ -172,7 +172,7 @@ var dirOnlyTestCases = []testCase{
 			"passlib":            "1.7.2",
 			"mypy":               "v0.770",
 			// common to image and directory
-			"Pygments":     "2.6.1",
+			"pygments":     "2.6.1",
 			"requests":     "2.22.0",
 			"somerequests": "3.22.0",
 			"someotherpkg": "3.19.0",

syft/pkg/cataloger/python/cataloger_test.go

@@ -119,9 +119,9 @@ func Test_PackageCataloger(t *testing.T) {
 				"test-fixtures/dist-info/direct_url.json",
 			},
 			expectedPackage: pkg.Package{
-				Name:     "Pygments",
+				Name:     "pygments",
 				Version:  "2.6.1",
-				PURL:     "pkg:pypi/Pygments@2.6.1?vcs_url=git%2Bhttps://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+				PURL:     "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 				Type:     pkg.PythonPkg,
 				Language: pkg.Python,
 				Licenses: pkg.NewLicenseSet(
@@ -161,9 +161,9 @@ func Test_PackageCataloger(t *testing.T) {
 				"test-fixtures/casesensitive/DIST-INFO/direct_url.json",
 			},
 			expectedPackage: pkg.Package{
-				Name:     "Pygments",
+				Name:     "pygments",
 				Version:  "2.6.1",
-				PURL:     "pkg:pypi/Pygments@2.6.1?vcs_url=git%2Bhttps://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+				PURL:     "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 				Type:     pkg.PythonPkg,
 				Language: pkg.Python,
 				Licenses: pkg.NewLicenseSet(
@@ -199,9 +199,9 @@ func Test_PackageCataloger(t *testing.T) {
 				"test-fixtures/malformed-record/dist-info/RECORD",
 			},
 			expectedPackage: pkg.Package{
-				Name:     "Pygments",
+				Name:     "pygments",
 				Version:  "2.6.1",
-				PURL:     "pkg:pypi/Pygments@2.6.1",
+				PURL:     "pkg:pypi/pygments@2.6.1",
 				Type:     pkg.PythonPkg,
 				Language: pkg.Python,
 				Licenses: pkg.NewLicenseSet(
@@ -231,9 +231,9 @@ func Test_PackageCataloger(t *testing.T) {
 			name:     "partial dist-info directory",
 			fixtures: []string{"test-fixtures/partial.dist-info/METADATA"},
 			expectedPackage: pkg.Package{
-				Name:     "Pygments",
+				Name:     "pygments",
 				Version:  "2.6.1",
-				PURL:     "pkg:pypi/Pygments@2.6.1",
+				PURL:     "pkg:pypi/pygments@2.6.1",
 				Type:     pkg.PythonPkg,
 				Language: pkg.Python,
 				Licenses: pkg.NewLicenseSet(

syft/pkg/cataloger/python/package.go

@@ -2,6 +2,8 @@ package python
 
 import (
 	"fmt"
+	"regexp"
+	"strings"
 
 	"github.com/anchore/packageurl-go"
 	"github.com/anchore/syft/internal/licenses"
@@ -10,7 +12,16 @@ import (
 	"github.com/anchore/syft/syft/pkg"
 )
 
+func normalize(name string) string {
+	// https://packaging.python.org/en/latest/specifications/name-normalization/
+	re := regexp.MustCompile(`[-_.]+`)
+	normalized := re.ReplaceAllString(name, "-")
+	return strings.ToLower(normalized)
+}
+
 func newPackageForIndex(name, version string, locations ...file.Location) pkg.Package {
+	name = normalize(name)
+
 	p := pkg.Package{
 		Name:      name,
 		Version:   version,
@@ -26,6 +37,8 @@ func newPackageForIndex(name, version string, locations ...file.Location) pkg.Pa
 }
 
 func newPackageForIndexWithMetadata(name, version string, metadata interface{}, locations ...file.Location) pkg.Package {
+	name = normalize(name)
+
 	p := pkg.Package{
 		Name:      name,
 		Version:   version,
@@ -42,6 +55,8 @@ func newPackageForIndexWithMetadata(name, version string, metadata interface{},
 }
 
 func newPackageForRequirementsWithMetadata(name, version string, metadata pkg.PythonRequirementsEntry, locations ...file.Location) pkg.Package {
+	name = normalize(name)
+
 	p := pkg.Package{
 		Name:      name,
 		Version:   version,
@@ -87,10 +102,12 @@ func newPackageForPackage(resolver file.Resolver, m parsedData, sources ...file.
 		}
 	}
 
+	name := normalize(m.Name)
+
 	p := pkg.Package{
-		Name:      m.Name,
+		Name:      name,
 		Version:   m.Version,
-		PURL:      packageURL(m.Name, m.Version, &m.PythonPackage),
+		PURL:      packageURL(name, m.Version, &m.PythonPackage),
 		Locations: file.NewLocationSet(sources...),
 		Licenses:  licenseSet,
 		Language:  pkg.Python,

syft/pkg/cataloger/python/package_test.go

@@ -44,3 +44,21 @@ func Test_packageURL(t *testing.T) {
 		})
 	}
 }
+
+func Test_normalization(t *testing.T) {
+	normalForm := "friendly-bard"
+	tests := []string{
+		normalForm,
+		"Friendly-Bard",
+		"FRIENDLY-BARD",
+		"friendly.bard",
+		"friendly_bard",
+		"friendly--bard",
+		"FrIeNdLy-._.-bArD",
+	}
+	for _, tt := range tests {
+		t.Run(tt, func(t *testing.T) {
+			assert.Equal(t, normalForm, normalize(tt))
+		})
+	}
+}

syft/pkg/cataloger/python/parse_requirements_test.go

@@ -41,9 +41,9 @@ func TestParseRequirementsTxt(t *testing.T) {
 			},
 		},
 		{
-			Name:      "SomeProject",
+			Name:      "someproject",
 			Version:   "5.4",
-			PURL:      "pkg:pypi/SomeProject@5.4",
+			PURL:      "pkg:pypi/someproject@5.4",
 			Locations: locations,
 			Language:  pkg.Python,
 			Type:      pkg.PythonPkg,
@@ -91,9 +91,9 @@ func TestParseRequirementsTxt(t *testing.T) {
 			},
 		},
 		{
-			Name:      "GithubSampleProject",
+			Name:      "githubsampleproject",
 			Version:   "3.7.1",
-			PURL:      "pkg:pypi/GithubSampleProject@3.7.1",
+			PURL:      "pkg:pypi/githubsampleproject@3.7.1",
 			Locations: locations,
 			Language:  pkg.Python,
 			Type:      pkg.PythonPkg,
@@ -103,6 +103,18 @@ func TestParseRequirementsTxt(t *testing.T) {
 				URL:               "git+https://github.com/owner/repo@releases/tag/v3.7.1",
 			},
 		},
+		{
+			Name:      "friendly-bard",
+			Version:   "1.0.0",
+			PURL:      "pkg:pypi/friendly-bard@1.0.0",
+			Locations: locations,
+			Language:  pkg.Python,
+			Type:      pkg.PythonPkg,
+			Metadata: pkg.PythonRequirementsEntry{
+				Name:              "FrIeNdLy-_-bArD",
+				VersionConstraint: "== 1.0.0",
+			},
+		},
 	}
 
 	var testCases = []struct {
@@ -128,9 +140,9 @@ func TestParseRequirementsTxt(t *testing.T) {
 			},
 			expectedPkgs: append([]pkg.Package{
 				{
-					Name:      "Mopidy-Dirble",
+					Name:      "mopidy-dirble",
 					Version:   "1.1",
-					PURL:      "pkg:pypi/Mopidy-Dirble@1.1",
+					PURL:      "pkg:pypi/mopidy-dirble@1.1",
 					Locations: locations,
 					Language:  pkg.Python,
 					Type:      pkg.PythonPkg,

syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt

@@ -22,3 +22,4 @@ argh==0.26.3 --hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971
 celery[redis, pytest] == 4.4.7 # should remove [redis, pytest]
 requests[security] == 2.8.* ; python_version < "2.7" and sys_platform == "linux"
 GithubSampleProject == 3.7.1 @ git+https://github.com/owner/repo@releases/tag/v3.7.1
+FrIeNdLy-_-bArD == 1.0.0