From 36f95d682895c1a95d949fc9328260861e116c00 Mon Sep 17 00:00:00 2001 From: mikcl <43545032+Mikcl@users.noreply.github.com> Date: Thu, 25 Jul 2024 18:54:13 +0100 Subject: [PATCH] python-cataloger: normalize package names (#3069) Signed-off-by: mikcl --- .../catalog_packages_cases_test.go | 4 ++-- syft/pkg/cataloger/python/cataloger_test.go | 16 ++++++------- syft/pkg/cataloger/python/package.go | 21 ++++++++++++++-- syft/pkg/cataloger/python/package_test.go | 18 ++++++++++++++ .../python/parse_requirements_test.go | 24 ++++++++++++++----- .../test-fixtures/requires/requirements.txt | 1 + 6 files changed, 66 insertions(+), 18 deletions(-) diff --git a/cmd/syft/internal/test/integration/catalog_packages_cases_test.go b/cmd/syft/internal/test/integration/catalog_packages_cases_test.go index 188b4ae45..ff21bd1c1 100644 --- a/cmd/syft/internal/test/integration/catalog_packages_cases_test.go +++ b/cmd/syft/internal/test/integration/catalog_packages_cases_test.go @@ -35,7 +35,7 @@ var imageOnlyTestCases = []testCase{ pkgType: pkg.PythonPkg, pkgLanguage: pkg.Python, pkgInfo: map[string]string{ - "Pygments": "2.6.1", + "pygments": "2.6.1", "requests": "2.22.0", "somerequests": "3.22.0", "someotherpkg": "3.19.0", @@ -172,7 +172,7 @@ var dirOnlyTestCases = []testCase{ "passlib": "1.7.2", "mypy": "v0.770", // common to image and directory - "Pygments": "2.6.1", + "pygments": "2.6.1", "requests": "2.22.0", "somerequests": "3.22.0", "someotherpkg": "3.19.0", diff --git a/syft/pkg/cataloger/python/cataloger_test.go b/syft/pkg/cataloger/python/cataloger_test.go index e1093a55a..f3fc06017 100644 --- a/syft/pkg/cataloger/python/cataloger_test.go +++ b/syft/pkg/cataloger/python/cataloger_test.go @@ -119,9 +119,9 @@ func Test_PackageCataloger(t *testing.T) { "test-fixtures/dist-info/direct_url.json", }, expectedPackage: pkg.Package{ - Name: "Pygments", + Name: "pygments", Version: "2.6.1", - PURL: "pkg:pypi/Pygments@2.6.1?vcs_url=git%2Bhttps://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Type: pkg.PythonPkg, Language: pkg.Python, Licenses: pkg.NewLicenseSet( @@ -161,9 +161,9 @@ func Test_PackageCataloger(t *testing.T) { "test-fixtures/casesensitive/DIST-INFO/direct_url.json", }, expectedPackage: pkg.Package{ - Name: "Pygments", + Name: "pygments", Version: "2.6.1", - PURL: "pkg:pypi/Pygments@2.6.1?vcs_url=git%2Bhttps://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Type: pkg.PythonPkg, Language: pkg.Python, Licenses: pkg.NewLicenseSet( @@ -199,9 +199,9 @@ func Test_PackageCataloger(t *testing.T) { "test-fixtures/malformed-record/dist-info/RECORD", }, expectedPackage: pkg.Package{ - Name: "Pygments", + Name: "pygments", Version: "2.6.1", - PURL: "pkg:pypi/Pygments@2.6.1", + PURL: "pkg:pypi/pygments@2.6.1", Type: pkg.PythonPkg, Language: pkg.Python, Licenses: pkg.NewLicenseSet( @@ -231,9 +231,9 @@ func Test_PackageCataloger(t *testing.T) { name: "partial dist-info directory", fixtures: []string{"test-fixtures/partial.dist-info/METADATA"}, expectedPackage: pkg.Package{ - Name: "Pygments", + Name: "pygments", Version: "2.6.1", - PURL: "pkg:pypi/Pygments@2.6.1", + PURL: "pkg:pypi/pygments@2.6.1", Type: pkg.PythonPkg, Language: pkg.Python, Licenses: pkg.NewLicenseSet( diff --git a/syft/pkg/cataloger/python/package.go b/syft/pkg/cataloger/python/package.go index f2d51ac32..e574d583a 100644 --- a/syft/pkg/cataloger/python/package.go +++ b/syft/pkg/cataloger/python/package.go @@ -2,6 +2,8 @@ package python import ( "fmt" + "regexp" + "strings" "github.com/anchore/packageurl-go" "github.com/anchore/syft/internal/licenses" @@ -10,7 +12,16 @@ import ( "github.com/anchore/syft/syft/pkg" ) +func normalize(name string) string { + // https://packaging.python.org/en/latest/specifications/name-normalization/ + re := regexp.MustCompile(`[-_.]+`) + normalized := re.ReplaceAllString(name, "-") + return strings.ToLower(normalized) +} + func newPackageForIndex(name, version string, locations ...file.Location) pkg.Package { + name = normalize(name) + p := pkg.Package{ Name: name, Version: version, @@ -26,6 +37,8 @@ func newPackageForIndex(name, version string, locations ...file.Location) pkg.Pa } func newPackageForIndexWithMetadata(name, version string, metadata interface{}, locations ...file.Location) pkg.Package { + name = normalize(name) + p := pkg.Package{ Name: name, Version: version, @@ -42,6 +55,8 @@ func newPackageForIndexWithMetadata(name, version string, metadata interface{}, } func newPackageForRequirementsWithMetadata(name, version string, metadata pkg.PythonRequirementsEntry, locations ...file.Location) pkg.Package { + name = normalize(name) + p := pkg.Package{ Name: name, Version: version, @@ -87,10 +102,12 @@ func newPackageForPackage(resolver file.Resolver, m parsedData, sources ...file. } } + name := normalize(m.Name) + p := pkg.Package{ - Name: m.Name, + Name: name, Version: m.Version, - PURL: packageURL(m.Name, m.Version, &m.PythonPackage), + PURL: packageURL(name, m.Version, &m.PythonPackage), Locations: file.NewLocationSet(sources...), Licenses: licenseSet, Language: pkg.Python, diff --git a/syft/pkg/cataloger/python/package_test.go b/syft/pkg/cataloger/python/package_test.go index 0b88336a7..a9b9238ab 100644 --- a/syft/pkg/cataloger/python/package_test.go +++ b/syft/pkg/cataloger/python/package_test.go @@ -44,3 +44,21 @@ func Test_packageURL(t *testing.T) { }) } } + +func Test_normalization(t *testing.T) { + normalForm := "friendly-bard" + tests := []string{ + normalForm, + "Friendly-Bard", + "FRIENDLY-BARD", + "friendly.bard", + "friendly_bard", + "friendly--bard", + "FrIeNdLy-._.-bArD", + } + for _, tt := range tests { + t.Run(tt, func(t *testing.T) { + assert.Equal(t, normalForm, normalize(tt)) + }) + } +} diff --git a/syft/pkg/cataloger/python/parse_requirements_test.go b/syft/pkg/cataloger/python/parse_requirements_test.go index fe6d8c343..ebd67549a 100644 --- a/syft/pkg/cataloger/python/parse_requirements_test.go +++ b/syft/pkg/cataloger/python/parse_requirements_test.go @@ -41,9 +41,9 @@ func TestParseRequirementsTxt(t *testing.T) { }, }, { - Name: "SomeProject", + Name: "someproject", Version: "5.4", - PURL: "pkg:pypi/SomeProject@5.4", + PURL: "pkg:pypi/someproject@5.4", Locations: locations, Language: pkg.Python, Type: pkg.PythonPkg, @@ -91,9 +91,9 @@ func TestParseRequirementsTxt(t *testing.T) { }, }, { - Name: "GithubSampleProject", + Name: "githubsampleproject", Version: "3.7.1", - PURL: "pkg:pypi/GithubSampleProject@3.7.1", + PURL: "pkg:pypi/githubsampleproject@3.7.1", Locations: locations, Language: pkg.Python, Type: pkg.PythonPkg, @@ -103,6 +103,18 @@ func TestParseRequirementsTxt(t *testing.T) { URL: "git+https://github.com/owner/repo@releases/tag/v3.7.1", }, }, + { + Name: "friendly-bard", + Version: "1.0.0", + PURL: "pkg:pypi/friendly-bard@1.0.0", + Locations: locations, + Language: pkg.Python, + Type: pkg.PythonPkg, + Metadata: pkg.PythonRequirementsEntry{ + Name: "FrIeNdLy-_-bArD", + VersionConstraint: "== 1.0.0", + }, + }, } var testCases = []struct { @@ -128,9 +140,9 @@ func TestParseRequirementsTxt(t *testing.T) { }, expectedPkgs: append([]pkg.Package{ { - Name: "Mopidy-Dirble", + Name: "mopidy-dirble", Version: "1.1", - PURL: "pkg:pypi/Mopidy-Dirble@1.1", + PURL: "pkg:pypi/mopidy-dirble@1.1", Locations: locations, Language: pkg.Python, Type: pkg.PythonPkg, diff --git a/syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt b/syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt index 188258bb7..15d60947a 100644 --- a/syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt +++ b/syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt @@ -22,3 +22,4 @@ argh==0.26.3 --hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971 celery[redis, pytest] == 4.4.7 # should remove [redis, pytest] requests[security] == 2.8.* ; python_version < "2.7" and sys_platform == "linux" GithubSampleProject == 3.7.1 @ git+https://github.com/owner/repo@releases/tag/v3.7.1 +FrIeNdLy-_-bArD == 1.0.0