From f85108566819be4bda6d8e575bb3e210432da486 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 3 Apr 2025 10:31:02 -0400 Subject: [PATCH] Expand python license scanning to cover unclaimed files (#3779) * expand python license scanning to cover unclaimed files Signed-off-by: Alex Goodman * speed up tests using the license scanner Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- .../internal/pkgtest/test_generic_parser.go | 33 +- syft/pkg/cataloger/python/cataloger_test.go | 589 +++++++++++------- syft/pkg/cataloger/python/package.go | 35 -- syft/pkg/cataloger/python/parse_wheel_egg.go | 99 +++ .../python/parse_wheel_egg_metadata.go | 17 +- .../python/parse_wheel_egg_metadata_test.go | 45 +- .../python/parse_wheel_egg_record_test.go | 5 +- .../LICENSE.txt | 28 + .../METADATA | 53 ++ .../RECORD | 7 + .../direct_url.json | 0 .../top_level.txt | 0 .../LICENSE.txt | 28 + .../METADATA | 52 ++ .../RECORD | 7 + .../direct_url.json | 0 .../top_level.txt | 0 .../malformed-record/dist-info}/METADATA | 0 .../malformed-record/dist-info/RECORD | 0 .../nested/dist-name/dist-info/LICENSE.txt | 28 + .../nested/dist-name}/dist-info/METADATA | 0 .../nested/dist-name}/dist-info/RECORD | 1 + .../dist-name/dist-info/direct_url.json | 1 + .../nested/dist-name/dist-info/top_level.txt | 2 + .../nested/egg-name/egg-info}/PKG-INFO | 0 .../egg-name/egg-info}/PKG-INFO-INVALID | 0 .../nested/egg-name/egg-info}/RECORD | 0 .../nested/egg-name/egg-info}/top_level.txt | 0 .../no-version}/no-version-py3.8.egg-info | 0 .../partial.dist-info}/METADATA | 0 .../{ => site-packages/test}/test.egg-info | 0 .../uppercase/dist-name/DIST-INFO}/METADATA | 0 .../uppercase/dist-name}/DIST-INFO/RECORD | 0 .../dist-name/DIST-INFO/direct_url.json | 1 + .../dist-name/DIST-INFO/top_level.txt | 2 + .../uppercase/egg-name/EGG-INFO}/PKG-INFO | 0 .../egg-name/EGG-INFO}/PKG-INFO-INVALID | 0 .../uppercase/egg-name/EGG-INFO}/RECORD | 0 .../egg-name/EGG-INFO}/top_level.txt | 0 39 files changed, 730 insertions(+), 303 deletions(-) create mode 100644 syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/LICENSE.txt create mode 100644 syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/METADATA create mode 100644 syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/RECORD rename syft/pkg/cataloger/python/test-fixtures/{casesensitive/DIST-INFO => site-packages/license/with-license-file-declared.dist-info}/direct_url.json (100%) rename syft/pkg/cataloger/python/test-fixtures/{casesensitive/DIST-INFO => site-packages/license/with-license-file-declared.dist-info}/top_level.txt (100%) create mode 100644 syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/LICENSE.txt create mode 100644 syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/METADATA create mode 100644 syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/RECORD rename syft/pkg/cataloger/python/test-fixtures/{dist-info => site-packages/license/without-license-file-declared.dist-info}/direct_url.json (100%) rename syft/pkg/cataloger/python/test-fixtures/{dist-info => site-packages/license/without-license-file-declared.dist-info}/top_level.txt (100%) rename syft/pkg/cataloger/python/test-fixtures/{casesensitive/DIST-INFO => site-packages/malformed-record/dist-info}/METADATA (100%) rename syft/pkg/cataloger/python/test-fixtures/{ => site-packages}/malformed-record/dist-info/RECORD (100%) create mode 100644 syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/LICENSE.txt rename syft/pkg/cataloger/python/test-fixtures/{ => site-packages/nested/dist-name}/dist-info/METADATA (100%) rename syft/pkg/cataloger/python/test-fixtures/{ => site-packages/nested/dist-name}/dist-info/RECORD (81%) create mode 100644 syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/direct_url.json create mode 100644 syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/top_level.txt rename syft/pkg/cataloger/python/test-fixtures/{casesensitive/EGG-INFO => site-packages/nested/egg-name/egg-info}/PKG-INFO (100%) rename syft/pkg/cataloger/python/test-fixtures/{casesensitive/EGG-INFO => site-packages/nested/egg-name/egg-info}/PKG-INFO-INVALID (100%) rename syft/pkg/cataloger/python/test-fixtures/{casesensitive/EGG-INFO => site-packages/nested/egg-name/egg-info}/RECORD (100%) rename syft/pkg/cataloger/python/test-fixtures/{casesensitive/EGG-INFO => site-packages/nested/egg-name/egg-info}/top_level.txt (100%) rename syft/pkg/cataloger/python/test-fixtures/{ => site-packages/no-version}/no-version-py3.8.egg-info (100%) rename syft/pkg/cataloger/python/test-fixtures/{malformed-record/dist-info => site-packages/partial.dist-info}/METADATA (100%) rename syft/pkg/cataloger/python/test-fixtures/{ => site-packages/test}/test.egg-info (100%) rename syft/pkg/cataloger/python/test-fixtures/{partial.dist-info => site-packages/uppercase/dist-name/DIST-INFO}/METADATA (100%) rename syft/pkg/cataloger/python/test-fixtures/{casesensitive => site-packages/uppercase/dist-name}/DIST-INFO/RECORD (100%) create mode 100644 syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/direct_url.json create mode 100644 syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/top_level.txt rename syft/pkg/cataloger/python/test-fixtures/{egg-info => site-packages/uppercase/egg-name/EGG-INFO}/PKG-INFO (100%) rename syft/pkg/cataloger/python/test-fixtures/{egg-info => site-packages/uppercase/egg-name/EGG-INFO}/PKG-INFO-INVALID (100%) rename syft/pkg/cataloger/python/test-fixtures/{egg-info => site-packages/uppercase/egg-name/EGG-INFO}/RECORD (100%) rename syft/pkg/cataloger/python/test-fixtures/{egg-info => site-packages/uppercase/egg-name/EGG-INFO}/top_level.txt (100%) diff --git a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go index 6397d4d04..72a50e59e 100644 --- a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go +++ b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go @@ -7,15 +7,18 @@ import ( "os" "sort" "strings" + "sync" "testing" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/licensecheck" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/internal/cmptest" + "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/relationship" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -27,6 +30,11 @@ import ( "github.com/anchore/syft/syft/source/stereoscopesource" ) +var ( + once sync.Once + licenseScanner *licenses.Scanner +) + type CatalogTester struct { expectedPkgs []pkg.Package expectedRelationships []artifact.Relationship @@ -44,10 +52,26 @@ type CatalogTester struct { licenseComparer cmptest.LicenseComparer packageStringer func(pkg.Package) string customAssertions []func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship) + context context.Context +} + +func Context() context.Context { + once.Do(func() { + // most of the time in testing is initializing the scanner. Let's do that just once + sc := &licenses.ScannerConfig{Scanner: licensecheck.Scan, CoverageThreshold: 75} + scanner, err := licenses.NewScanner(sc) + if err != nil { + panic("unable to setup licences scanner for testing") + } + licenseScanner = &scanner + }) + + return licenses.SetContextLicenseScanner(context.Background(), *licenseScanner) } func NewCatalogTester() *CatalogTester { return &CatalogTester{ + context: Context(), locationComparer: cmptest.DefaultLocationComparer, licenseComparer: cmptest.DefaultLicenseComparer, packageStringer: stringPackage, @@ -64,6 +88,11 @@ func NewCatalogTester() *CatalogTester { } } +func (p *CatalogTester) WithContext(ctx context.Context) *CatalogTester { + p.context = ctx + return p +} + func (p *CatalogTester) FromDirectory(t *testing.T, path string) *CatalogTester { t.Helper() @@ -202,7 +231,7 @@ func (p *CatalogTester) IgnoreUnfulfilledPathResponses(paths ...string) *Catalog func (p *CatalogTester) TestParser(t *testing.T, parser generic.Parser) { t.Helper() - pkgs, relationships, err := parser(context.Background(), p.resolver, p.env, p.reader) + pkgs, relationships, err := parser(p.context, p.resolver, p.env, p.reader) // only test for errors if explicitly requested if p.wantErr != nil { p.wantErr(t, err) @@ -215,7 +244,7 @@ func (p *CatalogTester) TestCataloger(t *testing.T, cataloger pkg.Cataloger) { resolver := NewObservingResolver(p.resolver) - pkgs, relationships, err := cataloger.Catalog(context.Background(), resolver) + pkgs, relationships, err := cataloger.Catalog(p.context, resolver) // this is a minimum set, the resolver may return more that just this list for _, path := range p.expectedPathResponses { diff --git a/syft/pkg/cataloger/python/cataloger_test.go b/syft/pkg/cataloger/python/cataloger_test.go index e2a2d2a29..474042055 100644 --- a/syft/pkg/cataloger/python/cataloger_test.go +++ b/syft/pkg/cataloger/python/cataloger_test.go @@ -1,7 +1,6 @@ package python import ( - "context" "fmt" "path" "testing" @@ -15,264 +14,383 @@ import ( func Test_PackageCataloger(t *testing.T) { tests := []struct { - name string - fixtures []string - expectedPackage pkg.Package + name string + fixture string + expectedPackages []pkg.Package }{ { - name: "egg-file-no-version", - fixtures: []string{"test-fixtures/no-version-py3.8.egg-info"}, - expectedPackage: pkg.Package{ - Name: "no-version", - PURL: "pkg:pypi/no-version", - Type: pkg.PythonPkg, - Language: pkg.Python, - FoundBy: "python-installed-package-cataloger", - Metadata: pkg.PythonPackage{ - Name: "no-version", - SitePackagesRootPath: "test-fixtures", - }, - }, - }, - { - name: "egg-info directory", - fixtures: []string{ - "test-fixtures/egg-info/PKG-INFO", - "test-fixtures/egg-info/RECORD", - "test-fixtures/egg-info/top_level.txt", - }, - expectedPackage: pkg.Package{ - Name: "requests", - Version: "2.22.0", - PURL: "pkg:pypi/requests@2.22.0", - Type: pkg.PythonPkg, - Language: pkg.Python, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/egg-info/PKG-INFO")), - ), - FoundBy: "python-installed-package-cataloger", - Metadata: pkg.PythonPackage{ - Name: "requests", - Version: "2.22.0", - Platform: "UNKNOWN", - Author: "Kenneth Reitz", - AuthorEmail: "me@kennethreitz.org", - SitePackagesRootPath: "test-fixtures", - Files: []pkg.PythonFileRecord{ - {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, - {Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, - {Path: "requests/__pycache__/__version__.cpython-38.pyc"}, - {Path: "requests/__pycache__/utils.cpython-38.pyc"}, - {Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, - {Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, + name: "egg-file-no-version", + fixture: "test-fixtures/site-packages/no-version", + expectedPackages: []pkg.Package{ + { + Name: "no-version", + Locations: file.NewLocationSet(file.NewLocation("no-version-py3.8.egg-info")), + PURL: "pkg:pypi/no-version", + Type: pkg.PythonPkg, + Language: pkg.Python, + FoundBy: "python-installed-package-cataloger", + Metadata: pkg.PythonPackage{ + Name: "no-version", + SitePackagesRootPath: ".", // requires scanning the grandparent directory to get a valid path }, - TopLevelPackages: []string{"requests"}, - RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", - ProvidesExtra: []string{"security", "socks"}, }, }, }, { - name: "egg-info directory case sensitive", - fixtures: []string{ - "test-fixtures/casesensitive/EGG-INFO/PKG-INFO", - "test-fixtures/casesensitive/EGG-INFO/RECORD", - "test-fixtures/casesensitive/EGG-INFO/top_level.txt", - }, - expectedPackage: pkg.Package{ - Name: "requests", - Version: "2.22.0", - PURL: "pkg:pypi/requests@2.22.0", - Type: pkg.PythonPkg, - Language: pkg.Python, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/casesensitive/EGG-INFO/PKG-INFO")), - ), - FoundBy: "python-installed-package-cataloger", - Metadata: pkg.PythonPackage{ - Name: "requests", - Version: "2.22.0", - Platform: "UNKNOWN", - Author: "Kenneth Reitz", - AuthorEmail: "me@kennethreitz.org", - SitePackagesRootPath: "test-fixtures/casesensitive", - Files: []pkg.PythonFileRecord{ - {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, - {Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, - {Path: "requests/__pycache__/__version__.cpython-38.pyc"}, - {Path: "requests/__pycache__/utils.cpython-38.pyc"}, - {Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, - {Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, - }, - TopLevelPackages: []string{"requests"}, - RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", - ProvidesExtra: []string{"security", "socks"}, - }, - }, - }, - { - name: "dist-info directory", - fixtures: []string{ - "test-fixtures/dist-info/METADATA", - "test-fixtures/dist-info/RECORD", - "test-fixtures/dist-info/top_level.txt", - "test-fixtures/dist-info/direct_url.json", - }, - expectedPackage: pkg.Package{ - Name: "pygments", - Version: "2.6.1", - PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - Type: pkg.PythonPkg, - Language: pkg.Python, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/dist-info/METADATA")), - ), - FoundBy: "python-installed-package-cataloger", - Metadata: pkg.PythonPackage{ - Name: "Pygments", - Version: "2.6.1", - Platform: "any", - Author: "Georg Brandl", - AuthorEmail: "georg@python.org", - SitePackagesRootPath: "test-fixtures", - Files: []pkg.PythonFileRecord{ - {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, - {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, - {Path: "Pygments-2.6.1.dist-info/RECORD"}, - {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, - {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, + name: "dist-info+egg-info site-packages directory", + fixture: "test-fixtures/site-packages/nested", + expectedPackages: []pkg.Package{ + { + Name: "pygments", + Version: "2.6.1", + PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Type: pkg.PythonPkg, + Language: pkg.Python, + Locations: file.NewLocationSet( + file.NewLocation("dist-name/dist-info/METADATA"), + file.NewLocation("dist-name/dist-info/RECORD"), + file.NewLocation("dist-name/dist-info/direct_url.json"), + file.NewLocation("dist-name/dist-info/top_level.txt"), + ), + Licenses: pkg.NewLicenseSet( + // here we only used the license that was declared in the METADATA file, we did not go searching for other licenses + // this is the better source of truth when there is no explicit LicenseFile given + pkg.NewLicenseFromLocations("BSD License", file.NewLocation("dist-name/dist-info/METADATA")), + ), + FoundBy: "python-installed-package-cataloger", + Metadata: pkg.PythonPackage{ + Name: "Pygments", + Version: "2.6.1", + Platform: "any", + Author: "Georg Brandl", + AuthorEmail: "georg@python.org", + SitePackagesRootPath: "dist-name", + Files: []pkg.PythonFileRecord{ + {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, + {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "Pygments-2.6.1.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "Pygments-2.6.1.dist-info/RECORD"}, + {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, + {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, - {Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"}, + {Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"}, + }, + TopLevelPackages: []string{"pygments", "something_else"}, + DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + RequiresPython: ">=3.5", + RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"}, + ProvidesExtra: []string{"html5lib", "lxml"}, + }, + }, + { + Name: "requests", + Version: "2.22.0", + PURL: "pkg:pypi/requests@2.22.0", + Type: pkg.PythonPkg, + Language: pkg.Python, + Locations: file.NewLocationSet( + file.NewLocation("egg-name/egg-info/PKG-INFO"), + file.NewLocation("egg-name/egg-info/RECORD"), + file.NewLocation("egg-name/egg-info/top_level.txt"), + ), + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("egg-name/egg-info/PKG-INFO")), + ), + FoundBy: "python-installed-package-cataloger", + Metadata: pkg.PythonPackage{ + Name: "requests", + Version: "2.22.0", + Platform: "UNKNOWN", + Author: "Kenneth Reitz", + AuthorEmail: "me@kennethreitz.org", + SitePackagesRootPath: "egg-name", + Files: []pkg.PythonFileRecord{ + {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, + {Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, + {Path: "requests/__pycache__/__version__.cpython-38.pyc"}, + {Path: "requests/__pycache__/utils.cpython-38.pyc"}, + {Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, + {Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, + }, + TopLevelPackages: []string{"requests"}, + RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", + ProvidesExtra: []string{"security", "socks"}, }, - TopLevelPackages: []string{"pygments", "something_else"}, - DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, - RequiresPython: ">=3.5", - RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"}, - ProvidesExtra: []string{"html5lib", "lxml"}, }, }, }, { - name: "dist-info directory case sensitive", - fixtures: []string{ - "test-fixtures/casesensitive/DIST-INFO/METADATA", - "test-fixtures/casesensitive/DIST-INFO/RECORD", - "test-fixtures/casesensitive/DIST-INFO/top_level.txt", - "test-fixtures/casesensitive/DIST-INFO/direct_url.json", - }, - expectedPackage: pkg.Package{ - Name: "pygments", - Version: "2.6.1", - PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - Type: pkg.PythonPkg, - Language: pkg.Python, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/casesensitive/DIST-INFO/METADATA")), - ), - FoundBy: "python-installed-package-cataloger", - Metadata: pkg.PythonPackage{ - Name: "Pygments", - Version: "2.6.1", - Platform: "any", - Author: "Georg Brandl", - AuthorEmail: "georg@python.org", - SitePackagesRootPath: "test-fixtures/casesensitive", - Files: []pkg.PythonFileRecord{ - {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, - {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, - {Path: "Pygments-2.6.1.dist-info/RECORD"}, - {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, - {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, + name: "DIST-INFO+EGG-INFO site-packages directory (case insensitive)", + fixture: "test-fixtures/site-packages/uppercase", + expectedPackages: []pkg.Package{ + { + Name: "pygments", + Version: "2.6.1", + PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Type: pkg.PythonPkg, + Language: pkg.Python, + Locations: file.NewLocationSet( + file.NewLocation("dist-name/DIST-INFO/METADATA"), + file.NewLocation("dist-name/DIST-INFO/RECORD"), + file.NewLocation("dist-name/DIST-INFO/direct_url.json"), + file.NewLocation("dist-name/DIST-INFO/top_level.txt"), + ), + Licenses: pkg.NewLicenseSet( + // here we only used the license that was declared in the METADATA file, we did not go searching for other licenses + // this is the better source of truth when there is no explicit LicenseFile given + pkg.NewLicenseFromLocations("BSD License", file.NewLocation("dist-name/DIST-INFO/METADATA")), + ), + FoundBy: "python-installed-package-cataloger", + Metadata: pkg.PythonPackage{ + Name: "Pygments", + Version: "2.6.1", + Platform: "any", + Author: "Georg Brandl", + AuthorEmail: "georg@python.org", + SitePackagesRootPath: "dist-name", + Files: []pkg.PythonFileRecord{ + {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, + {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "Pygments-2.6.1.dist-info/RECORD"}, + {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, + {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, - {Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"}, + {Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"}, + }, + TopLevelPackages: []string{"pygments", "something_else"}, + DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + RequiresPython: ">=3.5", + }, + }, + { + Name: "requests", + Version: "2.22.0", + PURL: "pkg:pypi/requests@2.22.0", + Type: pkg.PythonPkg, + Language: pkg.Python, + Locations: file.NewLocationSet( + file.NewLocation("egg-name/EGG-INFO/PKG-INFO"), + file.NewLocation("egg-name/EGG-INFO/RECORD"), + file.NewLocation("egg-name/EGG-INFO/top_level.txt"), + ), + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("egg-name/EGG-INFO/PKG-INFO")), + ), + FoundBy: "python-installed-package-cataloger", + Metadata: pkg.PythonPackage{ + Name: "requests", + Version: "2.22.0", + Platform: "UNKNOWN", + Author: "Kenneth Reitz", + AuthorEmail: "me@kennethreitz.org", + SitePackagesRootPath: "egg-name", + Files: []pkg.PythonFileRecord{ + {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, + {Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, + {Path: "requests/__pycache__/__version__.cpython-38.pyc"}, + {Path: "requests/__pycache__/utils.cpython-38.pyc"}, + {Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"}, + {Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"}, + }, + TopLevelPackages: []string{"requests"}, + RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", + ProvidesExtra: []string{"security", "socks"}, }, - TopLevelPackages: []string{"pygments", "something_else"}, - DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, - RequiresPython: ">=3.5", }, }, }, { - name: "malformed-record", - fixtures: []string{ - "test-fixtures/malformed-record/dist-info/METADATA", - "test-fixtures/malformed-record/dist-info/RECORD", - }, - expectedPackage: pkg.Package{ - Name: "pygments", - Version: "2.6.1", - PURL: "pkg:pypi/pygments@2.6.1", - Type: pkg.PythonPkg, - Language: pkg.Python, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/malformed-record/dist-info/METADATA")), - ), - FoundBy: "python-installed-package-cataloger", - Metadata: pkg.PythonPackage{ - Name: "Pygments", - Version: "2.6.1", - Platform: "any", - Author: "Georg Brandl", - AuthorEmail: "georg@python.org", - SitePackagesRootPath: "test-fixtures/malformed-record", - Files: []pkg.PythonFileRecord{ - {Path: "flask/json/tag.py", Digest: &pkg.PythonFileDigest{"sha256", "9ehzrmt5k7hxf7ZEK0NOs3swvQyU9fWNe-pnYe69N60"}, Size: "8223"}, - {Path: "../../Scripts/flask.exe", Digest: &pkg.PythonFileDigest{"sha256", "mPrbVeZCDX20himZ_bRai1nCs_tgr7jHIOGZlcgn-T4"}, Size: "93063"}, - {Path: "../../Scripts/flask.exe", Size: "89470", Digest: &pkg.PythonFileDigest{"sha256", "jvqh4N3qOqXLlq40i6ZOLCY9tAOwfwdzIpLDYhRjoqQ"}}, - {Path: "Flask-1.0.2.dist-info/INSTALLER", Size: "4", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}}, + name: "detect licenses", + fixture: "test-fixtures/site-packages/license", + expectedPackages: []pkg.Package{ + { + Name: "pygments", + Version: "2.6.1", + PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Type: pkg.PythonPkg, + Language: pkg.Python, + Locations: file.NewLocationSet( + file.NewLocation("with-license-file-declared.dist-info/METADATA"), // the LicenseFile is declared in the METADATA file + file.NewLocation("with-license-file-declared.dist-info/RECORD"), + file.NewLocation("with-license-file-declared.dist-info/top_level.txt"), + file.NewLocation("with-license-file-declared.dist-info/direct_url.json"), + ), + Licenses: pkg.NewLicenseSet( + pkg.License{ + Value: "BSD-3-Clause", + SPDXExpression: "BSD-3-Clause", + Type: "concluded", + // we read the path from the LicenseFile field in the METADATA file, then read the license file directly + Locations: file.NewLocationSet(file.NewLocation("with-license-file-declared.dist-info/LICENSE.txt")), + }, + ), + FoundBy: "python-installed-package-cataloger", + Metadata: pkg.PythonPackage{ + Name: "Pygments", + Version: "2.6.1", + Platform: "any", + Author: "Georg Brandl", + AuthorEmail: "georg@python.org", + SitePackagesRootPath: ".", + Files: []pkg.PythonFileRecord{ + {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, + {Path: "with-license-file-declared.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "with-license-file-declared.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "with-license-file-declared.dist-info/RECORD"}, + {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, + {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, + + {Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"}, + }, + TopLevelPackages: []string{"pygments", "something_else"}, + DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + RequiresPython: ">=3.5", + RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"}, + ProvidesExtra: []string{"html5lib", "lxml"}, + }, + }, + { + Name: "pygments", + Version: "2.6.1", + PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Type: pkg.PythonPkg, + Language: pkg.Python, + Locations: file.NewLocationSet( + file.NewLocation("without-license-file-declared.dist-info/METADATA"), // the LicenseFile is declared in the METADATA file + file.NewLocation("without-license-file-declared.dist-info/RECORD"), + file.NewLocation("without-license-file-declared.dist-info/top_level.txt"), + file.NewLocation("without-license-file-declared.dist-info/direct_url.json"), + ), + Licenses: pkg.NewLicenseSet( + pkg.License{ + Value: "BSD-3-Clause", + SPDXExpression: "BSD-3-Clause", + Type: "concluded", + // we discover license files automatically + Locations: file.NewLocationSet(file.NewLocation("without-license-file-declared.dist-info/LICENSE.txt")), + }, + ), + FoundBy: "python-installed-package-cataloger", + Metadata: pkg.PythonPackage{ + Name: "Pygments", + Version: "2.6.1", + Platform: "any", + Author: "Georg Brandl", + AuthorEmail: "georg@python.org", + SitePackagesRootPath: ".", + Files: []pkg.PythonFileRecord{ + {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, + {Path: "without-license-file-declared.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "without-license-file-declared.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "without-license-file-declared.dist-info/RECORD"}, + {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, + {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, + + {Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"}, + }, + TopLevelPackages: []string{"pygments", "something_else"}, + DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + RequiresPython: ">=3.5", + RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"}, + ProvidesExtra: []string{"html5lib", "lxml"}, + }, + }, + }, + }, + { + name: "malformed-record", + fixture: "test-fixtures/site-packages/malformed-record", + expectedPackages: []pkg.Package{ + { + Name: "pygments", + Version: "2.6.1", + PURL: "pkg:pypi/pygments@2.6.1", + Type: pkg.PythonPkg, + Language: pkg.Python, + Locations: file.NewLocationSet( + file.NewLocation("dist-info/METADATA"), + file.NewLocation("dist-info/RECORD"), + ), + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("BSD License", file.NewLocation("dist-info/METADATA")), + ), + FoundBy: "python-installed-package-cataloger", + Metadata: pkg.PythonPackage{ + Name: "Pygments", + Version: "2.6.1", + Platform: "any", + Author: "Georg Brandl", + AuthorEmail: "georg@python.org", + SitePackagesRootPath: ".", + Files: []pkg.PythonFileRecord{ + {Path: "flask/json/tag.py", Digest: &pkg.PythonFileDigest{"sha256", "9ehzrmt5k7hxf7ZEK0NOs3swvQyU9fWNe-pnYe69N60"}, Size: "8223"}, + {Path: "../../Scripts/flask.exe", Digest: &pkg.PythonFileDigest{"sha256", "mPrbVeZCDX20himZ_bRai1nCs_tgr7jHIOGZlcgn-T4"}, Size: "93063"}, + {Path: "../../Scripts/flask.exe", Size: "89470", Digest: &pkg.PythonFileDigest{"sha256", "jvqh4N3qOqXLlq40i6ZOLCY9tAOwfwdzIpLDYhRjoqQ"}}, + {Path: "Flask-1.0.2.dist-info/INSTALLER", Size: "4", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}}, + }, + RequiresPython: ">=3.5", }, - RequiresPython: ">=3.5", }, }, }, { // in cases where the metadata file is available and the record is not we should still record there is a package // additionally empty top_level.txt files should not result in an error - name: "partial dist-info directory", - fixtures: []string{"test-fixtures/partial.dist-info/METADATA"}, - expectedPackage: pkg.Package{ - Name: "pygments", - Version: "2.6.1", - PURL: "pkg:pypi/pygments@2.6.1", - Type: pkg.PythonPkg, - Language: pkg.Python, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/partial.dist-info/METADATA")), - ), - FoundBy: "python-installed-package-cataloger", - Metadata: pkg.PythonPackage{ - Name: "Pygments", - Version: "2.6.1", - Platform: "any", - Author: "Georg Brandl", - AuthorEmail: "georg@python.org", - SitePackagesRootPath: "test-fixtures", - RequiresPython: ">=3.5", + name: "partial dist-info directory", + fixture: "test-fixtures/site-packages/partial.dist-info", + expectedPackages: []pkg.Package{ + { + Name: "pygments", + Version: "2.6.1", + PURL: "pkg:pypi/pygments@2.6.1", + Type: pkg.PythonPkg, + Language: pkg.Python, + Locations: file.NewLocationSet( + file.NewLocation("METADATA"), + ), + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("BSD License", file.NewLocation("METADATA")), + ), + FoundBy: "python-installed-package-cataloger", + Metadata: pkg.PythonPackage{ + Name: "Pygments", + Version: "2.6.1", + Platform: "any", + Author: "Georg Brandl", + AuthorEmail: "georg@python.org", + SitePackagesRootPath: ".", + RequiresPython: ">=3.5", + }, }, }, }, { - name: "egg-info regular file", - fixtures: []string{"test-fixtures/test.egg-info"}, - expectedPackage: pkg.Package{ - Name: "requests", - Version: "2.22.0", - PURL: "pkg:pypi/requests@2.22.0", - Type: pkg.PythonPkg, - Language: pkg.Python, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/test.egg-info")), - ), - FoundBy: "python-installed-package-cataloger", - Metadata: pkg.PythonPackage{ - Name: "requests", - Version: "2.22.0", - Platform: "UNKNOWN", - Author: "Kenneth Reitz", - AuthorEmail: "me@kennethreitz.org", - SitePackagesRootPath: "test-fixtures", - RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", - ProvidesExtra: []string{"security", "socks"}, + name: "egg-info regular file", + fixture: "test-fixtures/site-packages/test", + expectedPackages: []pkg.Package{ + { + Name: "requests", + Version: "2.22.0", + PURL: "pkg:pypi/requests@2.22.0", + Type: pkg.PythonPkg, + Language: pkg.Python, + Locations: file.NewLocationSet( + file.NewLocation("test.egg-info"), + ), + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test.egg-info")), + ), + FoundBy: "python-installed-package-cataloger", + Metadata: pkg.PythonPackage{ + Name: "requests", + Version: "2.22.0", + Platform: "UNKNOWN", + Author: "Kenneth Reitz", + AuthorEmail: "me@kennethreitz.org", + SitePackagesRootPath: ".", + RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", + ProvidesExtra: []string{"security", "socks"}, + }, }, }, }, @@ -280,16 +398,9 @@ func Test_PackageCataloger(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - resolver := file.NewMockResolverForPaths(test.fixtures...) - - locations, err := resolver.FilesByPath(test.fixtures...) - require.NoError(t, err) - - test.expectedPackage.Locations = file.NewLocationSet(locations...) - pkgtest.NewCatalogTester(). - WithResolver(resolver). - Expects([]pkg.Package{test.expectedPackage}, nil). + FromDirectory(t, test.fixture). + Expects(test.expectedPackages, nil). TestCataloger(t, NewInstalledPackageCataloger()) }) } @@ -311,7 +422,7 @@ func Test_PackageCataloger_IgnorePackage(t *testing.T) { t.Run(test.MetadataFixture, func(t *testing.T) { resolver := file.NewMockResolverForPaths(test.MetadataFixture) - actual, _, err := NewInstalledPackageCataloger().Catalog(context.Background(), resolver) + actual, _, err := NewInstalledPackageCataloger().Catalog(pkgtest.Context(), resolver) require.NoError(t, err) if len(actual) != 0 { diff --git a/syft/pkg/cataloger/python/package.go b/syft/pkg/cataloger/python/package.go index a0c146494..0c724ca81 100644 --- a/syft/pkg/cataloger/python/package.go +++ b/syft/pkg/cataloger/python/package.go @@ -1,14 +1,11 @@ package python import ( - "context" "fmt" "regexp" "strings" "github.com/anchore/packageurl-go" - "github.com/anchore/syft/internal/licenses" - "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" ) @@ -92,38 +89,6 @@ func newPackageForPackage(m parsedData, licenses pkg.LicenseSet, sources ...file return p } -func findLicenses(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, m parsedData) pkg.LicenseSet { - var licenseSet pkg.LicenseSet - - switch { - case m.LicenseExpression != "": - licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(m.LicenseLocation, m.LicenseExpression)...) - case m.Licenses != "": - licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(m.LicenseLocation, m.Licenses)...) - case m.LicenseLocation.Path() != "": - // If we have a license file then resolve and parse it - found, err := resolver.FilesByPath(m.LicenseLocation.Path()) - if err != nil { - log.WithFields("error", err, "path", m.LicenseLocation.Path()).Trace("unable to resolve python license") - } - if len(found) > 0 { - metadataContents, err := resolver.FileContentsByLocation(found[0]) - if err == nil { - parsed, err := scanner.PkgSearch(ctx, file.NewLocationReadCloser(m.LicenseLocation, metadataContents)) - if err != nil { - log.WithFields("error", err, "path", m.LicenseLocation.Path()).Trace("unable to parse a license from the file") - } - if len(parsed) > 0 { - licenseSet = pkg.NewLicenseSet(parsed...) - } - } else { - log.WithFields("error", err, "path", m.LicenseLocation.Path()).Trace("unable to read file contents") - } - } - } - return licenseSet -} - func packageURL(name, version string, m *pkg.PythonPackage) string { // generate a purl from the package data pURL := packageurl.NewPackageURL( diff --git a/syft/pkg/cataloger/python/parse_wheel_egg.go b/syft/pkg/cataloger/python/parse_wheel_egg.go index a593778eb..e01139a79 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg.go @@ -6,7 +6,12 @@ import ( "encoding/json" "fmt" "io" + "path" "path/filepath" + "sort" + "strings" + + "github.com/scylladb/go-set/strset" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/licenses" @@ -247,3 +252,97 @@ func assembleEggOrWheelMetadata(resolver file.Resolver, metadataLocation file.Lo pd.DirectURLOrigin = d return &pd, sources, nil } + +func findLicenses(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, m parsedData) pkg.LicenseSet { + var licenseSet pkg.LicenseSet + + licenseLocations := file.NewLocationSet() + if m.LicenseFilePath != "" { + locs, err := resolver.FilesByPath(m.LicenseFilePath) + if err != nil { + log.WithFields("error", err, "path", m.LicenseFilePath).Trace("unable to resolve python license file") + } else { + licenseLocations.Add(locs...) + } + } + + switch { + case m.LicenseExpression != "" || m.Licenses != "": + licenseSet = getLicenseSetFromValues(licenseLocations.ToSlice(), m.LicenseExpression, m.Licenses) + case !licenseLocations.Empty(): + licenseSet = getLicenseSetFromFiles(ctx, scanner, resolver, licenseLocations.ToSlice()...) + + default: + // search for known license paths from RECORDS file + licenseNames := strset.New() + for _, n := range licenses.FileNames() { + licenseNames.Add(strings.ToLower(n)) + } + parent := path.Base(path.Dir(m.DistInfoLocation.Path())) + candidatePaths := strset.New() + for _, f := range m.Files { + if !strings.HasPrefix(f.Path, parent) || strings.Count(f.Path, "/") > 1 { + continue + } + + if licenseNames.Has(strings.ToLower(filepath.Base(f.Path))) { + candidatePaths.Add(path.Join(m.SitePackagesRootPath, f.Path)) + } + } + + paths := candidatePaths.List() + sort.Strings(paths) + locationSet := file.NewLocationSet() + for _, p := range paths { + locs, err := resolver.FilesByPath(p) + if err != nil { + log.WithFields("error", err, "path", p).Trace("unable to resolve python license in dist-info") + continue + } + locationSet.Add(locs...) + } + + licenseSet = getLicenseSetFromFiles(ctx, scanner, resolver, locationSet.ToSlice()...) + } + return licenseSet +} + +func getLicenseSetFromValues(locations []file.Location, licenseValues ...string) pkg.LicenseSet { + if len(locations) == 0 { + return pkg.NewLicenseSet(pkg.NewLicensesFromValues(licenseValues...)...) + } + + licenseSet := pkg.NewLicenseSet() + for _, value := range licenseValues { + if value == "" { + continue + } + + licenseSet.Add(pkg.NewLicenseFromLocations(value, locations...)) + } + return licenseSet +} + +func getLicenseSetFromFiles(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, locations ...file.Location) pkg.LicenseSet { + licenseSet := pkg.NewLicenseSet() + for _, loc := range locations { + licenseSet.Add(getLicenseSetFromFile(ctx, scanner, resolver, loc)...) + } + return licenseSet +} + +func getLicenseSetFromFile(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, location file.Location) []pkg.License { + metadataContents, err := resolver.FileContentsByLocation(location) + if err != nil { + log.WithFields("error", err, "path", location.Path()).Trace("unable to read file contents") + return nil + } + defer internal.CloseAndLogError(metadataContents, location.Path()) + parsed, err := scanner.PkgSearch(ctx, file.NewLocationReadCloser(location, metadataContents)) + if err != nil { + log.WithFields("error", err, "path", location.Path()).Trace("unable to parse a license from the file") + return nil + } + + return parsed +} diff --git a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go index ac5d44207..5d11d3e52 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go @@ -15,11 +15,18 @@ import ( ) type parsedData struct { + // core info + + // DistInfoLocation is the location of the METADATA file within the .dist-info directory where we obtained the python package information + DistInfoLocation file.Location + pkg.PythonPackage `mapstructure:",squash"` + + // license info + Licenses string `mapstructure:"License"` LicenseFile string `mapstructure:"LicenseFile"` LicenseExpression string `mapstructure:"LicenseExpression"` - LicenseLocation file.Location - pkg.PythonPackage `mapstructure:",squash"` + LicenseFilePath string } var pluralFields = map[string]bool{ @@ -45,11 +52,13 @@ func parseWheelOrEggMetadata(locationReader file.LocationReadCloser) (parsedData pd.SitePackagesRootPath = determineSitePackagesRootPath(path) if pd.Licenses != "" || pd.LicenseExpression != "" { - pd.LicenseLocation = file.NewLocation(path) + pd.LicenseFilePath = path } else if pd.LicenseFile != "" { - pd.LicenseLocation = file.NewLocation(filepath.Join(filepath.Dir(path), pd.LicenseFile)) + pd.LicenseFilePath = filepath.Join(filepath.Dir(path), pd.LicenseFile) } + pd.DistInfoLocation = locationReader.Location + return pd, nil } diff --git a/syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go b/syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go index 57fbf65ee..9c1e901b1 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go @@ -22,38 +22,40 @@ func TestParseWheelEggMetadata(t *testing.T) { ExpectedMetadata parsedData }{ { - Fixture: "test-fixtures/egg-info/PKG-INFO", + Fixture: "test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO", ExpectedMetadata: parsedData{ - "Apache 2.0", - "", - "", - file.NewLocation("test-fixtures/egg-info/PKG-INFO"), - pkg.PythonPackage{ + DistInfoLocation: file.NewLocation("test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO"), + Licenses: "Apache 2.0", + LicenseFile: "", + LicenseExpression: "", + LicenseFilePath: "test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO", + PythonPackage: pkg.PythonPackage{ Name: "requests", Version: "2.22.0", Platform: "UNKNOWN", Author: "Kenneth Reitz", AuthorEmail: "me@kennethreitz.org", - SitePackagesRootPath: "test-fixtures", + SitePackagesRootPath: "test-fixtures/site-packages/nested/egg-name", RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", ProvidesExtra: []string{"security", "socks"}, }, }, }, { - Fixture: "test-fixtures/dist-info/METADATA", + Fixture: "test-fixtures/site-packages/nested/dist-name/dist-info/METADATA", ExpectedMetadata: parsedData{ - "BSD License", - "", - "", - file.NewLocation("test-fixtures/dist-info/METADATA"), - pkg.PythonPackage{ + DistInfoLocation: file.NewLocation("test-fixtures/site-packages/nested/dist-name/dist-info/METADATA"), + Licenses: "BSD License", + LicenseFile: "", + LicenseExpression: "", + LicenseFilePath: "test-fixtures/site-packages/nested/dist-name/dist-info/METADATA", + PythonPackage: pkg.PythonPackage{ Name: "Pygments", Version: "2.6.1", Platform: "any", Author: "Georg Brandl", AuthorEmail: "georg@python.org", - SitePackagesRootPath: "test-fixtures", + SitePackagesRootPath: "test-fixtures/site-packages/nested/dist-name", RequiresPython: ">=3.5", RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"}, ProvidesExtra: []string{"html5lib", "lxml"}, @@ -149,16 +151,17 @@ func TestParseWheelEggMetadataInvalid(t *testing.T) { ExpectedMetadata parsedData }{ { - Fixture: "test-fixtures/egg-info/PKG-INFO-INVALID", + Fixture: "test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO-INVALID", ExpectedMetadata: parsedData{ - "", - "", - "", - file.Location{}, - pkg.PythonPackage{ + DistInfoLocation: file.NewLocation("test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO-INVALID"), + Licenses: "", + LicenseExpression: "", + LicenseFile: "", + LicenseFilePath: "", + PythonPackage: pkg.PythonPackage{ Name: "mxnet", Version: "1.8.0", - SitePackagesRootPath: "test-fixtures", + SitePackagesRootPath: "test-fixtures/site-packages/nested/egg-name", }, }, }, diff --git a/syft/pkg/cataloger/python/parse_wheel_egg_record_test.go b/syft/pkg/cataloger/python/parse_wheel_egg_record_test.go index 7ac938008..0815c06b0 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg_record_test.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg_record_test.go @@ -15,7 +15,7 @@ func TestParseWheelEggRecord(t *testing.T) { ExpectedMetadata []pkg.PythonFileRecord }{ { - Fixture: "test-fixtures/egg-info/RECORD", + Fixture: "test-fixtures/site-packages/nested/egg-name/egg-info/RECORD", ExpectedMetadata: []pkg.PythonFileRecord{ {Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"}, {Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"}, @@ -26,10 +26,11 @@ func TestParseWheelEggRecord(t *testing.T) { }, }, { - Fixture: "test-fixtures/dist-info/RECORD", + Fixture: "test-fixtures/site-packages/nested/dist-name/dist-info/RECORD", ExpectedMetadata: []pkg.PythonFileRecord{ {Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"}, {Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"}, + {Path: "Pygments-2.6.1.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"}, {Path: "Pygments-2.6.1.dist-info/RECORD"}, {Path: "pygments/__pycache__/__init__.cpython-38.pyc"}, {Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"}, diff --git a/syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/LICENSE.txt b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/LICENSE.txt new file mode 100644 index 000000000..9d227a0cc --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/LICENSE.txt @@ -0,0 +1,28 @@ +Copyright 2010 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/METADATA b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/METADATA new file mode 100644 index 000000000..da1a3ab42 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/METADATA @@ -0,0 +1,53 @@ +Metadata-Version: 2.1 +Name: Pygments +Version: 2.6.1 +Summary: Pygments is a syntax highlighting package written in Python. +Home-page: https://pygments.org/ +Author: Georg Brandl +Author-email: georg@python.org +LicenseFile: LICENSE.txt +Keywords: syntax highlighting +Platform: any +Classifier: License :: OSI Approved :: BSD License +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: End Users/Desktop +Classifier: Intended Audience :: System Administrators +Classifier: Development Status :: 6 - Mature +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Operating System :: OS Independent +Classifier: Topic :: Text Processing :: Filters +Classifier: Topic :: Utilities +Requires-Python: >=3.5 +Description-Content-Type: text/markdown +Requires-Dist: soupsieve (>1.2) +Provides-Extra: html5lib +Requires-Dist: html5lib ; extra == 'html5lib' +Provides-Extra: lxml +Requires-Dist: lxml ; extra == 'lxml' + + +Pygments +~~~~~~~~ + +Pygments is a syntax highlighting package written in Python. + +It is a generic syntax highlighter suitable for use in code hosting, forums, +wikis or other applications that need to prettify source code. Highlights +are: + +* a wide range of over 500 languages and other text formats is supported +* special attention is paid to details, increasing quality by a fair amount +* support for new languages and formats are added easily +* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image formats that PIL supports and ANSI sequences +* it is usable as a command-line tool and as a library + +:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. +:license: BSD, see LICENSE for details. + diff --git a/syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/RECORD b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/RECORD new file mode 100644 index 000000000..4dec7195d --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/RECORD @@ -0,0 +1,7 @@ +../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220 +with-license-file-declared.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449 +with-license-file-declared.dist-info/LICENSE.txt,sha256=utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY,8449 +with-license-file-declared.dist-info/RECORD,, +pygments/__pycache__/__init__.cpython-38.pyc,, +pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778 +pygments/x_util.py,sha256=qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI=,10778 diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/direct_url.json b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/direct_url.json similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/direct_url.json rename to syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/direct_url.json diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/top_level.txt similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/top_level.txt rename to syft/pkg/cataloger/python/test-fixtures/site-packages/license/with-license-file-declared.dist-info/top_level.txt diff --git a/syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/LICENSE.txt b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/LICENSE.txt new file mode 100644 index 000000000..9d227a0cc --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/LICENSE.txt @@ -0,0 +1,28 @@ +Copyright 2010 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/METADATA b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/METADATA new file mode 100644 index 000000000..79c9faef9 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/METADATA @@ -0,0 +1,52 @@ +Metadata-Version: 2.1 +Name: Pygments +Version: 2.6.1 +Summary: Pygments is a syntax highlighting package written in Python. +Home-page: https://pygments.org/ +Author: Georg Brandl +Author-email: georg@python.org +Keywords: syntax highlighting +Platform: any +Classifier: License :: OSI Approved :: BSD License +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: End Users/Desktop +Classifier: Intended Audience :: System Administrators +Classifier: Development Status :: 6 - Mature +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Operating System :: OS Independent +Classifier: Topic :: Text Processing :: Filters +Classifier: Topic :: Utilities +Requires-Python: >=3.5 +Description-Content-Type: text/markdown +Requires-Dist: soupsieve (>1.2) +Provides-Extra: html5lib +Requires-Dist: html5lib ; extra == 'html5lib' +Provides-Extra: lxml +Requires-Dist: lxml ; extra == 'lxml' + + +Pygments +~~~~~~~~ + +Pygments is a syntax highlighting package written in Python. + +It is a generic syntax highlighter suitable for use in code hosting, forums, +wikis or other applications that need to prettify source code. Highlights +are: + +* a wide range of over 500 languages and other text formats is supported +* special attention is paid to details, increasing quality by a fair amount +* support for new languages and formats are added easily +* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image formats that PIL supports and ANSI sequences +* it is usable as a command-line tool and as a library + +:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. +:license: BSD, see LICENSE for details. + diff --git a/syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/RECORD b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/RECORD new file mode 100644 index 000000000..c8f2e05f1 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/RECORD @@ -0,0 +1,7 @@ +../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220 +without-license-file-declared.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449 +without-license-file-declared.dist-info/LICENSE.txt,sha256=utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY,8449 +without-license-file-declared.dist-info/RECORD,, +pygments/__pycache__/__init__.cpython-38.pyc,, +pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778 +pygments/x_util.py,sha256=qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI=,10778 diff --git a/syft/pkg/cataloger/python/test-fixtures/dist-info/direct_url.json b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/direct_url.json similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/dist-info/direct_url.json rename to syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/direct_url.json diff --git a/syft/pkg/cataloger/python/test-fixtures/dist-info/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/top_level.txt similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/dist-info/top_level.txt rename to syft/pkg/cataloger/python/test-fixtures/site-packages/license/without-license-file-declared.dist-info/top_level.txt diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/METADATA b/syft/pkg/cataloger/python/test-fixtures/site-packages/malformed-record/dist-info/METADATA similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/METADATA rename to syft/pkg/cataloger/python/test-fixtures/site-packages/malformed-record/dist-info/METADATA diff --git a/syft/pkg/cataloger/python/test-fixtures/malformed-record/dist-info/RECORD b/syft/pkg/cataloger/python/test-fixtures/site-packages/malformed-record/dist-info/RECORD similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/malformed-record/dist-info/RECORD rename to syft/pkg/cataloger/python/test-fixtures/site-packages/malformed-record/dist-info/RECORD diff --git a/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/LICENSE.txt b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/LICENSE.txt new file mode 100644 index 000000000..9d227a0cc --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/LICENSE.txt @@ -0,0 +1,28 @@ +Copyright 2010 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/syft/pkg/cataloger/python/test-fixtures/dist-info/METADATA b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/METADATA similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/dist-info/METADATA rename to syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/METADATA diff --git a/syft/pkg/cataloger/python/test-fixtures/dist-info/RECORD b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/RECORD similarity index 81% rename from syft/pkg/cataloger/python/test-fixtures/dist-info/RECORD rename to syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/RECORD index b7beac37e..d8f3f9e5e 100644 --- a/syft/pkg/cataloger/python/test-fixtures/dist-info/RECORD +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/RECORD @@ -1,5 +1,6 @@ ../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220 Pygments-2.6.1.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449 +Pygments-2.6.1.dist-info/LICENSE.txt,sha256=utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY,8449 Pygments-2.6.1.dist-info/RECORD,, pygments/__pycache__/__init__.cpython-38.pyc,, pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778 diff --git a/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/direct_url.json b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/direct_url.json new file mode 100644 index 000000000..5d42a36e3 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/direct_url.json @@ -0,0 +1 @@ +{"url": "https://github.com/python-test/test.git", "vcs_info": {"commit_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "vcs": "git"}} \ No newline at end of file diff --git a/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/top_level.txt new file mode 100644 index 000000000..2c30fc0c4 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/dist-name/dist-info/top_level.txt @@ -0,0 +1,2 @@ +pygments +something_else \ No newline at end of file diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO rename to syft/pkg/cataloger/python/test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO-INVALID b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO-INVALID similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO-INVALID rename to syft/pkg/cataloger/python/test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO-INVALID diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/RECORD b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/egg-name/egg-info/RECORD similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/RECORD rename to syft/pkg/cataloger/python/test-fixtures/site-packages/nested/egg-name/egg-info/RECORD diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/site-packages/nested/egg-name/egg-info/top_level.txt similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/top_level.txt rename to syft/pkg/cataloger/python/test-fixtures/site-packages/nested/egg-name/egg-info/top_level.txt diff --git a/syft/pkg/cataloger/python/test-fixtures/no-version-py3.8.egg-info b/syft/pkg/cataloger/python/test-fixtures/site-packages/no-version/no-version-py3.8.egg-info similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/no-version-py3.8.egg-info rename to syft/pkg/cataloger/python/test-fixtures/site-packages/no-version/no-version-py3.8.egg-info diff --git a/syft/pkg/cataloger/python/test-fixtures/malformed-record/dist-info/METADATA b/syft/pkg/cataloger/python/test-fixtures/site-packages/partial.dist-info/METADATA similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/malformed-record/dist-info/METADATA rename to syft/pkg/cataloger/python/test-fixtures/site-packages/partial.dist-info/METADATA diff --git a/syft/pkg/cataloger/python/test-fixtures/test.egg-info b/syft/pkg/cataloger/python/test-fixtures/site-packages/test/test.egg-info similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/test.egg-info rename to syft/pkg/cataloger/python/test-fixtures/site-packages/test/test.egg-info diff --git a/syft/pkg/cataloger/python/test-fixtures/partial.dist-info/METADATA b/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/METADATA similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/partial.dist-info/METADATA rename to syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/METADATA diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/RECORD b/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/RECORD similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/RECORD rename to syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/RECORD diff --git a/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/direct_url.json b/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/direct_url.json new file mode 100644 index 000000000..5d42a36e3 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/direct_url.json @@ -0,0 +1 @@ +{"url": "https://github.com/python-test/test.git", "vcs_info": {"commit_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "vcs": "git"}} \ No newline at end of file diff --git a/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/top_level.txt new file mode 100644 index 000000000..2c30fc0c4 --- /dev/null +++ b/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/dist-name/DIST-INFO/top_level.txt @@ -0,0 +1,2 @@ +pygments +something_else \ No newline at end of file diff --git a/syft/pkg/cataloger/python/test-fixtures/egg-info/PKG-INFO b/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/egg-name/EGG-INFO/PKG-INFO similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/egg-info/PKG-INFO rename to syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/egg-name/EGG-INFO/PKG-INFO diff --git a/syft/pkg/cataloger/python/test-fixtures/egg-info/PKG-INFO-INVALID b/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/egg-name/EGG-INFO/PKG-INFO-INVALID similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/egg-info/PKG-INFO-INVALID rename to syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/egg-name/EGG-INFO/PKG-INFO-INVALID diff --git a/syft/pkg/cataloger/python/test-fixtures/egg-info/RECORD b/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/egg-name/EGG-INFO/RECORD similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/egg-info/RECORD rename to syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/egg-name/EGG-INFO/RECORD diff --git a/syft/pkg/cataloger/python/test-fixtures/egg-info/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/egg-name/EGG-INFO/top_level.txt similarity index 100% rename from syft/pkg/cataloger/python/test-fixtures/egg-info/top_level.txt rename to syft/pkg/cataloger/python/test-fixtures/site-packages/uppercase/egg-name/EGG-INFO/top_level.txt