diff --git a/Makefile b/Makefile index 04b3913c4..8871c8965 100644 --- a/Makefile +++ b/Makefile @@ -122,7 +122,7 @@ validate-cyclonedx-schema: .PHONY: unit unit: fixtures ## Run unit tests (with coverage) $(call title,Running unit tests) - go test -coverprofile $(COVER_REPORT) ./... + go test -coverprofile $(COVER_REPORT) $(shell go list ./... | grep -v anchore/syft/test) @go tool cover -func $(COVER_REPORT) | grep total | awk '{print substr($$3, 1, length($$3)-1)}' > $(COVER_TOTAL) @echo "Coverage: $$(cat $(COVER_TOTAL))" @if [ $$(echo "$$(cat $(COVER_TOTAL)) >= $(COVERAGE_THRESHOLD)" | bc -l) -ne 1 ]; then echo "$(RED)$(BOLD)Failed coverage quality gate (> $(COVERAGE_THRESHOLD)%)$(RESET)" && false; fi diff --git a/syft/cataloger/cataloger.go b/syft/cataloger/cataloger.go index e10de959b..0ee9f5be2 100644 --- a/syft/cataloger/cataloger.go +++ b/syft/cataloger/cataloger.go @@ -32,7 +32,7 @@ type Cataloger interface { func ImageCatalogers() []Cataloger { return []Cataloger{ ruby.NewGemSpecCataloger(), - python.NewPythonCataloger(), // TODO: split and replace me + python.NewPythonPackageCataloger(), javascript.NewJavascriptPackageCataloger(), deb.NewDpkgdbCataloger(), rpmdb.NewRpmdbCataloger(), @@ -46,7 +46,8 @@ func ImageCatalogers() []Cataloger { func DirectoryCatalogers() []Cataloger { return []Cataloger{ ruby.NewGemFileLockCataloger(), - python.NewPythonCataloger(), // TODO: split and replace me + python.NewPythonIndexCataloger(), + python.NewPythonPackageCataloger(), javascript.NewJavascriptLockCataloger(), deb.NewDpkgdbCataloger(), rpmdb.NewRpmdbCataloger(), diff --git a/syft/cataloger/python/cataloger.go b/syft/cataloger/python/cataloger.go index b4e9f1329..65faac0b1 100644 --- a/syft/cataloger/python/cataloger.go +++ b/syft/cataloger/python/cataloger.go @@ -7,15 +7,23 @@ import ( "github.com/anchore/syft/syft/cataloger/common" ) -// NewPythonCataloger returns a new Python cataloger object. -func NewPythonCataloger() *common.GenericCataloger { +// NewPythonPackageCataloger returns a new cataloger for python packages within egg or wheel installation directories. +func NewPythonPackageCataloger() *common.GenericCataloger { globParsers := map[string]common.ParserFn{ - "**/*egg-info/PKG-INFO": parseEggMetadata, - "**/*dist-info/METADATA": parseWheelMetadata, - "**/*requirements*.txt": parseRequirementsTxt, - "**/poetry.lock": parsePoetryLock, - "**/setup.py": parseSetup, + "**/*egg-info/PKG-INFO": parseWheelOrEggMetadata, + "**/*dist-info/METADATA": parseWheelOrEggMetadata, } - return common.NewGenericCataloger(nil, globParsers, "python-cataloger") + return common.NewGenericCataloger(nil, globParsers, "python-package-cataloger") +} + +// NewPythonIndexCataloger returns a new cataloger for python packages referenced from poetry lock files, requirements.txt files, and setup.py files. +func NewPythonIndexCataloger() *common.GenericCataloger { + globParsers := map[string]common.ParserFn{ + "**/*requirements*.txt": parseRequirementsTxt, + "**/poetry.lock": parsePoetryLock, + "**/setup.py": parseSetup, + } + + return common.NewGenericCataloger(nil, globParsers, "python-index-cataloger") } diff --git a/syft/cataloger/python/parse_poetry_lock_test.go b/syft/cataloger/python/parse_poetry_lock_test.go index e96c47836..80cc6b625 100644 --- a/syft/cataloger/python/parse_poetry_lock_test.go +++ b/syft/cataloger/python/parse_poetry_lock_test.go @@ -1,10 +1,11 @@ package python import ( - "github.com/anchore/syft/syft/pkg" - "github.com/go-test/deep" "os" "testing" + + "github.com/anchore/syft/syft/pkg" + "github.com/go-test/deep" ) func TestParsePoetryLock(t *testing.T) { @@ -13,28 +14,28 @@ func TestParsePoetryLock(t *testing.T) { Name: "added-value", Version: "0.14.2", Language: pkg.Python, - Type: pkg.PoetryPkg, + Type: pkg.PythonPkg, Licenses: nil, }, { Name: "alabaster", Version: "0.7.12", Language: pkg.Python, - Type: pkg.PoetryPkg, + Type: pkg.PythonPkg, Licenses: nil, }, { Name: "appnope", Version: "0.1.0", Language: pkg.Python, - Type: pkg.PoetryPkg, + Type: pkg.PythonPkg, Licenses: nil, }, { Name: "asciitree", Version: "0.3.3", Language: pkg.Python, - Type: pkg.PoetryPkg, + Type: pkg.PythonPkg, Licenses: nil, }, } diff --git a/syft/cataloger/python/parse_requirements.go b/syft/cataloger/python/parse_requirements.go index 80c95eecc..27ec43cc8 100644 --- a/syft/cataloger/python/parse_requirements.go +++ b/syft/cataloger/python/parse_requirements.go @@ -47,7 +47,7 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) { Name: name, Version: version, Language: pkg.Python, - Type: pkg.PythonRequirementsPkg, + Type: pkg.PythonPkg, }) default: continue diff --git a/syft/cataloger/python/parse_requirements_test.go b/syft/cataloger/python/parse_requirements_test.go index 96fd429af..a5c0f79bf 100644 --- a/syft/cataloger/python/parse_requirements_test.go +++ b/syft/cataloger/python/parse_requirements_test.go @@ -13,14 +13,14 @@ func TestParseRequirementsTxt(t *testing.T) { Name: "foo", Version: "1.0.0", Language: pkg.Python, - Type: pkg.PythonRequirementsPkg, + Type: pkg.PythonPkg, Licenses: []string{}, }, "flask": { Name: "flask", Version: "4.0.0", Language: pkg.Python, - Type: pkg.PythonRequirementsPkg, + Type: pkg.PythonPkg, Licenses: []string{}, }, } diff --git a/syft/cataloger/python/parse_setup.go b/syft/cataloger/python/parse_setup.go index 7851ccb81..337c436e7 100644 --- a/syft/cataloger/python/parse_setup.go +++ b/syft/cataloger/python/parse_setup.go @@ -41,7 +41,7 @@ func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) { Name: strings.Trim(name, "'\""), Version: strings.Trim(version, "'\""), Language: pkg.Python, - Type: pkg.PythonSetupPkg, + Type: pkg.PythonPkg, }) } } diff --git a/syft/cataloger/python/parse_setup_test.go b/syft/cataloger/python/parse_setup_test.go index 6ac911f8c..7369c42a4 100644 --- a/syft/cataloger/python/parse_setup_test.go +++ b/syft/cataloger/python/parse_setup_test.go @@ -13,35 +13,35 @@ func TestParseSetup(t *testing.T) { Name: "pathlib3", Version: "2.2.0", Language: pkg.Python, - Type: pkg.PythonSetupPkg, + Type: pkg.PythonPkg, Licenses: []string{}, }, "mypy": { Name: "mypy", Version: "v0.770", Language: pkg.Python, - Type: pkg.PythonSetupPkg, + Type: pkg.PythonPkg, Licenses: []string{}, }, "mypy1": { Name: "mypy1", Version: "v0.770", Language: pkg.Python, - Type: pkg.PythonSetupPkg, + Type: pkg.PythonPkg, Licenses: []string{}, }, "mypy2": { Name: "mypy2", Version: "v0.770", Language: pkg.Python, - Type: pkg.PythonSetupPkg, + Type: pkg.PythonPkg, Licenses: []string{}, }, "mypy3": { Name: "mypy3", Version: "v0.770", Language: pkg.Python, - Type: pkg.PythonSetupPkg, + Type: pkg.PythonPkg, Licenses: []string{}, }, } diff --git a/syft/cataloger/python/parse_wheel_egg.go b/syft/cataloger/python/parse_wheel_egg.go index fc5100687..c678a7c71 100644 --- a/syft/cataloger/python/parse_wheel_egg.go +++ b/syft/cataloger/python/parse_wheel_egg.go @@ -11,39 +11,17 @@ import ( ) // integrity check -var _ common.ParserFn = parseWheelMetadata -var _ common.ParserFn = parseEggMetadata - -// parseWheelMetadata is a parser function for individual Python Wheel metadata file contents, returning all Python -// packages listed. -func parseWheelMetadata(_ string, reader io.Reader) ([]pkg.Package, error) { - packages, err := parseWheelOrEggMetadata(reader) - for idx := range packages { - packages[idx].Type = pkg.WheelPkg - } - return packages, err -} - -// parseEggMetadata is a parser function for individual Python Egg metadata file contents, returning all Python -// packages listed. -func parseEggMetadata(_ string, reader io.Reader) ([]pkg.Package, error) { - packages, err := parseWheelOrEggMetadata(reader) - for idx := range packages { - packages[idx].Type = pkg.EggPkg - } - return packages, err -} +var _ common.ParserFn = parseWheelOrEggMetadata // parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes), // returning all Python packages listed. -func parseWheelOrEggMetadata(reader io.Reader) ([]pkg.Package, error) { +func parseWheelOrEggMetadata(_ string, reader io.Reader) ([]pkg.Package, error) { fields := make(map[string]string) var key string scanner := bufio.NewScanner(reader) for scanner.Scan() { line := scanner.Text() - line = strings.TrimRight(line, "\n") // empty line indicates end of entry @@ -90,6 +68,7 @@ func parseWheelOrEggMetadata(reader io.Reader) ([]pkg.Package, error) { Name: fields["Name"], Version: fields["Version"], Language: pkg.Python, + Type: pkg.PythonPkg, } if license, ok := fields["License"]; ok && license != "" { diff --git a/syft/cataloger/python/parse_wheel_egg_test.go b/syft/cataloger/python/parse_wheel_egg_test.go index 118573562..829dd49c0 100644 --- a/syft/cataloger/python/parse_wheel_egg_test.go +++ b/syft/cataloger/python/parse_wheel_egg_test.go @@ -52,7 +52,7 @@ func TestParseEggMetadata(t *testing.T) { Name: "requests", Version: "2.22.0", Language: pkg.Python, - Type: pkg.EggPkg, + Type: pkg.PythonPkg, Licenses: []string{"Apache 2.0"}, }, } @@ -61,7 +61,7 @@ func TestParseEggMetadata(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseEggMetadata(fixture.Name(), fixture) + actual, err := parseWheelOrEggMetadata(fixture.Name(), fixture) if err != nil { t.Fatalf("failed to parse egg-info: %+v", err) } @@ -75,7 +75,7 @@ func TestParseWheelMetadata(t *testing.T) { Name: "Pygments", Version: "2.6.1", Language: pkg.Python, - Type: pkg.WheelPkg, + Type: pkg.PythonPkg, Licenses: []string{"BSD License"}, }, } @@ -84,7 +84,7 @@ func TestParseWheelMetadata(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseWheelMetadata(fixture.Name(), fixture) + actual, err := parseWheelOrEggMetadata(fixture.Name(), fixture) if err != nil { t.Fatalf("failed to parse dist-info: %+v", err) } diff --git a/syft/cataloger/python/poetry_metadata_package.go b/syft/cataloger/python/poetry_metadata_package.go index 20a42ae05..5e9e454b1 100644 --- a/syft/cataloger/python/poetry_metadata_package.go +++ b/syft/cataloger/python/poetry_metadata_package.go @@ -16,6 +16,6 @@ func (p PoetryMetadataPackage) Pkg() pkg.Package { Name: p.Name, Version: p.Version, Language: pkg.Python, - Type: pkg.PoetryPkg, + Type: pkg.PythonPkg, } } diff --git a/syft/pkg/package_test.go b/syft/pkg/package_test.go index 1fec05b4b..94cdfcfc6 100644 --- a/syft/pkg/package_test.go +++ b/syft/pkg/package_test.go @@ -25,7 +25,7 @@ func TestPackage_pURL(t *testing.T) { pkg: Package{ Name: "name", Version: "v0.1.0", - Type: WheelPkg, + Type: PythonPkg, }, expected: "pkg:pypi/name@v0.1.0", }, @@ -33,7 +33,7 @@ func TestPackage_pURL(t *testing.T) { pkg: Package{ Name: "name", Version: "v0.1.0", - Type: EggPkg, + Type: PythonPkg, }, expected: "pkg:pypi/name@v0.1.0", }, @@ -41,7 +41,7 @@ func TestPackage_pURL(t *testing.T) { pkg: Package{ Name: "name", Version: "v0.1.0", - Type: PythonSetupPkg, + Type: PythonPkg, }, expected: "pkg:pypi/name@v0.1.0", }, @@ -49,7 +49,7 @@ func TestPackage_pURL(t *testing.T) { pkg: Package{ Name: "name", Version: "v0.1.0", - Type: PythonRequirementsPkg, + Type: PythonPkg, }, expected: "pkg:pypi/name@v0.1.0", }, diff --git a/syft/pkg/type.go b/syft/pkg/type.go index 3d7ff672b..85dc83159 100644 --- a/syft/pkg/type.go +++ b/syft/pkg/type.go @@ -6,32 +6,25 @@ import "github.com/package-url/packageurl-go" type Type string const ( - UnknownPkg Type = "UnknownPackage" - ApkPkg Type = "apk" - GemPkg Type = "gem" - DebPkg Type = "deb" - EggPkg Type = "egg" - RpmPkg Type = "rpm" - WheelPkg Type = "wheel" - PoetryPkg Type = "poetry" - NpmPkg Type = "npm" - PythonRequirementsPkg Type = "python-requirements" - PythonSetupPkg Type = "python-setup" - JavaPkg Type = "java-archive" - JenkinsPluginPkg Type = "jenkins-plugin" - GoModulePkg Type = "go-module" + UnknownPkg Type = "UnknownPackage" + ApkPkg Type = "apk" + GemPkg Type = "gem" + DebPkg Type = "deb" + RpmPkg Type = "rpm" + NpmPkg Type = "npm" + PythonPkg Type = "python" + JavaPkg Type = "java-archive" + JenkinsPluginPkg Type = "jenkins-plugin" + GoModulePkg Type = "go-module" ) var AllPkgs = []Type{ ApkPkg, GemPkg, DebPkg, - EggPkg, RpmPkg, - WheelPkg, NpmPkg, - PythonRequirementsPkg, - PythonSetupPkg, + PythonPkg, JavaPkg, JenkinsPluginPkg, GoModulePkg, @@ -45,7 +38,7 @@ func (t Type) PackageURLType() string { return packageurl.TypeGem case DebPkg: return "deb" - case EggPkg, WheelPkg, PythonRequirementsPkg, PythonSetupPkg: + case PythonPkg: return packageurl.TypePyPi case NpmPkg: return packageurl.TypeNPM diff --git a/test/integration/pkg_cases_test.go b/test/integration/pkg_cases_test.go index 3eb83e45e..26727d688 100644 --- a/test/integration/pkg_cases_test.go +++ b/test/integration/pkg_cases_test.go @@ -26,6 +26,17 @@ var imageOnlyTestCases = []testCase{ "npm": "6.14.6", }, }, + { + name: "find python egg & wheel packages", + pkgType: pkg.PythonPkg, + pkgLanguage: pkg.Python, + pkgInfo: map[string]string{ + "Pygments": "2.6.1", + "requests": "2.22.0", + "somerequests": "3.22.0", + "someotherpkg": "3.19.0", + }, + }, } var dirOnlyTestCases = []testCase{ @@ -96,6 +107,26 @@ var dirOnlyTestCases = []testCase{ "get-stdin": "8.0.0", }, }, + { + name: "find python requirements.txt & setup.py package references", + pkgType: pkg.PythonPkg, + pkgLanguage: pkg.Python, + pkgInfo: map[string]string{ + // dir specific test cases + "flask": "4.0.0", + "python-dateutil": "2.8.1", + "python-swiftclient": "3.8.1", + "pytz": "2019.3", + "jsonschema": "2.6.0", + "passlib": "1.7.2", + "mypy": "v0.770", + // common to image and directory + "Pygments": "2.6.1", + "requests": "2.22.0", + "somerequests": "3.22.0", + "someotherpkg": "3.19.0", + }, + }, } var commonTestCases = []testCase{ @@ -131,46 +162,6 @@ var commonTestCases = []testCase{ "example-jenkins-plugin": "1.0-SNAPSHOT", }, }, - { - name: "find python wheel packages", - pkgType: pkg.WheelPkg, - pkgLanguage: pkg.Python, - pkgInfo: map[string]string{ - "Pygments": "2.6.1", - "requests": "2.10.0", - }, - }, - { - name: "find python egg packages", - pkgType: pkg.EggPkg, - pkgLanguage: pkg.Python, - pkgInfo: map[string]string{ - "requests": "2.22.0", - "otherpkg": "2.19.0", - }, - }, - { - name: "find python requirements.txt packages", - pkgType: pkg.PythonRequirementsPkg, - pkgLanguage: pkg.Python, - pkgInfo: map[string]string{ - "flask": "4.0.0", - "python-dateutil": "2.8.1", - "python-swiftclient": "3.8.1", - "pytz": "2019.3", - "jsonschema": "2.6.0", - "passlib": "1.7.2", - "pathlib": "1.0.1", - }, - }, - { - name: "find python setup.py packages", - pkgType: pkg.PythonSetupPkg, - pkgLanguage: pkg.Python, - pkgInfo: map[string]string{ - "mypy": "v0.770", - }, - }, { name: "find apkdb packages", diff --git a/test/integration/test-fixtures/image-pkg-coverage/python/requires/requirements-dev.txt b/test/integration/test-fixtures/image-pkg-coverage/python/requires/requirements-dev.txt index 4b9365fe9..16c74e8b5 100644 --- a/test/integration/test-fixtures/image-pkg-coverage/python/requires/requirements-dev.txt +++ b/test/integration/test-fixtures/image-pkg-coverage/python/requires/requirements-dev.txt @@ -1,3 +1,2 @@ jsonschema==2.6.0 -passlib==1.7.2 -pathlib==1.0.1 \ No newline at end of file +passlib==1.7.2 \ No newline at end of file diff --git a/test/integration/test-fixtures/image-pkg-coverage/python/otherpkg-2.19.0-py3.8.egg-info/PKG-INFO b/test/integration/test-fixtures/image-pkg-coverage/python/someotherpkg-2.19.0-py3.8.egg-info/PKG-INFO similarity index 82% rename from test/integration/test-fixtures/image-pkg-coverage/python/otherpkg-2.19.0-py3.8.egg-info/PKG-INFO rename to test/integration/test-fixtures/image-pkg-coverage/python/someotherpkg-2.19.0-py3.8.egg-info/PKG-INFO index 05a01e1c8..6d4758610 100644 --- a/test/integration/test-fixtures/image-pkg-coverage/python/otherpkg-2.19.0-py3.8.egg-info/PKG-INFO +++ b/test/integration/test-fixtures/image-pkg-coverage/python/someotherpkg-2.19.0-py3.8.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 -Name: otherpkg -Version: 2.19.0 +Name: someotherpkg +Version: 3.19.0 Summary: Python HTTP for Humans. Home-page: http://python-requests.org Author: Kenneth Reitz diff --git a/test/integration/test-fixtures/image-pkg-coverage/python/requests-2.10.0.dist-info/METADATA b/test/integration/test-fixtures/image-pkg-coverage/python/somerequests-3.22.0.dist-info/METADATA similarity index 96% rename from test/integration/test-fixtures/image-pkg-coverage/python/requests-2.10.0.dist-info/METADATA rename to test/integration/test-fixtures/image-pkg-coverage/python/somerequests-3.22.0.dist-info/METADATA index cef498e16..ddc02219b 100644 --- a/test/integration/test-fixtures/image-pkg-coverage/python/requests-2.10.0.dist-info/METADATA +++ b/test/integration/test-fixtures/image-pkg-coverage/python/somerequests-3.22.0.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 -Name: requests -Version: 2.10.0 +Name: somerequests +Version: 3.22.0 Summary: stuff Home-page: stuff Author: Georg Brandl