split python package catalogers by image vs directory

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Makefile

@@ -122,7 +122,7 @@ validate-cyclonedx-schema:
 .PHONY: unit
 unit: fixtures ## Run unit tests (with coverage)
 	$(call title,Running unit tests)
-	go test -coverprofile $(COVER_REPORT) ./...
+	go test -coverprofile $(COVER_REPORT) $(shell go list ./... | grep -v anchore/syft/test)
 	@go tool cover -func $(COVER_REPORT) | grep total |  awk '{print substr($$3, 1, length($$3)-1)}' > $(COVER_TOTAL)
 	@echo "Coverage: $$(cat $(COVER_TOTAL))"
 	@if [ $$(echo "$$(cat $(COVER_TOTAL)) >= $(COVERAGE_THRESHOLD)" | bc -l) -ne 1 ]; then echo "$(RED)$(BOLD)Failed coverage quality gate (> $(COVERAGE_THRESHOLD)%)$(RESET)" && false; fi

syft/cataloger/cataloger.go

@@ -32,7 +32,7 @@ type Cataloger interface {
 func ImageCatalogers() []Cataloger {
 	return []Cataloger{
 		ruby.NewGemSpecCataloger(),
-		python.NewPythonCataloger(), // TODO: split and replace me
+		python.NewPythonPackageCataloger(),
 		javascript.NewJavascriptPackageCataloger(),
 		deb.NewDpkgdbCataloger(),
 		rpmdb.NewRpmdbCataloger(),
@@ -46,7 +46,8 @@ func ImageCatalogers() []Cataloger {
 func DirectoryCatalogers() []Cataloger {
 	return []Cataloger{
 		ruby.NewGemFileLockCataloger(),
-		python.NewPythonCataloger(), // TODO: split and replace me
+		python.NewPythonIndexCataloger(),
+		python.NewPythonPackageCataloger(),
 		javascript.NewJavascriptLockCataloger(),
 		deb.NewDpkgdbCataloger(),
 		rpmdb.NewRpmdbCataloger(),

syft/cataloger/python/cataloger.go

@@ -7,15 +7,23 @@ import (
 	"github.com/anchore/syft/syft/cataloger/common"
 )
 
-// NewPythonCataloger returns a new Python cataloger object.
+// NewPythonPackageCataloger returns a new cataloger for python packages within egg or wheel installation directories.
-func NewPythonCataloger() *common.GenericCataloger {
+func NewPythonPackageCataloger() *common.GenericCataloger {
 	globParsers := map[string]common.ParserFn{
-		"**/*egg-info/PKG-INFO":  parseEggMetadata,
+		"**/*egg-info/PKG-INFO":  parseWheelOrEggMetadata,
-		"**/*dist-info/METADATA": parseWheelMetadata,
+		"**/*dist-info/METADATA": parseWheelOrEggMetadata,
-		"**/*requirements*.txt":  parseRequirementsTxt,
-		"**/poetry.lock":         parsePoetryLock,
-		"**/setup.py":            parseSetup,
 	}
 
-	return common.NewGenericCataloger(nil, globParsers, "python-cataloger")
+	return common.NewGenericCataloger(nil, globParsers, "python-package-cataloger")
+}
+
+// NewPythonIndexCataloger returns a new cataloger for python packages referenced from poetry lock files, requirements.txt files, and setup.py files.
+func NewPythonIndexCataloger() *common.GenericCataloger {
+	globParsers := map[string]common.ParserFn{
+		"**/*requirements*.txt": parseRequirementsTxt,
+		"**/poetry.lock":        parsePoetryLock,
+		"**/setup.py":           parseSetup,
+	}
+
+	return common.NewGenericCataloger(nil, globParsers, "python-index-cataloger")
 }

syft/cataloger/python/parse_poetry_lock_test.go

@@ -1,10 +1,11 @@
 package python
 
 import (
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/go-test/deep"
 	"os"
 	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
 )
 
 func TestParsePoetryLock(t *testing.T) {
@@ -13,28 +14,28 @@ func TestParsePoetryLock(t *testing.T) {
 			Name:     "added-value",
 			Version:  "0.14.2",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 		{
 			Name:     "alabaster",
 			Version:  "0.7.12",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 		{
 			Name:     "appnope",
 			Version:  "0.1.0",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 		{
 			Name:     "asciitree",
 			Version:  "0.3.3",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 	}

syft/cataloger/python/parse_requirements.go

@@ -47,7 +47,7 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) {
 				Name:     name,
 				Version:  version,
 				Language: pkg.Python,
-				Type:     pkg.PythonRequirementsPkg,
+				Type:     pkg.PythonPkg,
 			})
 		default:
 			continue

syft/cataloger/python/parse_requirements_test.go

@@ -13,14 +13,14 @@ func TestParseRequirementsTxt(t *testing.T) {
 			Name:     "foo",
 			Version:  "1.0.0",
 			Language: pkg.Python,
-			Type:     pkg.PythonRequirementsPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: []string{},
 		},
 		"flask": {
 			Name:     "flask",
 			Version:  "4.0.0",
 			Language: pkg.Python,
-			Type:     pkg.PythonRequirementsPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: []string{},
 		},
 	}

syft/cataloger/python/parse_setup.go

@@ -41,7 +41,7 @@ func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) {
 				Name:     strings.Trim(name, "'\""),
 				Version:  strings.Trim(version, "'\""),
 				Language: pkg.Python,
-				Type:     pkg.PythonSetupPkg,
+				Type:     pkg.PythonPkg,
 			})
 		}
 	}

syft/cataloger/python/parse_setup_test.go

@@ -13,35 +13,35 @@ func TestParseSetup(t *testing.T) {
 			Name:     "pathlib3",
 			Version:  "2.2.0",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: []string{},
 		},
 		"mypy": {
 			Name:     "mypy",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: []string{},
 		},
 		"mypy1": {
 			Name:     "mypy1",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: []string{},
 		},
 		"mypy2": {
 			Name:     "mypy2",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: []string{},
 		},
 		"mypy3": {
 			Name:     "mypy3",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: []string{},
 		},
 	}

syft/cataloger/python/parse_wheel_egg.go

@@ -11,39 +11,17 @@ import (
 )
 
 // integrity check
-var _ common.ParserFn = parseWheelMetadata
+var _ common.ParserFn = parseWheelOrEggMetadata
-var _ common.ParserFn = parseEggMetadata
-
-// parseWheelMetadata is a parser function for individual Python Wheel metadata file contents, returning all Python
-// packages listed.
-func parseWheelMetadata(_ string, reader io.Reader) ([]pkg.Package, error) {
-	packages, err := parseWheelOrEggMetadata(reader)
-	for idx := range packages {
-		packages[idx].Type = pkg.WheelPkg
-	}
-	return packages, err
-}
-
-// parseEggMetadata is a parser function for individual Python Egg metadata file contents, returning all Python
-// packages listed.
-func parseEggMetadata(_ string, reader io.Reader) ([]pkg.Package, error) {
-	packages, err := parseWheelOrEggMetadata(reader)
-	for idx := range packages {
-		packages[idx].Type = pkg.EggPkg
-	}
-	return packages, err
-}
 
 // parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
 // returning all Python packages listed.
-func parseWheelOrEggMetadata(reader io.Reader) ([]pkg.Package, error) {
+func parseWheelOrEggMetadata(_ string, reader io.Reader) ([]pkg.Package, error) {
 	fields := make(map[string]string)
 	var key string
 
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
 		line := scanner.Text()
-
 		line = strings.TrimRight(line, "\n")
 
 		// empty line indicates end of entry
@@ -90,6 +68,7 @@ func parseWheelOrEggMetadata(reader io.Reader) ([]pkg.Package, error) {
 		Name:     fields["Name"],
 		Version:  fields["Version"],
 		Language: pkg.Python,
+		Type:     pkg.PythonPkg,
 	}
 
 	if license, ok := fields["License"]; ok && license != "" {

syft/cataloger/python/parse_wheel_egg_test.go

@@ -52,7 +52,7 @@ func TestParseEggMetadata(t *testing.T) {
 			Name:     "requests",
 			Version:  "2.22.0",
 			Language: pkg.Python,
-			Type:     pkg.EggPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: []string{"Apache 2.0"},
 		},
 	}
@@ -61,7 +61,7 @@ func TestParseEggMetadata(t *testing.T) {
 		t.Fatalf("failed to open fixture: %+v", err)
 	}
 
-	actual, err := parseEggMetadata(fixture.Name(), fixture)
+	actual, err := parseWheelOrEggMetadata(fixture.Name(), fixture)
 	if err != nil {
 		t.Fatalf("failed to parse egg-info: %+v", err)
 	}
@@ -75,7 +75,7 @@ func TestParseWheelMetadata(t *testing.T) {
 			Name:     "Pygments",
 			Version:  "2.6.1",
 			Language: pkg.Python,
-			Type:     pkg.WheelPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: []string{"BSD License"},
 		},
 	}
@@ -84,7 +84,7 @@ func TestParseWheelMetadata(t *testing.T) {
 		t.Fatalf("failed to open fixture: %+v", err)
 	}
 
-	actual, err := parseWheelMetadata(fixture.Name(), fixture)
+	actual, err := parseWheelOrEggMetadata(fixture.Name(), fixture)
 	if err != nil {
 		t.Fatalf("failed to parse dist-info: %+v", err)
 	}

syft/cataloger/python/poetry_metadata_package.go

@@ -16,6 +16,6 @@ func (p PoetryMetadataPackage) Pkg() pkg.Package {
 		Name:     p.Name,
 		Version:  p.Version,
 		Language: pkg.Python,
-		Type:     pkg.PoetryPkg,
+		Type:     pkg.PythonPkg,
 	}
 }

syft/pkg/package_test.go

@@ -25,7 +25,7 @@ func TestPackage_pURL(t *testing.T) {
 			pkg: Package{
 				Name:    "name",
 				Version: "v0.1.0",
-				Type:    WheelPkg,
+				Type:    PythonPkg,
 			},
 			expected: "pkg:pypi/name@v0.1.0",
 		},
@@ -33,7 +33,7 @@ func TestPackage_pURL(t *testing.T) {
 			pkg: Package{
 				Name:    "name",
 				Version: "v0.1.0",
-				Type:    EggPkg,
+				Type:    PythonPkg,
 			},
 			expected: "pkg:pypi/name@v0.1.0",
 		},
@@ -41,7 +41,7 @@ func TestPackage_pURL(t *testing.T) {
 			pkg: Package{
 				Name:    "name",
 				Version: "v0.1.0",
-				Type:    PythonSetupPkg,
+				Type:    PythonPkg,
 			},
 			expected: "pkg:pypi/name@v0.1.0",
 		},
@@ -49,7 +49,7 @@ func TestPackage_pURL(t *testing.T) {
 			pkg: Package{
 				Name:    "name",
 				Version: "v0.1.0",
-				Type:    PythonRequirementsPkg,
+				Type:    PythonPkg,
 			},
 			expected: "pkg:pypi/name@v0.1.0",
 		},

syft/pkg/type.go

@@ -6,32 +6,25 @@ import "github.com/package-url/packageurl-go"
 type Type string
 
 const (
-	UnknownPkg            Type = "UnknownPackage"
+	UnknownPkg       Type = "UnknownPackage"
-	ApkPkg                Type = "apk"
+	ApkPkg           Type = "apk"
-	GemPkg                Type = "gem"
+	GemPkg           Type = "gem"
-	DebPkg                Type = "deb"
+	DebPkg           Type = "deb"
-	EggPkg                Type = "egg"
+	RpmPkg           Type = "rpm"
-	RpmPkg                Type = "rpm"
+	NpmPkg           Type = "npm"
-	WheelPkg              Type = "wheel"
+	PythonPkg        Type = "python"
-	PoetryPkg             Type = "poetry"
+	JavaPkg          Type = "java-archive"
-	NpmPkg                Type = "npm"
+	JenkinsPluginPkg Type = "jenkins-plugin"
-	PythonRequirementsPkg Type = "python-requirements"
+	GoModulePkg      Type = "go-module"
-	PythonSetupPkg        Type = "python-setup"
-	JavaPkg               Type = "java-archive"
-	JenkinsPluginPkg      Type = "jenkins-plugin"
-	GoModulePkg           Type = "go-module"
 )
 
 var AllPkgs = []Type{
 	ApkPkg,
 	GemPkg,
 	DebPkg,
-	EggPkg,
 	RpmPkg,
-	WheelPkg,
 	NpmPkg,
-	PythonRequirementsPkg,
+	PythonPkg,
-	PythonSetupPkg,
 	JavaPkg,
 	JenkinsPluginPkg,
 	GoModulePkg,
@@ -45,7 +38,7 @@ func (t Type) PackageURLType() string {
 		return packageurl.TypeGem
 	case DebPkg:
 		return "deb"
-	case EggPkg, WheelPkg, PythonRequirementsPkg, PythonSetupPkg:
+	case PythonPkg:
 		return packageurl.TypePyPi
 	case NpmPkg:
 		return packageurl.TypeNPM

test/integration/pkg_cases_test.go

@@ -26,6 +26,17 @@ var imageOnlyTestCases = []testCase{
 			"npm": "6.14.6",
 		},
 	},
+	{
+		name:        "find python egg & wheel packages",
+		pkgType:     pkg.PythonPkg,
+		pkgLanguage: pkg.Python,
+		pkgInfo: map[string]string{
+			"Pygments":     "2.6.1",
+			"requests":     "2.22.0",
+			"somerequests": "3.22.0",
+			"someotherpkg": "3.19.0",
+		},
+	},
 }
 
 var dirOnlyTestCases = []testCase{
@@ -96,6 +107,26 @@ var dirOnlyTestCases = []testCase{
 			"get-stdin":         "8.0.0",
 		},
 	},
+	{
+		name:        "find python requirements.txt & setup.py package references",
+		pkgType:     pkg.PythonPkg,
+		pkgLanguage: pkg.Python,
+		pkgInfo: map[string]string{
+			// dir specific test cases
+			"flask":              "4.0.0",
+			"python-dateutil":    "2.8.1",
+			"python-swiftclient": "3.8.1",
+			"pytz":               "2019.3",
+			"jsonschema":         "2.6.0",
+			"passlib":            "1.7.2",
+			"mypy":               "v0.770",
+			// common to image and directory
+			"Pygments":     "2.6.1",
+			"requests":     "2.22.0",
+			"somerequests": "3.22.0",
+			"someotherpkg": "3.19.0",
+		},
+	},
 }
 
 var commonTestCases = []testCase{
@@ -131,46 +162,6 @@ var commonTestCases = []testCase{
 			"example-jenkins-plugin": "1.0-SNAPSHOT",
 		},
 	},
-	{
-		name:        "find python wheel packages",
-		pkgType:     pkg.WheelPkg,
-		pkgLanguage: pkg.Python,
-		pkgInfo: map[string]string{
-			"Pygments": "2.6.1",
-			"requests": "2.10.0",
-		},
-	},
-	{
-		name:        "find python egg packages",
-		pkgType:     pkg.EggPkg,
-		pkgLanguage: pkg.Python,
-		pkgInfo: map[string]string{
-			"requests": "2.22.0",
-			"otherpkg": "2.19.0",
-		},
-	},
-	{
-		name:        "find python requirements.txt packages",
-		pkgType:     pkg.PythonRequirementsPkg,
-		pkgLanguage: pkg.Python,
-		pkgInfo: map[string]string{
-			"flask":              "4.0.0",
-			"python-dateutil":    "2.8.1",
-			"python-swiftclient": "3.8.1",
-			"pytz":               "2019.3",
-			"jsonschema":         "2.6.0",
-			"passlib":            "1.7.2",
-			"pathlib":            "1.0.1",
-		},
-	},
-	{
-		name:        "find python setup.py packages",
-		pkgType:     pkg.PythonSetupPkg,
-		pkgLanguage: pkg.Python,
-		pkgInfo: map[string]string{
-			"mypy": "v0.770",
-		},
-	},
 	{
 
 		name:    "find apkdb packages",

test/integration/test-fixtures/image-pkg-coverage/python/requires/requirements-dev.txt

@@ -1,3 +1,2 @@
 jsonschema==2.6.0
-passlib==1.7.2
+passlib==1.7.2
-pathlib==1.0.1

test/integration/test-fixtures/image-pkg-coverage/python/someotherpkg-2.19.0-py3.8.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: otherpkg
+Name: someotherpkg
-Version: 2.19.0
+Version: 3.19.0
 Summary: Python HTTP for Humans.
 Home-page: http://python-requests.org
 Author: Kenneth Reitz

test/integration/test-fixtures/image-pkg-coverage/python/somerequests-3.22.0.dist-info/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: requests
+Name: somerequests
-Version: 2.10.0
+Version: 3.22.0
 Summary: stuff
 Home-page: stuff
 Author: Georg Brandl