Performance improvements around package ID (#698)

* set package ID in catalogers and improve hashing performance

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update setting ID + tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

syft/pkg/cataloger/python/package_cataloger.go

@@ -74,7 +74,7 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver source.FileResolver, metad
 		licenses = []string{metadata.License}
 	}
 
-	return &pkg.Package{
+	p := &pkg.Package{
 		Name:         metadata.Name,
 		Version:      metadata.Version,
 		FoundBy:      c.Name(),
@@ -84,7 +84,11 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver source.FileResolver, metad
 		Type:         pkg.PythonPkg,
 		MetadataType: pkg.PythonPackageMetadataType,
 		Metadata:     *metadata,
-	}, nil
+	}
+
+	p.SetID()
+
+	return p, nil
 }
 
 // fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.

syft/pkg/cataloger/python/parse_pipfile_lock.go

@@ -38,8 +38,8 @@ type Dependency struct {
 var _ common.ParserFn = parsePipfileLock
 
 // parsePipfileLock is a parser function for Pipfile.lock contents, returning "Default" python packages discovered.
-func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parsePipfileLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 	dec := json.NewDecoder(reader)
 
 	for {
@@ -51,7 +51,7 @@ func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Rel
 		}
 		for name, pkgMeta := range lock.Default {
 			version := strings.TrimPrefix(pkgMeta.Version, "==")
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  version,
 				Language: pkg.Python,

syft/pkg/cataloger/python/parse_poetry_lock.go

@@ -14,7 +14,7 @@ import (
 var _ common.ParserFn = parsePoetryLock
 
 // parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered.
-func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parsePoetryLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	tree, err := toml.LoadReader(reader)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err)

syft/pkg/cataloger/python/parse_poetry_lock_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestParsePoetryLock(t *testing.T) {
-	expected := []pkg.Package{
+	expected := []*pkg.Package{
 		{
 			Name:     "added-value",
 			Version:  "0.14.2",

syft/pkg/cataloger/python/parse_requirements.go

@@ -16,8 +16,8 @@ var _ common.ParserFn = parseRequirementsTxt
 
 // parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a
 // specific version.
-func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parseRequirementsTxt(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
@@ -44,7 +44,7 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, []artifact
 			parts := strings.Split(uncommented, "==")
 			name := strings.TrimSpace(parts[0])
 			version := strings.TrimSpace(parts[1])
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  version,
 				Language: pkg.Python,

syft/pkg/cataloger/python/parse_requirements_test.go

@@ -4,12 +4,14 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/go-test/deep"
 
 	"github.com/anchore/syft/syft/pkg"
 )
 
-func assertPackagesEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
+func assertPackagesEqual(t *testing.T, actual []*pkg.Package, expected map[string]pkg.Package) {
 	t.Helper()
 	if len(actual) != len(expected) {
 		for _, a := range actual {
@@ -20,11 +22,9 @@ func assertPackagesEqual(t *testing.T, actual []pkg.Package, expected map[string
 
 	for _, a := range actual {
 		expectedPkg, ok := expected[a.Name]
-		if !ok {
-			t.Errorf("unexpected package found: '%s'", a.Name)
-		}
+		assert.True(t, ok)
 
-		for _, d := range deep.Equal(a, expectedPkg) {
+		for _, d := range deep.Equal(a, &expectedPkg) {
 			t.Errorf("diff: %+v", d)
 		}
 	}

syft/pkg/cataloger/python/parse_setup.go

@@ -20,8 +20,8 @@ var _ common.ParserFn = parseSetup
 //		" mypy2 == v0.770", ' mypy3== v0.770',  --> match(name=mypy2 version=v0.770), match(name=mypy3, version=v0.770)
 var pinnedDependency = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w\.]*)`)
 
-func parseSetup(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parseSetup(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 
 	scanner := bufio.NewScanner(reader)
 
@@ -38,7 +38,7 @@ func parseSetup(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relations
 			name = strings.TrimSpace(name)
 
 			version := strings.TrimSpace(parts[len(parts)-1])
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     strings.Trim(name, "'\""),
 				Version:  strings.Trim(version, "'\""),
 				Language: pkg.Python,

syft/pkg/cataloger/python/poetry_metadata.go

@@ -7,8 +7,8 @@ type PoetryMetadata struct {
 }
 
 // Pkgs returns all of the packages referenced within the poetry.lock metadata.
-func (m PoetryMetadata) Pkgs() []pkg.Package {
-	pkgs := make([]pkg.Package, 0)
+func (m PoetryMetadata) Pkgs() []*pkg.Package {
+	pkgs := make([]*pkg.Package, 0)
 
 	for _, p := range m.Packages {
 		pkgs = append(pkgs, p.Pkg())

syft/pkg/cataloger/python/poetry_metadata_package.go

@@ -11,8 +11,8 @@ type PoetryMetadataPackage struct {
 }
 
 // Pkg returns the standard `pkg.Package` representation of the package referenced within the poetry.lock metadata.
-func (p PoetryMetadataPackage) Pkg() pkg.Package {
-	return pkg.Package{
+func (p PoetryMetadataPackage) Pkg() *pkg.Package {
+	return &pkg.Package{
 		Name:     p.Name,
 		Version:  p.Version,
 		Language: pkg.Python,