fix: improve CPE and upstream generation logic for Alpine packages (#1567)

* fix: improved CPE-generation logic for alpine packages

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>

* fix: improved alpine upstream name generation

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>

* fix: improve CPE vendor for alpine

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>

* fix: python vendor CPE gen

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>

* fix: alpine cpe gen logic

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>

* fix: apk CPE update for nodejs-current

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>

* fix: CPE update for python pip

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>

* fix: CPE update for some ruby packages

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>

* fix linting

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>

---------

Signed-off-by: Weston Steimel <weston.steimel@anchore.com>

syft/pkg/cataloger/apkdb/package.go

@@ -9,6 +9,10 @@ import (
 	"github.com/anchore/syft/syft/source"
 )
 
+var (
+	prefixes = []string{"py-", "py2-", "py3-", "ruby-"}
+)
+
 func newPackage(d pkg.ApkMetadata, release *linux.Release, locations ...source.Location) pkg.Package {
 	p := pkg.Package{
 		Name:         d.Package,
@@ -26,6 +30,20 @@ func newPackage(d pkg.ApkMetadata, release *linux.Release, locations ...source.L
 	return p
 }
 
+func generateUpstream(m pkg.ApkMetadata) string {
+	if m.OriginPackage != "" && m.OriginPackage != m.Package {
+		return m.OriginPackage
+	}
+
+	for _, p := range prefixes {
+		if strings.HasPrefix(m.Package, p) {
+			return strings.TrimPrefix(m.Package, p)
+		}
+	}
+
+	return m.Package
+}
+
 // packageURL returns the PURL for the specific Alpine package (see https://github.com/package-url/purl-spec)
 func packageURL(m pkg.ApkMetadata, distro *linux.Release) string {
 	if distro == nil || distro.ID != "alpine" {
@@ -38,7 +56,7 @@ func packageURL(m pkg.ApkMetadata, distro *linux.Release) string {
 	}
 
 	if m.OriginPackage != "" {
-		qualifiers[pkg.PURLQualifierUpstream] = m.OriginPackage
+		qualifiers[pkg.PURLQualifierUpstream] = generateUpstream(m)
 	}
 
 	return packageurl.NewPackageURL(

syft/pkg/cataloger/apkdb/package_test.go

@@ -96,6 +96,34 @@ func Test_PackageURL(t *testing.T) {
 			},
 			expected: "pkg:apk/alpine/p@v?arch=a&upstream=origin&distro=alpine-3.4.6",
 		},
+		{
+			name: "upstream python package information as qualifier",
+			metadata: pkg.ApkMetadata{
+				Package:       "py3-potatoes",
+				Version:       "v",
+				Architecture:  "a",
+				OriginPackage: "py3-potatoes",
+			},
+			distro: linux.Release{
+				ID:        "alpine",
+				VersionID: "3.4.6",
+			},
+			expected: "pkg:apk/alpine/py3-potatoes@v?arch=a&upstream=potatoes&distro=alpine-3.4.6",
+		},
+		{
+			name: "python package with origin package as upstream",
+			metadata: pkg.ApkMetadata{
+				Package:       "py3-non-existant",
+				Version:       "v",
+				Architecture:  "a",
+				OriginPackage: "abcdefg",
+			},
+			distro: linux.Release{
+				ID:        "alpine",
+				VersionID: "3.4.6",
+			},
+			expected: "pkg:apk/alpine/py3-non-existant@v?arch=a&upstream=abcdefg&distro=alpine-3.4.6",
+		},
 	}
 
 	for _, test := range tests {

syft/pkg/cataloger/common/cpe/apk.go

@@ -0,0 +1,173 @@
+package cpe
+
+import (
+	"strings"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+var (
+	pythonPrefixes = []string{"py-", "py2-", "py3-"}
+	rubyPrefixes   = []string{"ruby-"}
+)
+
+func pythonCandidateVendorsFromName(v string) fieldCandidateSet {
+	vendors := newFieldCandidateSet()
+	vendors.add(fieldCandidate{
+		value:                       v,
+		disallowSubSelections:       true,
+		disallowDelimiterVariations: true,
+	})
+
+	vendors.addValue(findAdditionalVendors(defaultCandidateAdditions, pkg.PythonPkg, v, v)...)
+	vendors.removeByValue(findVendorsToRemove(defaultCandidateRemovals, pkg.PythonPkg, v)...)
+
+	for _, av := range additionalVendorsForPython(v) {
+		vendors.add(fieldCandidate{
+			value:                       av,
+			disallowSubSelections:       true,
+			disallowDelimiterVariations: true,
+		})
+		vendors.addValue(findAdditionalVendors(defaultCandidateAdditions, pkg.PythonPkg, av, av)...)
+		vendors.removeByValue(findVendorsToRemove(defaultCandidateRemovals, pkg.PythonPkg, av)...)
+	}
+
+	return vendors
+}
+
+func pythonCandidateVendorsFromAPK(m pkg.ApkMetadata) fieldCandidateSet {
+	vendors := newFieldCandidateSet()
+
+	for _, p := range pythonPrefixes {
+		if strings.HasPrefix(m.Package, p) {
+			t := strings.ToLower(strings.TrimPrefix(m.Package, p))
+			vendors.union(pythonCandidateVendorsFromName(t))
+		}
+
+		if m.OriginPackage != m.Package && strings.HasPrefix(m.OriginPackage, p) {
+			t := strings.ToLower(strings.TrimPrefix(m.OriginPackage, p))
+			vendors.union(pythonCandidateVendorsFromName(t))
+		}
+	}
+
+	return vendors
+}
+
+func pythonCandidateProductsFromName(p string) fieldCandidateSet {
+	products := newFieldCandidateSet()
+	products.add(fieldCandidate{
+		value:                       p,
+		disallowSubSelections:       true,
+		disallowDelimiterVariations: true,
+	})
+
+	products.addValue(findAdditionalProducts(defaultCandidateAdditions, pkg.PythonPkg, p)...)
+	products.removeByValue(findProductsToRemove(defaultCandidateRemovals, pkg.PythonPkg, p)...)
+	return products
+}
+
+func pythonCandidateProductsFromAPK(m pkg.ApkMetadata) fieldCandidateSet {
+	products := newFieldCandidateSet()
+
+	for _, p := range pythonPrefixes {
+		if strings.HasPrefix(m.Package, p) {
+			t := strings.ToLower(strings.TrimPrefix(m.Package, p))
+			products.union(pythonCandidateProductsFromName(t))
+		}
+
+		if m.OriginPackage != m.Package && strings.HasPrefix(m.OriginPackage, p) {
+			t := strings.ToLower(strings.TrimPrefix(m.OriginPackage, p))
+			products.union(pythonCandidateProductsFromName(t))
+		}
+	}
+
+	return products
+}
+
+func rubyCandidateVendorsFromName(v string) fieldCandidateSet {
+	vendors := newFieldCandidateSet()
+	vendors.add(fieldCandidate{
+		value:                       v,
+		disallowSubSelections:       true,
+		disallowDelimiterVariations: true,
+	})
+
+	vendors.addValue(findAdditionalVendors(defaultCandidateAdditions, pkg.GemPkg, v, v)...)
+	vendors.removeByValue(findVendorsToRemove(defaultCandidateRemovals, pkg.GemPkg, v)...)
+	return vendors
+}
+
+func rubyCandidateVendorsFromAPK(m pkg.ApkMetadata) fieldCandidateSet {
+	vendors := newFieldCandidateSet()
+
+	for _, p := range rubyPrefixes {
+		if strings.HasPrefix(m.Package, p) {
+			t := strings.ToLower(strings.TrimPrefix(m.Package, p))
+			vendors.union(rubyCandidateVendorsFromName(t))
+		}
+
+		if m.OriginPackage != m.Package && strings.HasPrefix(m.OriginPackage, p) {
+			t := strings.ToLower(strings.TrimPrefix(m.OriginPackage, p))
+			vendors.union(rubyCandidateVendorsFromName(t))
+		}
+	}
+
+	return vendors
+}
+
+func rubyCandidateProductsFromName(p string) fieldCandidateSet {
+	products := newFieldCandidateSet()
+	products.add(fieldCandidate{
+		value:                       p,
+		disallowSubSelections:       true,
+		disallowDelimiterVariations: true,
+	})
+
+	products.addValue(findAdditionalProducts(defaultCandidateAdditions, pkg.GemPkg, p)...)
+	products.removeByValue(findProductsToRemove(defaultCandidateRemovals, pkg.GemPkg, p)...)
+	return products
+}
+
+func rubyCandidateProductsFromAPK(m pkg.ApkMetadata) fieldCandidateSet {
+	products := newFieldCandidateSet()
+
+	for _, p := range rubyPrefixes {
+		if strings.HasPrefix(m.Package, p) {
+			t := strings.ToLower(strings.TrimPrefix(m.Package, p))
+			products.union(rubyCandidateProductsFromName(t))
+		}
+
+		if m.OriginPackage != m.Package && strings.HasPrefix(m.OriginPackage, p) {
+			t := strings.ToLower(strings.TrimPrefix(m.OriginPackage, p))
+			products.union(rubyCandidateProductsFromName(t))
+		}
+	}
+
+	return products
+}
+
+func candidateVendorsForAPK(p pkg.Package) fieldCandidateSet {
+	metadata, ok := p.Metadata.(pkg.ApkMetadata)
+	if !ok {
+		return nil
+	}
+
+	vendors := newFieldCandidateSet()
+	vendors.union(pythonCandidateVendorsFromAPK(metadata))
+	vendors.union(rubyCandidateVendorsFromAPK(metadata))
+
+	return vendors
+}
+
+func candidateProductsForAPK(p pkg.Package) fieldCandidateSet {
+	metadata, ok := p.Metadata.(pkg.ApkMetadata)
+	if !ok {
+		return nil
+	}
+
+	products := newFieldCandidateSet()
+	products.union(pythonCandidateProductsFromAPK(metadata))
+	products.union(rubyCandidateProductsFromAPK(metadata))
+
+	return products
+}

syft/pkg/cataloger/common/cpe/apk_test.go

@@ -0,0 +1,91 @@
+package cpe
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func Test_candidateVendorsForAPK(t *testing.T) {
+	tests := []struct {
+		name     string
+		pkg      pkg.Package
+		expected []string
+	}{
+		{
+			name: "py3-cryptography Package",
+			pkg: pkg.Package{
+				Metadata: pkg.ApkMetadata{
+					Package: "py3-cryptography",
+				},
+			},
+			expected: []string{"python-cryptography_project", "cryptography", "cryptographyproject", "cryptography_project"},
+		},
+		{
+			name: "py2-pypdf OriginPackage",
+			pkg: pkg.Package{
+				Metadata: pkg.ApkMetadata{
+					OriginPackage: "py2-pypdf",
+				},
+			},
+			expected: []string{"pypdf", "pypdfproject", "pypdf_project"},
+		},
+		{
+			name: "ruby-armadillo Package",
+			pkg: pkg.Package{
+				Metadata: pkg.ApkMetadata{
+					Package: "ruby-armadillo",
+				},
+			},
+			expected: []string{"armadillo"},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assert.ElementsMatch(t, test.expected, candidateVendorsForAPK(test.pkg).values(), "different vendors")
+		})
+	}
+}
+
+func Test_candidateProductsForAPK(t *testing.T) {
+	tests := []struct {
+		name     string
+		pkg      pkg.Package
+		expected []string
+	}{
+		{
+			name: "py3-cryptography Package",
+			pkg: pkg.Package{
+				Metadata: pkg.ApkMetadata{
+					Package: "py3-cryptography",
+				},
+			},
+			expected: []string{"cryptography", "python-cryptography"},
+		},
+		{
+			name: "py2-pypdf OriginPackage",
+			pkg: pkg.Package{
+				Metadata: pkg.ApkMetadata{
+					OriginPackage: "py2-pypdf",
+				},
+			},
+			expected: []string{"pypdf"},
+		},
+		{
+			name: "ruby-armadillo Package",
+			pkg: pkg.Package{
+				Metadata: pkg.ApkMetadata{
+					Package: "ruby-armadillo",
+				},
+			},
+			expected: []string{"armadillo"},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assert.ElementsMatch(t, test.expected, candidateProductsForAPK(test.pkg).values(), "different products")
+		})
+	}
+}

syft/pkg/cataloger/common/cpe/candidate_by_package_type.go

@@ -121,12 +121,62 @@ var defaultCandidateAdditions = buildCandidateLookup(
 			candidateKey{PkgName: "yajl-ruby"},
 			candidateAddition{AdditionalProducts: []string{"yajl-ruby_gem"}},
 		},
+		{
+			pkg.GemPkg,
+			candidateKey{PkgName: "cgi"},
+			candidateAddition{AdditionalVendors: []string{"ruby-lang"}},
+		},
+		{
+			pkg.GemPkg,
+			candidateKey{PkgName: "date"},
+			candidateAddition{AdditionalVendors: []string{"ruby-lang"}},
+		},
+		{
+			pkg.GemPkg,
+			candidateKey{PkgName: "openssl"},
+			candidateAddition{AdditionalVendors: []string{"ruby-lang"}},
+		},
+		{
+			pkg.GemPkg,
+			candidateKey{PkgName: "rake"},
+			candidateAddition{AdditionalVendors: []string{"ruby-lang"}},
+		},
+		{
+			pkg.GemPkg,
+			candidateKey{PkgName: "rdoc"},
+			candidateAddition{AdditionalVendors: []string{"ruby-lang"}},
+		},
+		{
+			pkg.GemPkg,
+			candidateKey{PkgName: "rexml"},
+			candidateAddition{AdditionalVendors: []string{"ruby-lang"}},
+		},
+		{
+			pkg.GemPkg,
+			candidateKey{PkgName: "trunk"},
+			candidateAddition{AdditionalVendors: []string{"ruby-lang"}},
+		},
+		{
+			pkg.GemPkg,
+			candidateKey{PkgName: "webrick"},
+			candidateAddition{AdditionalVendors: []string{"ruby-lang"}},
+		},
 		// Python packages
 		{
 			pkg.PythonPkg,
 			candidateKey{PkgName: "python-rrdtool"},
 			candidateAddition{AdditionalProducts: []string{"rrdtool"}},
 		},
+		{
+			pkg.PythonPkg,
+			candidateKey{PkgName: "cryptography"},
+			candidateAddition{AdditionalProducts: []string{"python-cryptography"}, AdditionalVendors: []string{"python-cryptography_project"}},
+		},
+		{
+			pkg.PythonPkg,
+			candidateKey{PkgName: "pip"},
+			candidateAddition{AdditionalVendors: []string{"pypa"}},
+		},
 		// Alpine packages
 		{
 			pkg.ApkPkg,
@@ -143,6 +193,11 @@ var defaultCandidateAdditions = buildCandidateLookup(
 			candidateKey{PkgName: "nodejs"},
 			candidateAddition{AdditionalProducts: []string{"node.js"}},
 		},
+		{
+			pkg.ApkPkg,
+			candidateKey{PkgName: "nodejs-current"},
+			candidateAddition{AdditionalProducts: []string{"node.js"}},
+		},
 		// Binary packages
 		{
 			pkg.BinaryPkg,

syft/pkg/cataloger/common/cpe/generate.go

@@ -109,6 +109,8 @@ func candidateVendors(p pkg.Package) []string {
 		vendors.union(candidateVendorsForPython(p))
 	case pkg.JavaMetadataType:
 		vendors.union(candidateVendorsForJava(p))
+	case pkg.ApkMetadataType:
+		vendors.union(candidateVendorsForAPK(p))
 	}
 
 	// try swapping hyphens for underscores, vice versa, and removing separators altogether
@@ -156,6 +158,11 @@ func candidateProducts(p pkg.Package) []string {
 			products.addValue(prod)
 		}
 	}
+
+	if p.MetadataType == pkg.ApkMetadataType {
+		products.union(candidateProductsForAPK(p))
+	}
+
 	// it is never OK to have candidates with these values ["" and "*"] (since CPEs will match any other value)
 	products.removeByValue("")
 	products.removeByValue("*")

syft/pkg/cataloger/common/cpe/generate_test.go

@@ -100,6 +100,18 @@ func TestGeneratePackageCPEs(t *testing.T) {
 				"cpe:2.3:a:william_goodman:name:3.2:*:*:*:*:*:*:*",
 				"cpe:2.3:a:william_goodman:python-name:3.2:*:*:*:*:*:*:*",
 				"cpe:2.3:a:william_goodman:python_name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:alex_goodman_project:python_name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:alex_goodman_project:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:alex_goodman_project:python-name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:alex_goodmanproject:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:alex_goodmanproject:python-name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:alex_goodmanproject:python_name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:william_goodman_project:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:william_goodman_project:python-name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:william_goodman_project:python_name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:william_goodmanproject:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:william_goodmanproject:python-name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:william_goodmanproject:python_name:3.2:*:*:*:*:*:*:*",
 			},
 		},
 		{

syft/pkg/cataloger/common/cpe/python.go

@@ -1,6 +1,19 @@
 package cpe
 
-import "github.com/anchore/syft/syft/pkg"
+import (
+	"fmt"
+	"strings"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func additionalVendorsForPython(v string) (vendors []string) {
+	if !strings.HasSuffix(v, "project") {
+		vendors = append(vendors, fmt.Sprintf("%sproject", v), fmt.Sprintf("%s_project", v))
+	}
+
+	return vendors
+}
 
 func candidateVendorsForPython(p pkg.Package) fieldCandidateSet {
 	metadata, ok := p.Metadata.(pkg.PythonPackageMetadata)
@@ -11,18 +24,36 @@ func candidateVendorsForPython(p pkg.Package) fieldCandidateSet {
 	vendors := newFieldCandidateSet()
 
 	if metadata.Author != "" {
+		name := normalizePersonName(metadata.Author)
 		vendors.add(fieldCandidate{
-			value:                       normalizePersonName(metadata.Author),
+			value:                       name,
+			disallowSubSelections:       true,
+			disallowDelimiterVariations: true,
+		})
+
+		for _, v := range additionalVendorsForPython(name) {
+			vendors.add(fieldCandidate{
+				value:                       v,
 				disallowSubSelections:       true,
 				disallowDelimiterVariations: true,
 			})
 		}
+	}
 
 	if metadata.AuthorEmail != "" {
+		name := normalizePersonName(stripEmailSuffix(metadata.AuthorEmail))
 		vendors.add(fieldCandidate{
-			value:                 normalizePersonName(stripEmailSuffix(metadata.AuthorEmail)),
+			value:                 name,
 			disallowSubSelections: true,
 		})
+
+		for _, v := range additionalVendorsForPython(name) {
+			vendors.add(fieldCandidate{
+				value:                       v,
+				disallowSubSelections:       true,
+				disallowDelimiterVariations: true,
+			})
+		}
 	}
 
 	return vendors