Modify CPE vendor candidate generation approach (#484)

* consider additional vendor candidates for ruby, python, rpm, npm, and java

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add java pom.xml processing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* allow for downstream transform control in cpe generation processing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* migrate CPE generation logic to dedicated package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* split java manifest groupID extraction into two tiers

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* extract groupID from pom parent project during CPE generation

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update java groupID processing tests to cover multi-tier approach

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix constructor names for cpe.fieldCandidate

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* rename helper function to startsWithTopLevelDomain

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add nil changes for java manifest sections

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update comment to reflect parsing maven files

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* split out java description parsing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* split out pom parent processing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* simplify vendorsFromGroupIDs and associated tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* simplify test type for vendorsFromGroupIDs

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* copy candidate varidations to new instances

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* rename CPE generation string util functions

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add an explanation around fieldCandidate

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* simplify type for the cpe.fieldCandidateSet

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* make CPE filter function names more readable

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update groupIDsFromJavaManifest to use a guard clause

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* extract groupID extraction from artifactID fields into a separate function

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* bump goreleaser version to combat failure

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2021-09-03 14:21:25 -04:00
committed by GitHub
parent 0799fd9d46
commit abbba3fc19
37 changed files with 3208 additions and 1763 deletions
@@ -0,0 +1,18 @@
package cpe
import "github.com/anchore/syft/syft/pkg"
// this is a static mapping of known package names (keys) to official cpe names for each package
type candidatesByPackageType map[pkg.Type]map[string][]string
func (s candidatesByPackageType) getCandidates(t pkg.Type, key string) []string {
if _, ok := s[t]; !ok {
return nil
}
value, ok := s[t][key]
if !ok {
return nil
}
return value
}
@@ -0,0 +1,83 @@
package cpe
import (
"github.com/scylladb/go-set/strset"
)
// fieldCandidate represents a single "guess" for a specific field in a future CPE (vendor, product, target SW, etc).
// When generating these candidates depending on the field the value was sourced from there may be only a subset of
// transforms that should be applied (downstream of extraction). Expressing candidates in this struct allows for this
// flexibility such that downstream transforms can be elected into or skipped over.
type fieldCandidate struct {
value string
disallowSubSelections bool
disallowDelimiterVariations bool
}
type fieldCandidateSet map[fieldCandidate]struct{}
func newFieldCandidateFromSets(sets ...fieldCandidateSet) fieldCandidateSet {
s := newFieldCandidateSet()
for _, set := range sets {
s.add(set.list()...)
}
return s
}
func newFieldCandidateSet(values ...string) fieldCandidateSet {
s := make(fieldCandidateSet)
s.addValue(values...)
return s
}
func (s fieldCandidateSet) addValue(values ...string) {
for _, value := range values {
// default candidate as an allow-all
candidate := fieldCandidate{
value: value,
}
s[candidate] = struct{}{}
}
}
func (s fieldCandidateSet) add(candidates ...fieldCandidate) {
for _, candidate := range candidates {
s[candidate] = struct{}{}
}
}
func (s fieldCandidateSet) clear() {
for k := range s {
delete(s, k)
}
}
func (s fieldCandidateSet) union(others ...fieldCandidateSet) {
for _, other := range others {
s.add(other.list()...)
}
}
func (s fieldCandidateSet) list(filters ...filterFieldCandidateFn) (results []fieldCandidate) {
candidateLoop:
for c := range s {
for _, fn := range filters {
if fn(c) {
continue candidateLoop
}
}
results = append(results, c)
}
return results
}
func (s fieldCandidateSet) values(filters ...filterFieldCandidateFn) (results []string) {
for _, c := range s.list(filters...) {
results = append(results, c.value)
}
return results
}
func (s fieldCandidateSet) uniqueValues(filters ...filterFieldCandidateFn) []string {
return strset.New(s.values(filters...)...).List()
}
@@ -0,0 +1,12 @@
package cpe
// filterFieldCandidateFn instances should return true if the given fieldCandidate should be removed from a collection
type filterFieldCandidateFn func(fieldCandidate) bool
func filterOutBySubselection(c fieldCandidate) bool {
return c.disallowSubSelections
}
func filterOutByDelimiterVariations(c fieldCandidate) bool {
return c.disallowDelimiterVariations
}
@@ -0,0 +1,263 @@
package cpe
import (
"testing"
"github.com/stretchr/testify/assert"
)
func Test_cpeCandidateValues_filter(t *testing.T) {
tests := []struct {
name string
input []fieldCandidate
filters []filterFieldCandidateFn
expect []string
}{
{
name: "gocase",
input: []fieldCandidate{
{
value: "allow anything",
},
{
value: "no-sub-selections",
disallowSubSelections: true,
},
{
value: "no-delimiter-variations",
disallowDelimiterVariations: true,
},
{
value: "allow nothing",
disallowSubSelections: true,
disallowDelimiterVariations: true,
},
},
expect: []string{
"allow anything",
"no-sub-selections",
"no-delimiter-variations",
"allow nothing",
},
},
{
name: "filter out sub-selections",
input: []fieldCandidate{
{
value: "allow anything",
},
{
value: "no-sub-selections",
disallowSubSelections: true,
},
{
value: "no-delimiter-variations",
disallowDelimiterVariations: true,
},
{
value: "allow nothing",
disallowSubSelections: true,
disallowDelimiterVariations: true,
},
},
filters: []filterFieldCandidateFn{
filterOutBySubselection,
},
expect: []string{
"allow anything",
"no-delimiter-variations",
},
},
{
name: "filter out delimiter-variations",
input: []fieldCandidate{
{
value: "allow anything",
},
{
value: "no-sub-selections",
disallowSubSelections: true,
},
{
value: "no-delimiter-variations",
disallowDelimiterVariations: true,
},
{
value: "allow nothing",
disallowSubSelections: true,
disallowDelimiterVariations: true,
},
},
filters: []filterFieldCandidateFn{
filterOutByDelimiterVariations,
},
expect: []string{
"allow anything",
"no-sub-selections",
},
},
{
name: "all filters",
input: []fieldCandidate{
{
value: "allow anything",
},
{
value: "no-sub-selections",
disallowSubSelections: true,
},
{
value: "no-delimiter-variations",
disallowDelimiterVariations: true,
},
{
value: "allow nothing",
disallowSubSelections: true,
disallowDelimiterVariations: true,
},
},
filters: []filterFieldCandidateFn{
filterOutByDelimiterVariations,
filterOutBySubselection,
},
expect: []string{
"allow anything",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
set := newFieldCandidateSet()
set.add(test.input...)
assert.ElementsMatch(t, test.expect, set.values(test.filters...))
})
}
}
func Test_cpeFieldCandidateSet_clear(t *testing.T) {
s := newFieldCandidateSet("1", "2")
assert.NotEmpty(t, s.values())
s.clear()
assert.Empty(t, s.values())
}
func Test_cpeFieldCandidateSet_union(t *testing.T) {
s1 := newFieldCandidateSet("1", "2")
assert.Len(t, s1.list(), 2)
s2 := newFieldCandidateSet("2", "3", "4")
assert.Len(t, s2.list(), 3)
s3 := newFieldCandidateSet()
s3.add(
fieldCandidate{
value: "1",
disallowSubSelections: true,
disallowDelimiterVariations: false,
},
fieldCandidate{
value: "4",
disallowSubSelections: false,
disallowDelimiterVariations: true,
},
fieldCandidate{
value: "5",
disallowSubSelections: true,
disallowDelimiterVariations: true,
},
)
assert.Len(t, s3.list(), 3)
s1.union(s2, s3)
// 1 & 4 have duplicate entries since there are candidate conditions set
assert.ElementsMatch(t, s1.values(), []string{"1", "1", "2", "3", "4", "4", "5"})
assert.ElementsMatch(t, s1.list(), []fieldCandidate{
{
value: "1",
},
{
value: "1",
disallowSubSelections: true,
disallowDelimiterVariations: false,
},
{
value: "2",
},
{
value: "3",
},
{
value: "4",
},
{
value: "4",
disallowSubSelections: false,
disallowDelimiterVariations: true,
},
{
value: "5",
disallowSubSelections: true,
disallowDelimiterVariations: true,
},
})
}
func Test_cpeFieldCandidateSet_union_byValue(t *testing.T) {
s1 := newFieldCandidateSet("1", "2")
assert.Len(t, s1.list(), 2)
s2 := newFieldCandidateSet("2", "3", "4")
assert.Len(t, s2.list(), 3)
s3 := newFieldCandidateSet("1", "4", "5")
assert.Len(t, s3.list(), 3)
s1.union(s2, s3)
assert.ElementsMatch(t, s1.values(), []string{"1", "2", "3", "4", "5"})
assert.ElementsMatch(t, s1.list(), []fieldCandidate{
{
value: "1",
},
{
value: "2",
},
{
value: "3",
},
{
value: "4",
},
{
value: "5",
},
})
}
func Test_cpeFieldCandidateSet_uniqueValues(t *testing.T) {
set := newFieldCandidateSet()
set.add(
fieldCandidate{
value: "1",
},
fieldCandidate{
value: "1",
disallowSubSelections: true,
},
fieldCandidate{
value: "2",
disallowDelimiterVariations: true,
},
fieldCandidate{
value: "2",
},
fieldCandidate{
value: "3",
disallowSubSelections: true,
disallowDelimiterVariations: true,
},
)
assert.ElementsMatch(t, []string{"1", "2", "3"}, set.uniqueValues())
}
+63
View File
@@ -0,0 +1,63 @@
package cpe
import (
"strings"
"github.com/anchore/syft/syft/pkg"
"github.com/facebookincubator/nvdtools/wfn"
)
const jenkinsName = "jenkins"
// filterFn instances should return true if the given CPE should be removed from a collection for the given package
type filterFn func(cpe pkg.CPE, p pkg.Package) bool
var cpeFilters = []filterFn{
disallowJiraClientServerMismatch,
disallowJenkinsServerCPEForPluginPackage,
disallowJenkinsCPEsNotAssociatedWithJenkins,
}
func filter(cpes []pkg.CPE, p pkg.Package, filters ...filterFn) (result []pkg.CPE) {
cpeLoop:
for _, cpe := range cpes {
for _, fn := range filters {
if fn(cpe, p) {
continue cpeLoop
}
}
// all filter functions passed on filtering this CPE
result = append(result, cpe)
}
return result
}
// jenkins plugins should not match against jenkins
func disallowJenkinsServerCPEForPluginPackage(cpe pkg.CPE, p pkg.Package) bool {
if p.Type == pkg.JenkinsPluginPkg && cpe.Product == jenkinsName {
return true
}
return false
}
// filter to account that packages that are not for jenkins but have a CPE generated that will match against jenkins
func disallowJenkinsCPEsNotAssociatedWithJenkins(cpe pkg.CPE, p pkg.Package) bool {
// jenkins server should only match against a product with the name jenkins
if cpe.Product == jenkinsName && !strings.Contains(strings.ToLower(p.Name), jenkinsName) {
if cpe.Vendor == wfn.Any || cpe.Vendor == jenkinsName || cpe.Vendor == "cloudbees" {
return true
}
}
return false
}
// filter to account for packages which are jira client packages but have a CPE that will match against jira
func disallowJiraClientServerMismatch(cpe pkg.CPE, p pkg.Package) bool {
// jira / atlassian should not apply to clients
if cpe.Product == "jira" && strings.Contains(strings.ToLower(p.Name), "client") {
if cpe.Vendor == wfn.Any || cpe.Vendor == "jira" || cpe.Vendor == "atlassian" {
return true
}
}
return false
}
@@ -0,0 +1,167 @@
package cpe
import (
"testing"
"github.com/anchore/syft/syft/pkg"
"github.com/stretchr/testify/assert"
)
func Test_disallowJenkinsServerCPEForPluginPackage(t *testing.T) {
tests := []struct {
name string
cpe pkg.CPE
pkg pkg.Package
expected bool
}{
{
name: "go case (filter out)",
cpe: mustCPE("cpe:2.3:a:name:jenkins:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Type: pkg.JenkinsPluginPkg,
},
expected: true,
},
{
name: "ignore jenkins plugins with unique name",
cpe: mustCPE("cpe:2.3:a:name:ci-jenkins:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Type: pkg.JenkinsPluginPkg,
},
expected: false,
},
{
name: "ignore java packages",
cpe: mustCPE("cpe:2.3:a:name:jenkins:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Type: pkg.JavaPkg,
},
expected: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, disallowJenkinsServerCPEForPluginPackage(test.cpe, test.pkg))
})
}
}
func Test_disallowJenkinsCPEsNotAssociatedWithJenkins(t *testing.T) {
tests := []struct {
name string
cpe pkg.CPE
pkg pkg.Package
expected bool
}{
{
name: "filter out mismatched name (cloudbees vendor)",
cpe: mustCPE("cpe:2.3:a:cloudbees:jenkins:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Name: "not-j*nkins",
Type: pkg.JavaPkg,
},
expected: true,
},
{
name: "filter out mismatched name (jenkins vendor)",
cpe: mustCPE("cpe:2.3:a:jenkins:jenkins:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Name: "not-j*nkins",
Type: pkg.JavaPkg,
},
expected: true,
},
{
name: "filter out mismatched name (any vendor)",
cpe: mustCPE("cpe:2.3:a:*:jenkins:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Name: "not-j*nkins",
Type: pkg.JavaPkg,
},
expected: true,
},
{
name: "ignore packages with the name jenkins",
cpe: mustCPE("cpe:2.3:a:*:jenkins:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Name: "jenkins-thing",
Type: pkg.JavaPkg,
},
expected: false,
},
{
name: "ignore product names that are not exactly 'jenkins'",
cpe: mustCPE("cpe:2.3:a:*:jenkins-something-else:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Name: "not-j*nkins",
Type: pkg.JavaPkg,
},
expected: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, disallowJenkinsCPEsNotAssociatedWithJenkins(test.cpe, test.pkg))
})
}
}
func Test_disallowJiraClientServerMismatch(t *testing.T) {
tests := []struct {
name string
cpe pkg.CPE
pkg pkg.Package
expected bool
}{
{
name: "filter out mismatched name (atlassian vendor)",
cpe: mustCPE("cpe:2.3:a:atlassian:jira:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Name: "something-client",
Type: pkg.JavaPkg,
},
expected: true,
},
{
name: "filter out mismatched name (jira vendor)",
cpe: mustCPE("cpe:2.3:a:jira:jira:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Name: "something-client",
Type: pkg.JavaPkg,
},
expected: true,
},
{
name: "filter out mismatched name (any vendor)",
cpe: mustCPE("cpe:2.3:a:*:jira:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Name: "something-client",
Type: pkg.JavaPkg,
},
expected: true,
},
{
name: "ignore package names that do not have 'client'",
cpe: mustCPE("cpe:2.3:a:*:jira:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Name: "jira-thing",
Type: pkg.JavaPkg,
},
expected: false,
},
{
name: "ignore product names that are not exactly 'jira'",
cpe: mustCPE("cpe:2.3:a:*:jira-something-else:3.2:*:*:*:*:*:*:*"),
pkg: pkg.Package{
Name: "not-j*ra",
Type: pkg.JavaPkg,
},
expected: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, disallowJiraClientServerMismatch(test.cpe, test.pkg))
})
}
}
+247
View File
@@ -0,0 +1,247 @@
package cpe
import (
"bufio"
"bytes"
"fmt"
"sort"
"strings"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/pkg"
"github.com/facebookincubator/nvdtools/wfn"
)
var productCandidatesByPkgType = candidatesByPackageType{
pkg.JavaPkg: {
"springframework": []string{"spring_framework", "springsource_spring_framework"},
"spring-core": []string{"spring_framework", "springsource_spring_framework"},
},
pkg.NpmPkg: {
"hapi": []string{"hapi_server_framework"},
"handlebars.js": []string{"handlebars"},
"is-my-json-valid": []string{"is_my_json_valid"},
"mustache": []string{"mustache.js"},
},
pkg.GemPkg: {
"Arabic-Prawn": []string{"arabic_prawn"},
"bio-basespace-sdk": []string{"basespace_ruby_sdk"},
"cremefraiche": []string{"creme_fraiche"},
"html-sanitizer": []string{"html_sanitizer"},
"sentry-raven": []string{"raven-ruby"},
"RedCloth": []string{"redcloth_library"},
"VladTheEnterprising": []string{"vladtheenterprising"},
"yajl-ruby": []string{"yajl-ruby_gem"},
},
pkg.PythonPkg: {
"python-rrdtool": []string{"rrdtool"},
},
}
func newCPE(product, vendor, version, targetSW string) wfn.Attributes {
cpe := *(wfn.NewAttributesWithAny())
cpe.Part = "a"
cpe.Product = product
cpe.Vendor = vendor
cpe.Version = version
cpe.TargetSW = targetSW
return cpe
}
// Generate Create a list of CPEs for a given package, trying to guess the vendor, product tuple. We should be trying to
// generate the minimal set of representative CPEs, which implies that optional fields should not be included
// (such as target SW).
func Generate(p pkg.Package) []pkg.CPE {
vendors := candidateVendors(p)
products := candidateProducts(p)
if len(products) == 0 {
return nil
}
keys := internal.NewStringSet()
cpes := make([]pkg.CPE, 0)
for _, product := range products {
for _, vendor := range vendors {
// prevent duplicate entries...
key := fmt.Sprintf("%s|%s|%s", product, vendor, p.Version)
if keys.Contains(key) {
continue
}
keys.Add(key)
// add a new entry...
cpes = append(cpes, newCPE(product, vendor, p.Version, wfn.Any))
}
}
// filter out any known combinations that don't accurately represent this package
cpes = filter(cpes, p, cpeFilters...)
sort.Sort(BySpecificity(cpes))
return cpes
}
func candidateVendors(p pkg.Package) []string {
// in ecosystems where the packaging metadata does not have a clear field to indicate a vendor (or a field that
// could be interpreted indirectly as such) the project name tends to be a common stand in. Examples of this
// are the elasticsearch gem, xstream jar, and rack gem... all of these cases you can find vulnerabilities
// with CPEs where the vendor is the product name and doesn't appear to be derived from any available package
// metadata.
vendors := newFieldCandidateSet(candidateProducts(p)...)
switch p.Language {
case pkg.Ruby:
vendors.addValue("ruby-lang")
case pkg.Go:
// replace all candidates with only the golang-specific helper
vendors.clear()
vendor := candidateVendorForGo(p.Name)
if vendor != "" {
vendors.addValue(vendor)
}
}
// some ecosystems do not have enough metadata to determine the vendor accurately, in which case we selectively
// allow * as a candidate. Note: do NOT allow Java packages to have * vendors.
switch p.Language {
case pkg.Ruby, pkg.JavaScript:
vendors.addValue("*")
}
switch p.MetadataType {
case pkg.RpmdbMetadataType:
vendors.union(candidateVendorsForRPM(p))
case pkg.GemMetadataType:
vendors.union(candidateVendorsForRuby(p))
case pkg.PythonPackageMetadataType:
vendors.union(candidateVendorsForPython(p))
case pkg.JavaMetadataType:
vendors.union(candidateVendorsForJava(p))
}
// try swapping hyphens for underscores, vice versa, and removing separators altogether
addDelimiterVariations(vendors)
// generate sub-selections of each candidate based on separators (e.g. jenkins-ci -> [jenkins, jenkins-ci])
addAllSubSelections(vendors)
return vendors.uniqueValues()
}
func candidateProducts(p pkg.Package) []string {
products := newFieldCandidateSet(p.Name)
switch {
case p.Language == pkg.Python:
if !strings.HasPrefix(p.Name, "python") {
products.addValue("python-" + p.Name)
}
case p.Language == pkg.Java || p.MetadataType == pkg.JavaMetadataType:
products.addValue(candidateProductsForJava(p)...)
case p.Language == pkg.Go:
// replace all candidates with only the golang-specific helper
products.clear()
prod := candidateProductForGo(p.Name)
if prod != "" {
products.addValue(prod)
}
}
// try swapping hyphens for underscores, vice versa, and removing separators altogether
addDelimiterVariations(products)
// prepend any known product names for the given package type and name (note: this is not a replacement)
return append(productCandidatesByPkgType.getCandidates(p.Type, p.Name), products.uniqueValues()...)
}
func addAllSubSelections(set fieldCandidateSet) {
for _, candidate := range set.values(filterOutBySubselection) {
set.addValue(generateSubSelections(candidate)...)
}
}
// generateSubSelections attempts to split a field by hyphens and underscores and return a list of sensible sub-selections
// that can be used as product or vendor candidates. E.g. jenkins-ci-tools -> [jenkins-ci-tools, jenkins-ci, jenkins].
func generateSubSelections(field string) (results []string) {
scanner := bufio.NewScanner(strings.NewReader(field))
scanner.Split(scanByHyphenOrUnderscore)
var lastToken uint8
for scanner.Scan() {
rawCandidate := scanner.Text()
if len(rawCandidate) == 0 {
break
}
// trim any number of hyphen or underscore that is prefixed/suffixed on the given candidate. Since
// scanByHyphenOrUnderscore preserves delimiters (hyphens and underscores) they are guaranteed to be at least
// prefixed.
candidate := strings.TrimFunc(rawCandidate, trimHyphenOrUnderscore)
// capture the result (if there is content)
if len(candidate) > 0 {
if len(results) > 0 {
results = append(results, results[len(results)-1]+string(lastToken)+candidate)
} else {
results = append(results, candidate)
}
}
// keep track of the trailing separator for the next loop
lastToken = rawCandidate[len(rawCandidate)-1]
}
return results
}
// trimHyphenOrUnderscore is a character filter function for use with strings.TrimFunc in order to remove any hyphen or underscores.
func trimHyphenOrUnderscore(r rune) bool {
switch r {
case '-', '_':
return true
}
return false
}
// scanByHyphenOrUnderscore splits on hyphen or underscore and includes the separator in the split
func scanByHyphenOrUnderscore(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
if i := bytes.IndexAny(data, "-_"); i >= 0 {
return i + 1, data[0 : i+1], nil
}
if atEOF {
return len(data), data, nil
}
return 0, nil, nil
}
func addDelimiterVariations(fields fieldCandidateSet) {
for _, candidate := range fields.list(filterOutByDelimiterVariations) {
field := candidate.value
hasHyphen := strings.Contains(field, "-")
hasUnderscore := strings.Contains(field, "_")
if hasHyphen {
// provide variations of hyphen candidates with an underscore
newValue := strings.ReplaceAll(field, "-", "_")
underscoreCandidate := candidate
underscoreCandidate.value = newValue
fields.add(underscoreCandidate)
}
if hasUnderscore {
// provide variations of underscore candidates with a hyphen
newValue := strings.ReplaceAll(field, "_", "-")
hyphenCandidate := candidate
hyphenCandidate.value = newValue
fields.add(hyphenCandidate)
}
}
}
@@ -0,0 +1,658 @@
package cpe
import (
"fmt"
"sort"
"strings"
"testing"
"github.com/anchore/syft/syft/pkg"
"github.com/scylladb/go-set"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
)
func TestGeneratePackageCPEs(t *testing.T) {
tests := []struct {
name string
p pkg.Package
expected []string
}{
{
name: "hyphen replacement",
p: pkg.Package{
Name: "name-part",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Python,
Type: pkg.DebPkg,
},
expected: []string{
"cpe:2.3:a:name-part:name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name-part:name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name-part:python-name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name-part:python_name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:python-name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:python_name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name_part:name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name_part:name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name_part:python-name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name_part:python_name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name-part:name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name-part:name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name-part:python-name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name-part:python_name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name:name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name:name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name:python-name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name:python_name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python:name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python:name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python:python-name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python:python_name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python_name:name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python_name:name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python_name:python-name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python_name:python_name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python_name_part:name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python_name_part:name_part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python_name_part:python-name-part:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python_name_part:python_name_part:3.2:*:*:*:*:*:*:*",
},
},
{
name: "python language",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Python,
Type: pkg.DebPkg,
MetadataType: pkg.PythonPackageMetadataType,
Metadata: pkg.PythonPackageMetadata{
Author: "alex goodman",
AuthorEmail: "william.goodman@anchore.com",
},
},
expected: []string{
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:python-name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:python_name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name:python-name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name:python_name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python:python-name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python:python_name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python_name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python_name:python-name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python_name:python_name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:alex_goodman:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:alex_goodman:python-name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:alex_goodman:python_name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:william-goodman:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:william-goodman:python-name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:william-goodman:python_name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:william_goodman:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:william_goodman:python-name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:william_goodman:python_name:3.2:*:*:*:*:*:*:*",
},
},
{
name: "javascript language",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.JavaScript,
Type: pkg.DebPkg,
},
expected: []string{
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
},
},
{
name: "ruby language",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Ruby,
Type: pkg.DebPkg,
MetadataType: pkg.GemMetadataType,
Metadata: pkg.GemMetadata{
Authors: []string{
"someones name",
"someones.elses.name@gmail.com",
},
},
},
expected: []string{
"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:ruby-lang:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:ruby:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:ruby_lang:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:someones-elses-name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:someones-name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:someones_elses_name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:someones_name:name:3.2:*:*:*:*:*:*:*",
},
},
{
name: "java language",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JavaPkg,
},
expected: []string{
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
},
},
{
name: "java language with groupID",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JavaPkg,
MetadataType: pkg.JavaMetadataType,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "org.sonatype.nexus",
},
},
},
expected: []string{
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:nexus:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:nexus:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:nexus:nexus:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:sonatype:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:sonatype:nexus:3.2:*:*:*:*:*:*:*",
},
},
{
name: "jenkins package identified via pkg type",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JenkinsPluginPkg,
},
expected: []string{
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
},
},
{
name: "java language - multi tier manifest fields",
p: pkg.Package{
Name: "cxf-rt-bindings-xml",
Version: "3.3.10",
FoundBy: "java-cataloger",
Language: pkg.Java,
Type: pkg.JavaPkg,
MetadataType: pkg.JavaMetadataType,
Metadata: pkg.JavaMetadata{
VirtualPath: "/opt/jboss/keycloak/modules/system/layers/base/org/apache/cxf/impl/main/cxf-rt-bindings-xml-3.3.10.jar",
Manifest: &pkg.JavaManifest{
Main: map[string]string{
"Automatic-Module-Name": "org.apache.cxf.binding.xml",
"Bnd-LastModified": "1615836524860",
"Build-Jdk": "1.8.0_261",
"Built-By": "dkulp",
"Bundle-ActivationPolicy": "lazy",
"Bundle-Description": "Apache CXF Runtime XML Binding",
"Bundle-DocURL": "http://cxf.apache.org",
"Bundle-License": "https://www.apache.org/licenses/LICENSE-2.0.txt",
"Bundle-ManifestVersion": "2",
"Bundle-Name": "Apache CXF Runtime XML Binding",
"Bundle-SymbolicName": "org.apache.cxf.cxf-rt-bindings-xml",
"Bundle-Vendor": "The Apache Software Foundation",
"Bundle-Version": "3.3.10",
"Created-By": "Apache Maven Bundle Plugin",
"Export-Package": "org.apache.cxf.binding.xml;version=\"3.3.10\",org.apache.cxf.binding.xml.wsdl11;version=\"3.3.10\",org.apache.cxf.binding.xml.interceptor;version=\"3.3.10\",org.apache.cxf.bindings.xformat;version=\"3.3.10\"",
"Implementation-Vendor": "The Apache Software Foundation",
"Implementation-Vendor-Id": "org.apache",
"Implementation-Version": "3.3.10",
"Import-Package": "javax.xml.bind;version=\"[0,3)\",javax.xml.bind.annotation;version=\"[0,3)\",javax.wsdl;resolution:=optional,javax.wsdl.extensions;resolution:=optional,javax.wsdl.extensions.http;resolution:=optional,javax.xml.namespace,javax.xml.stream,org.apache.cxf;version=\"[3.3,4)\",org.apache.cxf.binding;version=\"[3.3,4)\",org.apache.cxf.binding.xml,org.apache.cxf.binding.xml.interceptor,org.apache.cxf.bindings.xformat,org.apache.cxf.common.i18n;version=\"[3.3,4)\",org.apache.cxf.common.injection;version=\"[3.3,4)\",org.apache.cxf.common.logging;version=\"[3.3,4)\",org.apache.cxf.common.util;version=\"[3.3,4)\",org.apache.cxf.endpoint;version=\"[3.3,4)\",org.apache.cxf.helpers;version=\"[3.3,4)\",org.apache.cxf.interceptor;version=\"[3.3,4)\",org.apache.cxf.message;version=\"[3.3,4)\",org.apache.cxf.service.model;version=\"[3.3,4)\",org.apache.cxf.staxutils;version=\"[3.3,4)\",org.apache.cxf.tools.common;version=\"[3.3,4)\";resolution:=optional,org.apache.cxf.tools.validator;version=\"[3.3,4)\";resolution:=optional,org.apache.cxf.transport;version=\"[3.3,4)\",org.apache.cxf.wsdl;version=\"[3.3,4)\";resolution:=optional,org.apache.cxf.wsdl.http;version=\"[3.3,4)\",org.apache.cxf.wsdl.interceptors;version=\"[3.3,4)\";resolution:=optional,org.w3c.dom",
"Manifest-Version": "1.0",
"Require-Capability": "osgi.ee;filter:=\"(&(osgi.ee=JavaSE)(version=1.8))\"",
"Specification-Vendor": "The Apache Software Foundation",
"Specification-Version": "3.3.10",
"Tool": "Bnd-4.2.0.201903051501",
},
},
PomProperties: &pkg.PomProperties{
Path: "META-INF/maven/org.apache.cxf/cxf-rt-bindings-xml/pom.properties",
GroupID: "org.apache.cxf",
ArtifactID: "cxf-rt-bindings-xml",
Version: "3.3.10",
},
},
},
expected: []string{
"cpe:2.3:a:apache-software-foundation:cxf-rt-bindings-xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:apache-software-foundation:cxf:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:apache-software-foundation:cxf_rt_bindings_xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:apache:cxf-rt-bindings-xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:apache:cxf:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:apache:cxf_rt_bindings_xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:apache_software_foundation:cxf-rt-bindings-xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:apache_software_foundation:cxf:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:apache_software_foundation:cxf_rt_bindings_xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf-rt-bindings-xml:cxf-rt-bindings-xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf-rt-bindings-xml:cxf:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf-rt-bindings-xml:cxf_rt_bindings_xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf-rt-bindings:cxf-rt-bindings-xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf-rt-bindings:cxf:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf-rt-bindings:cxf_rt_bindings_xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf-rt:cxf-rt-bindings-xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf-rt:cxf:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf-rt:cxf_rt_bindings_xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf:cxf-rt-bindings-xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf:cxf:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf:cxf_rt_bindings_xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf_rt:cxf-rt-bindings-xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf_rt:cxf:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf_rt:cxf_rt_bindings_xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf_rt_bindings:cxf-rt-bindings-xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf_rt_bindings:cxf:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf_rt_bindings:cxf_rt_bindings_xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf_rt_bindings_xml:cxf-rt-bindings-xml:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf_rt_bindings_xml:cxf:3.3.10:*:*:*:*:*:*:*",
"cpe:2.3:a:cxf_rt_bindings_xml:cxf_rt_bindings_xml:3.3.10:*:*:*:*:*:*:*",
},
},
{
name: "rpm vendor selection",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Type: pkg.RpmPkg,
MetadataType: pkg.RpmdbMetadataType,
Metadata: pkg.RpmdbMetadata{
Vendor: "some-vendor",
},
},
expected: []string{
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:some-vendor:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:some_vendor:name:3.2:*:*:*:*:*:*:*",
},
},
{
name: "cloudbees jenkins package identified via groupId",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JenkinsPluginPkg,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "com.cloudbees.jenkins.plugins",
},
},
},
expected: []string{
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jenkins:name:3.2:*:*:*:*:*:*:*",
},
},
{
name: "jenkins.io package identified via groupId prefix",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JenkinsPluginPkg,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "io.jenkins.plugins.name.something",
},
},
},
expected: []string{
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:something:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:something:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:something:something:3.2:*:*:*:*:*:*:*",
},
},
{
name: "jenkins.io package identified via groupId",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JenkinsPluginPkg,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "io.jenkins.plugins",
},
},
},
expected: []string{
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
},
},
{
name: "jenkins-ci.io package identified via groupId",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JenkinsPluginPkg,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "io.jenkins-ci.plugins",
},
},
},
expected: []string{
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
},
},
{
name: "jenkins-ci.org package identified via groupId",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JenkinsPluginPkg,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "org.jenkins-ci.plugins",
},
},
},
expected: []string{
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
},
},
{
name: "jira-atlassian filtering",
p: pkg.Package{
Name: "jira_client_core",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JavaPkg,
MetadataType: pkg.JavaMetadataType,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "org.atlassian.jira",
ArtifactID: "jira_client_core",
},
},
},
expected: []string{
"cpe:2.3:a:atlassian:jira-client-core:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:atlassian:jira_client_core:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira-client-core:jira-client-core:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira-client-core:jira:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira-client-core:jira_client_core:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira-client:jira-client-core:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira-client:jira:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira-client:jira_client_core:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira:jira-client-core:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira:jira_client_core:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira_client:jira-client-core:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira_client:jira:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira_client:jira_client_core:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira_client_core:jira-client-core:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira_client_core:jira:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:jira_client_core:jira_client_core:3.2:*:*:*:*:*:*:*",
},
},
{
name: "jenkins filtering",
p: pkg.Package{
Name: "cloudbees-installation-manager",
Version: "2.89.0.33",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JavaPkg,
MetadataType: pkg.JavaMetadataType,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "com.cloudbees.jenkins.modules",
ArtifactID: "cloudbees-installation-manager",
},
},
},
expected: []string{
"cpe:2.3:a:cloudbees-installation-manager:cloudbees-installation-manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:cloudbees-installation-manager:cloudbees_installation_manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:cloudbees-installation:cloudbees-installation-manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:cloudbees-installation:cloudbees_installation_manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:cloudbees:cloudbees-installation-manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:cloudbees:cloudbees_installation_manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:cloudbees_installation:cloudbees-installation-manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:cloudbees_installation:cloudbees_installation_manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:cloudbees_installation_manager:cloudbees-installation-manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:cloudbees_installation_manager:cloudbees_installation_manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:jenkins:cloudbees-installation-manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:jenkins:cloudbees_installation_manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:modules:cloudbees-installation-manager:2.89.0.33:*:*:*:*:*:*:*",
"cpe:2.3:a:modules:cloudbees_installation_manager:2.89.0.33:*:*:*:*:*:*:*",
},
},
{
name: "go product and vendor candidates are wired up",
p: pkg.Package{
Name: "github.com/someone/something",
Version: "3.2",
FoundBy: "go-cataloger",
Language: pkg.Go,
Type: pkg.GoModulePkg,
},
expected: []string{
"cpe:2.3:a:someone:something:3.2:*:*:*:*:*:*:*",
},
},
{
name: "generate no CPEs for indeterminate golang package name",
p: pkg.Package{
Name: "github.com/what",
Version: "3.2",
FoundBy: "go-cataloger",
Language: pkg.Go,
Type: pkg.GoModulePkg,
},
expected: []string{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual := Generate(test.p)
expectedCpeSet := set.NewStringSet(test.expected...)
actualCpeSet := set.NewStringSet()
for _, a := range actual {
actualCpeSet.Add(a.BindToFmtString())
}
extra := strset.Difference(expectedCpeSet, actualCpeSet).List()
sort.Strings(extra)
if len(extra) > 0 {
t.Errorf("found extra CPEs:")
}
for _, d := range extra {
fmt.Printf(" %q,\n", d)
}
missing := strset.Difference(actualCpeSet, expectedCpeSet).List()
sort.Strings(missing)
if len(missing) > 0 {
t.Errorf("missing CPEs:")
}
for _, d := range missing {
fmt.Printf(" %q,\n", d)
}
})
}
}
func TestCandidateProducts(t *testing.T) {
tests := []struct {
p pkg.Package
expected []string
}{
{
p: pkg.Package{
Name: "springframework",
Type: pkg.JavaPkg,
},
expected: []string{"spring_framework", "springsource_spring_framework" /* <-- known good names | default guess --> */, "springframework"},
},
{
p: pkg.Package{
Name: "some-java-package-with-group-id",
Type: pkg.JavaPkg,
Language: pkg.Java,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "com.apple.itunes",
},
},
},
expected: []string{"itunes", "some-java-package-with-group-id", "some_java_package_with_group_id"},
},
{
p: pkg.Package{
Name: "some-jenkins-plugin",
Type: pkg.JenkinsPluginPkg,
Language: pkg.Java,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "com.cloudbees.jenkins.plugins",
},
},
},
expected: []string{"some-jenkins-plugin", "some_jenkins_plugin", "jenkins"},
},
{
p: pkg.Package{
Name: "handlebars.js",
Type: pkg.NpmPkg,
},
expected: []string{"handlebars" /* <-- known good names | default guess --> */, "handlebars.js"},
},
{
p: pkg.Package{
Name: "RedCloth",
Type: pkg.GemPkg,
},
expected: []string{"redcloth_library" /* <-- known good names | default guess --> */, "RedCloth"},
},
{
p: pkg.Package{
Name: "python-rrdtool",
Type: pkg.PythonPkg,
},
expected: []string{"rrdtool" /* <-- known good names | default guess --> */, "python-rrdtool", "python_rrdtool"},
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%+v %+v", test.p, test.expected), func(t *testing.T) {
assert.ElementsMatch(t, test.expected, candidateProducts(test.p))
})
}
}
func Test_generateSubSelections(t *testing.T) {
tests := []struct {
field string
expected []string
}{
{
field: "jenkins",
expected: []string{"jenkins"},
},
{
field: "jenkins-ci",
expected: []string{"jenkins", "jenkins-ci"},
},
{
field: "jenkins--ci",
expected: []string{"jenkins", "jenkins-ci"},
},
{
field: "jenkins_ci_tools",
expected: []string{"jenkins", "jenkins_ci", "jenkins_ci_tools"},
},
{
field: "-jenkins",
expected: []string{"jenkins"},
},
{
field: "jenkins_",
expected: []string{"jenkins"},
},
{
field: "",
expected: nil,
},
{
field: "-",
expected: nil,
},
{
field: "_",
expected: nil,
},
}
for _, test := range tests {
t.Run(test.field, func(t *testing.T) {
assert.ElementsMatch(t, test.expected, generateSubSelections(test.field))
})
}
}
func Test_addSeparatorVariations(t *testing.T) {
tests := []struct {
input []string
expected []string
}{
{
input: []string{"jenkins-ci"},
expected: []string{"jenkins-ci", "jenkins_ci"}, //, "jenkinsci"},
},
{
input: []string{"jenkins_ci"},
expected: []string{"jenkins_ci", "jenkins-ci"}, //, "jenkinsci"},
},
{
input: []string{"jenkins"},
expected: []string{"jenkins"},
},
{
input: []string{"jenkins-ci", "circle-ci"},
expected: []string{"jenkins-ci", "jenkins_ci", "circle-ci", "circle_ci"}, //, "jenkinsci", "circleci"},
},
}
for _, test := range tests {
t.Run(strings.Join(test.input, ","), func(t *testing.T) {
val := newFieldCandidateSet(test.input...)
addDelimiterVariations(val)
assert.ElementsMatch(t, test.expected, val.values())
})
}
}
+59
View File
@@ -0,0 +1,59 @@
package cpe
import (
"net/url"
"strings"
)
// candidateProductForGo attempts to find a single product name in a best-effort attempt. This implementation prefers
// to return no vendor over returning potentially nonsensical results.
func candidateProductForGo(name string) string {
// note: url.Parse requires a scheme for correct processing, which a golang module will not have, so one is provided.
u, err := url.Parse("http://" + name)
if err != nil {
return ""
}
cleanPath := strings.Trim(u.Path, "/")
pathElements := strings.Split(cleanPath, "/")
switch u.Host {
case "golang.org", "gopkg.in":
return cleanPath
case "google.golang.org":
return pathElements[0]
}
if len(pathElements) < 2 {
return ""
}
return pathElements[1]
}
// candidateVendorForGo attempts to find a single vendor name in a best-effort attempt. This implementation prefers
// to return no vendor over returning potentially nonsensical results.
func candidateVendorForGo(name string) string {
// note: url.Parse requires a scheme for correct processing, which a golang module will not have, so one is provided.
u, err := url.Parse("http://" + name)
if err != nil {
return ""
}
cleanPath := strings.Trim(u.Path, "/")
switch u.Host {
case "google.golang.org":
return "google"
case "golang.org":
return "golang"
case "gopkg.in":
return ""
}
pathElements := strings.Split(cleanPath, "/")
if len(pathElements) < 2 {
return ""
}
return pathElements[0]
}
+99
View File
@@ -0,0 +1,99 @@
package cpe
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestCandidateProductForGo(t *testing.T) {
tests := []struct {
pkg string
expected string
}{
{
pkg: "github.com/someone/something",
expected: "something",
},
{
pkg: "golang.org/x/xerrors",
expected: "x/xerrors",
},
{
pkg: "gopkg.in/yaml.v2",
expected: "yaml.v2",
},
{
pkg: "place",
expected: "",
},
{
pkg: "place.com/",
expected: "",
},
{
pkg: "place.com/someone-or-thing",
expected: "",
},
{
pkg: "google.golang.org/genproto/googleapis/rpc/status",
expected: "genproto",
},
{
pkg: "github.com/someone/something/long/package/name",
expected: "something",
},
}
for _, test := range tests {
t.Run(test.pkg, func(t *testing.T) {
assert.Equal(t, test.expected, candidateProductForGo(test.pkg))
})
}
}
func TestCandidateVendorForGo(t *testing.T) {
tests := []struct {
pkg string
expected string
}{
{
pkg: "github.com/someone/something",
expected: "someone",
},
{
pkg: "golang.org/x/xerrors",
expected: "golang",
},
{
pkg: "gopkg.in/yaml.v2",
expected: "",
},
{
pkg: "place",
expected: "",
},
{
pkg: "place.com/",
expected: "",
},
{
pkg: "place.com/someone-or-thing",
expected: "",
},
{
pkg: "google.golang.org/genproto/googleapis/rpc/status",
expected: "google",
},
{
pkg: "github.com/someone/something/long/package/name",
expected: "someone",
},
}
for _, test := range tests {
t.Run(test.pkg, func(t *testing.T) {
assert.Equal(t, test.expected, candidateVendorForGo(test.pkg))
})
}
}
+283
View File
@@ -0,0 +1,283 @@
package cpe
import (
"strings"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/pkg"
"github.com/scylladb/go-set/strset"
)
var (
forbiddenProductGroupIDFields = strset.New("plugin", "plugins", "client")
forbiddenVendorGroupIDFields = strset.New("plugin", "plugins")
domains = []string{
"com",
"org",
"net",
"io",
}
primaryJavaManifestGroupIDFields = []string{
"Extension-Name",
"Specification-Vendor",
"Implementation-Vendor",
"Bundle-SymbolicName",
"Implementation-Vendor-Id",
"Implementation-Title",
"Bundle-Activator",
}
secondaryJavaManifestGroupIDFields = []string{
"Automatic-Module-Name",
"Main-Class",
"Package",
}
javaManifestNameFields = []string{
"Specification-Vendor",
"Implementation-Vendor",
}
)
func candidateProductsForJava(p pkg.Package) []string {
return productsFromArtifactAndGroupIDs(artifactIDFromJavaPackage(p), groupIDsFromJavaPackage(p))
}
func candidateVendorsForJava(p pkg.Package) fieldCandidateSet {
gidVendors := vendorsFromGroupIDs(groupIDsFromJavaPackage(p))
nameVendors := vendorsFromJavaManifestNames(p)
return newFieldCandidateFromSets(gidVendors, nameVendors)
}
func vendorsFromJavaManifestNames(p pkg.Package) fieldCandidateSet {
vendors := newFieldCandidateSet()
metadata, ok := p.Metadata.(pkg.JavaMetadata)
if !ok {
return vendors
}
if metadata.Manifest == nil {
return vendors
}
for _, name := range javaManifestNameFields {
if metadata.Manifest.Main != nil {
if value, exists := metadata.Manifest.Main[name]; exists {
if !startsWithTopLevelDomain(value) {
vendors.add(fieldCandidate{
value: normalizePersonName(value),
disallowSubSelections: true,
})
}
}
}
if metadata.Manifest.NamedSections != nil {
for _, section := range metadata.Manifest.NamedSections {
if section == nil {
continue
}
if value, exists := section[name]; exists {
if !startsWithTopLevelDomain(value) {
vendors.add(fieldCandidate{
value: normalizePersonName(value),
disallowSubSelections: true,
})
}
}
}
}
}
return vendors
}
func vendorsFromGroupIDs(groupIDs []string) fieldCandidateSet {
vendors := newFieldCandidateSet()
for _, groupID := range groupIDs {
for i, field := range strings.Split(groupID, ".") {
field = strings.TrimSpace(field)
if len(field) == 0 {
continue
}
if forbiddenVendorGroupIDFields.Has(strings.ToLower(field)) {
continue
}
if i == 0 {
continue
}
vendors.addValue(field)
}
}
return vendors
}
func productsFromArtifactAndGroupIDs(artifactID string, groupIDs []string) []string {
products := strset.New()
if artifactID != "" {
products.Add(artifactID)
}
for _, groupID := range groupIDs {
isPlugin := strings.Contains(artifactID, "plugin") || strings.Contains(groupID, "plugin")
for i, field := range strings.Split(groupID, ".") {
field = strings.TrimSpace(field)
if len(field) == 0 {
continue
}
// don't add this field as a name if the name is implying the package is a plugin or client
if forbiddenProductGroupIDFields.Has(strings.ToLower(field)) {
continue
}
if i <= 1 {
continue
}
// umbrella projects tend to have sub components that either start or end with the project name. We expect
// to identify fields that may represent the umbrella project, and not fields that indicate auxiliary
// information about the package.
couldBeProjectName := strings.HasPrefix(artifactID, field) || strings.HasSuffix(artifactID, field)
if artifactID == "" || (couldBeProjectName && !isPlugin) {
products.Add(field)
}
}
}
return products.List()
}
func artifactIDFromJavaPackage(p pkg.Package) string {
metadata, ok := p.Metadata.(pkg.JavaMetadata)
if !ok {
return ""
}
if metadata.PomProperties == nil {
return ""
}
artifactID := strings.TrimSpace(metadata.PomProperties.ArtifactID)
if startsWithTopLevelDomain(artifactID) && len(strings.Split(artifactID, ".")) > 1 {
// there is a strong indication that the artifact ID is really a group ID, don't use it
return ""
}
return artifactID
}
func groupIDsFromJavaPackage(p pkg.Package) (groupIDs []string) {
metadata, ok := p.Metadata.(pkg.JavaMetadata)
if !ok {
return nil
}
groupIDs = append(groupIDs, groupIDsFromPomProperties(metadata.PomProperties)...)
groupIDs = append(groupIDs, groupIDsFromPomProject(metadata.PomProject)...)
groupIDs = append(groupIDs, groupIDsFromJavaManifest(metadata.Manifest)...)
return groupIDs
}
func groupIDsFromPomProperties(properties *pkg.PomProperties) (groupIDs []string) {
if properties == nil {
return nil
}
if startsWithTopLevelDomain(properties.GroupID) {
groupIDs = append(groupIDs, strings.TrimSpace(properties.GroupID))
}
// sometimes the publisher puts the group ID in the artifact ID field unintentionally
if startsWithTopLevelDomain(properties.ArtifactID) && len(strings.Split(properties.ArtifactID, ".")) > 1 {
// there is a strong indication that the artifact ID is really a group ID
groupIDs = append(groupIDs, strings.TrimSpace(properties.ArtifactID))
}
return groupIDs
}
func groupIDsFromPomProject(project *pkg.PomProject) (groupIDs []string) {
if project == nil {
return nil
}
// extract the project info...
groupIDs = addGroupIDsFromGroupIDsAndArtifactID(project.GroupID, project.ArtifactID)
if project.Parent == nil {
return groupIDs
}
// extract the parent project info...
groupIDs = append(groupIDs, addGroupIDsFromGroupIDsAndArtifactID(project.Parent.GroupID, project.Parent.ArtifactID)...)
return groupIDs
}
func addGroupIDsFromGroupIDsAndArtifactID(groupID, artifactID string) (groupIDs []string) {
if startsWithTopLevelDomain(groupID) {
groupIDs = append(groupIDs, strings.TrimSpace(groupID))
}
// sometimes the publisher puts the group ID in the artifact ID field unintentionally
if startsWithTopLevelDomain(artifactID) && len(strings.Split(artifactID, ".")) > 1 {
// there is a strong indication that the artifact ID is really a group ID
groupIDs = append(groupIDs, strings.TrimSpace(artifactID))
}
return groupIDs
}
func groupIDsFromJavaManifest(manifest *pkg.JavaManifest) []string {
if manifest == nil {
return nil
}
// try the common manifest fields first for a set of candidates
groupIDs := getManifestFieldGroupIDs(manifest, primaryJavaManifestGroupIDFields)
if len(groupIDs) != 0 {
return groupIDs
}
// if we haven't found anything yet, let's try a last ditch effort:
// attempt to get group-id-like info from the MANIFEST.MF "Automatic-Module-Name" and "Extension-Name" field.
// for more info see pkg:maven/commons-io/commons-io@2.8.0 within cloudbees/cloudbees-core-mm:2.263.4.2
// at /usr/share/jenkins/jenkins.war:WEB-INF/plugins/analysis-model-api.hpi:WEB-INF/lib/commons-io-2.8.0.jar
// as well as the ant package from cloudbees/cloudbees-core-mm:2.277.2.4-ra.
return getManifestFieldGroupIDs(manifest, secondaryJavaManifestGroupIDFields)
}
func getManifestFieldGroupIDs(manifest *pkg.JavaManifest, fields []string) (groupIDs []string) {
if manifest == nil {
return nil
}
for _, name := range fields {
if value, exists := manifest.Main[name]; exists {
if startsWithTopLevelDomain(value) {
groupIDs = append(groupIDs, value)
}
}
for _, section := range manifest.NamedSections {
if value, exists := section[name]; exists {
if startsWithTopLevelDomain(value) {
groupIDs = append(groupIDs, value)
}
}
}
}
return groupIDs
}
func startsWithTopLevelDomain(value string) bool {
return internal.HasAnyOfPrefixes(value, domains...)
}
+428
View File
@@ -0,0 +1,428 @@
package cpe
import (
"strings"
"testing"
"github.com/anchore/syft/syft/pkg"
"github.com/stretchr/testify/assert"
)
func Test_productsFromArtifactAndGroupIDs(t *testing.T) {
tests := []struct {
groupIDs []string
artifactID string
expected []string
}{
{
groupIDs: []string{"org.sonatype.nexus"},
artifactID: "nexus-extender",
expected: []string{"nexus", "nexus-extender"},
},
{
groupIDs: []string{"org.sonatype.nexus"},
expected: []string{"nexus"},
},
{
groupIDs: []string{"org.jenkins-ci.plugins"},
artifactID: "ant",
expected: []string{"ant"},
},
{
groupIDs: []string{"org.jenkins-ci.plugins"},
artifactID: "antisamy-markup-formatter",
expected: []string{"antisamy-markup-formatter"},
},
{
groupIDs: []string{"io.jenkins.plugins"},
artifactID: "aws-global-configuration",
expected: []string{"aws-global-configuration"},
},
{
groupIDs: []string{"com.cloudbees.jenkins.plugins"},
artifactID: "cloudbees-servicenow-jenkins-plugin",
expected: []string{"cloudbees-servicenow-jenkins-plugin"},
},
{
groupIDs: []string{"com.atlassian.confluence.plugins"},
artifactID: "confluence-mobile-plugin",
expected: []string{"confluence-mobile-plugin"},
},
{
groupIDs: []string{"com.atlassian.confluence.plugins"},
artifactID: "confluence-view-file-macro",
expected: []string{"confluence-view-file-macro"},
},
{
groupIDs: []string{"com.google.guava"},
artifactID: "failureaccess",
expected: []string{"failureaccess"},
},
}
for _, test := range tests {
t.Run(strings.Join(test.groupIDs, ",")+":"+test.artifactID, func(t *testing.T) {
actual := productsFromArtifactAndGroupIDs(test.artifactID, test.groupIDs)
assert.ElementsMatch(t, test.expected, actual, "different products")
})
}
}
func Test_candidateProductsForJava(t *testing.T) {
tests := []struct {
name string
pkg pkg.Package
expected []string
}{
{
name: "duplicate groupID in artifactID field",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "org.sonatype.nexus",
ArtifactID: "org.sonatype.nexus",
},
},
},
expected: []string{"nexus"},
},
{
name: "detect groupID-like value in artifactID field",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
ArtifactID: "org.sonatype.nexus",
},
},
},
expected: []string{"nexus"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual := candidateProductsForJava(test.pkg)
assert.ElementsMatch(t, test.expected, actual, "different products")
})
}
}
func Test_vendorsFromGroupIDs(t *testing.T) {
tests := []struct {
groupID string
expected []string
}{
{
groupID: "org.sonatype.nexus",
expected: []string{"sonatype", "nexus"},
},
{
groupID: "org.jenkins-ci.plugins",
expected: []string{"jenkins-ci"},
},
{
groupID: "io.jenkins.plugins",
expected: []string{"jenkins"},
},
{
groupID: "com.cloudbees.jenkins.plugins",
expected: []string{"cloudbees", "jenkins"},
},
{
groupID: "com.atlassian.confluence.plugins",
expected: []string{"atlassian", "confluence"},
},
{
groupID: "com.google.guava",
expected: []string{"google", "guava"},
},
}
for _, test := range tests {
t.Run(test.groupID, func(t *testing.T) {
assert.ElementsMatch(t, test.expected, vendorsFromGroupIDs([]string{test.groupID}).values(), "different vendors")
})
}
}
func Test_groupIDsFromJavaPackage(t *testing.T) {
tests := []struct {
name string
pkg pkg.Package
expects []string
}{
{
name: "go case",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "io.jenkins-ci.plugin.thing",
},
},
},
expects: []string{"io.jenkins-ci.plugin.thing"},
},
{
name: "from artifactID",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
ArtifactID: "io.jenkins-ci.plugin.thing",
},
},
},
expects: []string{"io.jenkins-ci.plugin.thing"},
},
{
name: "from main Extension-Name field",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
Manifest: &pkg.JavaManifest{
Main: map[string]string{
"Extension-Name": "io.jenkins-ci.plugin.thing",
},
},
},
},
expects: []string{"io.jenkins-ci.plugin.thing"},
},
{
name: "from named section Extension-Name field",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
Manifest: &pkg.JavaManifest{
NamedSections: map[string]map[string]string{
"section": {
"Extension-Name": "io.jenkins-ci.plugin.thing",
},
},
},
},
},
expects: []string{"io.jenkins-ci.plugin.thing"},
},
{
name: "from main field - tier 1",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
Manifest: &pkg.JavaManifest{
Main: map[string]string{
// positive cases
// tier 1
"Extension-Name": "io.jenkins-ci.plugin.1",
"Specification-Vendor": "io.jenkins-ci.plugin.2",
"Implementation-Vendor": "io.jenkins-ci.plugin.3",
"Bundle-SymbolicName": "io.jenkins-ci.plugin.4",
"Implementation-Vendor-Id": "io.jenkins-ci.plugin.5",
"Implementation-Title": "io.jenkins-ci.plugin.6",
"Bundle-Activator": "io.jenkins-ci.plugin.7",
// tier 2
"Automatic-Module-Name": "io.jenkins-ci.plugin.8",
"Main-Class": "io.jenkins-ci.plugin.9",
"Package": "io.jenkins-ci.plugin.10",
},
},
},
},
expects: []string{
"io.jenkins-ci.plugin.1",
"io.jenkins-ci.plugin.2",
"io.jenkins-ci.plugin.3",
"io.jenkins-ci.plugin.4",
"io.jenkins-ci.plugin.5",
"io.jenkins-ci.plugin.6",
"io.jenkins-ci.plugin.7",
},
},
{
name: "from main field - tier 2",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
Manifest: &pkg.JavaManifest{
Main: map[string]string{
// positive cases
"Automatic-Module-Name": "io.jenkins-ci.plugin.8",
"Main-Class": "io.jenkins-ci.plugin.9",
"Package": "io.jenkins-ci.plugin.10",
},
},
},
},
expects: []string{
"io.jenkins-ci.plugin.8",
"io.jenkins-ci.plugin.9",
"io.jenkins-ci.plugin.10",
},
},
{
name: "from main field - negative cases",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
Manifest: &pkg.JavaManifest{
Main: map[string]string{
// negative cases
"Extension-Name": "not.a-group.id",
"bogus": "io.jenkins-ci.plugin.please-dont-find-me",
},
},
},
},
expects: nil,
},
{
name: "from named section field - tier 1",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
Manifest: &pkg.JavaManifest{
NamedSections: map[string]map[string]string{
"section": {
// positive cases
// tier 1
"Extension-Name": "io.jenkins-ci.plugin.1",
"Specification-Vendor": "io.jenkins-ci.plugin.2",
"Implementation-Vendor": "io.jenkins-ci.plugin.3",
"Bundle-SymbolicName": "io.jenkins-ci.plugin.4",
"Implementation-Vendor-Id": "io.jenkins-ci.plugin.5",
"Implementation-Title": "io.jenkins-ci.plugin.6",
"Bundle-Activator": "io.jenkins-ci.plugin.7",
// tier 2
"Automatic-Module-Name": "io.jenkins-ci.plugin.8",
"Main-Class": "io.jenkins-ci.plugin.9",
"Package": "io.jenkins-ci.plugin.10",
},
},
},
},
},
expects: []string{
"io.jenkins-ci.plugin.1",
"io.jenkins-ci.plugin.2",
"io.jenkins-ci.plugin.3",
"io.jenkins-ci.plugin.4",
"io.jenkins-ci.plugin.5",
"io.jenkins-ci.plugin.6",
"io.jenkins-ci.plugin.7",
},
},
{
name: "from named section field - negative cases",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
Manifest: &pkg.JavaManifest{
NamedSections: map[string]map[string]string{
"section": {
// negative cases
"Extension-Name": "not.a-group.id",
"bogus": "io.jenkins-ci.plugin.please-dont-find-me",
},
},
},
},
},
expects: nil,
},
{
name: "no manifest or pom info",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{},
},
expects: nil,
},
{
name: "no java info",
pkg: pkg.Package{},
expects: nil,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.ElementsMatch(t, test.expects, groupIDsFromJavaPackage(test.pkg))
})
}
}
func Test_artifactIDFromJavaPackage(t *testing.T) {
tests := []struct {
name string
pkg pkg.Package
expects string
}{
{
name: "go case",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
ArtifactID: "cloudbees-installation-manager",
},
},
},
expects: "cloudbees-installation-manager",
},
{
name: "ignore groupID-like things",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
ArtifactID: "io.jenkins-ci.plugin.thing",
},
},
},
expects: "",
},
{
name: "no java info",
pkg: pkg.Package{},
expects: "",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expects, artifactIDFromJavaPackage(test.pkg))
})
}
}
func Test_vendorsFromJavaManifestNames(t *testing.T) {
tests := []struct {
name string
pkg pkg.Package
expects []string
}{
{
name: "from manifest named section fields",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
Manifest: &pkg.JavaManifest{
NamedSections: map[string]map[string]string{
"section": {
// positive cases
"Specification-Vendor": "Alex Goodman",
"Implementation-Vendor": "William Goodman",
},
},
},
},
},
expects: []string{"alex_goodman", "william_goodman"},
},
{
name: "from manifest named section fields - negative cases",
pkg: pkg.Package{
Metadata: pkg.JavaMetadata{
Manifest: &pkg.JavaManifest{
NamedSections: map[string]map[string]string{
"section": {
// negative cases
"Specification-Vendor": "io.jenkins-ci.plugin.thing",
"Implementation-Vendor-ID": "William Goodman",
},
},
},
},
},
expects: nil,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.ElementsMatch(t, test.expects, vendorsFromJavaManifestNames(test.pkg).values())
})
}
}
+29
View File
@@ -0,0 +1,29 @@
package cpe
import "github.com/anchore/syft/syft/pkg"
func candidateVendorsForPython(p pkg.Package) fieldCandidateSet {
metadata, ok := p.Metadata.(pkg.PythonPackageMetadata)
if !ok {
return nil
}
vendors := newFieldCandidateSet()
if metadata.Author != "" {
vendors.add(fieldCandidate{
value: normalizePersonName(metadata.Author),
disallowSubSelections: true,
disallowDelimiterVariations: true,
})
}
if metadata.AuthorEmail != "" {
vendors.add(fieldCandidate{
value: normalizePersonName(stripEmailSuffix(metadata.AuthorEmail)),
disallowSubSelections: true,
})
}
return vendors
}
+21
View File
@@ -0,0 +1,21 @@
package cpe
import "github.com/anchore/syft/syft/pkg"
func candidateVendorsForRPM(p pkg.Package) fieldCandidateSet {
metadata, ok := p.Metadata.(pkg.RpmdbMetadata)
if !ok {
return nil
}
vendors := newFieldCandidateSet()
if metadata.Vendor != "" {
vendors.add(fieldCandidate{
value: normalizeName(metadata.Vendor),
disallowSubSelections: true,
})
}
return vendors
}
+21
View File
@@ -0,0 +1,21 @@
package cpe
import "github.com/anchore/syft/syft/pkg"
func candidateVendorsForRuby(p pkg.Package) fieldCandidateSet {
metadata, ok := p.Metadata.(pkg.GemMetadata)
if !ok {
return nil
}
vendors := newFieldCandidateSet()
for _, author := range metadata.Authors {
// author could be a name or an email
vendors.add(fieldCandidate{
value: normalizePersonName(stripEmailSuffix(author)),
disallowSubSelections: true,
})
}
return vendors
}
@@ -0,0 +1,49 @@
package cpe
import (
"sort"
"github.com/facebookincubator/nvdtools/wfn"
)
var _ sort.Interface = (*BySpecificity)(nil)
type BySpecificity []wfn.Attributes
func (c BySpecificity) Len() int { return len(c) }
func (c BySpecificity) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c BySpecificity) Less(i, j int) bool {
iScore := weightedCountForSpecifiedFields(c[i])
jScore := weightedCountForSpecifiedFields(c[j])
if iScore == jScore {
return countFieldLength(c[i]) > countFieldLength(c[j])
}
return iScore > jScore
}
func countFieldLength(cpe wfn.Attributes) int {
return len(cpe.Part + cpe.Vendor + cpe.Product + cpe.Version + cpe.TargetSW)
}
func weightedCountForSpecifiedFields(cpe wfn.Attributes) int {
checksForSpecifiedField := []func(cpe wfn.Attributes) (bool, int){
func(cpe wfn.Attributes) (bool, int) { return cpe.Part != "", 2 },
func(cpe wfn.Attributes) (bool, int) { return cpe.Vendor != "", 3 },
func(cpe wfn.Attributes) (bool, int) { return cpe.Product != "", 4 },
func(cpe wfn.Attributes) (bool, int) { return cpe.Version != "", 1 },
func(cpe wfn.Attributes) (bool, int) { return cpe.TargetSW != "", 1 },
}
weightedCount := 0
for _, fieldIsSpecified := range checksForSpecifiedField {
isSpecified, weight := fieldIsSpecified(cpe)
if isSpecified {
weightedCount += weight
}
}
return weightedCount
}
@@ -0,0 +1,92 @@
package cpe
import (
"sort"
"testing"
"github.com/anchore/syft/syft/pkg"
"github.com/stretchr/testify/assert"
)
func mustCPE(c string) pkg.CPE {
return must(pkg.NewCPE(c))
}
func must(c pkg.CPE, e error) pkg.CPE {
if e != nil {
panic(e)
}
return c
}
func TestCPESpecificity(t *testing.T) {
tests := []struct {
name string
input []pkg.CPE
expected []pkg.CPE
}{
{
name: "sort strictly by wfn *",
input: []pkg.CPE{
mustCPE("cpe:2.3:a:*:package:1:*:*:*:*:*:*:*"),
mustCPE("cpe:2.3:a:some:package:1:*:*:*:*:*:*:*"),
mustCPE("cpe:2.3:a:*:package:1:*:*:*:*:some:*:*"),
mustCPE("cpe:2.3:a:some:package:1:*:*:*:*:some:*:*"),
mustCPE("cpe:2.3:a:some:package:*:*:*:*:*:*:*:*"),
},
expected: []pkg.CPE{
mustCPE("cpe:2.3:a:some:package:1:*:*:*:*:some:*:*"),
mustCPE("cpe:2.3:a:some:package:1:*:*:*:*:*:*:*"),
mustCPE("cpe:2.3:a:some:package:*:*:*:*:*:*:*:*"),
mustCPE("cpe:2.3:a:*:package:1:*:*:*:*:some:*:*"),
mustCPE("cpe:2.3:a:*:package:1:*:*:*:*:*:*:*"),
},
},
{
name: "sort strictly by field length",
input: []pkg.CPE{
mustCPE("cpe:2.3:a:1:22:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:55555:1:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:1:1:333:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:1:666666:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:1:1:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:1:1:1:*:*:*:*:4444:*:*"),
},
expected: []pkg.CPE{
mustCPE("cpe:2.3:a:1:666666:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:55555:1:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:1:1:1:*:*:*:*:4444:*:*"),
mustCPE("cpe:2.3:a:1:1:333:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:1:22:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:1:1:1:*:*:*:*:1:*:*"),
},
},
{
name: "sort by mix of field length and specificity",
input: []pkg.CPE{
mustCPE("cpe:2.3:a:1:666666:*:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:*:1:1:*:*:*:*:4444:*:*"),
mustCPE("cpe:2.3:a:1:*:333:*:*:*:*:*:*:*"),
mustCPE("cpe:2.3:a:1:1:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:1:22:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:55555:1:1:*:*:*:*:1:*:*"),
},
expected: []pkg.CPE{
mustCPE("cpe:2.3:a:55555:1:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:1:22:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:1:1:1:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:1:666666:*:*:*:*:*:1:*:*"),
mustCPE("cpe:2.3:a:*:1:1:*:*:*:*:4444:*:*"),
mustCPE("cpe:2.3:a:1:*:333:*:*:*:*:*:*:*"),
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
sort.Sort(BySpecificity(test.input))
assert.Equal(t, test.expected, test.input)
})
}
}
+21
View File
@@ -0,0 +1,21 @@
package cpe
import "strings"
func stripEmailSuffix(email string) string {
return strings.Split(email, "@")[0]
}
func normalizePersonName(name string) string {
name = strings.TrimSpace(strings.ToLower(name))
for _, value := range []string{"-", " ", "."} {
name = strings.ReplaceAll(name, value, "_")
}
return strings.TrimPrefix(name, "the_")
}
func normalizeName(name string) string {
name = strings.Split(name, ",")[0]
name = strings.TrimSpace(strings.ToLower(name))
return strings.ReplaceAll(name, " ", "")
}
@@ -0,0 +1,71 @@
package cpe
import (
"testing"
"github.com/stretchr/testify/assert"
)
func Test_normalizeName(t *testing.T) {
tests := []struct {
input string
expects string
}{
{
// note: extra spaces
input: " Alex Goodman ",
expects: "alexgoodman",
},
{
input: "Alex Goodman, LLC",
expects: "alexgoodman",
},
{
input: "alex.goodman",
expects: "alex.goodman",
},
}
for _, test := range tests {
t.Run(test.input, func(t *testing.T) {
assert.Equal(t, test.expects, normalizeName(test.input))
})
}
}
func Test_normalizePersonName(t *testing.T) {
tests := []struct {
input string
expects string
}{
{
// note: extra spaces
input: " Alex Goodman ",
expects: "alex_goodman",
},
{
input: "Alex Goodman",
expects: "alex_goodman",
},
{
input: "Alex.Goodman",
expects: "alex_goodman",
},
{
input: "Alex.Goodman",
expects: "alex_goodman",
},
{
input: "AlexGoodman",
expects: "alexgoodman",
},
{
input: "The Apache Software Foundation",
expects: "apache_software_foundation",
},
}
for _, test := range tests {
t.Run(test.input, func(t *testing.T) {
assert.Equal(t, test.expects, normalizePersonName(test.input))
})
}
}