chore: refactor basic CPE functionality to its own package (#1436)

syft/cpe/by_specificity.go

@@ -0,0 +1,60 @@
+package cpe
+
+import (
+	"sort"
+
+	"github.com/facebookincubator/nvdtools/wfn"
+)
+
+var _ sort.Interface = (*BySpecificity)(nil)
+
+type BySpecificity []wfn.Attributes
+
+func (c BySpecificity) Len() int { return len(c) }
+
+func (c BySpecificity) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
+
+func (c BySpecificity) Less(i, j int) bool {
+	iScore := weightedCountForSpecifiedFields(c[i])
+	jScore := weightedCountForSpecifiedFields(c[j])
+
+	// check weighted sort first
+	if iScore != jScore {
+		return iScore > jScore
+	}
+
+	// sort longer fields to top
+	if countFieldLength(c[i]) != countFieldLength(c[j]) {
+		return countFieldLength(c[i]) > countFieldLength(c[j])
+	}
+
+	// if score and length are equal then text sort
+	// note that we are not using String from the syft pkg
+	// as we are not encoding/decoding this CPE string so we don't
+	// need the proper quoted version of the CPE.
+	return c[i].BindToFmtString() < c[j].BindToFmtString()
+}
+
+func countFieldLength(cpe wfn.Attributes) int {
+	return len(cpe.Part + cpe.Vendor + cpe.Product + cpe.Version + cpe.TargetSW)
+}
+
+func weightedCountForSpecifiedFields(cpe wfn.Attributes) int {
+	checksForSpecifiedField := []func(cpe wfn.Attributes) (bool, int){
+		func(cpe wfn.Attributes) (bool, int) { return cpe.Part != "", 2 },
+		func(cpe wfn.Attributes) (bool, int) { return cpe.Vendor != "", 3 },
+		func(cpe wfn.Attributes) (bool, int) { return cpe.Product != "", 4 },
+		func(cpe wfn.Attributes) (bool, int) { return cpe.Version != "", 1 },
+		func(cpe wfn.Attributes) (bool, int) { return cpe.TargetSW != "", 1 },
+	}
+
+	weightedCount := 0
+	for _, fieldIsSpecified := range checksForSpecifiedField {
+		isSpecified, weight := fieldIsSpecified(cpe)
+		if isSpecified {
+			weightedCount += weight
+		}
+	}
+
+	return weightedCount
+}

syft/cpe/by_specificity_test.go

@@ -0,0 +1,98 @@
+package cpe
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_BySpecificity(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []CPE
+		expected []CPE
+	}{
+		{
+			name: "sort strictly by wfn *",
+			input: []CPE{
+				Must("cpe:2.3:a:*:package:1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:some:package:1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:*:package:1:*:*:*:*:some:*:*"),
+				Must("cpe:2.3:a:some:package:1:*:*:*:*:some:*:*"),
+				Must("cpe:2.3:a:some:package:*:*:*:*:*:*:*:*"),
+			},
+			expected: []CPE{
+				Must("cpe:2.3:a:some:package:1:*:*:*:*:some:*:*"),
+				Must("cpe:2.3:a:some:package:1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:some:package:*:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:*:package:1:*:*:*:*:some:*:*"),
+				Must("cpe:2.3:a:*:package:1:*:*:*:*:*:*:*"),
+			},
+		},
+		{
+			name: "sort strictly by field length",
+			input: []CPE{
+				Must("cpe:2.3:a:1:22:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:55555:1:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:1:1:333:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:1:666666:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:1:1:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:1:1:1:*:*:*:*:4444:*:*"),
+			},
+			expected: []CPE{
+				Must("cpe:2.3:a:1:666666:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:55555:1:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:1:1:1:*:*:*:*:4444:*:*"),
+				Must("cpe:2.3:a:1:1:333:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:1:22:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:1:1:1:*:*:*:*:1:*:*"),
+			},
+		},
+		{
+			name: "sort by mix of field length and specificity",
+			input: []CPE{
+				Must("cpe:2.3:a:1:666666:*:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:*:1:1:*:*:*:*:4444:*:*"),
+				Must("cpe:2.3:a:1:*:333:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:1:1:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:1:22:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:55555:1:1:*:*:*:*:1:*:*"),
+			},
+			expected: []CPE{
+				Must("cpe:2.3:a:55555:1:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:1:22:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:1:1:1:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:1:666666:*:*:*:*:*:1:*:*"),
+				Must("cpe:2.3:a:*:1:1:*:*:*:*:4444:*:*"),
+				Must("cpe:2.3:a:1:*:333:*:*:*:*:*:*:*"),
+			},
+		},
+		{
+			name: "sort by mix of field length, specificity, dash",
+			input: []CPE{
+				Must("cpe:2.3:a:alpine:alpine_keys:2.3-r1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:alpine_keys:alpine_keys:2.3-r1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:alpine-keys:alpine_keys:2.3-r1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:alpine:alpine-keys:2.3-r1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:alpine-keys:alpine-keys:2.3-r1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:alpine_keys:alpine-keys:2.3-r1:*:*:*:*:*:*:*"),
+			},
+			expected: []CPE{
+				Must("cpe:2.3:a:alpine-keys:alpine-keys:2.3-r1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:alpine-keys:alpine_keys:2.3-r1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:alpine_keys:alpine-keys:2.3-r1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:alpine_keys:alpine_keys:2.3-r1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:alpine:alpine-keys:2.3-r1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:alpine:alpine_keys:2.3-r1:*:*:*:*:*:*:*"),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			sort.Sort(BySpecificity(test.input))
+			assert.Equal(t, test.expected, test.input)
+		})
+	}
+}

syft/cpe/cpe.go

@@ -0,0 +1,155 @@
+package cpe
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/facebookincubator/nvdtools/wfn"
+)
+
+type CPE = wfn.Attributes
+
+const (
+	allowedCPEPunctuation = "-!\"#$%&'()+,./:;<=>@[]^`{|}~"
+)
+
+// This regex string is taken from
+// https://csrc.nist.gov/schema/cpe/2.3/cpe-naming_2.3.xsd which has the official cpe spec
+// This first part matches CPE urls and the second part matches binding strings
+const cpeRegexString = ((`^([c][pP][eE]:/[AHOaho]?(:[A-Za-z0-9\._\-~%]*){0,6})`) +
+	// Or match the CPE binding string
+	// Note that we had to replace '`' with '\x60' to escape the backticks
+	`|(cpe:2\.3:[aho\*\-](:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&'\(\)\+,/:;<=>@\[\]\^\x60\{\|}~]))+(\?*|\*?))|[\*\-])){5}(:(([a-zA-Z]{2,3}(-([a-zA-Z]{2}|[0-9]{3}))?)|[\*\-]))(:(((\?*|\*?)([a-zA-Z0-9\-\._]|(\\[\\\*\?!"#$$%&'\(\)\+,/:;<=>@\[\]\^\x60\{\|}~]))+(\?*|\*?))|[\*\-])){4})$`)
+
+var cpeRegex = regexp.MustCompile(cpeRegexString)
+
+// New will parse a formatted CPE string and return a CPE object. Some input, such as the existence of whitespace
+// characters is allowed, however, a more strict validation is done after this sanitization process.
+func New(cpeStr string) (CPE, error) {
+	// get a CPE object based on the given string --don't validate yet since it may be possible to escape select cases on the callers behalf
+	c, err := newWithoutValidation(cpeStr)
+	if err != nil {
+		return CPE{}, fmt.Errorf("unable to parse CPE string: %w", err)
+	}
+
+	// ensure that this CPE can be validated after being fully sanitized
+	if ValidateString(String(c)) != nil {
+		return CPE{}, err
+	}
+
+	// we don't return the sanitized string, as this is a concern for later when creating CPE strings. In fact, since
+	// sanitization is lossy (whitespace is replaced, not escaped) it's important that the raw values are left as.
+	return c, nil
+}
+
+// Must returns a CPE or panics if the provided string is not valid
+func Must(cpeStr string) CPE {
+	c, err := New(cpeStr)
+	if err != nil {
+		panic(err)
+	}
+	return c
+}
+
+func ValidateString(cpeStr string) error {
+	// We should filter out all CPEs that do not match the official CPE regex
+	// The facebook nvdtools parser can sometimes incorrectly parse invalid CPE strings
+	if !cpeRegex.MatchString(cpeStr) {
+		return fmt.Errorf("failed to parse CPE=%q as it doesn't match the regex=%s", cpeStr, cpeRegexString)
+	}
+	return nil
+}
+
+func newWithoutValidation(cpeStr string) (CPE, error) {
+	value, err := wfn.Parse(cpeStr)
+	if err != nil {
+		return CPE{}, fmt.Errorf("failed to parse CPE=%q: %w", cpeStr, err)
+	}
+
+	if value == nil {
+		return CPE{}, fmt.Errorf("failed to parse CPE=%q", cpeStr)
+	}
+
+	// we need to compare the raw data since we are constructing CPEs in other locations
+	value.Vendor = normalizeField(value.Vendor)
+	value.Product = normalizeField(value.Product)
+	value.Language = normalizeField(value.Language)
+	value.Version = normalizeField(value.Version)
+	value.TargetSW = normalizeField(value.TargetSW)
+	value.Part = normalizeField(value.Part)
+	value.Edition = normalizeField(value.Edition)
+	value.Other = normalizeField(value.Other)
+	value.SWEdition = normalizeField(value.SWEdition)
+	value.TargetHW = normalizeField(value.TargetHW)
+	value.Update = normalizeField(value.Update)
+
+	return *value, nil
+}
+
+func normalizeField(field string) string {
+	// replace spaces with underscores (per section 5.3.2 of the CPE spec v 2.3)
+	field = strings.ReplaceAll(field, " ", "_")
+
+	// keep dashes and forward slashes unescaped
+	if field == "*" {
+		return wfn.Any
+	}
+	return stripSlashes(field)
+}
+
+// stripSlashes is a reverse of the sanitize function below.
+// It correctly removes slashes that are followed by allowed puncts.
+// This is to allow for a correct round trip parsing of cpes with quoted characters.
+func stripSlashes(s string) string {
+	sb := strings.Builder{}
+	for i, c := range s {
+		if c == '\\' && i+1 < len(s) && strings.ContainsRune(allowedCPEPunctuation, rune(s[i+1])) {
+			continue
+		} else {
+			sb.WriteRune(c)
+		}
+	}
+	return sb.String()
+}
+
+func String(c CPE) string {
+	output := CPE{}
+	output.Vendor = sanitize(c.Vendor)
+	output.Product = sanitize(c.Product)
+	output.Language = sanitize(c.Language)
+	output.Version = sanitize(c.Version)
+	output.TargetSW = sanitize(c.TargetSW)
+	output.Part = sanitize(c.Part)
+	output.Edition = sanitize(c.Edition)
+	output.Other = sanitize(c.Other)
+	output.SWEdition = sanitize(c.SWEdition)
+	output.TargetHW = sanitize(c.TargetHW)
+	output.Update = sanitize(c.Update)
+	return output.BindToFmtString()
+}
+
+// sanitize is a modified version of WFNize function from nvdtools
+// that quotes all the allowed punctation chars with a slash and replaces
+// spaces with underscores. It differs from the upstream implmentation as
+// it does not use the buggy nvdtools implementation, specifically the "addSlashesAt" part of the
+// function which stops the loop as soon as it encounters ":" a valid
+// character for a WFN attribute after quoting, but the way nvdtools
+// handles it causes it to truncate strings that container ":". As a result
+// strings like "prefix:1.2" which would have been quoted as "prefix\:1.2"
+// end up becoming "prefix" instead causing loss of information and
+// incorrect CPEs being generated.
+func sanitize(s string) string {
+	// replace spaces with underscores
+	in := strings.ReplaceAll(s, " ", "_")
+
+	// escape allowable punctuation per section 5.3.2 in the CPE 2.3 spec
+	sb := strings.Builder{}
+	for _, c := range in {
+		if strings.ContainsRune(allowedCPEPunctuation, c) {
+			sb.WriteRune('\\')
+		}
+		sb.WriteRune(c)
+	}
+	return sb.String()
+}

syft/cpe/cpe_test.go

@@ -0,0 +1,227 @@
+package cpe
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_New(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected CPE
+	}{
+		{
+			name:     "gocase",
+			input:    `cpe:/a:10web:form_maker:1.0.0::~~~wordpress~~`,
+			expected: Must(`cpe:2.3:a:10web:form_maker:1.0.0:*:*:*:*:wordpress:*:*`),
+		},
+		{
+			name:     "dashes",
+			input:    `cpe:/a:7-zip:7-zip:4.56:beta:~~~windows~~`,
+			expected: Must(`cpe:2.3:a:7-zip:7-zip:4.56:beta:*:*:*:windows:*:*`),
+		},
+		{
+			name:     "URL escape characters",
+			input:    `cpe:/a:%240.99_kindle_books_project:%240.99_kindle_books:6::~~~android~~`,
+			expected: Must(`cpe:2.3:a:\$0.99_kindle_books_project:\$0.99_kindle_books:6:*:*:*:*:android:*:*`),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			actual, err := New(test.input)
+			if err != nil {
+				t.Fatalf("got an error while creating CPE: %+v", err)
+			}
+
+			if String(actual) != String(test.expected) {
+				t.Errorf("mismatched entries:\n\texpected:%+v\n\t  actual:%+v\n", String(test.expected), String(actual))
+			}
+
+		})
+	}
+}
+
+func Test_normalizeCpeField(t *testing.T) {
+
+	tests := []struct {
+		field    string
+		expected string
+	}{
+		{
+			field:    "something",
+			expected: "something",
+		},
+		{
+			field:    "some\\thing",
+			expected: `some\thing`,
+		},
+		{
+			field:    "*",
+			expected: "",
+		},
+		{
+			field:    "",
+			expected: "",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.field, func(t *testing.T) {
+			assert.Equal(t, test.expected, normalizeField(test.field))
+		})
+	}
+}
+
+func Test_CPEParser(t *testing.T) {
+	testCases := []struct {
+		CPEString string `json:"cpe-string"`
+		CPEUrl    string `json:"cpe-url"`
+		WFN       CPE    `json:"wfn"`
+	}{}
+	out, err := ioutil.ReadFile("test-fixtures/cpe-data.json")
+	require.NoError(t, err)
+	require.NoError(t, json.Unmarshal(out, &testCases))
+
+	for _, test := range testCases {
+		t.Run(test.CPEString, func(t *testing.T) {
+			c1, err := New(test.CPEString)
+			assert.NoError(t, err)
+			c2, err := New(test.CPEUrl)
+			assert.NoError(t, err)
+			assert.Equal(t, c1, c2)
+			assert.Equal(t, c1, test.WFN)
+			assert.Equal(t, c2, test.WFN)
+			assert.Equal(t, String(test.WFN), test.CPEString)
+		})
+	}
+}
+
+func Test_InvalidCPE(t *testing.T) {
+	type testcase struct {
+		name        string
+		in          string
+		expected    string
+		expectedErr bool
+	}
+
+	tests := []testcase{
+		{
+			// 5.3.2: The underscore (x5f) MAY be used, and it SHOULD be used in place of whitespace characters (which SHALL NOT be used)
+			name:     "translates spaces",
+			in:       "cpe:2.3:a:some-vendor:name:1 2:*:*:*:*:*:*:*",
+			expected: "cpe:2.3:a:some-vendor:name:1_2:*:*:*:*:*:*:*",
+		},
+		{
+			// it isn't easily possible in the string formatted string to detect improper escaping of : (it will fail parsing)
+			name:        "unescaped ':' cannot be helped -- too many fields",
+			in:          "cpe:2.3:a:some-vendor:name:::*:*:*:*:*:*:*",
+			expectedErr: true,
+		},
+		{
+			name:     "too few fields",
+			in:       "cpe:2.3:a:some-vendor:name:*:*:*:*:*:*:*",
+			expected: "cpe:2.3:a:some-vendor:name:*:*:*:*:*:*:*:*",
+		},
+		// Note: though the CPE spec does not allow for ? and * as escaped character input, these seem to be allowed in
+		// the NVD CPE validator for this reason these edge cases were removed
+	}
+
+	// the wfn library does not account for escapes of . and -
+	exceptions := ".-"
+	// it isn't easily possible in the string formatted string to detect improper escaping of : (it will fail parsing)
+	skip := ":"
+
+	// make escape exceptions for section 5.3.2 of the CPE spec (2.3)
+	for _, char := range allowedCPEPunctuation {
+		if strings.Contains(skip, string(char)) {
+			continue
+		}
+
+		in := fmt.Sprintf("cpe:2.3:a:some-vendor:name:*:%s:*:*:*:*:*:*", string(char))
+		exp := fmt.Sprintf(`cpe:2.3:a:some-vendor:name:*:\%s:*:*:*:*:*:*`, string(char))
+		if strings.Contains(exceptions, string(char)) {
+			exp = in
+		}
+
+		tests = append(tests, testcase{
+			name:        fmt.Sprintf("allowes future escape of character (%s)", string(char)),
+			in:          in,
+			expected:    exp,
+			expectedErr: false,
+		})
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			c, err := New(test.in)
+			if test.expectedErr {
+				assert.Error(t, err)
+				if t.Failed() {
+					t.Logf("got CPE: %q details: %+v", String(c), c)
+				}
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, test.expected, String(c))
+		})
+	}
+}
+
+func Test_RoundTrip(t *testing.T) {
+	tests := []struct {
+		name      string
+		cpe       string
+		parsedCPE CPE
+	}{
+		{
+			name: "normal",
+			cpe:  "cpe:2.3:a:some-vendor:name:3.2:*:*:*:*:*:*:*",
+			parsedCPE: CPE{
+				Part:    "a",
+				Vendor:  "some-vendor",
+				Product: "name",
+				Version: "3.2",
+			},
+		},
+		{
+			name: "escaped colon",
+			cpe:  "cpe:2.3:a:some-vendor:name:1\\:3.2:*:*:*:*:*:*:*",
+			parsedCPE: CPE{
+				Part:    "a",
+				Vendor:  "some-vendor",
+				Product: "name",
+				Version: "1:3.2",
+			},
+		},
+		{
+			name: "escaped forward slash",
+			cpe:  "cpe:2.3:a:test\\/some-vendor:name:3.2:*:*:*:*:*:*:*",
+			parsedCPE: CPE{
+				Part:    "a",
+				Vendor:  "test/some-vendor",
+				Product: "name",
+				Version: "3.2",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// CPE string must be preserved through a round trip
+			assert.Equal(t, test.cpe, String(Must(test.cpe)))
+			// The parsed CPE must be the same after a round trip
+			assert.Equal(t, Must(test.cpe), Must(String(Must(test.cpe))))
+			// The test case parsed CPE must be the same after parsing the input string
+			assert.Equal(t, test.parsedCPE, Must(test.cpe))
+			// The test case parsed CPE must produce the same string as the input cpe
+			assert.Equal(t, String(test.parsedCPE), test.cpe)
+		})
+	}
+}

syft/cpe/merge_cpes.go

@@ -0,0 +1,25 @@
+package cpe
+
+import (
+	"sort"
+)
+
+func Merge(a, b []CPE) (result []CPE) {
+	aCPEs := make(map[string]CPE)
+
+	// keep all CPEs from a and create a quick string-based lookup
+	for _, aCPE := range a {
+		aCPEs[aCPE.BindToFmtString()] = aCPE
+		result = append(result, aCPE)
+	}
+
+	// keep all unique CPEs from b
+	for _, bCPE := range b {
+		if _, exists := aCPEs[bCPE.BindToFmtString()]; !exists {
+			result = append(result, bCPE)
+		}
+	}
+
+	sort.Sort(BySpecificity(result))
+	return result
+}

syft/cpe/merge_cpes_test.go

@@ -0,0 +1,41 @@
+package cpe
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Merge(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    [][]CPE
+		expected []CPE
+	}{
+		{
+			name: "merge, removing duplicates and ordered",
+			input: [][]CPE{
+				{
+					Must("cpe:2.3:a:*:package:1:*:*:*:*:*:*:*"),
+					Must("cpe:2.3:a:some:package:*:*:*:*:*:*:*:*"),
+				},
+				{
+					Must("cpe:2.3:a:some:package:1:*:*:*:*:*:*:*"),
+					Must("cpe:2.3:a:some:package:*:*:*:*:*:*:*:*"),
+				},
+			},
+			expected: []CPE{
+				Must("cpe:2.3:a:some:package:1:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:some:package:*:*:*:*:*:*:*:*"),
+				Must("cpe:2.3:a:*:package:1:*:*:*:*:*:*:*"),
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			out := Merge(test.input[0], test.input[1])
+			assert.Equal(t, test.expected, out)
+		})
+	}
+}