mirror of
https://github.com/anchore/syft.git
synced 2026-11-03 07:56:36 +01:00
fix(cpe): bound CPE candidate generation for pathological package metadata (#5356)
* fix(cpe): cap sub-selection prefixes to bound candidate cross-product a name with thousands of `-`/`_` separators produced one vendor and product candidate per segment, and the vendor x product cross-product made CPE generation roughly cubic in time and memory (8KB name: ~250s, 4.8GB RSS). `generateSubSelections` now stops after 10 prefixes. The full name is still a candidate, so real package names are unaffected. Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix(cpe): bound candidate count and field length before the cross-product the prefix cap only covered one way to inflate the vendor x product cross-product in `FromPackageAttributes`. Java groupIDs add a vendor (and, with no artifactID, a product) per dot segment, and .NET executables add one of each per file, so a 1000-segment `Bundle-SymbolicName` still produced ~1M CPEs and several GB. - vendor and product candidates are each capped at 64 (shortest first, so truncation is deterministic) - names, versions, and candidates over 512 bytes are skipped, since every byte is paid once per CPE in key building and regex validation real packages produce around a dozen candidates with names well under 100 bytes, so output for real inputs is unchanged. Also corrects the earlier comment: prefixes only become vendor candidates and the cost was quadratic, not cubic. Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
@@ -3,10 +3,12 @@ package cpegenerate
|
|||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"cmp"
|
||||||
_ "embed"
|
_ "embed"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -145,8 +147,14 @@ func FromPackageAttributes(p pkg.Package) []cpe.CPE {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
vendors := candidateVendors(p)
|
// real CPE fields are short, and every byte of a field is paid once per generated CPE (key building and regex
|
||||||
products := candidateProducts(p)
|
// validation), so a hostile name or version is not worth generating candidates for at all.
|
||||||
|
if len(p.Name) > maxCandidateFieldLength || len(p.Version) > maxCandidateFieldLength {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
vendors := boundCandidates(candidateVendors(p))
|
||||||
|
products := boundCandidates(candidateProducts(p))
|
||||||
targetSWs := candidateTargetSw(p)
|
targetSWs := candidateTargetSw(p)
|
||||||
if len(products) == 0 {
|
if len(products) == 0 {
|
||||||
return nil
|
return nil
|
||||||
@@ -183,6 +191,32 @@ func FromPackageAttributes(p pkg.Package) []cpe.CPE {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// maxCandidateFieldLength bounds the name, version, and each vendor and product candidate that reaches the
|
||||||
|
// cross-product in FromPackageAttributes. Real values are well under this.
|
||||||
|
const maxCandidateFieldLength = 512
|
||||||
|
|
||||||
|
// maxCandidates bounds how many vendor and product candidates each feed the cross-product in FromPackageAttributes,
|
||||||
|
// so a single package yields at most maxCandidates² CPEs. Some sources add a candidate per input element with no bound
|
||||||
|
// of their own (e.g. one vendor per java groupID segment, one product per .NET executable), and real packages produce
|
||||||
|
// a dozen or so candidates at most.
|
||||||
|
const maxCandidates = 64
|
||||||
|
|
||||||
|
// boundCandidates drops candidates longer than maxCandidateFieldLength and keeps at most maxCandidates of the rest.
|
||||||
|
// Candidates are ordered shortest first (then lexically) so that truncation is deterministic and keeps the short,
|
||||||
|
// generic values that tend to be real vendors and products.
|
||||||
|
func boundCandidates(candidates []string) []string {
|
||||||
|
candidates = slices.DeleteFunc(candidates, func(c string) bool {
|
||||||
|
return len(c) > maxCandidateFieldLength
|
||||||
|
})
|
||||||
|
if len(candidates) <= maxCandidates {
|
||||||
|
return candidates
|
||||||
|
}
|
||||||
|
slices.SortFunc(candidates, func(a, b string) int {
|
||||||
|
return cmp.Or(cmp.Compare(len(a), len(b)), strings.Compare(a, b))
|
||||||
|
})
|
||||||
|
return candidates[:maxCandidates]
|
||||||
|
}
|
||||||
|
|
||||||
func candidateTargetSw(p pkg.Package) []string {
|
func candidateTargetSw(p pkg.Package) []string {
|
||||||
switch p.Type {
|
switch p.Type {
|
||||||
case pkg.WordpressPluginPkg:
|
case pkg.WordpressPluginPkg:
|
||||||
@@ -359,13 +393,19 @@ func addAllSubSelections(fields fieldCandidateSet) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// maxSubSelections bounds how many prefixes generateSubSelections returns. Each prefix becomes a vendor candidate, so
|
||||||
|
// without a bound a name with N separators yields O(N) vendors of O(N) length each (quadratic in time and memory).
|
||||||
|
// Real package names rarely exceed a handful of segments.
|
||||||
|
const maxSubSelections = 10
|
||||||
|
|
||||||
// generateSubSelections attempts to split a field by hyphens and underscores and return a list of sensible sub-selections
|
// generateSubSelections attempts to split a field by hyphens and underscores and return a list of sensible sub-selections
|
||||||
// that can be used as product or vendor candidates. E.g. jenkins-ci-tools -> [jenkins-ci-tools, jenkins-ci, jenkins].
|
// that can be used as product or vendor candidates. E.g. jenkins-ci-tools -> [jenkins, jenkins-ci, jenkins-ci-tools].
|
||||||
|
// At most maxSubSelections prefixes are returned, shortest first.
|
||||||
func generateSubSelections(field string) (results []string) {
|
func generateSubSelections(field string) (results []string) {
|
||||||
scanner := bufio.NewScanner(strings.NewReader(field))
|
scanner := bufio.NewScanner(strings.NewReader(field))
|
||||||
scanner.Split(scanByHyphenOrUnderscore)
|
scanner.Split(scanByHyphenOrUnderscore)
|
||||||
var lastToken uint8
|
var lastToken uint8
|
||||||
for scanner.Scan() {
|
for len(results) < maxSubSelections && scanner.Scan() {
|
||||||
rawCandidate := scanner.Text()
|
rawCandidate := scanner.Text()
|
||||||
if len(rawCandidate) == 0 {
|
if len(rawCandidate) == 0 {
|
||||||
break
|
break
|
||||||
|
|||||||
@@ -1121,6 +1121,11 @@ func Test_generateSubSelections(t *testing.T) {
|
|||||||
field: "_",
|
field: "_",
|
||||||
expected: nil,
|
expected: nil,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
// capped at maxSubSelections, keeping the shortest prefixes (the full name is not returned)
|
||||||
|
field: "a_b_c_d_e_f_g_h_i_j_k_l",
|
||||||
|
expected: []string{"a", "a_b", "a_b_c", "a_b_c_d", "a_b_c_d_e", "a_b_c_d_e_f", "a_b_c_d_e_f_g", "a_b_c_d_e_f_g_h", "a_b_c_d_e_f_g_h_i", "a_b_c_d_e_f_g_h_i_j"},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.field, func(t *testing.T) {
|
t.Run(test.field, func(t *testing.T) {
|
||||||
@@ -1265,3 +1270,59 @@ func TestAddBinaryPackageDigitVariations(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFromPackageAttributes_boundsSubSelectionBlowup(t *testing.T) {
|
||||||
|
// a name with many separators used to produce a vendor candidate per segment, each O(name length), which was
|
||||||
|
// quadratic in time and memory.
|
||||||
|
name := strings.Repeat("a_", 200) + "a"
|
||||||
|
p := pkg.Package{Name: name, Version: "1.0", Type: pkg.PythonPkg, Language: pkg.Python}
|
||||||
|
|
||||||
|
cpes := FromPackageAttributes(p)
|
||||||
|
assert.NotEmpty(t, cpes)
|
||||||
|
assert.LessOrEqual(t, len(cpes), 1000)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFromPackageAttributes_boundsCandidateCrossProduct(t *testing.T) {
|
||||||
|
// every groupID segment becomes a vendor, and with no artifactID every segment past the second also becomes a
|
||||||
|
// product, so the vendor x product cross-product is quadratic in segment count without a bound.
|
||||||
|
var segments []string
|
||||||
|
for i := range 1000 {
|
||||||
|
segments = append(segments, fmt.Sprintf("s%d", i))
|
||||||
|
}
|
||||||
|
p := pkg.Package{
|
||||||
|
Name: "thing",
|
||||||
|
Version: "1.0",
|
||||||
|
Type: pkg.JavaPkg,
|
||||||
|
Language: pkg.Java,
|
||||||
|
Metadata: pkg.JavaArchive{
|
||||||
|
Manifest: &pkg.JavaManifest{
|
||||||
|
Main: pkg.KeyValues{{Key: "Bundle-SymbolicName", Value: "com.x." + strings.Join(segments, ".")}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cpes := FromPackageAttributes(p)
|
||||||
|
assert.NotEmpty(t, cpes)
|
||||||
|
assert.LessOrEqual(t, len(cpes), maxCandidates*maxCandidates)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFromPackageAttributes_skipsOversizedFields(t *testing.T) {
|
||||||
|
long := strings.Repeat("a", maxCandidateFieldLength+1)
|
||||||
|
|
||||||
|
assert.Empty(t, FromPackageAttributes(pkg.Package{Name: long, Version: "1.0", Type: pkg.PythonPkg}))
|
||||||
|
assert.Empty(t, FromPackageAttributes(pkg.Package{Name: "thing", Version: long, Type: pkg.PythonPkg}))
|
||||||
|
assert.NotEmpty(t, FromPackageAttributes(pkg.Package{Name: "thing", Version: "1.0", Type: pkg.PythonPkg}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_boundCandidates(t *testing.T) {
|
||||||
|
long := strings.Repeat("a", maxCandidateFieldLength+1)
|
||||||
|
assert.Equal(t, []string{"b", "a"}, boundCandidates([]string{"b", long, "a"}), "under the cap order is untouched")
|
||||||
|
|
||||||
|
var many []string
|
||||||
|
for i := range maxCandidates + 10 {
|
||||||
|
many = append(many, strings.Repeat("x", maxCandidates+10-i))
|
||||||
|
}
|
||||||
|
got := boundCandidates(many)
|
||||||
|
assert.Len(t, got, maxCandidates)
|
||||||
|
assert.Equal(t, "x", got[0], "shortest candidates are kept")
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user