oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2025-11-17 16:33:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix: improvements to known CPE index construction (#2801)

Browse Source 
* fix: stop pre-filtering potential known CPE URLs

Previously when building the known CPE index, there was logic to
de-duplicate processing based on the normalized CPE name; however, this
means a significant number of known CPE's don't get indexed because the
first instance of that name didn't have a supported collection url but a
later one did.  This isn't code that executes at runtime in syft so
de-duplicating the processing for performance isn't really necessary
here and it doesn't add much to the total runtime anyways

Signed-off-by: Weston Steimel <commits@weston.slmail.me>

* fix: CPE index builder should extract and consider all reference urls

Previously the struct definition for CpeItem caused only the last URL
reference in the list to be kept and processed for inclusion in the
index

Signed-off-by: Weston Steimel <commits@weston.slmail.me>

---------

Signed-off-by: Weston Steimel <commits@weston.slmail.me>
This commit is contained in:
Weston Steimel 2024-04-23 14:28:18 +01:00 committed by GitHub
parent f7d3d552ce
commit 891e61a2ef
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 700 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

696
syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json
View File

File diff suppressed because it is too large Load Diff

14
syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/generate.go
View File

@ -52,11 +52,9 @@ func generateIndexedDictionaryJSON(rawGzipData io.Reader) ([]byte, error) {
func filterCpeList(cpeList CpeList) CpeList {
var processedCpeList CpeList
seen := make(map[string]struct{})
for _, cpeItem := range cpeList.CpeItems {
// Skip CPE items that don't have any references.
if len(cpeItem.References) == 0 {
if len(cpeItem.References.Reference) == 0 {
continue
}
@ -64,6 +62,7 @@ func filterCpeList(cpeList CpeList) CpeList {
parsedName, err := wfn.Parse(cpeItem.Name)
if err != nil {
log.Printf("unable to parse CPE URI %q: %s", cpeItem.Name, err)
continue
}
if slices.Contains([]string{"h", "o"}, parsedName.Part) {
@ -71,15 +70,12 @@ func filterCpeList(cpeList CpeList) CpeList {
}
normalizedName := normalizeCPE(parsedName).BindToURI()
if _, ok := seen[normalizedName]; ok {
continue
}
seen[normalizedName] = struct{}{}
cpeItem.Name = normalizedName
parsedCPE, err := wfn.Parse(cpeItem.Cpe23Item.Name)
if err != nil {
log.Printf("unable to parse CPE value %q: %s", cpeItem.Cpe23Item.Name, err)
continue
}
cpeItem.Cpe23Item.Name = normalizeCPE(parsedCPE).BindToFmtString()
@ -123,8 +119,8 @@ func indexCPEList(list CpeList) *dictionary.Indexed {
for _, cpeItem := range list.CpeItems {
cpeItemName := cpeItem.Cpe23Item.Name
for _, reference := range cpeItem.References {
ref := reference.Reference.Href
for _, reference := range cpeItem.References.Reference {
ref := reference.Href
switch {
case strings.HasPrefix(ref, prefixForNPMPackages):

4
syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/nvd.go
View File

@ -3,8 +3,8 @@ package main
type CpeItem struct {
Name string `xml:"name,attr"`
Title string `xml:"title"`
References []struct {
Reference struct {
References struct {
Reference []struct {
Href string `xml:"href,attr"`
Body string `xml:",chardata"`
} `xml:"reference"`

6
syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/testdata/expected-cpe-index.json vendored
View File

@ -5,12 +5,15 @@
},
"npm": {
"merge-recursive": "cpe:2.3:a:umbraengineering:merge-recursive:*:*:*:*:*:node.js:*:*",
"ps": "cpe:2.3:a:umbraengineering:ps:*:*:*:*:*:node.js:*:*",
"static-dev-server": "cpe:2.3:a:static-dev-server_project:static-dev-server:*:*:*:*:*:node.js:*:*",
"umount": "cpe:2.3:a:umount_project:umount:*:*:*:*:*:node.js:*:*",
"undefsafe": "cpe:2.3:a:undefsafe_project:undefsafe:*:*:*:*:*:node.js:*:*",
"underscore": "cpe:2.3:a:underscorejs:underscore:*:*:*:*:*:node.js:*:*",
"underscore-99xp": "cpe:2.3:a:underscore-99xp_project:underscore-99xp:*:*:*:*:*:node.js:*:*",
"ungit": "cpe:2.3:a:ungit_project:ungit:*:*:*:*:*:node.js:*:*",
"unicode": "cpe:2.3:a:unicode_project:unicode:*:*:*:*:*:node.js:*:*",
"unicode-json": "cpe:2.3:a:unicode:unicode-json:*:*:*:*:*:node.js:*:*",
"unicorn-list": "cpe:2.3:a:unicorn-list_project:unicorn-list:*:*:*:*:*:node.js:*:*"
},
"php_pear": {
@ -23,6 +26,9 @@
"memcached": "cpe:2.3:a:php:memcached:*:*:*:*:*:*:*:*",
"xhprof": "cpe:2.3:a:php:xhprof:*:*:*:*:*:*:*:*"
},
"pypi": {
"vault-cli": "cpe:2.3:a:ukg:vault-cli:*:*:*:*:*:python:*:*"
},
"rubygems": {
"openssl": "cpe:2.3:a:ruby-lang:openssl:*:*:*:*:*:*:*:*"
},

2
syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/testdata/official-cpe-dictionary_v2.3.xml vendored
View File

@ -24972,6 +24972,7 @@
<references>
<reference href="http://pear.php.net/package/HTML_QuickForm/">Project</reference>
<reference href="http://pear.php.net/package/HTML_QuickForm/download">Version</reference>
<reference href="http://some-other-unrelated-reference">Test</reference>
</references>
<cpe-23:cpe23-item name="cpe:2.3:a:html_quickform_project:html_quickform:2.0:*:*:*:*:*:*:*"/>
</cpe-item>
@ -24980,6 +24981,7 @@
<references>
<reference href="http://pear.php.net/package/HTML_QuickForm/">Project</reference>
<reference href="http://pear.php.net/package/HTML_QuickForm/download">Version</reference>
<reference href="http://some-other-unrelated-reference">Test</reference>
</references>
<cpe-23:cpe23-item name="cpe:2.3:a:html_quickform_project:html_quickform:2.1:*:*:*:*:*:*:*"/>
</cpe-item>