Compare commits

...
5 Commits
Author SHA1 Message Date
Alex Goodman f87c59b4eb Merge pull request #286 from anchore/rm-tree-catalog-sync-check
Bump stereoscope to remove tree-catalog syft check
2020-12-03 16:32:24 -05:00
Alex Goodman 3ce7eabc98 bump stereoscope to remove tree-catalog syft check
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2020-12-03 16:20:17 -05:00
Alex Goodman 6f9ded60ed Merge pull request #279 from anchore/enhance-java-cpe-by-group-id
Include CPEs with elements from POM GroupId fields
2020-12-02 07:50:31 -05:00
Dan Luhring 65cbacd135 Clarify python wheel parsing process (#281)
Signed-off-by: Dan Luhring <dan.luhring@anchore.com>
2020-12-01 16:30:30 -05:00
Alex Goodman 2989d3d975 include CPEs with elementds from POM GroupId fields
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2020-12-01 08:00:49 -05:00
6 changed files with 175 additions and 83 deletions
+2 -1
View File
@@ -8,7 +8,7 @@ require (
github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
github.com/anchore/stereoscope v0.0.0-20201130153727-b3f1fad856b0
github.com/anchore/stereoscope v0.0.0-20201203153145-3f9a05a624d7
github.com/bmatcuk/doublestar v1.3.3
github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
github.com/dustin/go-humanize v1.0.0
@@ -23,6 +23,7 @@ require (
github.com/olekukonko/tablewriter v0.0.4
github.com/package-url/packageurl-go v0.1.0
github.com/pelletier/go-toml v1.8.0
github.com/scylladb/go-set v1.0.2
github.com/sergi/go-diff v1.1.0
github.com/sirupsen/logrus v1.6.0
github.com/spf13/afero v1.2.2
+5
View File
@@ -136,6 +136,8 @@ github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409 h1:xKSpDRjmYrE
github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
github.com/anchore/stereoscope v0.0.0-20201130153727-b3f1fad856b0 h1:wa0hdnvBeCpI+rmzDbPG7k5SKlsGkot7aZ8Az1i/vws=
github.com/anchore/stereoscope v0.0.0-20201130153727-b3f1fad856b0/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
github.com/anchore/stereoscope v0.0.0-20201203153145-3f9a05a624d7 h1:G3LnRqHL/IIeQZTAMtDOJNYfSYsXLNCZX4DCiS0R0FY=
github.com/anchore/stereoscope v0.0.0-20201203153145-3f9a05a624d7/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ=
github.com/apex/log v1.3.0 h1:1fyfbPvUwD10nMoh3hY6MXzvZShJQn9/ck7ATgAt5pA=
@@ -263,6 +265,7 @@ github.com/facebookincubator/nvdtools v0.1.4/go.mod h1:0/FIVnSEl9YHXLq3tKBPpKaI0
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -726,6 +729,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
github.com/scylladb/go-set v1.0.2 h1:SkvlMCKhP0wyyct6j+0IHJkBkSZL+TDzZ4E7f7BCcRE=
github.com/scylladb/go-set v1.0.2/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7AzSq3/kywjUDxSNq2SJ27RxCz2un0H3ePqE=
github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989 h1:rq2/kILQnPtq5oL4+IAjgVOjh5e2yj2aaCYi7squEvI=
+35 -6
View File
@@ -2,6 +2,7 @@ package cataloger
import (
"fmt"
"strings"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/pkg"
@@ -31,6 +32,7 @@ func generatePackageCPEs(p pkg.Package) []pkg.CPE {
// add a new entry...
candidateCpe := wfn.NewAttributesWithAny()
candidateCpe.Part = "a"
candidateCpe.Product = product
candidateCpe.Vendor = vendor
candidateCpe.Version = p.Version
@@ -45,8 +47,6 @@ func generatePackageCPEs(p pkg.Package) []pkg.CPE {
}
func candidateTargetSoftwareAttrs(p pkg.Package) []string {
// TODO: expand with package metadata (from type assert)
// TODO: would be great to allow these to be overridden by user data/config
var targetSw []string
switch p.Language {
@@ -68,14 +68,43 @@ func candidateTargetSoftwareAttrs(p pkg.Package) []string {
}
func candidateVendors(p pkg.Package) []string {
// TODO: expand with package metadata (from type assert)
vendors := []string{p.Name}
if p.Language == pkg.Python {
vendors := candidateProducts(p)
switch p.Language {
case pkg.Python:
vendors = append(vendors, fmt.Sprintf("python-%s", p.Name))
case pkg.Java:
if p.MetadataType == pkg.JavaMetadataType {
if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok && metadata.PomProperties != nil {
// derive the vendor from the groupID (e.g. org.sonatype.nexus --> sonatype)
if strings.HasPrefix(metadata.PomProperties.GroupID, "org.") || strings.HasPrefix(metadata.PomProperties.GroupID, "com.") {
fields := strings.Split(metadata.PomProperties.GroupID, ".")
if len(fields) >= 3 {
vendors = append(vendors, fields[1])
}
}
}
}
}
return vendors
}
func candidateProducts(p pkg.Package) []string {
return []string{p.Name}
var products = []string{p.Name}
switch p.Language {
case pkg.Java:
if p.MetadataType == pkg.JavaMetadataType {
if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok && metadata.PomProperties != nil {
// derive the product from the groupID (e.g. org.sonatype.nexus --> nexus)
if strings.HasPrefix(metadata.PomProperties.GroupID, "org.") || strings.HasPrefix(metadata.PomProperties.GroupID, "com.") {
fields := strings.Split(metadata.PomProperties.GroupID, ".")
if len(fields) >= 3 {
products = append(products, fields[2])
}
}
}
}
default:
return products
}
return products
}
+102 -57
View File
@@ -1,23 +1,21 @@
package cataloger
import (
"sort"
"testing"
"github.com/scylladb/go-set/strset"
"github.com/scylladb/go-set"
"github.com/anchore/syft/syft/pkg"
)
func must(c pkg.CPE, e error) pkg.CPE {
if e != nil {
panic(e)
}
return c
}
func TestGenerate(t *testing.T) {
tests := []struct {
name string
p pkg.Package
expected []pkg.CPE
expected []string
}{
{
name: "python language",
@@ -28,13 +26,13 @@ func TestGenerate(t *testing.T) {
Language: pkg.Python,
Type: pkg.DebPkg,
},
expected: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:python:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:python:*:*")),
must(pkg.NewCPE("cpe:2.3:*:python-name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:python-name:name:3.2:*:*:*:*:python:*:*")),
expected: []string{
"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:python:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:python:*:*",
"cpe:2.3:a:python-name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:python-name:name:3.2:*:*:*:*:python:*:*",
},
},
{
@@ -46,13 +44,13 @@ func TestGenerate(t *testing.T) {
Language: pkg.JavaScript,
Type: pkg.DebPkg,
},
expected: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:node.js:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:nodejs:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:node.js:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:nodejs:*:*")),
expected: []string{
"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:node.js:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:nodejs:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:node.js:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:nodejs:*:*",
},
},
{
@@ -64,13 +62,13 @@ func TestGenerate(t *testing.T) {
Language: pkg.Ruby,
Type: pkg.DebPkg,
},
expected: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:ruby:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:rails:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:ruby:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:rails:*:*")),
expected: []string{
"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:ruby:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:rails:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:ruby:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:rails:*:*",
},
},
{
@@ -82,13 +80,55 @@ func TestGenerate(t *testing.T) {
Language: pkg.Java,
Type: pkg.DebPkg,
},
expected: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:java:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:maven:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:java:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:maven:*:*")),
expected: []string{
"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:maven:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:maven:*:*",
},
},
{
name: "java language with groupID",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JavaPkg,
MetadataType: pkg.JavaMetadataType,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "org.sonatype.nexus",
},
},
},
expected: []string{
"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:maven:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:maven:*:*",
"cpe:2.3:a:sonatype:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:sonatype:name:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:sonatype:name:3.2:*:*:*:*:maven:*:*",
"cpe:2.3:a:*:nexus:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:*:nexus:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:*:nexus:3.2:*:*:*:*:maven:*:*",
"cpe:2.3:a:sonatype:nexus:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:sonatype:nexus:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:sonatype:nexus:3.2:*:*:*:*:maven:*:*",
"cpe:2.3:a:nexus:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:nexus:name:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:nexus:name:3.2:*:*:*:*:maven:*:*",
"cpe:2.3:a:name:nexus:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:nexus:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:name:nexus:3.2:*:*:*:*:maven:*:*",
"cpe:2.3:a:nexus:nexus:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:nexus:nexus:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:nexus:nexus:3.2:*:*:*:*:maven:*:*",
},
},
{
@@ -100,17 +140,17 @@ func TestGenerate(t *testing.T) {
Language: pkg.Java,
Type: pkg.JenkinsPluginPkg,
},
expected: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:java:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:maven:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:jenkins:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:cloudbees_jenkins:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:java:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:maven:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:jenkins:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:cloudbees_jenkins:*:*")),
expected: []string{
"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:maven:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:jenkins:*:*",
"cpe:2.3:a:*:name:3.2:*:*:*:*:cloudbees_jenkins:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:java:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:maven:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:jenkins:*:*",
"cpe:2.3:a:name:name:3.2:*:*:*:*:cloudbees_jenkins:*:*",
},
},
}
@@ -119,19 +159,24 @@ func TestGenerate(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
actual := generatePackageCPEs(test.p)
if len(actual) != len(test.expected) {
for _, e := range actual {
t.Errorf(" unexpected entry: %+v", e.BindToFmtString())
}
t.Fatalf("unexpected number of entries: %d", len(actual))
expectedCpeSet := set.NewStringSet(test.expected...)
actualCpeSet := set.NewStringSet()
for _, a := range actual {
actualCpeSet.Add(a.BindToFmtString())
}
for idx, a := range actual {
e := test.expected[idx]
if a.BindToFmtString() != e.BindToFmtString() {
t.Errorf("mismatched entries @ %d:\n\texpected:%+v\n\t actual:%+v\n", idx, e.BindToFmtString(), a.BindToFmtString())
}
extra := strset.Difference(expectedCpeSet, actualCpeSet).List()
sort.Strings(extra)
for _, d := range extra {
t.Errorf("extra CPE: %+v", d)
}
missing := strset.Difference(actualCpeSet, expectedCpeSet).List()
sort.Strings(missing)
for _, d := range missing {
t.Errorf("missing CPE: %+v", d)
}
})
}
}
@@ -23,29 +23,28 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
line := scanner.Text()
line = strings.TrimRight(line, "\n")
// empty line indicates end of entry
// An empty line means we are done parsing (either because there's no more data,
// or because a description follows as specified in
// https://packaging.python.org/specifications/core-metadata/#description;
// and at this time, we're not interested in the description).
if len(line) == 0 {
// if the entry has not started, keep parsing lines
if len(fields) == 0 {
continue
if len(fields) > 0 {
break
}
break
// however, if the field parsing has not started yet, keep scanning lines
continue
}
switch {
case strings.HasPrefix(line, " "):
// a field-body continuation
if len(key) == 0 {
return pkg.PythonPackageMetadata{}, fmt.Errorf("no match for continuation: line: '%s'", line)
updatedValue, err := handleFieldBodyContinuation(key, line, fields)
if err != nil {
return pkg.PythonPackageMetadata{}, err
}
val, ok := fields[key]
if !ok {
return pkg.PythonPackageMetadata{}, fmt.Errorf("no previous key exists, expecting: %s", key)
}
// concatenate onto previous value
val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line))
fields[key] = val
fields[key] = updatedValue
default:
// parse a new key (note, duplicate keys are overridden)
if i := strings.Index(line, ":"); i > 0 {
@@ -71,8 +70,23 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
// add additional metadata not stored in the egg/wheel metadata file
sitePackagesRoot := filepath.Clean(filepath.Join(filepath.Dir(path), ".."))
metadata.SitePackagesRootPath = sitePackagesRoot
metadata.SitePackagesRootPath = filepath.Clean(filepath.Join(filepath.Dir(path), ".."))
return metadata, nil
}
// handleFieldBodyContinuation returns the updated value for the specified field after processing the specified line.
// If the continuation cannot be processed, it returns an error.
func handleFieldBodyContinuation(key, line string, fields map[string]string) (string, error) {
if len(key) == 0 {
return "", fmt.Errorf("no match for continuation: line: '%s'", line)
}
val, ok := fields[key]
if !ok {
return "", fmt.Errorf("no previous key exists, expecting: %s", key)
}
// concatenate onto previous value
return fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line)), nil
}
+1 -3
View File
@@ -1,5 +1,3 @@
// +build integration
package integration
import (
@@ -21,7 +19,7 @@ func TestRegression212ApkBufferSize(t *testing.T) {
tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
defer cleanup()
catalog, _, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
_, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
if err != nil {
t.Fatalf("failed to catalog image: %+v", err)
}