oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2025-11-18 00:43:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #247 from anchore/syft-java-cataloger-integration

Browse Source 
General Java cataloger enhancements
This commit is contained in:
Alex Goodman 2020-10-30 13:55:16 -04:00 committed by GitHub
commit 6e98752c6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 410 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
schema/json/schema.json
View File

@ -126,7 +126,7 @@
},
"manifest": {
"properties": {
"extraFields": {
"main": {
"properties": {
"Archiver-Version": {
"type": "string"
@ -149,6 +149,12 @@
"Hudson-Version": {
"type": "string"
},
"Implementation-Title": {
"type": "string"
},
"Implementation-Version": {
"type": "string"
},
"Jenkins-Version": {
"type": "string"
},
@ -158,6 +164,9 @@
"Main-Class": {
"type": "string"
},
"Manifest-Version": {
"type": "string"
},
"Minimum-Java-Version": {
"type": "string"
},
@ -181,51 +190,23 @@
},
"Short-Name": {
"type": "string"
},
"Specification-Title": {
"type": "string"
}
},
"required": [
"Archiver-Version",
"Build-Jdk",
"Built-By",
"Created-By"
"Created-By",
"Manifest-Version"
],
"type": "object"
},
"implementationTitle": {
"type": "string"
},
"implementationVendor": {
"type": "string"
},
"implementationVersion": {
"type": "string"
},
"manifestVersion": {
"type": "string"
},
"name": {
"type": "string"
},
"specificationTitle": {
"type": "string"
},
"specificationVendor": {
"type": "string"
},
"specificationVersion": {
"type": "string"
}
},
"required": [
"extraFields",
"implementationTitle",
"implementationVendor",
"implementationVersion",
"manifestVersion",
"name",
"specificationTitle",
"specificationVendor",
"specificationVersion"
"main"
],
"type": "object"
},

23
syft/cataloger/java/archive_filename.go
View File

@ -10,11 +10,20 @@ import (
"github.com/anchore/syft/syft/pkg"
)
// match on versions and anything after the version. This is used to isolate the name from the version.
// match examples:
// wagon-webdav-1.0.2-rc1-hudson.jar ---> -1.0.2-rc1-hudson.jar
// windows-remote-command-1.0.jar ---> -1.0.jar
// wstx-asl-1-2.jar ---> -1-2.jar
// guava-rc0.jar ---> -rc0.jar
var versionAreaPattern = regexp.MustCompile(`-(?P<version>(\d+\.)?(\d+\.)?(r?c?\d+)(-[a-zA-Z0-9\-.]+)*)(?P<remaining>.*)$`)
// match on explicit versions. This is used for extracting version information.
// match examples:
// pkg-extra-field-4.3.2-rc1 --> match(name=pkg-extra-field version=4.3.2-rc1)
// pkg-extra-field-4.3-rc1 --> match(name=pkg-extra-field version=4.3-rc1)
// pkg-extra-field-4.3 --> match(name=pkg-extra-field version=4.3)
var versionPattern = regexp.MustCompile(`(?P<name>.+)-(?P<version>(\d+\.)?(\d+\.)?(\*|\d+)(-[a-zA-Z0-9\-\.]+)*)`)
var versionPattern = regexp.MustCompile(`-(?P<version>(\d+\.)?(\d+\.)?(r?c?\d+)(-[a-zA-Z0-9\-.]+)*)`)
type archiveFilename struct {
raw string
@ -70,14 +79,8 @@ func (a archiveFilename) version() string {
}
func (a archiveFilename) name() string {
for _, fieldSet := range a.fields {
if name, ok := fieldSet["name"]; ok {
// return the first name
return name
}
}
// derive the name from the archive name (no path or extension)
// derive the name from the archive name (no path or extension) and remove any versions found
basename := filepath.Base(a.raw)
return strings.TrimSuffix(basename, filepath.Ext(basename))
cleaned := strings.TrimSuffix(basename, filepath.Ext(basename))
return versionAreaPattern.ReplaceAllString(cleaned, "")
}

69
syft/cataloger/java/archive_filename_test.go
View File

@ -1,9 +1,10 @@
package java
import (
"testing"
"github.com/anchore/syft/syft/pkg"
"github.com/sergi/go-diff/diffmatchpatch"
"testing"
)
func TestExtractInfoFromJavaArchiveFilename(t *testing.T) {
@ -56,12 +57,78 @@ func TestExtractInfoFromJavaArchiveFilename(t *testing.T) {
name: "pkg-extra-field-maven",
ty: pkg.JenkinsPluginPkg,
},
{
filename: "/some/path-with-version-5.4.3/wagon-webdav-1.0.2-beta-2.2.3a-hudson.jar",
version: "1.0.2-beta-2.2.3a-hudson",
extension: "jar",
name: "wagon-webdav",
ty: pkg.JavaPkg,
},
{
filename: "/some/path-with-version-5.4.3/wagon-webdav-1.0.2-beta-2.2.3-hudson.jar",
version: "1.0.2-beta-2.2.3-hudson",
extension: "jar",
name: "wagon-webdav",
ty: pkg.JavaPkg,
},
{
filename: "/some/path-with-version-5.4.3/windows-remote-command-1.0.jar",
version: "1.0",
extension: "jar",
name: "windows-remote-command",
ty: pkg.JavaPkg,
},
{
filename: "/some/path-with-version-5.4.3/wagon-http-lightweight-1.0.5-beta-2.jar",
version: "1.0.5-beta-2",
extension: "jar",
name: "wagon-http-lightweight",
ty: pkg.JavaPkg,
},
{
filename: "/hudson.war:WEB-INF/lib/commons-jelly-1.1-hudson-20100305.jar",
version: "1.1-hudson-20100305",
extension: "jar",
name: "commons-jelly",
ty: pkg.JavaPkg,
},
{
filename: "/hudson.war:WEB-INF/lib/jtidy-4aug2000r7-dev-hudson-1.jar",
// I don't see how we can reliably account for this case
//version: "4aug2000r7-dev-hudson-1",
version: "",
extension: "jar",
name: "jtidy",
ty: pkg.JavaPkg,
},
{
filename: "/hudson.war:WEB-INF/lib/trilead-ssh2-build212-hudson-5.jar",
// I don't see how we can reliably account for this case
//version: "build212-hudson-5",
version: "5",
extension: "jar",
// name: "trilead-ssh2",
name: "trilead-ssh2-build212-hudson",
ty: pkg.JavaPkg,
},
{
filename: "/hudson.war:WEB-INF/lib/guava-r06.jar",
version: "r06",
extension: "jar",
name: "guava",
ty: pkg.JavaPkg,
},
}
for _, test := range tests {
t.Run(test.filename, func(t *testing.T) {
obj := newJavaArchiveFilename(test.filename)
ty := obj.pkgType()
if ty != test.ty {
t.Errorf("mismatched type: %+v != %v", ty, test.ty)
}
version := obj.version()
if version != test.version {
dmp := diffmatchpatch.New()

45
syft/cataloger/java/archive_parser.go
View File

@ -66,13 +66,17 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo
return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err)
}
// fetch the last element of the virtual path
virtualElements := strings.Split(virtualPath, ":")
currentFilepath := virtualElements[len(virtualElements)-1]
return &archiveParser{
discoveredPkgs: internal.NewStringSet(),
fileManifest: fileManifest,
virtualPath: virtualPath,
archivePath: archivePath,
contentPath: contentPath,
fileInfo: newJavaArchiveFilename(virtualPath),
fileInfo: newJavaArchiveFilename(currentFilepath),
detectNested: detectNested,
}, cleanupFn, nil
}
@ -136,7 +140,7 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
// parse the manifest file into a rich object
manifestContents := contents[manifestMatches[0]]
manifest, err := parseJavaManifest(strings.NewReader(manifestContents))
manifest, err := parseJavaManifest(j.archivePath, strings.NewReader(manifestContents))
if err != nil {
return nil, fmt.Errorf("failed to parse java manifest (%s): %w", j.virtualPath, err)
}
@ -156,6 +160,7 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
// discoverPkgsFromPomProperties parses Maven POM properties for a given parent package, returning all listed Java packages found and
// associating each discovered package to the given parent package.
// nolint:funlen,gocognit
func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([]pkg.Package, error) {
var pkgs = make([]pkg.Package, 0)
parentKey := uniquePkgKey(parentPkg)
@ -177,6 +182,13 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
if propsObj.Version != "" && propsObj.ArtifactID != "" {
// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package
// keep the artifact name within the virtual path if this package does not match the parent package
vPathSuffix := ""
if !strings.HasPrefix(propsObj.ArtifactID, parentPkg.Name) {
vPathSuffix += ":" + propsObj.ArtifactID
}
virtualPath := j.virtualPath + vPathSuffix
// discovered props = new package
p := pkg.Package{
Name: propsObj.ArtifactID,
@ -185,7 +197,7 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
Type: pkg.JavaPkg,
MetadataType: pkg.JavaMetadataType,
Metadata: pkg.JavaMetadata{
VirtualPath: j.virtualPath,
VirtualPath: virtualPath,
PomProperties: propsObj,
Parent: parentPkg,
},
@ -193,16 +205,35 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
pkgKey := uniquePkgKey(&p)
if !j.discoveredPkgs.Contains(pkgKey) {
// only keep packages we haven't seen yet
pkgs = append(pkgs, p)
} else if pkgKey == parentKey {
// the name/version pair matches...
matchesParentPkg := pkgKey == parentKey
// the virtual path matches...
matchesParentPkg = matchesParentPkg || parentPkg.Metadata.(pkg.JavaMetadata).VirtualPath == virtualPath
// the pom artifactId has the parent name or vice versa
if propsObj.ArtifactID != "" {
matchesParentPkg = matchesParentPkg || strings.Contains(parentPkg.Name, propsObj.ArtifactID) || strings.Contains(propsObj.ArtifactID, parentPkg.Name)
}
if matchesParentPkg {
// we've run across more information about our parent package, add this info to the parent package metadata
// the pom properties is typically a better source of information for name and version than the manifest
if p.Name != parentPkg.Name {
parentPkg.Name = p.Name
}
if p.Version != parentPkg.Version {
parentPkg.Version = p.Version
}
parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata)
if ok {
parentMetadata.PomProperties = propsObj
parentPkg.Metadata = parentMetadata
}
} else if !j.discoveredPkgs.Contains(pkgKey) {
// only keep packages we haven't seen yet (and are not related to the parent package)
pkgs = append(pkgs, p)
}
}
}

141
syft/cataloger/java/archive_parser_test.go
View File

@ -78,51 +78,6 @@ func generateJavaBuildFixture(t *testing.T, fixturePath string) {
}
}
func TestSelectName(t *testing.T) {
tests := []struct {
desc string
manifest pkg.JavaManifest
archive archiveFilename
expected string
}{
{
desc: "name from Implementation-Title",
archive: archiveFilename{},
manifest: pkg.JavaManifest{
Name: "",
SpecTitle: "",
ImplTitle: "maven-wrapper",
},
expected: "maven-wrapper",
},
{
desc: "Implementation-Title does not override",
manifest: pkg.JavaManifest{
Name: "Foo",
SpecTitle: "",
ImplTitle: "maven-wrapper",
},
archive: archiveFilename{
fields: []map[string]string{
{"name": "omg"},
},
},
expected: "omg",
},
}
for _, test := range tests {
t.Run(test.desc, func(t *testing.T) {
result := selectName(&test.manifest, test.archive)
if result != test.expected {
t.Errorf("mismatch in names: '%s' != '%s'", result, test.expected)
}
})
}
}
func TestParseJar(t *testing.T) {
tests := []struct {
fixture string
@ -145,11 +100,12 @@ func TestParseJar(t *testing.T) {
Metadata: pkg.JavaMetadata{
VirtualPath: "test-fixtures/java-builds/packages/example-jenkins-plugin.hpi",
Manifest: &pkg.JavaManifest{
ManifestVersion: "1.0",
SpecTitle: "The Jenkins Plugins Parent POM Project",
ImplTitle: "example-jenkins-plugin",
ImplVersion: "1.0-SNAPSHOT",
Extra: map[string]string{
Main: map[string]string{
"Manifest-Version": "1.0",
"Specification-Title": "The Jenkins Plugins Parent POM Project",
"Implementation-Title": "example-jenkins-plugin",
"Implementation-Version": "1.0-SNAPSHOT",
// extra fields...
"Archiver-Version": "Plexus Archiver",
"Plugin-License-Url": "https://opensource.org/licenses/MIT",
"Plugin-License-Name": "MIT License",
@ -191,7 +147,9 @@ func TestParseJar(t *testing.T) {
Metadata: pkg.JavaMetadata{
VirtualPath: "test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar",
Manifest: &pkg.JavaManifest{
ManifestVersion: "1.0",
Main: map[string]string{
"Manifest-Version": "1.0",
},
},
},
},
@ -212,8 +170,9 @@ func TestParseJar(t *testing.T) {
Metadata: pkg.JavaMetadata{
VirtualPath: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar",
Manifest: &pkg.JavaManifest{
ManifestVersion: "1.0",
Extra: map[string]string{
Main: map[string]string{
"Manifest-Version": "1.0",
// extra fields...
"Archiver-Version": "Plexus Archiver",
"Created-By": "Apache Maven 3.6.3",
"Built-By": "?",
@ -236,7 +195,9 @@ func TestParseJar(t *testing.T) {
Type: pkg.JavaPkg,
MetadataType: pkg.JavaMetadataType,
Metadata: pkg.JavaMetadata{
VirtualPath: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar",
// ensure that nested packages with different names than that of the parent are appended as
// a suffix on the virtual path
VirtualPath: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar:joda-time",
PomProperties: &pkg.PomProperties{
Path: "META-INF/maven/joda-time/joda-time/pom.properties",
GroupID: "joda-time",
@ -303,11 +264,11 @@ func TestParseJar(t *testing.T) {
metadata := a.Metadata.(pkg.JavaMetadata)
metadata.Parent = nil
// ignore select fields
// ignore select fields (only works for the main section)
for _, field := range test.ignoreExtras {
if metadata.Manifest != nil && metadata.Manifest.Extra != nil {
if _, ok := metadata.Manifest.Extra[field]; ok {
delete(metadata.Manifest.Extra, field)
if metadata.Manifest != nil && metadata.Manifest.Main != nil {
if _, ok := metadata.Manifest.Main[field]; ok {
delete(metadata.Manifest.Main, field)
}
}
}
@ -342,7 +303,7 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-boot-starter",
Version: "2.2.2.RELEASE",
Version: "2.2.2",
},
{
Name: "jul-to-slf4j",
@ -354,7 +315,7 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-boot-starter-validation",
Version: "2.2.2.RELEASE",
Version: "2.2.2",
},
{
Name: "hibernate-validator",
@ -366,7 +327,7 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-expression",
Version: "5.2.2.RELEASE",
Version: "5.2.2",
},
{
Name: "jakarta.validation-api",
@ -374,11 +335,11 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-web",
Version: "5.2.2.RELEASE",
Version: "5.2.2",
},
{
Name: "spring-boot-starter-actuator",
Version: "2.2.2.RELEASE",
Version: "2.2.2",
},
{
Name: "log4j-api",
@ -398,23 +359,23 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-aop",
Version: "5.2.2.RELEASE",
Version: "5.2.2",
},
{
Name: "spring-boot-actuator-autoconfigure",
Version: "2.2.2.RELEASE",
Version: "2.2.2",
},
{
Name: "spring-jcl",
Version: "5.2.2.RELEASE",
Version: "5.2.2",
},
{
Name: "spring-boot",
Version: "2.2.2.RELEASE",
Version: "2.2.2",
},
{
Name: "spring-boot-starter-logging",
Version: "2.2.2.RELEASE",
Version: "2.2.2",
},
{
Name: "jakarta.annotation-api",
@ -422,7 +383,7 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-webmvc",
Version: "5.2.2.RELEASE",
Version: "5.2.2",
},
{
Name: "HdrHistogram",
@ -430,7 +391,7 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-boot-starter-web",
Version: "2.2.2.RELEASE",
Version: "2.2.2",
},
{
Name: "logback-classic",
@ -442,7 +403,7 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-boot-starter-json",
Version: "2.2.2.RELEASE",
Version: "2.2.2",
},
{
Name: "jackson-databind",
@ -458,7 +419,7 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-boot-autoconfigure",
Version: "2.2.2.RELEASE",
Version: "2.2.2",
},
{
Name: "jackson-datatype-jdk8",
@ -474,11 +435,11 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-beans",
Version: "5.2.2.RELEASE",
Version: "5.2.2",
},
{
Name: "spring-boot-actuator",
Version: "2.2.2.RELEASE",
Version: "2.2.2",
},
{
Name: "slf4j-api",
@ -486,7 +447,7 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-core",
Version: "5.2.2.RELEASE",
Version: "5.2.2",
},
{
Name: "logback-core",
@ -506,7 +467,7 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-boot-starter-tomcat",
Version: "2.2.2.RELEASE",
Version: "2.2.2",
},
{
Name: "classmate",
@ -514,7 +475,7 @@ func TestParseNestedJar(t *testing.T) {
},
{
Name: "spring-context",
Version: "5.2.2.RELEASE",
Version: "5.2.2",
},
},
},
@ -535,7 +496,7 @@ func TestParseNestedJar(t *testing.T) {
t.Fatalf("failed to parse java archive: %+v", err)
}
nameVersionPairSet := internal.NewStringSet()
expectedNameVersionPairSet := internal.NewStringSet()
makeKey := func(p *pkg.Package) string {
if p == nil {
@ -545,20 +506,32 @@ func TestParseNestedJar(t *testing.T) {
}
for _, e := range test.expected {
nameVersionPairSet.Add(makeKey(&e))
expectedNameVersionPairSet.Add(makeKey(&e))
}
if len(actual) != len(nameVersionPairSet) {
if len(actual) != len(expectedNameVersionPairSet) {
actualNameVersionPairSet := internal.NewStringSet()
for _, a := range actual {
t.Log(" ", a)
key := makeKey(&a)
actualNameVersionPairSet.Add(key)
if !expectedNameVersionPairSet.Contains(key) {
t.Logf("extra package: %s", a)
}
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(nameVersionPairSet))
for _, key := range expectedNameVersionPairSet.ToSlice() {
if !actualNameVersionPairSet.Contains(key) {
t.Logf("missing package: %s", key)
}
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedNameVersionPairSet))
}
for _, a := range actual {
actualKey := makeKey(&a)
if !nameVersionPairSet.Contains(actualKey) {
if !expectedNameVersionPairSet.Contains(actualKey) {
t.Errorf("unexpected pkg: %q", actualKey)
}
@ -571,7 +544,7 @@ func TestParseNestedJar(t *testing.T) {
if metadata.Parent == nil {
t.Errorf("unassigned error for pkg=%q", actualKey)
} else if makeKey(metadata.Parent) != "spring-boot|0.0.1-SNAPSHOT" {
// NB: this is a hard-coded condition to simplify the test harness
// NB: this is a hard-coded condition to simplify the test harness to account for https://github.com/micrometer-metrics/micrometer/issues/1785
if a.Name == "pcollections" {
if metadata.Parent.Name != "micrometer-core" {
t.Errorf("nested 'pcollections' pkg has wrong parent: %q", metadata.Parent.Name)

76
syft/cataloger/java/java_manifest.go
View File

@ -4,16 +4,20 @@ import (
"bufio"
"fmt"
"io"
"strconv"
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/mitchellh/mapstructure"
)
const manifestGlob = "/META-INF/MANIFEST.MF"
// nolint:funlen
func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) {
// parseJavaManifest takes MANIFEST.MF file content and returns sections of parsed key/value pairs.
// For more information: https://docs.oracle.com/en/java/javase/11/docs/specs/jar/jar.html#jar-manifest
func parseJavaManifest(path string, reader io.Reader) (*pkg.JavaManifest, error) {
var manifest pkg.JavaManifest
sections := []map[string]string{
make(map[string]string),
@ -63,19 +67,24 @@ func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) {
return nil, fmt.Errorf("unable to read java manifest: %w", err)
}
if err := mapstructure.Decode(sections[0], &manifest); err != nil {
return nil, fmt.Errorf("unable to parse java manifest: %w", err)
}
// append on extra sections
if len(sections) > 1 {
manifest.Sections = sections[1:]
}
// clean select fields
if strings.Trim(manifest.ImplVersion, " ") != "" {
// transform versions with dates attached to just versions (e.g. "1.3 2244 October 5 2008" --> "1.3")
manifest.ImplVersion = strings.Split(manifest.ImplVersion, " ")[0]
if len(sections) > 0 {
manifest.Main = sections[0]
if len(sections) > 1 {
manifest.NamedSections = make(map[string]map[string]string)
for i, s := range sections[1:] {
name, ok := s["Name"]
if !ok {
// per the manifest spec (https://docs.oracle.com/en/java/javase/11/docs/specs/jar/jar.html#jar-manifest)
// this should never happen. If it does we want to know about it, but not necessarily stop
// cataloging entirely... for this reason we only log.
log.Errorf("java manifest section found without a name: %s", path)
name = strconv.Itoa(i)
} else {
delete(s, "Name")
}
manifest.NamedSections[name] = s
}
}
}
return &manifest, nil
@ -86,24 +95,21 @@ func selectName(manifest *pkg.JavaManifest, filenameObj archiveFilename) string
switch {
case filenameObj.name() != "":
name = filenameObj.name()
case manifest.Name != "":
case manifest.Main["Name"] != "":
// Manifest original spec...
name = manifest.Name
case manifest.Extra["Bundle-Name"] != "":
name = manifest.Main["Name"]
case manifest.Main["Bundle-Name"] != "":
// BND tooling...
name = manifest.Extra["Bundle-Name"]
case manifest.Extra["Short-Name"] != "":
name = manifest.Main["Bundle-Name"]
case manifest.Main["Short-Name"] != "":
// Jenkins...
name = manifest.Extra["Short-Name"]
case manifest.Extra["Extension-Name"] != "":
name = manifest.Main["Short-Name"]
case manifest.Main["Extension-Name"] != "":
// Jenkins...
name = manifest.Extra["Extension-Name"]
}
// in situations where we hit this point and no name was
// determined, look at the Implementation-Title
if name == "" && manifest.ImplTitle != "" {
name = manifest.ImplTitle
name = manifest.Main["Extension-Name"]
case manifest.Main["Implementation-Title"] != "":
// last ditch effort...
name = manifest.Main["Implementation-Title"]
}
return name
}
@ -111,14 +117,14 @@ func selectName(manifest *pkg.JavaManifest, filenameObj archiveFilename) string
func selectVersion(manifest *pkg.JavaManifest, filenameObj archiveFilename) string {
var version string
switch {
case manifest.ImplVersion != "":
version = manifest.ImplVersion
case filenameObj.version() != "":
version = filenameObj.version()
case manifest.SpecVersion != "":
version = manifest.SpecVersion
case manifest.Extra["Plugin-Version"] != "":
version = manifest.Extra["Plugin-Version"]
case manifest.Main["Implementation-Version"] != "":
version = manifest.Main["Implementation-Version"]
case manifest.Main["Specification-Version"] != "":
version = manifest.Main["Specification-Version"]
case manifest.Main["Plugin-Version"] != "":
version = manifest.Main["Plugin-Version"]
}
return version
}

89
syft/cataloger/java/java_manifest_test.go
View File

@ -17,35 +17,39 @@ func TestParseJavaManifest(t *testing.T) {
{
fixture: "test-fixtures/manifest/small",
expected: pkg.JavaManifest{
ManifestVersion: "1.0",
Main: map[string]string{
"Manifest-Version": "1.0",
},
},
},
{
fixture: "test-fixtures/manifest/standard-info",
expected: pkg.JavaManifest{
ManifestVersion: "1.0",
Name: "the-best-name",
SpecTitle: "the-spec-title",
SpecVersion: "the-spec-version",
SpecVendor: "the-spec-vendor",
ImplTitle: "the-impl-title",
ImplVersion: "the-impl-version",
ImplVendor: "the-impl-vendor",
Main: map[string]string{
"Name": "the-best-name",
"Manifest-Version": "1.0",
"Specification-Title": "the-spec-title",
"Specification-Version": "the-spec-version",
"Specification-Vendor": "the-spec-vendor",
"Implementation-Title": "the-impl-title",
"Implementation-Version": "the-impl-version",
"Implementation-Vendor": "the-impl-vendor",
},
},
},
{
fixture: "test-fixtures/manifest/extra-info",
expected: pkg.JavaManifest{
ManifestVersion: "1.0",
Extra: map[string]string{
Main: map[string]string{
"Manifest-Version": "1.0",
"Archiver-Version": "Plexus Archiver",
"Created-By": "Apache Maven 3.6.3",
},
Sections: []map[string]string{
{
NamedSections: map[string]map[string]string{
"thing-1": {
"Built-By": "?",
},
{
"1": {
"Build-Jdk": "14.0.1",
"Main-Class": "hello.HelloWorld",
},
@ -55,17 +59,20 @@ func TestParseJavaManifest(t *testing.T) {
{
fixture: "test-fixtures/manifest/continuation",
expected: pkg.JavaManifest{
ManifestVersion: "1.0",
Extra: map[string]string{
"Plugin-ScmUrl": "https://github.com/jenkinsci/plugin-pom/example-jenkins-plugin",
Main: map[string]string{
"Manifest-Version": "1.0",
"Plugin-ScmUrl": "https://github.com/jenkinsci/plugin-pom/example-jenkins-plugin",
},
},
},
{
// regression test, we should always keep the full version
fixture: "test-fixtures/manifest/version-with-date",
expected: pkg.JavaManifest{
ManifestVersion: "1.0",
ImplVersion: "1.3", // ensure the date is stripped off during processing
Main: map[string]string{
"Manifest-Version": "1.0",
"Implementation-Version": "1.3 2244 October 5 2005",
},
},
},
}
@ -77,7 +84,7 @@ func TestParseJavaManifest(t *testing.T) {
t.Fatalf("could not open fixture: %+v", err)
}
actual, err := parseJavaManifest(fixture)
actual, err := parseJavaManifest(test.fixture, fixture)
if err != nil {
t.Fatalf("failed to parse manifest: %+v", err)
}
@ -98,3 +105,45 @@ func TestParseJavaManifest(t *testing.T) {
})
}
}
func TestSelectName(t *testing.T) {
tests := []struct {
desc string
manifest pkg.JavaManifest
archive archiveFilename
expected string
}{
{
desc: "Get name from Implementation-Title",
archive: archiveFilename{},
manifest: pkg.JavaManifest{
Main: map[string]string{
"Implementation-Title": "maven-wrapper",
},
},
expected: "maven-wrapper",
},
{
desc: "Implementation-Title does not override name from filename",
manifest: pkg.JavaManifest{
Main: map[string]string{
"Name": "foo",
"Implementation-Title": "maven-wrapper",
},
},
archive: newJavaArchiveFilename("/something/omg.jar"),
expected: "omg",
},
}
for _, test := range tests {
t.Run(test.desc, func(t *testing.T) {
result := selectName(&test.manifest, test.archive)
if result != test.expected {
t.Errorf("mismatch in names: '%s' != '%s'", result, test.expected)
}
})
}
}

1
syft/cataloger/java/test-fixtures/manifest/extra-info
View File

@ -2,6 +2,7 @@ Manifest-Version: 1.0
Archiver-Version: Plexus Archiver
Created-By: Apache Maven 3.6.3
Name: thing-1
Built-By: ?
Build-Jdk: 14.0.1

12
syft/pkg/java_metadata.go
View File

@ -22,16 +22,8 @@ type PomProperties struct {
// JavaManifest represents the fields of interest extracted from a Java archive's META-INF/MANIFEST.MF file.
type JavaManifest struct {
Name string `mapstructure:"Name" json:"name"`
ManifestVersion string `mapstructure:"Manifest-Version" json:"manifestVersion"`
SpecTitle string `mapstructure:"Specification-Title" json:"specificationTitle"`
SpecVersion string `mapstructure:"Specification-Version" json:"specificationVersion"`
SpecVendor string `mapstructure:"Specification-Vendor" json:"specificationVendor"`
ImplTitle string `mapstructure:"Implementation-Title" json:"implementationTitle"`
ImplVersion string `mapstructure:"Implementation-Version" json:"implementationVersion"`
ImplVendor string `mapstructure:"Implementation-Vendor" json:"implementationVendor"`
Extra map[string]string `mapstructure:",remain" json:"extraFields"`
Sections []map[string]string `json:"sections,omitempty"`
Main map[string]string `json:"main,omitempty"`
NamedSections map[string]map[string]string `json:"namedSections,omitempty"`
}
func (m JavaMetadata) PackageURL() string {

3
test/inline-compare/compare-all.sh
View File

@ -1,8 +1,7 @@
#!/usr/bin/env bash
set -eu
# TODO: Add "alpine:3.12.0" back in when we've figured out how to handle the apk version field w/ and w/o release information (see issue: https://github.com/anchore/syft/pull/195)
images=("debian:10.5" "centos:8.2.2004" "rails:5.0.1")
images=("debian:10.5" "centos:8.2.2004" "rails:5.0.1" "alpine:3.12.0" "anchore/test_images:java" "anchore/test_images:py38" "anchore/anchore-engine:v0.8.2" "jenkins/jenkins:2.249.2-lts-jdk11" )
# gather all image analyses
for img in "${images[@]}"; do

134
test/inline-compare/compare.py
View File

@ -1,6 +1,7 @@
#!/usr/bin/env python3
import os
import sys
import difflib
import collections
import utils.package
@ -8,9 +9,15 @@ from utils.format import Colors, print_rows
from utils.inline import InlineScan
from utils.syft import Syft
QUALITY_GATE_THRESHOLD = 0.95
DEFAULT_QUALITY_GATE_THRESHOLD = 0.95
INDENT = " "
IMAGE_QUALITY_GATE = collections.defaultdict(lambda: QUALITY_GATE_THRESHOLD, **{})
PACKAGE_QUALITY_GATE = collections.defaultdict(lambda: DEFAULT_QUALITY_GATE_THRESHOLD, **{})
METADATA_QUALITY_GATE = collections.defaultdict(lambda: DEFAULT_QUALITY_GATE_THRESHOLD, **{
# syft is better at detecting package versions in specific cases, leading to a drop in matching metadata
"anchore/test_images:java": 0.61,
"jenkins/jenkins:2.249.2-lts-jdk11": 0.82,
})
# We additionally fail if an image is above a particular threshold. Why? We expect the lower threshold to be 90%,
# however additional functionality in grype is still being implemented, so this threshold may not be able to be met.
@ -18,10 +25,15 @@ IMAGE_QUALITY_GATE = collections.defaultdict(lambda: QUALITY_GATE_THRESHOLD, **{
# issues/enhancements are done we want to ensure that the lower threshold is bumped up to catch regression. The only way
# to do this is to select an upper threshold for images with known threshold values, so we have a failure that
# loudly indicates the lower threshold should be bumped.
IMAGE_UPPER_THRESHOLD = collections.defaultdict(lambda: 1, **{})
PACKAGE_UPPER_THRESHOLD = collections.defaultdict(lambda: 1, **{})
METADATA_UPPER_THRESHOLD = collections.defaultdict(lambda: 1, **{
# syft is better at detecting package versions in specific cases, leading to a drop in matching metadata
"anchore/test_images:java": 0.65,
"jenkins/jenkins:2.249.2-lts-jdk11": 0.84,
})
def report(analysis):
def report(image, analysis):
if analysis.extra_packages:
rows = []
print(
@ -47,7 +59,6 @@ def report(analysis):
print()
if analysis.missing_metadata:
rows = []
print(
Colors.bold + "Syft mismatched metadata:",
Colors.reset,
@ -58,25 +69,17 @@ def report(analysis):
if pkg not in analysis.syft_data.metadata[pkg.type]:
continue
syft_metadata_item = analysis.syft_data.metadata[pkg.type][pkg]
rows.append(
[
INDENT,
"for:",
repr(pkg),
":",
repr(syft_metadata_item),
"!=",
repr(metadata),
]
)
if rows:
print_rows(rows)
else:
diffs = difflib.ndiff([repr(syft_metadata_item)], [repr(metadata)])
print(INDENT + "for: " + repr(pkg), "(top is syft, bottom is inline)")
print(INDENT+INDENT+("\n"+INDENT+INDENT).join(list(diffs)))
if not analysis.missing_metadata:
print(
INDENT,
"There are mismatches, but only due to packages Syft did not find (but inline did).",
"There are mismatches, but only due to packages Syft did not find (but inline did).\n",
)
print()
if analysis.similar_missing_packages:
rows = []
@ -97,7 +100,9 @@ def report(analysis):
print_rows(rows)
print()
if analysis.unmatched_missing_packages and analysis.extra_packages:
show_probable_mismatches = analysis.unmatched_missing_packages and analysis.extra_packages and len(analysis.unmatched_missing_packages) != len(analysis.missing_packages)
if show_probable_mismatches:
rows = []
print(
Colors.bold + "Probably missed packages:",
@ -109,17 +114,17 @@ def report(analysis):
print_rows(rows)
print()
print(Colors.bold + "Summary:", Colors.reset)
print(Colors.bold + "Summary:", Colors.reset, image)
print(" Inline Packages : %d" % len(analysis.inline_data.packages))
print(" Syft Packages : %d" % len(analysis.syft_data.packages))
print(
" (extra) : %d (note: this is ignored in the analysis!)"
" (extra) : %d (note: this is ignored by the quality gate!)"
% len(analysis.extra_packages)
)
print(" (missing) : %d" % len(analysis.missing_packages))
print()
if analysis.unmatched_missing_packages and analysis.extra_packages:
if show_probable_mismatches:
print(
" Probable Package Matches : %d (matches not made, but were probably found by both Inline and Syft)"
% len(analysis.similar_missing_packages)
@ -155,12 +160,37 @@ def report(analysis):
)
)
overall_score = (
analysis.percent_overlapping_packages + analysis.percent_overlapping_metadata
) / 2.0
print(Colors.bold + " Overall Score: %2.1f %%" % overall_score, Colors.reset)
def enforce_quality_gate(title, actual_value, lower_gate_value, upper_gate_value):
if actual_value < lower_gate_value:
print(
Colors.bold
+ " %s Quality Gate:\t" % title
+ Colors.FG.red
+ "FAIL (is not >= %d %%)" % lower_gate_value,
Colors.reset,
)
return False
elif actual_value > upper_gate_value:
print(
Colors.bold
+ " %s Quality Gate:\t" % title
+ Colors.FG.orange
+ "FAIL (lower threshold is artificially low and should be updated)",
Colors.reset,
)
return False
print(
Colors.bold
+ " %s Quality Gate:\t" % title
+ Colors.FG.green
+ "Pass (>= %d %%)" % lower_gate_value,
Colors.reset,
)
return True
def main(image):
cwd = os.path.dirname(os.path.abspath(__file__))
@ -175,41 +205,27 @@ def main(image):
)
# show some useful report data for debugging / warm fuzzies
report(analysis)
report(image, analysis)
# enforce a quality gate based on the comparison of package values and metadata values
upper_gate_value = IMAGE_UPPER_THRESHOLD[image] * 100
lower_gate_value = IMAGE_QUALITY_GATE[image] * 100
if analysis.quality_gate_score < lower_gate_value:
print(
Colors.bold
+ " Quality Gate: "
+ Colors.FG.red
+ "FAILED (is not >= %d %%)\n" % lower_gate_value,
Colors.reset,
)
return 1
elif analysis.quality_gate_score > upper_gate_value:
print(
Colors.bold
+ " Quality Gate: "
+ Colors.FG.orange
+ "FAILED (lower threshold is artificially low and should be updated)\n",
Colors.reset,
)
return 1
else:
print(
Colors.bold
+ " Quality Gate: "
+ Colors.FG.green
+ "pass (>= %d %%)\n" % lower_gate_value,
Colors.reset,
)
success = True
success &= enforce_quality_gate(
title="Package",
actual_value=analysis.percent_overlapping_packages,
lower_gate_value=PACKAGE_QUALITY_GATE[image] * 100,
upper_gate_value=PACKAGE_UPPER_THRESHOLD[image] * 100
)
success &= enforce_quality_gate(
title="Metadata",
actual_value=analysis.percent_overlapping_metadata,
lower_gate_value=METADATA_QUALITY_GATE[image] * 100,
upper_gate_value=METADATA_UPPER_THRESHOLD[image] * 100
)
if not success:
return 1
return 0
if __name__ == "__main__":
if len(sys.argv) != 2:
sys.exit("provide an image")

15
test/inline-compare/utils/inline.py
View File

@ -1,4 +1,5 @@
import os
import re
import json
import collections
@ -66,13 +67,23 @@ class InlineScan:
elif pkg_type in ("java-jpi", "java-hpi"):
pkg_type = "java-?pi"
# this would usually be "package" but this would not be able to account for duplicate dependencies in
# nested jars of the same name. Fallback to the package name if there is no given location
name = entry["location"]
# replace fields with "N/A" with None
for k, v in dict(entry).items():
if v in ("", "N/A"):
entry[k] = None
pkg = utils.package.Package(
name=entry["package"],
name=name,
type=pkg_type,
)
packages.add(pkg)
metadata[pkg.type][pkg] = utils.package.Metadata(
version=entry["maven-version"]
version=entry["maven-version"],
)
return packages, metadata

10
test/inline-compare/utils/package.py
View File

@ -144,13 +144,3 @@ class Analysis:
float(len(self.overlapping_packages) + len(self.similar_missing_packages))
/ float(len(self.inline_data.packages))
) * 100.0
@property
def quality_gate_score(self):
"""
The result of the analysis in the form of an aggregated percentage; it is up to the caller to use this value
and enforce a quality gate.
"""
return (
self.percent_overlapping_packages + self.percent_overlapping_metadata
) / 2.0

18
test/inline-compare/utils/syft.py
View File

@ -43,12 +43,26 @@ class Syft:
elif pkg_type in ("apk",):
pkg_type = "apkg"
name = entry["name"]
version = entry["version"]
if "java" in pkg_type:
# we need to use the virtual path instead of the name to account for nested dependencies with the same
# package name (but potentially different metadata)
name = entry.get("metadata", {}).get("virtualPath")
elif pkg_type == "apkg":
# inline scan strips off the release from the version, which should be normalized here
fields = entry["version"].split("-")
version = "-".join(fields[:-1])
pkg = utils.package.Package(
name=entry["name"],
name=name,
type=pkg_type,
)
packages.add(pkg)
metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"])
metadata[pkg.type][pkg] = utils.package.Metadata(version=version)
return utils.package.Info(packages=frozenset(packages), metadata=metadata)