From 091fd1f0b05c59ef549cde38794a07dbb38bf64a Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Sun, 18 Apr 2021 08:01:22 -0400 Subject: [PATCH 1/5] Improve CPE generation for Jenkins/Jira plugins Signed-off-by: Dan Luhring --- syft/pkg/cataloger/cpe.go | 136 ++++++++++++++++++++++++++------- syft/pkg/cataloger/cpe_test.go | 35 +++++++-- 2 files changed, 136 insertions(+), 35 deletions(-) diff --git a/syft/pkg/cataloger/cpe.go b/syft/pkg/cataloger/cpe.go index a972cf535..b53bdfb99 100644 --- a/syft/pkg/cataloger/cpe.go +++ b/syft/pkg/cataloger/cpe.go @@ -54,6 +54,9 @@ func (s candidateStore) getCandidates(t pkg.Type, key string) []string { return value } +const pomPropertiesGroupIDJenkinsPlugins = "com.cloudbees.jenkins.plugins" +const pomPropertiesGroupIDJiraPlugins = "com.atlassian.jira.plugins" + func newCPE(product, vendor, version, targetSW string) wfn.Attributes { cpe := *(wfn.NewAttributesWithAny()) cpe.Part = "a" @@ -100,7 +103,7 @@ func candidateTargetSoftwareAttrs(p pkg.Package) []string { var targetSw []string switch p.Language { case pkg.Java: - targetSw = append(targetSw, "java", "maven") + targetSw = append(targetSw, candidateTargetSoftwareAttrsForJava(p)...) case pkg.JavaScript: targetSw = append(targetSw, "node.js", "nodejs") case pkg.Ruby: @@ -109,51 +112,130 @@ func candidateTargetSoftwareAttrs(p pkg.Package) []string { targetSw = append(targetSw, "python") } - if p.Type == pkg.JenkinsPluginPkg { - targetSw = append(targetSw, "jenkins", "cloudbees_jenkins") - } - return targetSw } +func isJenkinsPlugin(p pkg.Package) bool { + if p.Type == pkg.JenkinsPluginPkg { + return true + } + + if groupID := groupIDFromPomProperties(p); groupID == pomPropertiesGroupIDJenkinsPlugins { + return true + } + + return false +} + +func candidateTargetSoftwareAttrsForJava(p pkg.Package) []string { + // Use the more specific indicator if available + + if isJenkinsPlugin(p) { + return []string{"jenkins", "cloudbees_jenkins"} + } + + return []string{"java", "maven"} +} + func candidateVendors(p pkg.Package) []string { + // TODO: Confirm whether using products as vendors is helpful to the matching process vendors := candidateProducts(p) + switch p.Language { case pkg.Python: vendors = append(vendors, fmt.Sprintf("python-%s", p.Name)) case pkg.Java: if p.MetadataType == pkg.JavaMetadataType { - if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok && metadata.PomProperties != nil { - // derive the vendor from the groupID (e.g. org.sonatype.nexus --> sonatype) - if strings.HasPrefix(metadata.PomProperties.GroupID, "org.") || strings.HasPrefix(metadata.PomProperties.GroupID, "com.") { - fields := strings.Split(metadata.PomProperties.GroupID, ".") - if len(fields) >= 3 { - vendors = append(vendors, fields[1]) - } - } - } + vendors = append(vendors, candidateVendorsForJava(p)...) } } return vendors } func candidateProducts(p pkg.Package) []string { - var products = []string{p.Name} - + products := []string{p.Name} if p.Language == pkg.Java { - if p.MetadataType == pkg.JavaMetadataType { - if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok && metadata.PomProperties != nil { - // derive the product from the groupID (e.g. org.sonatype.nexus --> nexus) - if strings.HasPrefix(metadata.PomProperties.GroupID, "org.") || strings.HasPrefix(metadata.PomProperties.GroupID, "com.") { - fields := strings.Split(metadata.PomProperties.GroupID, ".") - if len(fields) >= 3 { - products = append(products, fields[2]) - } - } - } - } + products = append(products, candidateProductsForJava(p)...) } // return any known product name swaps prepended to the results return append(productCandidatesByPkgType.getCandidates(p.Type, p.Name), products...) } + +func candidateProductsForJava(p pkg.Package) []string { + if product, _ := productAndVendorFromPomPropertiesGroupID(p); product != "" { + return []string{product} + } + + return nil +} + +func candidateVendorsForJava(p pkg.Package) []string { + if _, vendor := productAndVendorFromPomPropertiesGroupID(p); vendor != "" { + return []string{vendor} + } + + return nil +} + +func productAndVendorFromPomPropertiesGroupID(p pkg.Package) (string, string) { + groupID := groupIDFromPomProperties(p) + if !shouldConsiderGroupID(groupID) { + return "", "" + } + + if !hasAnyOfPrefixes(groupID, "com", "org") { + return "", "" + } + + fields := strings.Split(groupID, ".") + if len(fields) < 3 { + return "", "" + } + + product := fields[2] + vendor := fields[1] + return product, vendor +} + +func groupIDFromPomProperties(p pkg.Package) string { + metadata, ok := p.Metadata.(pkg.JavaMetadata) + if !ok { + return "" + } + + if metadata.PomProperties == nil { + return "" + } + + return metadata.PomProperties.GroupID +} + +func shouldConsiderGroupID(groupID string) bool { + if groupID == "" { + return false + } + + excludedGroupIDs := []string{ + pomPropertiesGroupIDJiraPlugins, + pomPropertiesGroupIDJenkinsPlugins, + } + + for _, excludedGroupID := range excludedGroupIDs { + if groupID == excludedGroupID { + return false + } + } + + return true +} + +func hasAnyOfPrefixes(input string, prefixes ...string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(input, prefix) { + return true + } + } + + return false +} diff --git a/syft/pkg/cataloger/cpe_test.go b/syft/pkg/cataloger/cpe_test.go index e5a0be66c..560c574b0 100644 --- a/syft/pkg/cataloger/cpe_test.go +++ b/syft/pkg/cataloger/cpe_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestGenerate(t *testing.T) { +func TestGeneratePackageCPEs(t *testing.T) { tests := []struct { name string p pkg.Package @@ -132,7 +132,7 @@ func TestGenerate(t *testing.T) { }, }, { - name: "jenkins package", + name: "jenkins package identified via pkg type", p: pkg.Package{ Name: "name", Version: "3.2", @@ -142,13 +142,32 @@ func TestGenerate(t *testing.T) { }, expected: []string{ "cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*", - "cpe:2.3:a:*:name:3.2:*:*:*:*:java:*:*", - "cpe:2.3:a:*:name:3.2:*:*:*:*:maven:*:*", "cpe:2.3:a:*:name:3.2:*:*:*:*:jenkins:*:*", "cpe:2.3:a:*:name:3.2:*:*:*:*:cloudbees_jenkins:*:*", "cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*", - "cpe:2.3:a:name:name:3.2:*:*:*:*:java:*:*", - "cpe:2.3:a:name:name:3.2:*:*:*:*:maven:*:*", + "cpe:2.3:a:name:name:3.2:*:*:*:*:jenkins:*:*", + "cpe:2.3:a:name:name:3.2:*:*:*:*:cloudbees_jenkins:*:*", + }, + }, + { + name: "jenkins package identified via groupId", + p: pkg.Package{ + Name: "name", + Version: "3.2", + FoundBy: "some-analyzer", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaMetadata{ + PomProperties: &pkg.PomProperties{ + GroupID: "com.cloudbees.jenkins.plugins", + }, + }, + }, + expected: []string{ + "cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*", + "cpe:2.3:a:*:name:3.2:*:*:*:*:jenkins:*:*", + "cpe:2.3:a:*:name:3.2:*:*:*:*:cloudbees_jenkins:*:*", + "cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*", "cpe:2.3:a:name:name:3.2:*:*:*:*:jenkins:*:*", "cpe:2.3:a:name:name:3.2:*:*:*:*:cloudbees_jenkins:*:*", }, @@ -165,13 +184,13 @@ func TestGenerate(t *testing.T) { actualCpeSet.Add(a.BindToFmtString()) } - extra := strset.Difference(expectedCpeSet, actualCpeSet).List() + extra := strset.Difference(actualCpeSet, expectedCpeSet).List() sort.Strings(extra) for _, d := range extra { t.Errorf("extra CPE: %+v", d) } - missing := strset.Difference(actualCpeSet, expectedCpeSet).List() + missing := strset.Difference(expectedCpeSet, actualCpeSet).List() sort.Strings(missing) for _, d := range missing { t.Errorf("missing CPE: %+v", d) From 65e4e1759001f6d9fb7cdccb9e680a642f92b2ac Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Tue, 20 Apr 2021 11:02:01 -0400 Subject: [PATCH 2/5] Pin gradle builder container image Signed-off-by: Dan Luhring --- .../java-builds/build-example-java-app-gradle.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-java-app-gradle.sh b/syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-java-app-gradle.sh index c7c266051..345542c93 100755 --- a/syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-java-app-gradle.sh +++ b/syft/pkg/cataloger/java/test-fixtures/java-builds/build-example-java-app-gradle.sh @@ -4,7 +4,7 @@ set -uxe # note: this can be easily done in a 1-liner, however circle CI does NOT allow volume mounts from the host in docker executors (since they are on remote hosts, where the host files are inaccessible) PKGSDIR=$1 -CTRID=$(docker create -u "$(id -u):$(id -g)" -v /example-java-app -w /example-java-app gradle:jdk gradle build) +CTRID=$(docker create -u "$(id -u):$(id -g)" -v /example-java-app -w /example-java-app gradle:6.8.3-jdk gradle build) function cleanup() { docker rm "${CTRID}" @@ -16,4 +16,4 @@ set +e docker cp "$(pwd)/example-java-app" "${CTRID}:/" docker start -a "${CTRID}" mkdir -p "$PKGSDIR" -docker cp "${CTRID}:/example-java-app/build/libs/example-java-app-gradle-0.1.0.jar" "$PKGSDIR" \ No newline at end of file +docker cp "${CTRID}:/example-java-app/build/libs/example-java-app-gradle-0.1.0.jar" "$PKGSDIR" From fa7fd718cb344fee258b166d6f3bc02194a6032a Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Tue, 20 Apr 2021 16:06:04 -0400 Subject: [PATCH 3/5] Refactor Java archive parsing logic Signed-off-by: Dan Luhring --- syft/pkg/cataloger/java/archive_parser.go | 140 ++++++++++------------ 1 file changed, 65 insertions(+), 75 deletions(-) diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 03d9d5060..201e44c93 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -7,7 +7,6 @@ import ( "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" @@ -25,13 +24,12 @@ var archiveFormatGlobs = []string{ } type archiveParser struct { - discoveredPkgs internal.StringSet - fileManifest file.ZipFileManifest - virtualPath string - archivePath string - contentPath string - fileInfo archiveFilename - detectNested bool + fileManifest file.ZipFileManifest + virtualPath string + archivePath string + contentPath string + fileInfo archiveFilename + detectNested bool } // parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives. @@ -71,13 +69,12 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo currentFilepath := virtualElements[len(virtualElements)-1] return &archiveParser{ - discoveredPkgs: internal.NewStringSet(), - fileManifest: fileManifest, - virtualPath: virtualPath, - archivePath: archivePath, - contentPath: contentPath, - fileInfo: newJavaArchiveFilename(currentFilepath), - detectNested: detectNested, + fileManifest: fileManifest, + virtualPath: virtualPath, + archivePath: archivePath, + contentPath: contentPath, + fileInfo: newJavaArchiveFilename(currentFilepath), + detectNested: detectNested, }, cleanupFn, nil } @@ -91,25 +88,21 @@ func (j *archiveParser) parse() ([]pkg.Package, error) { return nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err) } - // don't add the parent package yet, we still may discover aux info to add to the metadata (but still track it as added to prevent duplicates) - parentKey := uniquePkgKey(parentPkg) - if parentKey != "" { - j.discoveredPkgs.Add(parentKey) - } - - // find aux packages from pom.properties + // find aux packages from pom.properties and potentially modify the existing parentPkg auxPkgs, err := j.discoverPkgsFromAllPomProperties(parentPkg) if err != nil { return nil, err } pkgs = append(pkgs, auxPkgs...) - // find nested java archive packages - nestedPkgs, err := j.discoverPkgsFromNestedArchives(parentPkg) - if err != nil { - return nil, err + if j.detectNested { + // find nested java archive packages + nestedPkgs, err := j.discoverPkgsFromNestedArchives(parentPkg) + if err != nil { + return nil, err + } + pkgs = append(pkgs, nestedPkgs...) } - pkgs = append(pkgs, nestedPkgs...) // lastly, add the parent package to the list (assuming the parent exists) if parentPkg != nil { @@ -159,69 +152,71 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) { }, nil } -// discoverPkgsFromAllPomProperties parses Maven POM properties for a given parent package, returning all listed Java packages found for each pom properties discovered. +// discoverPkgsFromAllPomProperties parses Maven POM properties for a given +// parent package, returning all listed Java packages found for each pom +// properties discovered and potentially updating the given parentPkg with new +// data. func (j *archiveParser) discoverPkgsFromAllPomProperties(parentPkg *pkg.Package) ([]pkg.Package, error) { - var pkgs = make([]pkg.Package, 0) + if parentPkg == nil { + return nil, nil + } + + var pkgs []pkg.Package // search and parse pom.properties files & fetch the contents - contents, err := file.ContentsFromZip(j.archivePath, j.fileManifest.GlobMatch(pomPropertiesGlob)...) + contentsOfPomPropertiesFiles, err := file.ContentsFromZip(j.archivePath, j.fileManifest.GlobMatch(pomPropertiesGlob)...) if err != nil { return nil, fmt.Errorf("unable to extract pom.properties: %w", err) } - // parse the manifest file into a rich object - for propsPath, propsContents := range contents { - propsObj, err := parsePomProperties(propsPath, strings.NewReader(propsContents)) + for filePath, fileContents := range contentsOfPomPropertiesFiles { + // parse the pom properties file into a rich object + pomProperties, err := parsePomProperties(filePath, strings.NewReader(fileContents)) if err != nil { log.Warnf("failed to parse pom.properties (%s): %+v", j.virtualPath, err) continue } - if propsObj == nil { + if pomProperties == nil { continue } - if propsObj.Version == "" || propsObj.ArtifactID == "" { + if pomProperties.Version == "" || pomProperties.ArtifactID == "" { // TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package continue } - if parentPkg == nil { - continue - } - - pkgs = append(pkgs, j.packagesFromPomProperties(propsObj, parentPkg)...) + pkgs = append(pkgs, j.packagesFromPomProperties(pomProperties, parentPkg)...) } return pkgs, nil } // packagesFromPomProperties processes a single Maven POM properties for a given parent package, returning all listed Java packages found and // associating each discovered package to the given parent package. -func (j *archiveParser) packagesFromPomProperties(propsObj *pkg.PomProperties, parentPkg *pkg.Package) (pkgs []pkg.Package) { - parentKey := uniquePkgKey(parentPkg) - +func (j *archiveParser) packagesFromPomProperties(pomProperties *pkg.PomProperties, parentPkg *pkg.Package) []pkg.Package { // keep the artifact name within the virtual path if this package does not match the parent package vPathSuffix := "" - if !strings.HasPrefix(propsObj.ArtifactID, parentPkg.Name) { - vPathSuffix += ":" + propsObj.ArtifactID + if !strings.HasPrefix(pomProperties.ArtifactID, parentPkg.Name) { + vPathSuffix += ":" + pomProperties.ArtifactID } virtualPath := j.virtualPath + vPathSuffix // discovered props = new package p := pkg.Package{ - Name: propsObj.ArtifactID, - Version: propsObj.Version, + Name: pomProperties.ArtifactID, + Version: pomProperties.Version, Language: pkg.Java, Type: pkg.JavaPkg, MetadataType: pkg.JavaMetadataType, Metadata: pkg.JavaMetadata{ VirtualPath: virtualPath, - PomProperties: propsObj, + PomProperties: pomProperties, Parent: parentPkg, }, } pkgKey := uniquePkgKey(&p) + parentKey := uniquePkgKey(parentPkg) // the name/version pair matches... matchesParentPkg := pkgKey == parentKey @@ -230,33 +225,32 @@ func (j *archiveParser) packagesFromPomProperties(propsObj *pkg.PomProperties, p matchesParentPkg = matchesParentPkg || parentPkg.Metadata.(pkg.JavaMetadata).VirtualPath == virtualPath // the pom artifactId has the parent name or vice versa - if propsObj.ArtifactID != "" { - matchesParentPkg = matchesParentPkg || strings.Contains(parentPkg.Name, propsObj.ArtifactID) || strings.Contains(propsObj.ArtifactID, parentPkg.Name) + if pomProperties.ArtifactID != "" { + matchesParentPkg = matchesParentPkg || strings.Contains(parentPkg.Name, pomProperties.ArtifactID) || strings.Contains(pomProperties.ArtifactID, parentPkg.Name) } - if matchesParentPkg { - // we've run across more information about our parent package, add this info to the parent package metadata - // the pom properties is typically a better source of information for name and version than the manifest - if parentPkg.Name == "" { - parentPkg.Name = p.Name - } - if parentPkg.Version == "" { - parentPkg.Version = p.Version - } - - // keep the pom properties, but don't overwrite existing pom properties - parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata) - if ok && parentMetadata.PomProperties == nil { - parentMetadata.PomProperties = propsObj - parentPkg.Metadata = parentMetadata - } - } - - if !matchesParentPkg && !j.discoveredPkgs.Contains(pkgKey) { + if !matchesParentPkg { // only keep packages we haven't seen yet (and are not related to the parent package) - pkgs = append(pkgs, p) + return []pkg.Package{p} } - return pkgs + + // we've run across more information about our parent package, add this info to the parent package metadata + // the pom properties is typically a better source of information for name and version than the manifest + if parentPkg.Name == "" { + parentPkg.Name = p.Name + } + if parentPkg.Version == "" { + parentPkg.Version = p.Version + } + + // keep the pom properties, but don't overwrite existing pom properties + parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata) + if ok && parentMetadata.PomProperties == nil { + parentMetadata.PomProperties = pomProperties + parentPkg.Metadata = parentMetadata + } + + return nil } // discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and @@ -264,10 +258,6 @@ func (j *archiveParser) packagesFromPomProperties(propsObj *pkg.PomProperties, p func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) { var pkgs = make([]pkg.Package, 0) - if !j.detectNested { - return pkgs, nil - } - // search and parse pom.properties files & fetch the contents openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...) if err != nil { From 33e6be0b74633fc3e6afbb6d579cd28f956a11fc Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Tue, 20 Apr 2021 19:25:19 -0400 Subject: [PATCH 4/5] Identify Jenkins plugin upstream of CPE generation Signed-off-by: Dan Luhring --- syft/pkg/cataloger/cpe.go | 22 +--- syft/pkg/cataloger/cpe_test.go | 2 +- syft/pkg/cataloger/java/archive_parser.go | 13 +- .../pkg/cataloger/java/archive_parser_test.go | 120 +++++++++++++++++- syft/pkg/java_metadata.go | 12 ++ syft/pkg/java_metadata_test.go | 39 ++++++ 6 files changed, 176 insertions(+), 32 deletions(-) diff --git a/syft/pkg/cataloger/cpe.go b/syft/pkg/cataloger/cpe.go index b53bdfb99..ad5ee4e04 100644 --- a/syft/pkg/cataloger/cpe.go +++ b/syft/pkg/cataloger/cpe.go @@ -54,9 +54,6 @@ func (s candidateStore) getCandidates(t pkg.Type, key string) []string { return value } -const pomPropertiesGroupIDJenkinsPlugins = "com.cloudbees.jenkins.plugins" -const pomPropertiesGroupIDJiraPlugins = "com.atlassian.jira.plugins" - func newCPE(product, vendor, version, targetSW string) wfn.Attributes { cpe := *(wfn.NewAttributesWithAny()) cpe.Part = "a" @@ -115,22 +112,9 @@ func candidateTargetSoftwareAttrs(p pkg.Package) []string { return targetSw } -func isJenkinsPlugin(p pkg.Package) bool { - if p.Type == pkg.JenkinsPluginPkg { - return true - } - - if groupID := groupIDFromPomProperties(p); groupID == pomPropertiesGroupIDJenkinsPlugins { - return true - } - - return false -} - func candidateTargetSoftwareAttrsForJava(p pkg.Package) []string { // Use the more specific indicator if available - - if isJenkinsPlugin(p) { + if p.Type == pkg.JenkinsPluginPkg { return []string{"jenkins", "cloudbees_jenkins"} } @@ -217,8 +201,8 @@ func shouldConsiderGroupID(groupID string) bool { } excludedGroupIDs := []string{ - pomPropertiesGroupIDJiraPlugins, - pomPropertiesGroupIDJenkinsPlugins, + pkg.PomPropertiesGroupIDJiraPlugins, + pkg.PomPropertiesGroupIDJenkinsPlugins, } for _, excludedGroupID := range excludedGroupIDs { diff --git a/syft/pkg/cataloger/cpe_test.go b/syft/pkg/cataloger/cpe_test.go index 560c574b0..678eed7aa 100644 --- a/syft/pkg/cataloger/cpe_test.go +++ b/syft/pkg/cataloger/cpe_test.go @@ -156,7 +156,7 @@ func TestGeneratePackageCPEs(t *testing.T) { Version: "3.2", FoundBy: "some-analyzer", Language: pkg.Java, - Type: pkg.JavaPkg, + Type: pkg.JenkinsPluginPkg, Metadata: pkg.JavaMetadata{ PomProperties: &pkg.PomProperties{ GroupID: "com.cloudbees.jenkins.plugins", diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 201e44c93..591f0d85f 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -186,14 +186,14 @@ func (j *archiveParser) discoverPkgsFromAllPomProperties(parentPkg *pkg.Package) continue } - pkgs = append(pkgs, j.packagesFromPomProperties(pomProperties, parentPkg)...) + pkgs = append(pkgs, j.packagesFromPomProperties(*pomProperties, parentPkg)...) } return pkgs, nil } // packagesFromPomProperties processes a single Maven POM properties for a given parent package, returning all listed Java packages found and // associating each discovered package to the given parent package. -func (j *archiveParser) packagesFromPomProperties(pomProperties *pkg.PomProperties, parentPkg *pkg.Package) []pkg.Package { +func (j *archiveParser) packagesFromPomProperties(pomProperties pkg.PomProperties, parentPkg *pkg.Package) []pkg.Package { // keep the artifact name within the virtual path if this package does not match the parent package vPathSuffix := "" if !strings.HasPrefix(pomProperties.ArtifactID, parentPkg.Name) { @@ -206,11 +206,11 @@ func (j *archiveParser) packagesFromPomProperties(pomProperties *pkg.PomProperti Name: pomProperties.ArtifactID, Version: pomProperties.Version, Language: pkg.Java, - Type: pkg.JavaPkg, + Type: pomProperties.PkgTypeIndicated(), MetadataType: pkg.JavaMetadataType, Metadata: pkg.JavaMetadata{ VirtualPath: virtualPath, - PomProperties: pomProperties, + PomProperties: &pomProperties, Parent: parentPkg, }, } @@ -243,10 +243,13 @@ func (j *archiveParser) packagesFromPomProperties(pomProperties *pkg.PomProperti parentPkg.Version = p.Version } + // We may have learned more about the type via data in the pom properties + parentPkg.Type = p.Type + // keep the pom properties, but don't overwrite existing pom properties parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata) if ok && parentMetadata.PomProperties == nil { - parentMetadata.PomProperties = pomProperties + parentMetadata.PomProperties = &pomProperties parentPkg.Metadata = parentMetadata } diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index e044c4e8e..8988bdaa5 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -629,12 +629,12 @@ func TestPackagesFromPomProperties(t *testing.T) { }, }, { - name: "child matches parent by key", + name: "single package from pom properties that's a Jenkins plugin", props: &pkg.PomProperties{ Name: "some-name", - GroupID: "some-group-id", - ArtifactID: "some-parent-name", // note: matches parent package - Version: "2.0", // note: matches parent package + GroupID: "com.cloudbees.jenkins.plugins", + ArtifactID: "some-artifact-id", + Version: "1.0", }, parent: &pkg.Package{ Name: "some-parent-name", @@ -650,6 +650,66 @@ func TestPackagesFromPomProperties(t *testing.T) { expectedParent: pkg.Package{ Name: "some-parent-name", Version: "2.0", + Metadata: pkg.JavaMetadata{ + VirtualPath: "some-parent-virtual-path", + Manifest: nil, + PomProperties: nil, + Parent: nil, + }, + }, + expectedPackages: []pkg.Package{ + { + Name: "some-artifact-id", + Version: "1.0", + Language: pkg.Java, + Type: pkg.JenkinsPluginPkg, + MetadataType: pkg.JavaMetadataType, + Metadata: pkg.JavaMetadata{ + VirtualPath: virtualPath + ":" + "some-artifact-id", + PomProperties: &pkg.PomProperties{ + Name: "some-name", + GroupID: "com.cloudbees.jenkins.plugins", + ArtifactID: "some-artifact-id", + Version: "1.0", + }, + Parent: &pkg.Package{ + Name: "some-parent-name", + Version: "2.0", + Metadata: pkg.JavaMetadata{ + VirtualPath: "some-parent-virtual-path", + Manifest: nil, + PomProperties: nil, + Parent: nil, + }, + }, + }, + }, + }, + }, + { + name: "child matches parent by key", + props: &pkg.PomProperties{ + Name: "some-name", + GroupID: "some-group-id", + ArtifactID: "some-parent-name", // note: matches parent package + Version: "2.0", // note: matches parent package + }, + parent: &pkg.Package{ + Name: "some-parent-name", + Version: "2.0", + Type: pkg.JavaPkg, + Metadata: pkg.JavaMetadata{ + VirtualPath: "some-parent-virtual-path", + Manifest: nil, + PomProperties: nil, + Parent: nil, + }, + }, + // note: the SAME as the original parent values + expectedParent: pkg.Package{ + Name: "some-parent-name", + Version: "2.0", + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ VirtualPath: "some-parent-virtual-path", Manifest: nil, @@ -665,6 +725,44 @@ func TestPackagesFromPomProperties(t *testing.T) { }, expectedPackages: nil, }, + { + name: "child matches parent by key and is Jenkins plugin", + props: &pkg.PomProperties{ + Name: "some-name", + GroupID: "com.cloudbees.jenkins.plugins", + ArtifactID: "some-parent-name", // note: matches parent package + Version: "2.0", // note: matches parent package + }, + parent: &pkg.Package{ + Name: "some-parent-name", + Version: "2.0", + Type: pkg.JavaPkg, + Metadata: pkg.JavaMetadata{ + VirtualPath: "some-parent-virtual-path", + Manifest: nil, + PomProperties: nil, + Parent: nil, + }, + }, + expectedParent: pkg.Package{ + Name: "some-parent-name", + Version: "2.0", + Type: pkg.JenkinsPluginPkg, + Metadata: pkg.JavaMetadata{ + VirtualPath: "some-parent-virtual-path", + Manifest: nil, + // note: we attach the discovered pom properties data + PomProperties: &pkg.PomProperties{ + Name: "some-name", + GroupID: "com.cloudbees.jenkins.plugins", + ArtifactID: "some-parent-name", // note: matches parent package + Version: "2.0", // note: matches parent package + }, + Parent: nil, + }, + }, + expectedPackages: nil, + }, { name: "child matches parent by virtual path", props: &pkg.PomProperties{ @@ -676,6 +774,7 @@ func TestPackagesFromPomProperties(t *testing.T) { parent: &pkg.Package{ Name: "some-parent-name", Version: "2.0", + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ VirtualPath: virtualPath + ":some-parent-name", // note: matching virtual path Manifest: nil, @@ -686,6 +785,7 @@ func TestPackagesFromPomProperties(t *testing.T) { expectedParent: pkg.Package{ Name: "some-parent-name", Version: "2.0", + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ VirtualPath: virtualPath + ":some-parent-name", // note: matching virtual path Manifest: nil, @@ -712,6 +812,7 @@ func TestPackagesFromPomProperties(t *testing.T) { parent: &pkg.Package{ Name: "", // note: empty Version: "", // note: empty + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ VirtualPath: virtualPath + ":some-parent-name", // note: matching virtual path Manifest: nil, @@ -722,6 +823,7 @@ func TestPackagesFromPomProperties(t *testing.T) { expectedParent: pkg.Package{ Name: "some-parent-name", Version: "3.0", + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ VirtualPath: virtualPath + ":some-parent-name", // note: matching virtual path Manifest: nil, @@ -748,11 +850,12 @@ func TestPackagesFromPomProperties(t *testing.T) { parent: &pkg.Package{ Name: "some-parent-name", Version: "2.0", + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ VirtualPath: virtualPath + ":some-parent-name", // note: matching virtual path Manifest: nil, PomProperties: &pkg.PomProperties{ - Name: "EXISTS", //note: this already exists and should not be overridden + Name: "EXISTS", // note: this already exists and should not be overridden }, Parent: nil, }, @@ -760,12 +863,13 @@ func TestPackagesFromPomProperties(t *testing.T) { expectedParent: pkg.Package{ Name: "some-parent-name", Version: "2.0", + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ VirtualPath: virtualPath + ":some-parent-name", // note: matching virtual path Manifest: nil, // note: we attach the discovered pom properties data PomProperties: &pkg.PomProperties{ - Name: "EXISTS", //note: this already exists and should not be overridden + Name: "EXISTS", // note: this already exists and should not be overridden }, Parent: nil, }, @@ -783,6 +887,7 @@ func TestPackagesFromPomProperties(t *testing.T) { parent: &pkg.Package{ Name: "some-parent-name", Version: "2.0", + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ VirtualPath: virtualPath + ":NEW_VIRTUAL_PATH", // note: DOES NOT match the existing virtual path Manifest: nil, @@ -794,6 +899,7 @@ func TestPackagesFromPomProperties(t *testing.T) { expectedParent: pkg.Package{ Name: "some-parent-name", Version: "2.0", + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ VirtualPath: virtualPath + ":NEW_VIRTUAL_PATH", Manifest: nil, @@ -823,7 +929,7 @@ func TestPackagesFromPomProperties(t *testing.T) { t.Cleanup(cleanup) // get the test data - actualPackages := parser.packagesFromPomProperties(test.props, test.parent) + actualPackages := parser.packagesFromPomProperties(*test.props, test.parent) assert.Equal(t, test.expectedPackages, actualPackages) assert.Equal(t, test.expectedParent, *test.parent) }) diff --git a/syft/pkg/java_metadata.go b/syft/pkg/java_metadata.go index beb03463f..8969b7c36 100644 --- a/syft/pkg/java_metadata.go +++ b/syft/pkg/java_metadata.go @@ -20,6 +20,15 @@ type PomProperties struct { Extra map[string]string `mapstructure:",remain" json:"extraFields"` } +// PkgTypeIndicated returns the package Type indicated by the data contained in the PomProperties. +func (p PomProperties) PkgTypeIndicated() Type { + if p.GroupID == PomPropertiesGroupIDJenkinsPlugins { + return JenkinsPluginPkg + } + + return JavaPkg +} + // JavaManifest represents the fields of interest extracted from a Java archive's META-INF/MANIFEST.MF file. type JavaManifest struct { Main map[string]string `json:"main,omitempty"` @@ -43,3 +52,6 @@ func (m JavaMetadata) PackageURL() string { return "" } + +const PomPropertiesGroupIDJenkinsPlugins = "com.cloudbees.jenkins.plugins" +const PomPropertiesGroupIDJiraPlugins = "com.atlassian.jira.plugins" diff --git a/syft/pkg/java_metadata_test.go b/syft/pkg/java_metadata_test.go index 446d30820..866686f79 100644 --- a/syft/pkg/java_metadata_test.go +++ b/syft/pkg/java_metadata_test.go @@ -2,9 +2,48 @@ package pkg import ( "github.com/sergi/go-diff/diffmatchpatch" + "github.com/stretchr/testify/assert" "testing" ) +func TestPomProperties_PkgTypeIndicated(t *testing.T) { + cases := []struct { + name string + pomProperties PomProperties + expectedType Type + }{ + { + name: "regular Java package", + pomProperties: PomProperties{ + Path: "some path", + Name: "some name", + GroupID: "some group ID", + ArtifactID: "some artifact ID", + Version: "1", + }, + expectedType: JavaPkg, + }, + { + name: "jenkins plugin", + pomProperties: PomProperties{ + Path: "some path", + Name: "some name", + GroupID: "com.cloudbees.jenkins.plugins", + ArtifactID: "some artifact ID", + Version: "1", + }, + expectedType: JenkinsPluginPkg, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + actual := tc.pomProperties.PkgTypeIndicated() + assert.Equal(t, tc.expectedType, actual) + }) + } +} + func TestJavaMetadata_pURL(t *testing.T) { tests := []struct { metadata JavaMetadata From 060e60b6dd1f07404510452832c4d87412de5a92 Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Tue, 20 Apr 2021 20:42:40 -0400 Subject: [PATCH 5/5] Add more tests to CPE generation Signed-off-by: Dan Luhring --- syft/pkg/cataloger/cpe.go | 1 + syft/pkg/cataloger/cpe_test.go | 89 +++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/syft/pkg/cataloger/cpe.go b/syft/pkg/cataloger/cpe.go index ad5ee4e04..0a8ad7c3a 100644 --- a/syft/pkg/cataloger/cpe.go +++ b/syft/pkg/cataloger/cpe.go @@ -138,6 +138,7 @@ func candidateVendors(p pkg.Package) []string { func candidateProducts(p pkg.Package) []string { products := []string{p.Name} + if p.Language == pkg.Java { products = append(products, candidateProductsForJava(p)...) } diff --git a/syft/pkg/cataloger/cpe_test.go b/syft/pkg/cataloger/cpe_test.go index 678eed7aa..17289344f 100644 --- a/syft/pkg/cataloger/cpe_test.go +++ b/syft/pkg/cataloger/cpe_test.go @@ -195,7 +195,6 @@ func TestGeneratePackageCPEs(t *testing.T) { for _, d := range missing { t.Errorf("missing CPE: %+v", d) } - }) } } @@ -212,6 +211,32 @@ func TestCandidateProducts(t *testing.T) { }, expected: []string{"spring_framework", "springsource_spring_framework" /* <-- known good names | default guess --> */, "springframework"}, }, + { + p: pkg.Package{ + Name: "some-java-package-with-group-id", + Type: pkg.JavaPkg, + Language: pkg.Java, + Metadata: pkg.JavaMetadata{ + PomProperties: &pkg.PomProperties{ + GroupID: "com.apple.itunes", + }, + }, + }, + expected: []string{"itunes", "some-java-package-with-group-id"}, + }, + { + p: pkg.Package{ + Name: "some-jenkins-plugin", + Type: pkg.JenkinsPluginPkg, + Language: pkg.Java, + Metadata: pkg.JavaMetadata{ + PomProperties: &pkg.PomProperties{ + GroupID: "com.cloudbees.jenkins.plugins", + }, + }, + }, + expected: []string{"some-jenkins-plugin"}, + }, { p: pkg.Package{ Name: "handlebars.js", @@ -237,7 +262,67 @@ func TestCandidateProducts(t *testing.T) { for _, test := range tests { t.Run(fmt.Sprintf("%+v %+v", test.p, test.expected), func(t *testing.T) { - assert.Equal(t, test.expected, candidateProducts(test.p)) + assert.ElementsMatch(t, test.expected, candidateProducts(test.p)) + }) + } +} + +func TestCandidateTargetSoftwareAttrs(t *testing.T) { + cases := []struct { + name string + p pkg.Package + expected []string + }{ + { + name: "Java", + p: pkg.Package{ + Language: pkg.Java, + Type: pkg.JavaPkg, + }, + expected: []string{"java", "maven"}, + }, + { + name: "Jenkins plugin", + p: pkg.Package{ + Language: pkg.Java, + Type: pkg.JenkinsPluginPkg, + }, + expected: []string{"jenkins", "cloudbees_jenkins"}, + }, + { + name: "JavaScript", + p: pkg.Package{ + Language: pkg.JavaScript, + }, + expected: []string{"node.js", "nodejs"}, + }, + { + name: "Ruby", + p: pkg.Package{ + Language: pkg.Ruby, + }, + expected: []string{"ruby", "rails"}, + }, + { + name: "Python", + p: pkg.Package{ + Language: pkg.Python, + }, + expected: []string{"python"}, + }, + { + name: "Other language", + p: pkg.Package{ + Language: pkg.Rust, + }, + expected: nil, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + actual := candidateTargetSoftwareAttrs(tc.p) + assert.Equal(t, tc.expected, actual) }) } }