From 1230650771dd1cb2f6739623946f727a0f02331b Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 29 Oct 2020 11:28:27 -0400 Subject: [PATCH] allow for java manifest data to be optional Signed-off-by: Alex Goodman --- syft/pkg/java_metadata.go | 18 +++++------ test/inline-compare/compare.py | 21 +++++-------- test/inline-compare/utils/inline.py | 22 ++++++++++---- test/inline-compare/utils/package.py | 2 +- test/inline-compare/utils/syft.py | 43 ++++++++++++++++++++++++++- test/inline-compare/utils/traverse.py | 21 +++++++++++++ 6 files changed, 98 insertions(+), 29 deletions(-) create mode 100644 test/inline-compare/utils/traverse.py diff --git a/syft/pkg/java_metadata.go b/syft/pkg/java_metadata.go index ad2a66079..3aa6fcb20 100644 --- a/syft/pkg/java_metadata.go +++ b/syft/pkg/java_metadata.go @@ -22,15 +22,15 @@ type PomProperties struct { // JavaManifest represents the fields of interest extracted from a Java archive's META-INF/MANIFEST.MF file. type JavaManifest struct { - Name string `mapstructure:"Name" json:"name"` - ManifestVersion string `mapstructure:"Manifest-Version" json:"manifestVersion"` - SpecTitle string `mapstructure:"Specification-Title" json:"specificationTitle"` - SpecVersion string `mapstructure:"Specification-Version" json:"specificationVersion"` - SpecVendor string `mapstructure:"Specification-Vendor" json:"specificationVendor"` - ImplTitle string `mapstructure:"Implementation-Title" json:"implementationTitle"` - ImplVersion string `mapstructure:"Implementation-Version" json:"implementationVersion"` - ImplVendor string `mapstructure:"Implementation-Vendor" json:"implementationVendor"` - Extra map[string]string `mapstructure:",remain" json:"extraFields"` + Name string `mapstructure:"Name" json:"name,omitempty"` + ManifestVersion string `mapstructure:"Manifest-Version" json:"manifestVersion,omitempty"` + SpecTitle string `mapstructure:"Specification-Title" json:"specificationTitle,omitempty"` + SpecVersion string `mapstructure:"Specification-Version" json:"specificationVersion,omitempty"` + SpecVendor string `mapstructure:"Specification-Vendor" json:"specificationVendor,omitempty"` + ImplTitle string `mapstructure:"Implementation-Title" json:"implementationTitle,omitempty"` + ImplVersion string `mapstructure:"Implementation-Version" json:"implementationVersion,omitempty"` + ImplVendor string `mapstructure:"Implementation-Vendor" json:"implementationVendor,omitempty"` + Extra map[string]string `mapstructure:",remain" json:"extraFields,omitempty"` Sections []map[string]string `json:"sections,omitempty"` } diff --git a/test/inline-compare/compare.py b/test/inline-compare/compare.py index 2e0220221..a2f6224a4 100755 --- a/test/inline-compare/compare.py +++ b/test/inline-compare/compare.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os import sys +import difflib import collections import utils.package @@ -58,19 +59,13 @@ def report(analysis): if pkg not in analysis.syft_data.metadata[pkg.type]: continue syft_metadata_item = analysis.syft_data.metadata[pkg.type][pkg] - rows.append( - [ - INDENT, - "for:", - repr(pkg), - ":", - repr(syft_metadata_item), - "!=", - repr(metadata), - ] - ) - if rows: - print_rows(rows) + + diffs = difflib.ndiff([repr(syft_metadata_item)], [repr(metadata)]) + + print(INDENT + "for: " + repr(pkg)) + print(INDENT+INDENT+("\n"+INDENT+INDENT).join(list(diffs))) + print() + else: print( INDENT, diff --git a/test/inline-compare/utils/inline.py b/test/inline-compare/utils/inline.py index bf58c1510..1e9d77276 100644 --- a/test/inline-compare/utils/inline.py +++ b/test/inline-compare/utils/inline.py @@ -71,8 +71,20 @@ class InlineScan: type=pkg_type, ) packages.add(pkg) + + extra = dict(entry) + extra.pop('type') + extra.pop('maven-version') + for k, v in dict(extra).items(): + if v in ("", "N/A"): + extra[k] = None + + # temp temp temp + extra.pop("location") + metadata[pkg.type][pkg] = utils.package.Metadata( - version=entry["maven-version"] + version=entry["maven-version"], + extra=tuple(sorted(extra.items())), ) return packages, metadata @@ -86,7 +98,7 @@ class InlineScan: type=entry["type"].lower(), ) packages.add(pkg) - metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"]) + metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"], extra=tuple()) return packages, metadata @@ -101,7 +113,7 @@ class InlineScan: type=entry["type"].lower(), ) packages.add(pkg) - metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"]) + metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"], extra=tuple()) return packages, metadata @@ -114,7 +126,7 @@ class InlineScan: type=entry["type"].lower(), ) packages.add(pkg) - metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"]) + metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"], extra=tuple()) return packages, metadata @@ -126,6 +138,6 @@ class InlineScan: name=entry["package"], type=entry["type"].lower() ) packages.add(pkg) - metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"]) + metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"], extra=tuple()) return packages, metadata diff --git a/test/inline-compare/utils/package.py b/test/inline-compare/utils/package.py index a6cb3353d..06d36b800 100644 --- a/test/inline-compare/utils/package.py +++ b/test/inline-compare/utils/package.py @@ -3,7 +3,7 @@ import collections import dataclasses from typing import Set, FrozenSet, Tuple, Any, List -Metadata = collections.namedtuple("Metadata", "version") +Metadata = collections.namedtuple("Metadata", "version extra") Package = collections.namedtuple("Package", "name type") Info = collections.namedtuple("Info", "packages metadata") diff --git a/test/inline-compare/utils/syft.py b/test/inline-compare/utils/syft.py index b6e66b4d6..3bd1ac59c 100644 --- a/test/inline-compare/utils/syft.py +++ b/test/inline-compare/utils/syft.py @@ -4,6 +4,7 @@ import collections import utils.package import utils.image +from utils.traverse import dig class Syft: @@ -28,6 +29,8 @@ class Syft: metadata = collections.defaultdict(dict) for entry in self._enumerate_section(section="artifacts"): + extra = {} + # normalize to inline pkg_type = entry["type"].lower() if pkg_type in ("wheel", "egg", "python"): @@ -49,6 +52,44 @@ class Syft: ) packages.add(pkg) - metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"]) + + if "java" in pkg_type: + # lets match what inline scan expects to output + + path = dig(entry, "locations", 0, "path") + specVendor = dig(entry, "metadata", "manifest", "specificationVendor") + implVendor = dig(entry, "metadata", "manifest", "implementationVendor") + + specVersion = dig(entry, "metadata", "manifest", "specificationVersion") or None + implVersion = dig(entry, "metadata", "manifest", "implementationVersion") or None + + extra = { + "implementation-version": implVersion, + "specification-version": specVersion, + "origin": dig(entry, "metadata", "pomProperties", "groupId"), + "location": path, + "package": dig(entry, "metadata", "pomProperties", "artifactId"), + } + + if dig(entry, "metadata", "parentPackage"): + extra['origin'] = dig(entry, "metadata", "pomProperties", "groupId") + else: + # this is a nested package... + if specVendor: + extra['origin'] = specVendor + elif implVendor: + extra['origin'] = implVendor + + pomPath = dig(entry, "metadata", "pomProperties", "Path") + if path and pomPath: + extra["location"] = "%s:%s" % (path, pomPath), + + # temp temp temp + extra.pop("location") + + elif pkg_type == "apkg": + entry["version"] = entry["version"].split("-")[0] + + metadata[pkg.type][pkg] = utils.package.Metadata(version=entry["version"], extra=tuple(sorted(extra.items()))) return utils.package.Info(packages=frozenset(packages), metadata=metadata) diff --git a/test/inline-compare/utils/traverse.py b/test/inline-compare/utils/traverse.py new file mode 100644 index 000000000..d98cf4ee7 --- /dev/null +++ b/test/inline-compare/utils/traverse.py @@ -0,0 +1,21 @@ + +def dig(target, *keys, **kwargs): + """ + Traverse a nested set of dictionaries, tuples, or lists similar to ruby's dig function. + """ + end_of_chain = target + for key in keys: + if isinstance(end_of_chain, dict) and key in end_of_chain: + end_of_chain = end_of_chain[key] + elif isinstance(end_of_chain, (list, tuple)) and isinstance(key, int): + end_of_chain = end_of_chain[key] + else: + if 'fail' in kwargs and kwargs['fail'] is True: + if isinstance(end_of_chain, dict): + raise KeyError + else: + raise IndexError + else: + return None + + return end_of_chain