oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-10-30 14:02:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add inline-comparison as acceptance test (#130)

Browse Source 
* add inline-compare as acceptance test

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add additional RPM metadata

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add comments and doc strings to the compare-* make targets

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2020-08-10 10:33:44 -04:00
committed by GitHub
parent 4c7784da62
commit 2d452bf59e
17 changed files with 274 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/acceptance/compare.py
+5 -6
View File
@@ -5,10 +5,9 @@ import collections
Metadata = collections.namedtuple("Metadata", "metadata sources")
Package = collections.namedtuple("Package", "name type version")
Vulnerability = collections.namedtuple("Vulnerability", "cve package")
class syft:
class Syft:
def __init__(self, report_path):
self.report_path = report_path
@@ -35,10 +34,10 @@ class syft:
def main(baseline_report, new_report):
report1_obj = syft(report_path=baseline_report)
report1_obj = Syft(report_path=baseline_report)
report1_packages, report1_metadata = report1_obj.packages()
report2_obj = syft(report_path=new_report)
report2_obj = Syft(report_path=new_report)
report2_packages, report2_metadata = report2_obj.packages()
if len(report2_packages) == 0 and len(report1_packages) == 0:
@@ -102,9 +101,9 @@ def main(baseline_report, new_report):
if __name__ == "__main__":
print("\nComparing two syft reports...\n")
print("\nComparing two Syft reports...\n")
if len(sys.argv) != 3:
sys.exit("please provide two syft json files")
sys.exit("please provide two Syft json files")
rc = main(sys.argv[1], sys.argv[2])
sys.exit(rc)
test/acceptance/compare.sh
-27
View File
@@ -1,27 +0,0 @@
#!/usr/bin/env bash
set -eu
BOLD="$(tput -T linux bold)"
RED="$(tput -T linux setaf 1)"
RESET="$(tput -T linux sgr0)"
FAIL="${BOLD}${RED}"
SUCCESS="${BOLD}"
JQ_ARGS="-S .artifacts"
if ! command -v jq &> /dev/null ;then
JQ_IMAGE="imega/jq:latest"
JQ_CMD="docker run --rm -i ${JQ_IMAGE} ${JQ_ARGS}"
docker pull "${JQ_IMAGE}"
else
JQ_CMD="jq ${JQ_ARGS}"
fi
if [[ $(cat $1 | ${JQ_CMD}) ]]; then
set -x
# compare the output of both results
diff <(cat $1 | ${JQ_CMD}) <(cat $2 | ${JQ_CMD})
set +x
echo "${SUCCESS}Comparison passed!${RESET}"
else
exit "${FAIL}Failing since one of the test files is empty ($1)${RESET}"
fi
test/inline-compare/Dockerfile
-7
View File
@@ -1,7 +0,0 @@
FROM python:3
WORKDIR /
COPY syft-reports /syft-reports
COPY inline-reports /inline-reports
COPY compare.py .
ENTRYPOINT ["/compare.py"]
test/inline-compare/Makefile
+29 -19
View File
@@ -1,12 +1,15 @@
IMAGE = "centos:8"
IMAGE_CLEAN = $(shell echo $(IMAGE) | tr ":" "_")
syft_DIR = syft-reports
syft_REPORT = $(syft_DIR)/$(IMAGE_CLEAN).json
ifndef SYFT_CMD
SYFT_CMD = go run ../../main.go
endif
IMAGE_CLEAN = $(shell echo $(COMPARE_IMAGE) | tr ":" "_")
SYFT_DIR = syft-reports
SYFT_REPORT = $(SYFT_DIR)/$(IMAGE_CLEAN).json
INLINE_DIR = inline-reports
INLINE_REPORT = $(INLINE_DIR)/$(IMAGE_CLEAN)-content-os.json
ifndef syft_DIR
$(error syft_DIR is not set)
ifndef SYFT_DIR
$(error SYFT_DIR is not set)
endif
ifndef INLINE_DIR
@@ -14,26 +17,33 @@ ifndef INLINE_DIR
endif
.PHONY: all
all: compare
.DEFAULT_GOAL :=
all: clean-syft
./compare-all.sh
.PHONY: compare
compare: $(INLINE_REPORT) $(syft_REPORT)
docker build -t compare-syft:latest .
docker run compare-syft:latest $(IMAGE)
.PHONY: compare-image
compare-image: $(SYFT_REPORT) $(INLINE_REPORT)
./compare.py $(COMPARE_IMAGE)
.PHONY: gather-iamge
gather-image: $(SYFT_REPORT) $(INLINE_REPORT)
$(INLINE_REPORT):
echo "Creating $(INLINE_REPORT)..."
mkdir -p $(INLINE_DIR)
curl -s https://ci-tools.anchore.io/inline_scan-v0.7.0 | bash -s -- -p -r $(IMAGE)
curl -s https://ci-tools.anchore.io/inline_scan-v0.7.0 | bash -s -- -p -r $(COMPARE_IMAGE)
mv anchore-reports/* $(INLINE_DIR)/
rmdir anchore-reports
$(syft_REPORT):
echo "Creating $(syft_REPORT)..."
mkdir -p $(syft_DIR)
docker pull $(IMAGE)
go run ../../main.go $(IMAGE) -o json > $(syft_REPORT)
$(SYFT_REPORT):
echo "Creating $(SYFT_REPORT)..."
mkdir -p $(SYFT_DIR)
$(SYFT_CMD) $(COMPARE_IMAGE) -o json > $(SYFT_REPORT)
.PHONY: clean
clean:
rm -f $(INLINE_DIR)/* $(syft_DIR)/*
clean: clean-syft
rm -f $(INLINE_DIR)/*
.PHONY: clean-syft
clean-syft:
rm -f $(SYFT_DIR)/*
test/inline-compare/compare-all.sh
Executable
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -eu
# TODO: add "alpine:3.12.0" back
images=("debian:10.5" "centos:8.2.2004" )
# gather all image analyses
for img in "${images[@]}"; do
echo "Gathering facts for $img"
COMPARE_IMAGE=${img} make gather-image
done
# compare all results
for img in "${images[@]}"; do
echo "Comparing results for $img"
COMPARE_IMAGE=${img} make compare-image
done
test/inline-compare/compare.py
+128 -27
View File
@@ -2,14 +2,25 @@
import os
import sys
import json
import functools
import collections
QUALITY_GATE_THRESHOLD = 0.9
QUALITY_GATE_THRESHOLD = 0.95
INDENT = " "
IMAGE_QUALITY_GATE = collections.defaultdict(lambda: QUALITY_GATE_THRESHOLD, **{
})
# We additionally fail if an image is above a particular threshold. Why? We expect the lower threshold to be 90%,
# however additional functionality in grype is still being implemented, so this threshold may not be able to be met.
# In these cases the IMAGE_QUALITY_GATE is set to a lower value to allow the test to pass for known issues. Once these
# issues/enhancements are done we want to ensure that the lower threshold is bumped up to catch regression. The only way
# to do this is to select an upper threshold for images with known threshold values, so we have a failure that
# loudly indicates the lower threshold should be bumped.
IMAGE_UPPER_THRESHOLD = collections.defaultdict(lambda: 1, **{
})
Metadata = collections.namedtuple("Metadata", "version")
Package = collections.namedtuple("Package", "name type")
Vulnerability = collections.namedtuple("Vulnerability", "cve package")
class InlineScan:
@@ -33,12 +44,17 @@ class InlineScan:
def _enumerate_section(self, report, section):
report_path = self._report_path(report=report)
os_report_path = self._report_path(report="content-os")
if os.path.exists(os_report_path) and not os.path.exists(report_path):
# if the OS report is there but the target report is not, that is engine's way of saying "no findings"
return
with open(report_path) as json_file:
data = json.load(json_file)
for entry in data[section]:
yield entry
@functools.lru_cache
def _python_packages(self):
packages = set()
metadata = collections.defaultdict(dict)
@@ -51,7 +67,6 @@ class InlineScan:
return packages, metadata
@functools.lru_cache
def _os_packages(self):
packages = set()
metadata = collections.defaultdict(dict)
@@ -63,7 +78,7 @@ class InlineScan:
return packages, metadata
class syft:
class Syft:
report_tmpl = "{image}.json"
@@ -78,31 +93,54 @@ class syft:
for entry in data[section]:
yield entry
@functools.lru_cache
def packages(self):
packages = set()
metadata = collections.defaultdict(dict)
for entry in self._enumerate_section(section="artifacts"):
# normalize to inline
pType = entry["type"].lower()
if pType in ("wheel", "egg"):
pType = "python"
pkg_type = entry["type"].lower()
if pkg_type in ("wheel", "egg"):
pkg_type = "python"
elif pkg_type in ("deb",):
pkg_type = "dpkg"
elif pkg_type in ("java-archive",):
pkg_type = "java"
elif pkg_type in ("apk",):
pkg_type = "apkg"
package = Package(name=entry["name"], type=pType,)
package = Package(name=entry["name"], type=pkg_type,)
packages.add(package)
metadata[package.type][package] = Metadata(version=entry["version"])
return packages, metadata
def print_rows(rows):
if not rows:
return
widths = []
for col, _ in enumerate(rows[0]):
width = max(len(row[col]) for row in rows) + 2 # padding
widths.append(width)
for row in rows:
print("".join(word.ljust(widths[col_idx]) for col_idx, word in enumerate(row)))
def main(image):
print(colors.bold+"Image:", image, colors.reset)
inline = InlineScan(image=image, report_dir="inline-reports")
inline_packages, inline_metadata = inline.packages()
syft = syft(image=image, report_dir="syft-reports")
syft = Syft(image=image, report_dir="syft-reports")
syft_packages, syft_metadata = syft.packages()
if len(inline_packages) == 0:
# we are purposefully selecting test images that are guaranteed to have packages, so this should never happen
print(colors.bold + colors.fg.red + "inline found no packages!", colors.reset)
return 1
if len(syft_packages) == 0 and len(inline_packages) == 0:
print("nobody found any packages")
return 0
@@ -120,51 +158,114 @@ def main(image):
metadata = inline_metadata[package.type][package]
inline_metadata_set.add((package, metadata))
syft_metadata_set = set()
syft_overlap_metadata_set = set()
for package in syft_packages:
metadata = syft_metadata[package.type][package]
syft_metadata_set.add((package, metadata))
# we only want to really count mismatched metadata for packages that are at least found by inline
if package in inline_metadata[package.type]:
syft_overlap_metadata_set.add((package, metadata))
same_metadata = syft_metadata_set & inline_metadata_set
same_metadata = syft_overlap_metadata_set & inline_metadata_set
percent_overlap_metadata = (
float(len(same_metadata)) / float(len(inline_metadata_set))
) * 100.0
missing_metadata = inline_metadata_set - same_metadata
if len(bonus_packages) > 0:
print("syft Bonus packages:")
rows = []
print(colors.bold + "Syft found extra packages:", colors.reset)
for package in sorted(list(bonus_packages)):
print(" " + repr(package))
rows.append([INDENT, repr(package)])
print_rows(rows)
print()
if len(missing_pacakges) > 0:
print("syft Missing packages:")
rows = []
print(colors.bold + "Syft missed packages:", colors.reset)
for package in sorted(list(missing_pacakges)):
print(" " + repr(package))
rows.append([INDENT, repr(package)])
print_rows(rows)
print()
print("Inline Packages: %d" % len(inline_packages))
print("syft Packages: %d" % len(syft_packages))
print()
if len(missing_metadata) > 0:
rows = []
print(colors.bold + "Syft mismatched metadata:", colors.reset)
for inline_metadata_pair in sorted(list(missing_metadata)):
pkg, metadata = inline_metadata_pair
if pkg in syft_metadata[pkg.type]:
syft_metadata_item = syft_metadata[pkg.type][pkg]
else:
syft_metadata_item = "--- MISSING ---"
rows.append([INDENT, "for:", repr(pkg), ":", repr(syft_metadata_item), "!=", repr(metadata)])
print_rows(rows)
print()
print(colors.bold+"Summary:", colors.reset)
print(" Image: %s" % image)
print(" Inline Packages: %d" % len(inline_packages))
print(" Syft Packages: %d" % len(syft_packages))
print(
"Baseline Packages Matched: %2.3f %% (%d/%d packages)"
" Baseline Packages Matched: %2.3f %% (%d/%d packages)"
% (percent_overlap_packages, len(same_packages), len(inline_packages))
)
print(
"Baseline Metadata Matched: %2.3f %% (%d/%d metadata)"
" Baseline Metadata Matched: %2.3f %% (%d/%d metadata)"
% (percent_overlap_metadata, len(same_metadata), len(inline_metadata_set))
)
overall_score = (percent_overlap_packages + percent_overlap_metadata) / 2.0
print("Overall Score: %2.3f %%" % overall_score)
print(colors.bold + " Overall Score: %2.1f %%" % overall_score, colors.reset)
if overall_score < (QUALITY_GATE_THRESHOLD * 100):
print("failed quality gate (>= %d %%)" % (QUALITY_GATE_THRESHOLD * 100))
upper_gate_value = IMAGE_UPPER_THRESHOLD[image] * 100
lower_gate_value = IMAGE_QUALITY_GATE[image] * 100
if overall_score < lower_gate_value:
print(colors.bold + " Quality Gate: " + colors.fg.red + "FAILED (is not >= %d %%)\n" % lower_gate_value, colors.reset)
return 1
elif overall_score > upper_gate_value:
print(colors.bold + " Quality Gate: " + colors.fg.orange + "FAILED (lower threshold is artificially low and should be updated)\n", colors.reset)
return 1
else:
print(colors.bold + " Quality Gate: " + colors.fg.green + "pass (>= %d %%)\n" % lower_gate_value, colors.reset)
return 0
class colors:
reset='\033[0m'
bold='\033[01m'
disable='\033[02m'
underline='\033[04m'
reverse='\033[07m'
strikethrough='\033[09m'
invisible='\033[08m'
class fg:
black='\033[30m'
red='\033[31m'
green='\033[32m'
orange='\033[33m'
blue='\033[34m'
purple='\033[35m'
cyan='\033[36m'
lightgrey='\033[37m'
darkgrey='\033[90m'
lightred='\033[91m'
lightgreen='\033[92m'
yellow='\033[93m'
lightblue='\033[94m'
pink='\033[95m'
lightcyan='\033[96m'
class bg:
black='\033[40m'
red='\033[41m'
green='\033[42m'
orange='\033[43m'
blue='\033[44m'
purple='\033[45m'
cyan='\033[46m'
lightgrey='\033[47m'
if __name__ == "__main__":
if len(sys.argv) != 2:
sys.exit("provide an image")
test/integration/pkg_cases.go
+1 -1
View File
@@ -14,7 +14,7 @@ var cases = []struct {
name: "find rpmdb packages",
pkgType: pkg.RpmPkg,
pkgInfo: map[string]string{
"dive": "0.9.2",
"dive": "0.9.2-1",
},
},
{