Fix acceptance tests (#158)

* ensure acceptance tests fail when results are piped

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix spelling in inline-compare python script

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

test/acceptance/compare.py

@@ -3,6 +3,9 @@ import sys
 import json
 import collections
 
+INDENT = "    "
+
+
 Metadata = collections.namedtuple("Metadata", "metadata sources")
 Package = collections.namedtuple("Package", "name type version")
 
@@ -28,11 +31,22 @@ class Syft:
             packages.add(package)
             metadata[package.type][package] = Metadata(
                 # note: the metadata entry is optional
-                metadata=repr(entry.get("metadata", "")), sources=repr(entry["sources"])
+                metadata=repr(entry.get("metadata", "")), sources=repr(entry["locations"])
             )
         return packages, metadata
 
 
+def print_rows(rows):
+    if not rows:
+        return
+    widths = []
+    for col, _ in enumerate(rows[0]):
+        width = max(len(row[col]) for row in rows) + 2  # padding
+        widths.append(width)
+    for row in rows:
+        print("".join(word.ljust(widths[col_idx]) for col_idx, word in enumerate(row)))
+
+
 def main(baseline_report, new_report):
     report1_obj = Syft(report_path=baseline_report)
     report1_packages, report1_metadata = report1_obj.packages()
@@ -40,9 +54,10 @@ def main(baseline_report, new_report):
     report2_obj = Syft(report_path=new_report)
     report2_packages, report2_metadata = report2_obj.packages()
 
-    if len(report2_packages) == 0 and len(report1_packages) == 0:
+    if len(report2_packages) == 0 or len(report1_packages) == 0:
-        print("nobody found any packages")
+        # we are purposefully selecting test images that are guaranteed to have packages, so this should never happen
-        return 0
+        print(colors.bold + colors.fg.red + "no packages found!", colors.reset)
+        return 1
 
     same_packages = report2_packages & report1_packages
     percent_overlap_packages = (
@@ -69,21 +84,25 @@ def main(baseline_report, new_report):
             float(len(same_metadata)) / float(len(report1_metadata_set))
         ) * 100.0
 
-    if len(extra_packages) > 0:
+    if extra_packages:
-        print("Extra packages:")
+        rows = []
+        print(colors.bold + "Extra packages:", colors.reset)
         for package in sorted(list(extra_packages)):
-            print("    " + repr(package))
+            rows.append([INDENT, repr(package)])
+        print_rows(rows)
         print()
 
-    if len(missing_packages) > 0:
+    if missing_packages:
-        print("Missing packages:")
+        rows = []
+        print(colors.bold + "Missing packages:", colors.reset)
         for package in sorted(list(missing_packages)):
-            print("    " + repr(package))
+            rows.append([INDENT, repr(package)])
+        print_rows(rows)
         print()
 
+    print(colors.bold+"Summary:", colors.reset)
     print("   Baseline Packages: %d" % len(report1_packages))
     print("   New Packages:      %d" % len(report2_packages))
-    print()
     print(
         "   Baseline Packages Matched: %.2f %% (%d/%d packages)"
         % (percent_overlap_packages, len(same_packages), len(report1_packages))
@@ -94,12 +113,48 @@ def main(baseline_report, new_report):
     )
 
     if len(report1_packages) != len(report2_packages):
-        print("failed quality gate: requires exact name & version match")
+        print(colors.bold + "   Quality Gate: " + colors.fg.red + "FAILED (requires exact name & version match)\n", colors.reset)
         return 1
-
+    else:
+        print(colors.bold + "   Quality Gate: " + colors.fg.green + "pass\n", colors.reset)
     return 0
 
 
+class colors:
+    reset='\033[0m'
+    bold='\033[01m'
+    disable='\033[02m'
+    underline='\033[04m'
+    reverse='\033[07m'
+    strikethrough='\033[09m'
+    invisible='\033[08m'
+    class fg:
+        black='\033[30m'
+        red='\033[31m'
+        green='\033[32m'
+        orange='\033[33m'
+        blue='\033[34m'
+        purple='\033[35m'
+        cyan='\033[36m'
+        lightgrey='\033[37m'
+        darkgrey='\033[90m'
+        lightred='\033[91m'
+        lightgreen='\033[92m'
+        yellow='\033[93m'
+        lightblue='\033[94m'
+        pink='\033[95m'
+        lightcyan='\033[96m'
+    class bg:
+        black='\033[40m'
+        red='\033[41m'
+        green='\033[42m'
+        orange='\033[43m'
+        blue='\033[44m'
+        purple='\033[45m'
+        cyan='\033[46m'
+        lightgrey='\033[47m'
+
+
 if __name__ == "__main__":
     print("\nComparing two Syft reports...\n")
     if len(sys.argv) != 3:

test/acceptance/deb.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -eux
+set -o pipefail
 
 DISTDIR=$1
 ACC_DIR=$2
@@ -19,7 +20,10 @@ if [[ ! "${WORK_DIR}" || ! -d "${WORK_DIR}" ]]; then
 fi
 
 function cleanup {
+  # we should still preserve previous failures
+  exit_code=$?
   rm -rf "${WORK_DIR}"
+  exit ${exit_code}
 }
 
 trap cleanup EXIT
@@ -32,13 +36,13 @@ docker run --rm \
     -v /var/run/docker.sock://var/run/docker.sock \
     -v /${PWD}:/src \
     -v ${WORK_DIR}:${WORK_DIR} \
+    -e SYFT_CHECK_FOR_APP_UPDATE=0 \
     -w /src \
     ubuntu:latest \
         /bin/bash -x -c "\
             DEBIAN_FRONTEND=noninteractive apt install ${DISTDIR}/syft_*_linux_amd64.deb -y && \
             syft version -v && \
-            syft ${TEST_IMAGE} -vv -o json > ${REPORT} && \
+            syft ${TEST_IMAGE} -vv -o json > ${REPORT} \
-            cat ${REPORT} \
         "
 
 # keep the generated report around

test/acceptance/mac.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -eux
+set -o pipefail
 
 DISTDIR=$1
 ACC_DIR=$2
@@ -20,7 +21,10 @@ if [[ ! "${WORK_DIR}" || ! -d "${WORK_DIR}" ]]; then
 fi
 
 function cleanup {
+  # we should still preserve previous failures
+  exit_code=$?
   rm -rf "${WORK_DIR}"
+  exit ${exit_code}
 }
 
 trap cleanup EXIT
@@ -35,8 +39,7 @@ ls -alh ${TEST_IMAGE_TAR}
 # run syft
 chmod 755 ${DISTDIR}/syft_darwin_amd64/syft
 ${DISTDIR}/syft_darwin_amd64/syft version -v
-${DISTDIR}/syft_darwin_amd64/syft docker-archive://${TEST_IMAGE_TAR} -vv -o json > ${REPORT}
+SYFT_CHECK_FOR_APP_UPDATE=0 ${DISTDIR}/syft_darwin_amd64/syft docker-archive://${TEST_IMAGE_TAR} -vv -o json > ${REPORT}
-cat ${REPORT}
 
 # keep the generated report around
 mkdir -p ${RESULTSDIR}

test/acceptance/rpm.sh

@@ -1,11 +1,13 @@
 #!/usr/bin/env bash
 set -eux
+set -o pipefail
 
 DISTDIR=$1
 ACC_DIR=$2
 TEST_IMAGE=$3
 RESULTSDIR=$4
 
+EXIT_CODE=1
 TEST_TYPE=rpm
 WORK_DIR=`mktemp -d -t "syft-acceptance-test-${TEST_TYPE}-XXXXXX"`
 NORMAL_TEST_IMAGE=$(echo ${TEST_IMAGE} | tr ':' '-' )
@@ -20,6 +22,7 @@ fi
 
 function cleanup {
   rm -rf "${WORK_DIR}"
+  exit ${EXIT_CODE}
 }
 
 trap cleanup EXIT
@@ -32,13 +35,13 @@ docker run --rm \
     -v /var/run/docker.sock://var/run/docker.sock \
     -v /${PWD}:/src \
     -v ${WORK_DIR}:${WORK_DIR} \
+    -e SYFT_CHECK_FOR_APP_UPDATE=0 \
     -w /src \
     centos:latest \
         /bin/bash -x -c "\
             rpm -ivh ${DISTDIR}/syft_*_linux_amd64.rpm && \
             syft version -v && \
-            syft ${TEST_IMAGE} -vv -o json > ${REPORT} && \
+            syft ${TEST_IMAGE} -vv -o json > ${REPORT} \
-            cat ${REPORT} \
         "
 
 # keep the generated report around
@@ -49,3 +52,5 @@ cp ${REPORT} ${RESULTSDIR}
 ${ACC_DIR}/compare.py \
     ${GOLDEN_REPORT} \
     ${REPORT} | tee ${RESULTSDIR}/acceptance-${TEST_TYPE}.txt
+
+EXIT_CODE=0

test/inline-compare/compare.py

@@ -151,7 +151,7 @@ def main(image):
     ) * 100.0
 
     bonus_packages = syft_packages - inline_packages
-    missing_pacakges = inline_packages - syft_packages
+    missing_packages = inline_packages - syft_packages
 
     inline_metadata_set = set()
     for package in inline_packages:
@@ -171,7 +171,7 @@ def main(image):
     ) * 100.0
     missing_metadata = inline_metadata_set - same_metadata
 
-    if len(bonus_packages) > 0:
+    if bonus_packages:
         rows = []
         print(colors.bold + "Syft found extra packages:", colors.reset)
         for package in sorted(list(bonus_packages)):
@@ -179,15 +179,15 @@ def main(image):
         print_rows(rows)
         print()
 
-    if len(missing_pacakges) > 0:
+    if missing_packages:
         rows = []
         print(colors.bold + "Syft missed packages:", colors.reset)
-        for package in sorted(list(missing_pacakges)):
+        for package in sorted(list(missing_packages)):
             rows.append([INDENT, repr(package)])
         print_rows(rows)
         print()
 
-    if len(missing_metadata) > 0:
+    if missing_metadata:
         rows = []
         print(colors.bold + "Syft mismatched metadata:", colors.reset)
         for inline_metadata_pair in sorted(list(missing_metadata)):