oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2025-11-18 08:53:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #107 from anchore/fix-acceptance-compare

Browse Source 
Fix acceptance test compare script & persist results
This commit is contained in:
Alex Goodman 2020-07-24 17:53:02 -04:00 committed by GitHub
commit 6b0ab2de4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 33 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Makefile
View File

@ -164,8 +164,9 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie
$(call title,Running acceptance test: Run on Mac) $(call title,Running acceptance test: Run on Mac)
$(ACC_DIR)/mac.sh \ $(ACC_DIR)/mac.sh \
$(SNAPSHOTDIR) \ $(SNAPSHOTDIR) \
$(ACC_DIR)\ $(ACC_DIR) \
$(ACC_TEST_IMAGE) $(ACC_TEST_IMAGE) \
$(RESULTSDIR)
.PHONY: acceptance-linux .PHONY: acceptance-linux
acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux) acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux)
@ -175,16 +176,18 @@ acceptance-test-deb-package-install: $(SNAPSHOTDIR)
$(call title,Running acceptance test: DEB install) $(call title,Running acceptance test: DEB install)
$(ACC_DIR)/deb.sh \ $(ACC_DIR)/deb.sh \
$(SNAPSHOTDIR) \ $(SNAPSHOTDIR) \
$(ACC_DIR)\ $(ACC_DIR) \
$(ACC_TEST_IMAGE) $(ACC_TEST_IMAGE) \
$(RESULTSDIR)
.PHONY: acceptance-test-rpm-package-install .PHONY: acceptance-test-rpm-package-install
acceptance-test-rpm-package-install: $(SNAPSHOTDIR) acceptance-test-rpm-package-install: $(SNAPSHOTDIR)
$(call title,Running acceptance test: RPM install) $(call title,Running acceptance test: RPM install)
$(ACC_DIR)/rpm.sh \ $(ACC_DIR)/rpm.sh \
$(SNAPSHOTDIR) \ $(SNAPSHOTDIR) \
$(ACC_DIR)\ $(ACC_DIR) \
$(ACC_TEST_IMAGE) $(ACC_TEST_IMAGE) \
$(RESULTSDIR)
# TODO: this is not releasing yet # TODO: this is not releasing yet
.PHONY: release .PHONY: release
@ -203,11 +206,11 @@ release: clean-dist ## Build and publish final binaries and packages
# TODO: add upload to bucket # TODO: add upload to bucket
.PHONY: clean .PHONY: clean
clean: clean-dist clean-shapshot ## Remove previous builds and result reports clean: clean-dist clean-snapshot ## Remove previous builds and result reports
rm -rf $(RESULTSDIR)/* rm -rf $(RESULTSDIR)/*
.PHONY: clean-shapshot .PHONY: clean-snapshot
clean-shapshot: clean-snapshot:
rm -rf $(SNAPSHOTDIR) $(TEMPDIR)/goreleaser.yaml rm -rf $(SNAPSHOTDIR) $(TEMPDIR)/goreleaser.yaml
.PHONY: clean-dist .PHONY: clean-dist

9
test/acceptance/compare.py
View File

@ -28,7 +28,8 @@ class syft:
packages.add(package) packages.add(package)
metadata[package.type][package] = Metadata( metadata[package.type][package] = Metadata(
metadata=repr(entry["metadata"]), sources=repr(entry["sources"]) # note: the metadata entry is optional
metadata=repr(entry.get("metadata")), sources=repr(entry["sources"])
) )
return packages, metadata return packages, metadata
@ -50,7 +51,7 @@ def main(baseline_report, new_report):
) * 100.0 ) * 100.0
extra_packages = report2_packages - report1_packages extra_packages = report2_packages - report1_packages
missing_pacakges = report1_packages - report2_packages missing_packages = report1_packages - report2_packages
report1_metadata_set = set() report1_metadata_set = set()
for package in report1_packages: for package in report1_packages:
@ -75,9 +76,9 @@ def main(baseline_report, new_report):
print(" " + repr(package)) print(" " + repr(package))
print() print()
if len(missing_pacakges) > 0: if len(missing_packages) > 0:
print("Missing packages:") print("Missing packages:")
for package in sorted(list(missing_pacakges)): for package in sorted(list(missing_packages)):
print(" " + repr(package)) print(" " + repr(package))
print() print()

6
test/acceptance/deb.sh
View File

@ -4,6 +4,7 @@ set -eux
DISTDIR=$1 DISTDIR=$1
ACC_DIR=$2 ACC_DIR=$2
TEST_IMAGE=$3 TEST_IMAGE=$3
RESULTSDIR=$4
TEST_TYPE=deb TEST_TYPE=deb
WORK_DIR=`mktemp -d -t "syft-acceptance-test-${TEST_TYPE}-XXXXXX"` WORK_DIR=`mktemp -d -t "syft-acceptance-test-${TEST_TYPE}-XXXXXX"`
@ -40,7 +41,10 @@ docker run --rm \
cat ${REPORT} \ cat ${REPORT} \
" "
# keep the generated report around
cp ${REPORT} ${RESULTSDIR}
# compare the results to a known good output # compare the results to a known good output
${ACC_DIR}/compare.py \ ${ACC_DIR}/compare.py \
${GOLDEN_REPORT} \ ${GOLDEN_REPORT} \
${REPORT} ${REPORT} | tee ${RESULTSDIR}/acceptance-${TEST_TYPE}.txt

6
test/acceptance/mac.sh
View File

@ -4,6 +4,7 @@ set -eux
DISTDIR=$1 DISTDIR=$1
ACC_DIR=$2 ACC_DIR=$2
TEST_IMAGE=$3 TEST_IMAGE=$3
RESULTSDIR=$4
TEST_IMAGE_TAR=/tmp/image.tar TEST_IMAGE_TAR=/tmp/image.tar
TEST_TYPE=mac TEST_TYPE=mac
@ -37,7 +38,10 @@ ${DISTDIR}/syft_darwin_amd64/syft version -v
${DISTDIR}/syft_darwin_amd64/syft docker-archive://${TEST_IMAGE_TAR} -vv -o json > ${REPORT} ${DISTDIR}/syft_darwin_amd64/syft docker-archive://${TEST_IMAGE_TAR} -vv -o json > ${REPORT}
cat ${REPORT} cat ${REPORT}
# keep the generated report around
cp ${REPORT} ${RESULTSDIR}
# compare the results to a known good output # compare the results to a known good output
${ACC_DIR}/compare.py \ ${ACC_DIR}/compare.py \
${GOLDEN_REPORT} \ ${GOLDEN_REPORT} \
${REPORT} ${REPORT} | tee ${RESULTSDIR}/acceptance-${TEST_TYPE}.txt

6
test/acceptance/rpm.sh
View File

@ -4,6 +4,7 @@ set -eux
DISTDIR=$1 DISTDIR=$1
ACC_DIR=$2 ACC_DIR=$2
TEST_IMAGE=$3 TEST_IMAGE=$3
RESULTSDIR=$4
TEST_TYPE=rpm TEST_TYPE=rpm
WORK_DIR=`mktemp -d -t "syft-acceptance-test-${TEST_TYPE}-XXXXXX"` WORK_DIR=`mktemp -d -t "syft-acceptance-test-${TEST_TYPE}-XXXXXX"`
@ -40,7 +41,10 @@ docker run --rm \
cat ${REPORT} \ cat ${REPORT} \
" "
# keep the generated report around
cp ${REPORT} ${RESULTSDIR}
# compare the results to a known good output # compare the results to a known good output
${ACC_DIR}/compare.py \ ${ACC_DIR}/compare.py \
${GOLDEN_REPORT} \ ${GOLDEN_REPORT} \
${REPORT} ${REPORT} | tee ${RESULTSDIR}/acceptance-${TEST_TYPE}.txt

3
test/acceptance/test-fixtures/acceptance-centos-8.2.2004.json
View File

File diff suppressed because one or more lines are too long