From 1d87f07da1ff91f70ed7ef09bf3f26a8d91a181b Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 18 Mar 2021 09:01:07 -0400 Subject: [PATCH] update pipeline with new levels of testing Signed-off-by: Alex Goodman --- .github/workflows/release.yaml | 42 +- .../workflows/static-unit-integration.yaml | 101 ----- .github/workflows/validations.yaml | 368 ++++++++++++++++++ .gitignore | 1 + Makefile | 85 +++- go.mod | 6 +- go.sum | 11 +- 7 files changed, 470 insertions(+), 144 deletions(-) delete mode 100644 .github/workflows/static-unit-integration.yaml create mode 100644 .github/workflows/validations.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6e9393c3a..1721d9152 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,7 +9,7 @@ on: - "v*" env: - GO_VERSION: "1.14.x" + GO_VERSION: "1.16.x" jobs: wait-for-checks: @@ -29,17 +29,26 @@ jobs: id: static-analysis with: token: ${{ secrets.GITHUB_TOKEN }} - # This check name is defined as the circle-ci workflow name (in .circleci/config.yaml) - checkName: "Static-Analysis (1.x, ubuntu-latest)" + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) + checkName: "Static-Analysis" ref: ${{ github.event.pull_request.head.sha || github.sha }} - - name: Check unit + integration results (latest go version) + - name: Check unit test results uses: fountainhead/action-wait-for-check@v1.0.0 - id: unit-integration + id: unit with: token: ${{ secrets.GITHUB_TOKEN }} - # This check name is defined as the circle-ci workflow name (in .circleci/config.yaml) - checkName: "Tests (1.x, ubuntu-latest)" + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) + checkName: "Unit-Test" + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Check integration test results + uses: fountainhead/action-wait-for-check@v1.0.0 + id: integration + with: + token: ${{ secrets.GITHUB_TOKEN }} + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) + checkName: "Integration-Test" ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Check acceptance test results (linux) @@ -47,7 +56,7 @@ jobs: id: acceptance-linux with: token: ${{ secrets.GITHUB_TOKEN }} - # This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml) + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) checkName: "Acceptance-Linux" ref: ${{ github.event.pull_request.head.sha || github.sha }} @@ -56,27 +65,28 @@ jobs: id: acceptance-mac with: token: ${{ secrets.GITHUB_TOKEN }} - # This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml) + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) checkName: "Acceptance-Mac" ref: ${{ github.event.pull_request.head.sha || github.sha }} - - name: Check inline comparison test results + - name: Check cli test results (linux) uses: fountainhead/action-wait-for-check@v1.0.0 - id: inline-compare + id: cli-linux with: token: ${{ secrets.GITHUB_TOKEN }} - # This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml) - checkName: "Inline-Compare" + # This check name is defined as the github action job name (in .github/workflows/testing.yaml) + checkName: "Cli-Linux" ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Quality gate - if: steps.static-analysis.outputs.conclusion != 'success' || steps.unit-integration.outputs.conclusion != 'success' || steps.inline-compare.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success' + if: steps.static-analysis.outputs.conclusion != 'success' || steps.unit.outputs.conclusion != 'success' || steps.integration.outputs.conclusion != 'success' || steps.cli-linux.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success' run: | echo "Static Analysis Status: ${{ steps.static-analysis.conclusion }}" - echo "Unit & Integration Test Status: ${{ steps.unit-integration.outputs.conclusion }}" + echo "Unit Test Status: ${{ steps.unit.outputs.conclusion }}" + echo "Integration Test Status: ${{ steps.integration.outputs.conclusion }}" echo "Acceptance Test (Linux) Status: ${{ steps.acceptance-linux.outputs.conclusion }}" echo "Acceptance Test (Mac) Status: ${{ steps.acceptance-mac.outputs.conclusion }}" - echo "Inline Compare Status: ${{ steps.inline-compare.outputs.conclusion }}" + echo "CLI Test (Linux) Status: ${{ steps.cli-linux.outputs.conclusion }}" false release: diff --git a/.github/workflows/static-unit-integration.yaml b/.github/workflows/static-unit-integration.yaml deleted file mode 100644 index eac581c30..000000000 --- a/.github/workflows/static-unit-integration.yaml +++ /dev/null @@ -1,101 +0,0 @@ -name: "Static Analysis + Unit + Integration" -on: - workflow_dispatch: - push: - pull_request: -jobs: - Static-Analysis: - strategy: - matrix: - go-version: [1.x] - platform: [ubuntu-latest] - runs-on: ${{ matrix.platform }} - steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - - uses: actions/checkout@v2 - - - name: Restore bootstrap cache - id: bootstrap-cache - uses: actions/cache@v2.1.3 - with: - path: | - ~/go/pkg/mod - ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}-${{ hashFiles('Makefile') }} - restore-keys: | - ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}- - ${{ runner.os }}-go-${{ matrix.go-version }}- - - - name: Bootstrap project dependencies - if: steps.bootstrap-cache.outputs.cache-hit != 'true' - run: make bootstrap - - - name: Bootstrap CI dependencies - run: make ci-bootstrap - - - name: Run static analysis - run: make static-analysis - - Tests: - strategy: - matrix: - # test the lower bounds of support, and the latest available - go-version: [1.13.x, 1.x] - platform: [ubuntu-latest] - runs-on: ${{ matrix.platform }} - steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - - uses: actions/checkout@v2 - - - name: Restore bootstrap cache - id: bootstrap-cache - uses: actions/cache@v2.1.3 - with: - path: | - ~/go/pkg/mod - ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}-${{ hashFiles('Makefile') }} - restore-keys: | - ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}- - ${{ runner.os }}-go-${{ matrix.go-version }}- - - - name: Bootstrap project dependencies - if: steps.bootstrap-cache.outputs.cache-hit != 'true' - run: make bootstrap - - - name: Bootstrap CI dependencies - run: make ci-bootstrap - - - name: Build cache key for java test-fixture blobs (for unit tests) - run: make java-packages-fingerprint - - - name: Restore Java test-fixture cache - id: unit-java-cache - uses: actions/cache@v2.1.3 - with: - path: syft/cataloger/java/test-fixtures/java-builds/packages - key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/cataloger/java/test-fixtures/java-builds/packages.fingerprint' ) }} - - - name: Run unit tests - run: make unit - - - name: Validate syft output against the CycloneDX schema - run: make validate-cyclonedx-schema - - - name: Build key for tar cache - run: make integration-fingerprint - - - name: Restore integration test cache - uses: actions/cache@v2.1.3 - with: - path: ${{ github.workspace }}/integration/test-fixtures/cache - key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('integration/test-fixtures/cache.fingerprint') }} - - - name: Run integration tests - run: make integration diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml new file mode 100644 index 000000000..5b8560d09 --- /dev/null +++ b/.github/workflows/validations.yaml @@ -0,0 +1,368 @@ +name: "Validations" +on: + workflow_dispatch: + push: + pull_request: + +env: + GO_VERSION: "1.16.x" + +jobs: + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Static-Analysis: + name: "Static analysis" + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/checkout@v2 + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap all project dependencies + if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap + + - name: Bootstrap CI environment dependencies + run: make ci-bootstrap + + - name: Run static analysis + run: make static-analysis + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Unit-Test: + name: "Unit tests" + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/checkout@v2 + + - name: Restore docker cache + uses: satackey/action-docker-layer-caching@v0.0.11 + continue-on-error: true + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap all project dependencies + if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap + + - name: Bootstrap CI environment dependencies + run: make ci-bootstrap + + - name: Build cache key for java test-fixture blobs (for unit tests) + run: make java-packages-fingerprint + + - name: Restore Java test-fixture cache + id: unit-java-cache + uses: actions/cache@v2.1.3 + with: + path: syft/pkg/cataloger/java/test-fixtures/java-builds/packages + key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/pkg/cataloger/java/test-fixtures/java-builds/packages.fingerprint' ) }} + + - name: Run unit tests + run: make unit + + - uses: actions/upload-artifact@v2 + with: + name: unit-test-results + path: test/results/**/* + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Integration-Test: + name: "Integration tests" + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/checkout@v2 + + - name: Restore docker cache + uses: satackey/action-docker-layer-caching@v0.0.11 + continue-on-error: true + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap all project dependencies + if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap + + - name: Bootstrap CI environment dependencies + run: make ci-bootstrap + + - name: Validate syft output against the CycloneDX schema + run: make validate-cyclonedx-schema + + - name: Build key for tar cache + run: make integration-fingerprint + + - name: Restore integration test cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/test/integration/test-fixtures/cache + key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('test/integration/test-fixtures/cache.fingerprint') }} + + - name: Run integration tests + run: make integration + + Benchmark-Test: + name: "Benchmark tests" + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/checkout@v2 + + - name: Restore docker cache + uses: satackey/action-docker-layer-caching@v0.0.11 + continue-on-error: true + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap all project dependencies + if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap + + - name: Bootstrap CI environment dependencies + run: make ci-bootstrap + + - name: Restore base benchmark result + uses: actions/cache@v2 + with: + path: test/results/benchmark-main.txt + # use base sha for PR or new commit hash for main push in benchmark result key + key: ${{ runner.os }}-bench-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }} + + - name: Run benchmark tests + id: benchmark + run: | + REF_NAME=${GITHUB_REF##*/} make benchmark + OUTPUT=$(make show-benchstat) + OUTPUT="${OUTPUT//'%'/'%25'}" + OUTPUT="${OUTPUT//$'\n'/'%0A'}" + OUTPUT="${OUTPUT//$'\r'/'%0D'}" + echo "::set-output name=result::$OUTPUT" + + - uses: actions/upload-artifact@v2 + with: + name: branchmark-test-results + path: test/results/**/* + + - name: Update PR benchmark results comment + uses: marocchino/sticky-pull-request-comment@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + header: benchmark + message: | + ### Benchmark Test Results + +
+ Benchmark results from the latest changes vs base branch + + ``` + ${{ steps.benchmark.outputs.result }} + ``` + +
+ + Build-Snapshot-Artifacts: + name: "Build snapshot artifacts" + runs-on: macos-latest # We're creating these snapshot builds on macOS to be consistent with our release workflow's build process, which also takes place on macOS (due to code signing requirements). + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/checkout@v2 + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap all project dependencies + if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap + + - name: Build snapshot artifacts + run: make snapshot + + - uses: actions/upload-artifact@v2 + with: + name: artifacts + path: snapshot/**/* + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Acceptance-Linux: + name: "Acceptance tests (Linux)" + needs: [Build-Snapshot-Artifacts] + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: artifacts + path: snapshot + + - name: Run Acceptance Tests (Linux) + run: make acceptance-linux + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Acceptance-Mac: + name: "Acceptance tests (Mac)" + needs: [Build-Snapshot-Artifacts] + runs-on: macos-latest + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: artifacts + path: snapshot + + - name: Run Acceptance Tests (Mac) + run: make acceptance-mac + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Cli-Linux: + name: "CLI tests (Linux)" + needs: [Build-Snapshot-Artifacts] + runs-on: ubuntu-20.04 + # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR) + # skip if this is a pull_request event on an internal PR (which is already covered by push events) + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + steps: + - uses: actions/checkout@v2 + + - name: Restore docker cache + uses: satackey/action-docker-layer-caching@v0.0.11 + continue-on-error: true + + - name: Restore go cache + id: go-cache + uses: actions/cache@v2.1.3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: (cache-miss) Bootstrap go dependencies + if: steps.go-cache.outputs.cache-hit != 'true' + run: make bootstrap-go + + - name: Build key for tar cache + run: make cli-fingerprint + + - name: Restore CLI test cache + uses: actions/cache@v2.1.3 + with: + path: ${{ github.workspace }}/test/cli/test-fixtures/cache + key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }} + + - uses: actions/download-artifact@v2 + with: + name: artifacts + path: snapshot + + - name: Run CLI Tests (Linux) + run: make cli-linux diff --git a/.gitignore b/.gitignore index 3edd0085e..5e4ca2bee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ CHANGELOG.md +/test/results /dist /snapshot .server/ diff --git a/Makefile b/Makefile index b085a7b76..0caeb4cc0 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ BIN = syft TEMPDIR = ./.tmp -RESULTSDIR = $(TEMPDIR)/results -COVER_REPORT = $(RESULTSDIR)/cover.report -COVER_TOTAL = $(RESULTSDIR)/cover.total +RESULTSDIR = test/results +COVER_REPORT = $(RESULTSDIR)/unit-coverage-details.txt +COVER_TOTAL = $(RESULTSDIR)/unit-coverage-summary.txt LINTCMD = $(TEMPDIR)/golangci-lint run --tests=false --config .golangci.yaml ACC_TEST_IMAGE = centos:8.2.2004 ACC_DIR = ./test/acceptance @@ -19,6 +19,7 @@ COVERAGE_THRESHOLD := 68 # CI cache busting values; change these if you want CI to not use previous stored cache COMPARE_CACHE_BUSTER="f7e689d76a9" INTEGRATION_CACHE_BUSTER="789bacdf" +CLI_CACHE_BUSTER="789bacdf" BOOTSTRAP_CACHE="789bacdf" ## Build variables @@ -57,6 +58,10 @@ ifndef SNAPSHOTDIR $(error SNAPSHOTDIR is not set) endif +ifndef REF_NAME + REF_NAME = $(VERSION) +endif + define title @printf '$(TITLE)$(1)$(RESET)\n' endef @@ -68,7 +73,7 @@ all: clean static-analysis test ## Run all linux-based checks (linting, license @printf '$(SUCCESS)All checks pass!$(RESET)\n' .PHONY: test -test: unit validate-cyclonedx-schema integration acceptance-linux ## Run all tests (currently unit, integration, and linux acceptance tests) +test: unit validate-cyclonedx-schema integration benchmark acceptance-linux ## Run all tests (currently unit, integration, and linux acceptance tests) .PHONY: help help: @@ -78,20 +83,31 @@ help: ci-bootstrap: DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils -.PHONY: bootstrap -bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir) - $(call title,Bootstrapping dependencies) - @pwd - # prep temp dirs - mkdir -p $(TEMPDIR) +.PHONY: +ci-bootstrap-mac: + github_changelog_generator --version || sudo gem install github_changelog_generator + +$(RESULTSDIR): mkdir -p $(RESULTSDIR) - # install go dependencies - go mod download - # install utilities + +$(TEMPDIR): + mkdir -p $(TEMPDIR) + +.PHONY: bootstrap-tools +bootstrap-tools: $(TEMPDIR) + [ -f "$(TEMPDIR)/benchstat" ] || GO111MODULE=off GOBIN=$(shell realpath $(TEMPDIR)) go get -u golang.org/x/perf/cmd/benchstat [ -f "$(TEMPDIR)/golangci" ] || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ v1.26.0 [ -f "$(TEMPDIR)/bouncer" ] || curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ v0.2.0 [ -f "$(TEMPDIR)/goreleaser" ] || curl -sfL https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh | sh -s -- -b $(TEMPDIR)/ v0.140.0 +.PHONY: bootstrap-go +bootstrap-go: + go mod download + +.PHONY: bootstrap +bootstrap: $(RESULTSDIR) bootstrap-go bootstrap-tools ## Download and install all go dependencies (+ prep tooling in the ./tmp dir) + $(call title,Bootstrapping dependencies) + .PHONY: static-analysis static-analysis: lint check-licenses @@ -124,33 +140,45 @@ validate-cyclonedx-schema: cd schema/cyclonedx && make .PHONY: unit -unit: fixtures ## Run unit tests (with coverage) +unit: $(RESULTSDIR) fixtures ## Run unit tests (with coverage) $(call title,Running unit tests) go test -coverprofile $(COVER_REPORT) $(shell go list ./... | grep -v anchore/syft/test) @go tool cover -func $(COVER_REPORT) | grep total | awk '{print substr($$3, 1, length($$3)-1)}' > $(COVER_TOTAL) @echo "Coverage: $$(cat $(COVER_TOTAL))" @if [ $$(echo "$$(cat $(COVER_TOTAL)) >= $(COVERAGE_THRESHOLD)" | bc -l) -ne 1 ]; then echo "$(RED)$(BOLD)Failed coverage quality gate (> $(COVERAGE_THRESHOLD)%)$(RESET)" && false; fi +.PHONY: benchmark +benchmark: $(RESULTSDIR) ## Run benchmark tests and compare against the baseline (if available) + $(call title,Running benchmark tests) + go test -p 1 -run=^Benchmark -bench=. -count=5 -benchmem ./... | tee $(RESULTSDIR)/benchmark-$(REF_NAME).txt + (test -s $(RESULTSDIR)/benchmark-main.txt && \ + $(TEMPDIR)/benchstat $(RESULTSDIR)/benchmark-main.txt $(RESULTSDIR)/benchmark-$(REF_NAME).txt || \ + $(TEMPDIR)/benchstat $(RESULTSDIR)/benchmark-$(REF_NAME).txt) \ + | tee $(RESULTSDIR)/benchstat.txt + +.PHONY: show-benchstat +show-benchstat: + @cat $(RESULTSDIR)/benchstat.txt + .PHONY: integration integration: ## Run integration tests $(call title,Running integration tests) go test -v ./test/integration - # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted integration-fingerprint: find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint .PHONY: java-packages-fingerprint java-packages-fingerprint: - @cd syft/cataloger/java/test-fixtures/java-builds && \ + @cd syft/pkg/cataloger/java/test-fixtures/java-builds && \ make packages.fingerprint .PHONY: fixtures fixtures: $(call title,Generating test fixtures) - cd syft/cataloger/java/test-fixtures/java-builds && make + cd syft/pkg/cataloger/java/test-fixtures/java-builds && make .PHONY: generate-json-schema generate-json-schema: ## Generate a new json schema @@ -175,7 +203,7 @@ $(SNAPSHOTDIR): ## Build snapshot release binaries and packages # note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately .PHONY: acceptance-mac -acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac) +acceptance-mac: $(RESULTSDIR) $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac) $(call title,Running acceptance test: Run on Mac) $(ACC_DIR)/mac.sh \ $(SNAPSHOTDIR) \ @@ -202,7 +230,7 @@ compare: ## Compare the reports of a run of a main-branch build of syft against @cd test/inline-compare && make .PHONY: acceptance-test-deb-package-install -acceptance-test-deb-package-install: $(SNAPSHOTDIR) +acceptance-test-deb-package-install: $(RESULTSDIR) $(SNAPSHOTDIR) $(call title,Running acceptance test: DEB install) $(ACC_DIR)/deb.sh \ $(SNAPSHOTDIR) \ @@ -211,7 +239,7 @@ acceptance-test-deb-package-install: $(SNAPSHOTDIR) $(RESULTSDIR) .PHONY: acceptance-test-rpm-package-install -acceptance-test-rpm-package-install: $(SNAPSHOTDIR) +acceptance-test-rpm-package-install: $(RESULTSDIR) $(SNAPSHOTDIR) $(call title,Running acceptance test: RPM install) $(ACC_DIR)/rpm.sh \ $(SNAPSHOTDIR) \ @@ -219,6 +247,23 @@ acceptance-test-rpm-package-install: $(SNAPSHOTDIR) $(ACC_TEST_IMAGE) \ $(RESULTSDIR) +# note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted +cli-fingerprint: + find test/cli/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/cli/test-fixtures/cache.fingerprint && echo "$(CLI_CACHE_BUSTER)" >> test/cli/test-fixtures/cache.fingerprint + +.PHONY: cli-linux +cli-linux: $(SNAPSHOTDIR) ## Run CLI tests for Linux executable + chmod 755 "$(SNAPSHOTDIR)/$(BIN)_linux_amd64/$(BIN)" + $(SNAPSHOTDIR)/$(BIN)_linux_amd64/$(BIN) version + SYFT_BINARY_LOCATION='$(SNAPSHOTDIR)/$(BIN)_linux_amd64/$(BIN)' \ + go test -count=1 -v ./test/cli + +.PHONY: cli-macos +cli-macos: $(SNAPSHOTDIR) ## Run CLI tests for macOS executable + $(SNAPSHOTDIR)/$(BIN)_linux_amd64/$(BIN) version + SYFT_BINARY_LOCATION='$(SNAPSHOTDIR)/$(BIN)-macos_darwin_amd64/$(BIN)' \ + go test -count=1 -v ./test/cli + .PHONY: changlog-release changelog-release: @echo "Last tag: $(SECOND_TO_LAST_TAG)" diff --git a/go.mod b/go.mod index 038fe513e..f2138d29d 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,14 @@ module github.com/anchore/syft go 1.14 require ( + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/adrg/xdg v0.2.1 - github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 + github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074 github.com/anchore/client-go v0.0.0-20210222170800-9c70f9b80bcf github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b - github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d + github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40 github.com/antihax/optional v1.0.0 github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible @@ -31,6 +32,7 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/spf13/afero v1.2.2 github.com/spf13/cobra v1.0.1-0.20200909172742-8a63648dd905 + github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.0 github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240 diff --git a/go.sum b/go.sum index 625136e08..ba334d43d 100644 --- a/go.sum +++ b/go.sum @@ -96,10 +96,12 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/adrg/xdg v0.2.1 h1:VSVdnH7cQ7V+B33qSJHTCRlNgra1607Q8PzEmnvb2Ic= github.com/adrg/xdg v0.2.1/go.mod h1:ZuOshBmzV4Ta+s23hdfFZnBsdzmoR3US0d7ErpqSbTQ= -github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 h1:T3+cD5fYvuH36h7EZq+TDpm+d8a6FSD4pQsbmuGGQ8o= -github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= +github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074 h1:Lw9q+WyJLFOR+AULchS5/2GKfM+6gOh4szzizdfH3MU= +github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -113,8 +115,8 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= -github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d h1:2hv5NOZ0fD8tPk1UdGiW9PHxmjBmBLL+sFlhLXjjKgo= -github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d/go.mod h1:lhSEYyGLXTXMIFHAz7Ls/MNQ5EjYd5ziLxovKZp1xOs= +github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40 h1:k3/JigkYl7NjMad9eDBBcsg9qiXJFreW6rKNgE0aMUI= +github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40/go.mod h1:T/OUHXgngXFlo2vknQsDx+n/jErCLPt5o0H1ZXFBpig= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -737,7 +739,6 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= github.com/vdemeester/k8s-pkg-credentialprovider v1.17.4/go.mod h1:inCTmtUdr5KJbreVojo06krnTgaeAz/Z7lynpPk/Q2c= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d h1:KOxOL6qpmqwoPloNwi+CEgc1ayjHNOFNrvoOmeDOjDg=