From 6195002ae54056131521f2226ee21a0822ab1019 Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Wed, 10 Mar 2021 13:25:31 -0500 Subject: [PATCH] Add docker image and refactor release pipeline (#310) * Create independent build targets for Mac and Linux Signed-off-by: Dan Luhring * Create targets for macOS signing and notarization Signed-off-by: Dan Luhring * Create target for Linux packaging Signed-off-by: Dan Luhring * Update release workflow and leverage new make targets Signed-off-by: Dan Luhring * Add release assets to release draft Signed-off-by: Dan Luhring * Add homebrew formula release follow-up and improve Makefile Signed-off-by: Dan Luhring * Add follow-up workflow for updating version check file Signed-off-by: Dan Luhring * Get rid of fetch depth 0 for checkout action Signed-off-by: Dan Luhring * Add follow-up workflow for Docker images Signed-off-by: Dan Luhring * Restore wait-for-checks job Signed-off-by: Dan Luhring * Replace make functions with shell functions Signed-off-by: Dan Luhring * Account for envsubst command in bootstrap-ci-linux Signed-off-by: Dan Luhring * move homebrew generation into script Signed-off-by: Alex Goodman * add release approval step; remove goreleaser; add docker image smoke testing in acceptance step Signed-off-by: Alex Goodman * replace homebrew formula template file with heredoc template Signed-off-by: Alex Goodman * update release documentation Signed-off-by: Alex Goodman Co-authored-by: Alex Goodman --- .dockerignore | 3 + .github/scripts/homebrew-formula-generate.sh | 61 ++++ .github/scripts/package-linux.sh | 47 +++ .github/scripts/update-version-file.sh | 8 +- .github/workflows/acceptance-test.yaml | 35 +- .github/workflows/release-follow-up.yaml | 80 +++++ .github/workflows/release.yaml | 146 ++++++-- .../workflows/static-unit-integration.yaml | 4 +- .goreleaser.yaml | 92 ----- Dockerfile | 17 + Makefile | 316 +++++++++++++----- RELEASE.md | 14 +- gon.hcl | 2 +- test/acceptance/mac.sh | 25 +- 14 files changed, 619 insertions(+), 231 deletions(-) create mode 100644 .dockerignore create mode 100755 .github/scripts/homebrew-formula-generate.sh create mode 100755 .github/scripts/package-linux.sh create mode 100644 .github/workflows/release-follow-up.yaml delete mode 100644 .goreleaser.yaml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..6a726b97f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +* +!dist/syft_linux_amd64/syft +!snapshot/syft_linux_amd64/syft diff --git a/.github/scripts/homebrew-formula-generate.sh b/.github/scripts/homebrew-formula-generate.sh new file mode 100755 index 000000000..5778a06b6 --- /dev/null +++ b/.github/scripts/homebrew-formula-generate.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -eu -o pipefail + +BIN="syft" +TEMPDIR=$(mktemp -d syft-homebrew-formula-generate-XXXXXX) +VERSION_TAG=$1 +HOMEBREW_FORMULA_FILE=$2 + +trap "rm -f ${TEMPDIR}/*; rmdir ${TEMPDIR};" EXIT + +# dependencies: curl, jq, openssl + +RELEASE_URL="https://api.github.com/repos/anchore/${BIN}/releases/tags/${VERSION_TAG}" +echo "Using release: ${RELEASE_URL}" +curl -sSL "${RELEASE_URL}" > "${TEMPDIR}/release.json" + +function asset_url() { + cat "${1}" | jq -r ".assets[] | select(.name | contains(\"${2}\")) | .browser_download_url" +} + +function sha256() { + openssl dgst -sha256 "${1}" | cut -d " " -f 2 +} + +export DARWIN_AMD64_ASSET_URL=$(asset_url "${TEMPDIR}/release.json" "darwin_amd64.zip") +curl -sSL "${DARWIN_AMD64_ASSET_URL}" > "${TEMPDIR}/darwin_amd64_asset" +export DARWIN_AMD64_ASSET_SHA256=$(sha256 "${TEMPDIR}/darwin_amd64_asset") + +export LINUX_AMD64_ASSET_URL=$(asset_url "${TEMPDIR}/release.json" "linux_amd64.tar.gz") +curl -sSL "${LINUX_AMD64_ASSET_URL}" > "${TEMPDIR}/linux_amd64_asset" +export LINUX_AMD64_ASSET_SHA256=$(sha256 "${TEMPDIR}/linux_amd64_asset") + +export VERSION=${VERSION_TAG#v} + +cat > ${HOMEBREW_FORMULA_FILE} <<-EOF +class Syft < Formula + desc "A tool that generates a Software Bill Of Materials (SBOM) from container images and filesystems" + homepage "https://github.com/anchore/syft" + version "$VERSION" + bottle :unneeded + + if OS.mac? + if Hardware::CPU.intel? + url "$DARWIN_AMD64_ASSET_URL" + sha256 "$DARWIN_AMD64_ASSET_SHA256" + end + elsif OS.linux? + if Hardware::CPU.intel? + url "$LINUX_AMD64_ASSET_URL" + sha256 "$LINUX_AMD64_ASSET_SHA256" + end + end + + def install + bin.install "syft" + end +end +EOF + +echo "Generated ${HOMEBREW_FORMULA_FILE}:" +cat ${HOMEBREW_FORMULA_FILE} \ No newline at end of file diff --git a/.github/scripts/package-linux.sh b/.github/scripts/package-linux.sh new file mode 100755 index 000000000..cf3e63c68 --- /dev/null +++ b/.github/scripts/package-linux.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -eu -o pipefail + +BIN="syft" +DISTDIR=$1 +VERSION=$2 +TEMPDIR=$3 + +SYFT_BIN_PATH=${DISTDIR}/${BIN}_linux_amd64/${BIN} + +# stage the release tar directory +WORK_DIR=$(mktemp -d -t "syft-packaging-XXXXXX") +trap "rm -f ${WORK_DIR}/*; rmdir ${WORK_DIR};" EXIT + +cp ./README.md ${WORK_DIR} +cp ./LICENSE ${WORK_DIR} +cp ${SYFT_BIN_PATH} ${WORK_DIR} + +# produce .tar.gz +tar -cvzf "${DISTDIR}/${BIN}_${VERSION}_linux_amd64.tar.gz" -C ${WORK_DIR} . + +# produce .deb, .rpm +NFPM_CONFIG=$(mktemp -t "syft-nfpm-cfg-XXXXXX") +cat > ${NFPM_CONFIG} <<-EOF +name: "syft" +license: "Apache 2.0" +maintainer: "Anchore, Inc" +homepage: "https://github.com/anchore/syft" +description: "A tool that generates a Software Bill Of Materials (SBOM) from container images and filesystems" +contents: + - src: ${SYFT_BIN_PATH} + dst: /usr/local/bin/syft +EOF + +for packager in "deb" "rpm"; do + ${TEMPDIR}/nfpm -f ${NFPM_CONFIG} pkg --packager="$packager" --target="${DISTDIR}/${BIN}_${VERSION}_linux_amd64.$packager" +done + +# produce integrity-check files (checksums.txt, checksums.txt.sig) +pushd "${DISTDIR}" + CHECKSUMS_FILE="${BIN}_${VERSION}_checksums.txt" + echo "" > "$CHECKSUMS_FILE" + for file in ./*linux*.*; do + openssl dgst -sha256 "$file" >> "$CHECKSUMS_FILE" + done + gpg --detach-sign "$CHECKSUMS_FILE" +popd diff --git a/.github/scripts/update-version-file.sh b/.github/scripts/update-version-file.sh index 9d416df79..009ec4de9 100755 --- a/.github/scripts/update-version-file.sh +++ b/.github/scripts/update-version-file.sh @@ -3,12 +3,12 @@ set -ue BIN="syft" DISTDIR=$1 -VERSION=$2 +VERSION_TAG=$2 # the source of truth as to whether we want to notify users of an update is if the release just created is NOT # flagged as a pre-release on github -if [[ "$(curl -SsL https://api.github.com/repos/anchore/${BIN}/releases/tags/${VERSION} | jq .prerelease)" == "true" ]] ; then - echo "skipping publishing a version file (this is a pre-release: ${VERSION})" +if [[ "$(curl -SsL https://api.github.com/repos/anchore/${BIN}/releases/tags/${VERSION_TAG} | jq .prerelease)" == "true" ]] ; then + echo "skipping publishing a version file (this is a pre-release: ${VERSION_TAG})" exit 0 fi @@ -16,7 +16,7 @@ echo "creating and publishing version file" # create a version file for version-update checks VERSION_FILE="${DISTDIR}/VERSION" -echo "${VERSION}" | tee "${VERSION_FILE}" +echo "${VERSION_TAG}" | tee "${VERSION_FILE}" # upload the version file that supports the application version update check export AWS_DEFAULT_REGION=us-west-2 diff --git a/.github/workflows/acceptance-test.yaml b/.github/workflows/acceptance-test.yaml index dfc857813..d90da319d 100644 --- a/.github/workflows/acceptance-test.yaml +++ b/.github/workflows/acceptance-test.yaml @@ -10,11 +10,11 @@ on: - v* env: - GO_VERSION: "1.14.x" + GO_VERSION: "1.15.x" jobs: Build-Snapshot-Artifacts: - runs-on: macos-latest # We're creating these snapshot builds on macOS to be consistent with our release workflow's build process, which also takes place on macOS (due to code signing requirements). + runs-on: ubuntu-latest steps: - uses: actions/setup-go@v2 with: @@ -52,7 +52,7 @@ jobs: echo "name: ${{ steps.import_gpg.outputs.name }}" echo "email: ${{ steps.import_gpg.outputs.email }}" - - name: Build snapshot artifacts + - name: Build & package snapshot artifacts run: make snapshot env: GPG_PRIVATE_KEY: ${{ secrets.SIGNING_GPG_PRIVATE_KEY }} @@ -75,7 +75,6 @@ jobs: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline Acceptance-Linux: needs: [Build-Snapshot-Artifacts] - # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 @@ -124,7 +123,6 @@ jobs: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline Inline-Compare: needs: [Build-Snapshot-Artifacts] - # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 @@ -155,3 +153,30 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }} if: ${{ failure() }} + + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + Smoke-Test-Container-Image: + needs: [Build-Snapshot-Artifacts] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: artifacts + path: snapshot + + - name: Load docker image + run: docker image load --input snapshot/image.tar + + - name: Test Docker images + run: make container-image-smoke-test + + - uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,workflow,job,commit,message,author + text: The syft acceptance tests have failed tragically! + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }} + if: ${{ failure() }} diff --git a/.github/workflows/release-follow-up.yaml b/.github/workflows/release-follow-up.yaml new file mode 100644 index 000000000..a3f24e97b --- /dev/null +++ b/.github/workflows/release-follow-up.yaml @@ -0,0 +1,80 @@ +name: "Release follow-up" +on: + release: + types: [published] + +jobs: + create-homebrew-formula: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Generate homebrew formula + run: make homebrew-formula-generate + + - uses: actions/upload-artifact@v2 + with: + path: "./dist/syft.rb" + + test-formula-linux: + runs-on: ubuntu-latest + needs: create-homebrew-formula + steps: + - uses: actions/download-artifact@v2 + with: + path: "dist" + + - name: Test homebrew formula + run: make homebrew-formula-test + + test-formula-mac: + runs-on: macos-latest + needs: create-homebrew-formula + steps: + - uses: actions/download-artifact@v2 + with: + path: "dist" + + - name: Test homebrew formula + run: make homebrew-formula-test + + publish-formula: + runs-on: ubuntu-latest + needs: [ test-formula-linux, test-formula-mac ] + steps: + - uses: actions/download-artifact@v2 + + - name: Publish updated homebrew formula + run: homebrew-formula-publish + + update_version_check_file: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Update version check file + run: make version-check-update + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }} + + build_and_push_container_image: + runs-on: ubuntu-latest + steps: + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.TOOLBOX_DOCKER_USER }} + password: ${{ secrets.TOOLBOX_DOCKER_PASS }} + + - name: Stage released artifacts + run: make stage-released-linux-artifact + + - name: Build and tag Docker images + run: make container-image-build + + - name: Smoke test Docker image + run: make container-image-smoke-test + + - name: Push Docker images + run: make container-image-push diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5d4f57d4e..47a83ded6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,12 +9,14 @@ on: - "v*" env: - GO_VERSION: "1.14.x" + GO_VERSION: "1.15.x" jobs: - wait-for-checks: + quality-gate: + environment: release runs-on: ubuntu-latest # This OS choice is arbitrary. None of the steps in this job are specific to either Linux or macOS. steps: + - uses: actions/checkout@v2 # we don't want to release commits that have been pushed and tagged, but not necessarily merged onto main @@ -69,27 +71,74 @@ jobs: checkName: "Inline-Compare" ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Check container image smoke test results + uses: fountainhead/action-wait-for-check@v1.0.0 + id: smoke-test-container-image + with: + token: ${{ secrets.GITHUB_TOKEN }} + # This check name is defined as the circle-ci workflow name (in .github/workflows/acceptance-test.yaml) + checkName: "Smoke-Test-Container-Image" + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Quality gate - if: steps.static-analysis.outputs.conclusion != 'success' || steps.unit-integration.outputs.conclusion != 'success' || steps.inline-compare.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success' + if: steps.static-analysis.outputs.conclusion != 'success' || steps.unit-integration.outputs.conclusion != 'success' || steps.inline-compare.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success' || steps.smoke-test-container-image.outputs.conclusion != 'success' run: | echo "Static Analysis Status: ${{ steps.static-analysis.conclusion }}" echo "Unit & Integration Test Status: ${{ steps.unit-integration.outputs.conclusion }}" echo "Acceptance Test (Linux) Status: ${{ steps.acceptance-linux.outputs.conclusion }}" echo "Acceptance Test (Mac) Status: ${{ steps.acceptance-mac.outputs.conclusion }}" echo "Inline Compare Status: ${{ steps.inline-compare.outputs.conclusion }}" + echo "Smoke Test Container Image Status: ${{ steps.smoke-test-container-image.outputs.conclusion }}" false - release: - needs: [wait-for-checks] - runs-on: macos-latest # Due to our code signing process, it's vital that we run our release steps on macOS. + build-assets-mac: + needs: [ quality-gate ] + runs-on: macos-latest # Due to our code signing process, it's vital that we run these release steps on macOS. steps: - uses: actions/setup-go@v2 with: go-version: ${{ env.GO_VERSION }} - uses: actions/checkout@v2 + + # We are expecting this cache to have been created during the "Build-Snapshot-Artifacts" job in the "Acceptance" workflow. + - name: Restore bootstrap cache + id: cache + uses: actions/cache@v2.1.3 with: - fetch-depth: 0 + path: | + ~/go/pkg/mod + ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}-${{ hashFiles('Makefile') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}- + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + - name: Build for macOS + run: make build-mac + + - name: Create macOS release assets + run: make package-mac + env: + APPLE_DEVELOPER_ID_CERT: ${{ secrets.APPLE_DEVELOPER_ID_CERT }} # Used during macOS code signing. + APPLE_DEVELOPER_ID_CERT_PASS: ${{ secrets.APPLE_DEVELOPER_ID_CERT_PASS }} # Used during macOS code signing. + AC_USERNAME: ${{ secrets.ENG_CI_APPLE_ID }} # Used during macOS notarization. + AC_PASSWORD: ${{ secrets.ENG_CI_APPLE_ID_PASS }} # Used during macOS notarization. + + - uses: actions/upload-artifact@v2 + with: + name: macOS-artifacts + path: "./dist/*_darwin_*.*" + + build-assets-linux: + needs: [ quality-gate ] + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/checkout@v2 # We are expecting this cache to have been created during the "Build-Snapshot-Artifacts" job in the "Acceptance" workflow. - name: Restore bootstrap cache @@ -118,19 +167,77 @@ jobs: echo "name: ${{ steps.import_gpg.outputs.name }}" echo "email: ${{ steps.import_gpg.outputs.email }}" - - name: Build & publish release artifacts - run: make release + - name: Build Linux assets + run: make build-linux + + - name: Package Linux release assets + run: make package-linux env: - GITHUB_TOKEN: ${{ secrets.ANCHORE_GIT_READ_TOKEN }} GPG_PRIVATE_KEY: ${{ secrets.SIGNING_GPG_PRIVATE_KEY }} PASSPHRASE: ${{ secrets.SIGNING_GPG_PASSPHRASE }} - SIGNING_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }} - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }} - APPLE_DEVELOPER_ID_CERT: ${{ secrets.APPLE_DEVELOPER_ID_CERT }} # Used during macOS code signing. - APPLE_DEVELOPER_ID_CERT_PASS: ${{ secrets.APPLE_DEVELOPER_ID_CERT_PASS }} # Used during macOS code signing. - AC_USERNAME: ${{ secrets.ENG_CI_APPLE_ID }} # Used during macOS notarization. - AC_PASSWORD: ${{ secrets.ENG_CI_APPLE_ID_PASS }} # Used during macOS notarization. + + - uses: actions/upload-artifact@v2 + with: + name: Linux-artifacts + path: | + ./dist/*_linux_*.* + ./dist/*_checksums.* + + draft-release: + needs: [ build-assets-mac, build-assets-linux ] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # fetch all history --this is necessary since we are referencing multiple tags during the release process (for changelog generation) + fetch-depth: 0 + + # We are expecting this cache to have been created during the "Build-Snapshot-Artifacts" job in the "Acceptance" workflow. + - name: Restore bootstrap cache + id: cache + uses: actions/cache@v2.1.3 + with: + path: | + ~/go/pkg/mod + ${{ github.workspace }}/.tmp + key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}-${{ hashFiles('Makefile') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}- + ${{ runner.os }}-go-${{ env.GO_VERSION }}- + + + - uses: actions/download-artifact@v2 # Downloads all artifacts + + - name: Generate changelog + run: make changelog-release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + + - name: Create draft release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + body_path: "./CHANGELOG.md" + draft: true + prerelease: false + + - name: Upload release assets + run: | # Solution found at https://github.com/actions/upload-release-asset/issues/28#issuecomment-617208601 after seeing that the native "actions/upload-release-asset" might not be actively maintained. + set -eux + assets=() + for asset in ./Linux-artifacts/*; do + assets+=("-a" "$asset") + done + for asset in ./macOS-artifacts/*; do + assets+=("-a" "$asset") + done + tag_name="${GITHUB_REF##*/}" + hub release edit -m "" "${assets[@]}" "$tag_name" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: 8398a7/action-slack@v3 with: @@ -140,8 +247,3 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }} if: ${{ success() }} - - - uses: actions/upload-artifact@v2 - with: - name: artifacts - path: dist/**/* diff --git a/.github/workflows/static-unit-integration.yaml b/.github/workflows/static-unit-integration.yaml index eac581c30..6ab3ba3a6 100644 --- a/.github/workflows/static-unit-integration.yaml +++ b/.github/workflows/static-unit-integration.yaml @@ -34,7 +34,7 @@ jobs: run: make bootstrap - name: Bootstrap CI dependencies - run: make ci-bootstrap + run: make bootstrap-ci-linux - name: Run static analysis run: make static-analysis @@ -70,7 +70,7 @@ jobs: run: make bootstrap - name: Bootstrap CI dependencies - run: make ci-bootstrap + run: make bootstrap-ci-linux - name: Build cache key for java test-fixture blobs (for unit tests) run: make java-packages-fingerprint diff --git a/.goreleaser.yaml b/.goreleaser.yaml deleted file mode 100644 index f1fd98345..000000000 --- a/.goreleaser.yaml +++ /dev/null @@ -1,92 +0,0 @@ -release: - # If set to auto, will mark the release as not ready for production in case there is an indicator for this in the - # tag e.g. v1.0.0-rc1 .If set to true, will mark the release as not ready for production. - prerelease: auto - - # If set to true, will not auto-publish the release. This is done to allow us to review the changelog before publishing. - draft: true - -builds: - - binary: syft - id: syft - env: - - CGO_ENABLED=0 - goos: - # windows not supported yet (due to jotframe) - # - windows - - linux - goarch: - - amd64 - # Set the modified timestamp on the output binary to the git timestamp (to ensure a reproducible build) - mod_timestamp: '{{ .CommitTimestamp }}' - ldflags: | - -w - -s - -extldflags '-static' - -X github.com/anchore/syft/internal/version.version={{.Version}} - -X github.com/anchore/syft/internal/version.gitCommit={{.Commit}} - -X github.com/anchore/syft/internal/version.buildDate={{.Date}} - -X github.com/anchore/syft/internal/version.gitTreeState={{.Env.BUILD_GIT_TREE_STATE}} - # For more info on this macOS build, see: https://github.com/mitchellh/gon#usage-with-goreleaser - - binary: syft - id: syft-macos - env: - - CGO_ENABLED=0 - goos: - - darwin - goarch: - - amd64 - # Set the modified timestamp on the output binary to the git timestamp (to ensure a reproducible build) - mod_timestamp: '{{ .CommitTimestamp }}' - ldflags: | - -w - -s - -extldflags '-static' - -X github.com/anchore/syft/internal/version.version={{.Version}} - -X github.com/anchore/syft/internal/version.gitCommit={{.Commit}} - -X github.com/anchore/syft/internal/version.buildDate={{.Date}} - -X github.com/anchore/syft/internal/version.gitTreeState={{.Env.BUILD_GIT_TREE_STATE}} - -archives: - - format: tar.gz - builds: - - syft # i.e. Linux only - - format: zip # This is a hack! We don't actually intend to use _this_ ZIP file, we just need goreleaser to consider the ZIP file produced by gon (which will have the same file name) to be an artifact so we can use it downstream in publishing (e.g. to a homebrew tap) - id: syft-zip - builds: - - syft-macos - -signs: - - artifacts: checksum - cmd: sh - args: - - '-c' - # we should not include the zip artifact, as the artifact is mutated throughout the next macOS notarization step - # note: sed -i is not portable - - 'sed "/.*\.zip/d" ${artifact} > tmpfile && mv tmpfile ${artifact} && gpg --output ${signature} --detach-sign ${artifact}' - - id: syft-macos-signing - ids: - - syft-macos - cmd: ./.github/scripts/mac-sign-and-notarize.sh - signature: "syft_${VERSION}_darwin_amd64.dmg" # This is somewhat unintuitive. This gets the DMG file recognized as an artifact. In fact, both a DMG and a ZIP file are being produced by this signing step. - args: - - "{{ .IsSnapshot }}" - - "gon.hcl" - - "./dist/syft_{{ .Version }}_darwin_amd64" - artifacts: all - -nfpms: - - license: "Apache 2.0" - maintainer: "Anchore, Inc" - homepage: &website "https://github.com/anchore/syft" - description: &description "A tool that generates a Software Bill Of Materials (SBOM) from container images and filesystems" - formats: - - rpm - - deb - -brews: - - tap: - owner: anchore - name: homebrew-syft - homepage: *website - description: *description diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..f340de886 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM alpine:latest AS build + +RUN apk --no-cache add ca-certificates +RUN mkdir -p /tmp + +# ————————————————————————————————————————————————————————————————————— +FROM scratch +ARG DIST_DIR=./dist + +# Needed for version check HTTPS request +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +# Needed for image content cache +COPY --from=build /tmp / + +COPY ${DIST_DIR}/syft_linux_amd64/syft / + +ENTRYPOINT ["/syft"] diff --git a/Makefile b/Makefile index e77a2d96c..0bdfba6f1 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,15 @@ -BIN = syft -TEMPDIR = ./.tmp +.SHELLFLAGS := -o pipefail -ec +SHELL := /bin/bash + +BIN := syft +TEMPDIR := ./.tmp RESULTSDIR = $(TEMPDIR)/results COVER_REPORT = $(RESULTSDIR)/cover.report COVER_TOTAL = $(RESULTSDIR)/cover.total LINTCMD = $(TEMPDIR)/golangci-lint run --tests=false --config .golangci.yaml -ACC_TEST_IMAGE = centos:8.2.2004 -ACC_DIR = ./test/acceptance +SNAPSHOT_CMD = $(shell realpath $(shell pwd)/$(SNAPSHOTDIR)/syft_linux_amd64/syft) +ACC_TEST_IMAGE := centos:8.2.2004 +ACC_DIR := ./test/acceptance BOLD := $(shell tput -T linux bold) PURPLE := $(shell tput -T linux setaf 5) GREEN := $(shell tput -T linux setaf 2) @@ -22,18 +26,39 @@ INTEGRATION_CACHE_BUSTER="789bacdf" BOOTSTRAP_CACHE="789bacdf" ## Build variables -DISTDIR=./dist -SNAPSHOTDIR=./snapshot -GITTREESTATE=$(if $(shell git status --porcelain),dirty,clean) -SNAPSHOT_CMD=$(shell realpath $(shell pwd)/$(SNAPSHOTDIR)/syft_linux_amd64/syft) +DISTDIR := ./dist +SNAPSHOTDIR := ./snapshot +COMMIT = $(shell git log --format=%H -n 1) +DATE = $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") +GITTREESTATE = $(if $(shell git status --porcelain),dirty,clean) -ifeq "$(strip $(VERSION))" "" - override VERSION = $(shell git describe --always --tags --dirty) +# Homebrew variables +HOMEBREW_FORMULA_FILE = "$(DISTDIR)/$(BIN).rb" +BREW_DIR = "$(TEMPDIR)/homebrew" +BREW_BIN_DIR = "$(BREW_DIR)/bin" +BREW_CMD = "$(BREW_BIN_DIR)/brew" + +ifeq "$(strip $(VERSION_TAG))" "" + override VERSION_TAG = $(shell git describe --always --tags --dirty) endif +# Version variables and functions +is_dirty = $(findstring dirty,$(1)) +get_version_from_version_tag = $(shell echo "$(1)" | tr -d 'v') +VERSION = $(call get_version_from_version_tag,$(VERSION_TAG)) +major = $(shell echo "$(1)" | cut -d '.' -f 1) +minor = $(shell echo "$(1)" | cut -d '.' -f 2) +patch = $(shell echo "$(1)" | cut -d '.' -f 3) + # used to generate the changelog from the second to last tag to the current tag (used in the release pipeline when the release tag is in place) -LAST_TAG := $(shell git describe --abbrev=0 --tags $(shell git rev-list --tags --max-count=1)) -SECOND_TO_LAST_TAG := $(shell git describe --abbrev=0 --tags $(shell git rev-list --tags --skip=1 --max-count=1)) +LAST_TAG = $(shell git describe --abbrev=0 --tags $(shell git rev-list --tags --max-count=1)) +SECOND_TO_LAST_TAG = $(shell git describe --abbrev=0 --tags $(shell git rev-list --tags --skip=1 --max-count=1)) + +CONTAINER_IMAGE_REPOSITORY := "anchore/$(BIN)" +CONTAINER_IMAGE_TAG_MAJOR := "$(CONTAINER_IMAGE_REPOSITORY):$(call major,$(VERSION))" +CONTAINER_IMAGE_TAG_MINOR := "$(CONTAINER_IMAGE_REPOSITORY):$(call major,$(VERSION)).$(call minor,$(VERSION))" +CONTAINER_IMAGE_TAG_PATCH := "$(CONTAINER_IMAGE_REPOSITORY):$(call major,$(VERSION)).$(call minor,$(VERSION)).$(call patch,$(VERSION))" +CONTAINER_IMAGE_TAG_LATEST := "$(CONTAINER_IMAGE_REPOSITORY):latest" ## Variable assertions @@ -53,14 +78,34 @@ ifndef DISTDIR $(error DISTDIR is not set) endif -ifndef SNAPSHOTDIR - $(error SNAPSHOTDIR is not set) -endif - define title @printf '$(TITLE)$(1)$(RESET)\n' endef +define build_binary + GOOS="$1" \ + GOARCH="$2" \ + CGO_ENABLED=0 \ + go build \ + -o "./$3/syft_$1_$2/syft" \ + -ldflags "-w -s -extldflags '-static' \ + -X github.com/anchore/syft/internal/version.version=$(VERSION) \ + -X github.com/anchore/syft/internal/version.gitCommit=$(COMMIT) \ + -X github.com/anchore/syft/internal/version.buildDate=$(DATE) \ + -X github.com/anchore/syft/internal/version.gitTreeState=$(BUILD_GIT_TREE_STATE)" +endef + +define build_container_image + tags=( \ + "-t $(CONTAINER_IMAGE_TAG_MAJOR)" \ + "-t $(CONTAINER_IMAGE_TAG_MINOR)" \ + "-t $(CONTAINER_IMAGE_TAG_PATCH)" \ + "-t $(CONTAINER_IMAGE_TAG_LATEST)" \ + ) && \ + DOCKER_BUILDKIT=1 docker build --build-arg DIST_DIR=$1 --no-cache $${tags[@]} -f "./Dockerfile" . + # Using buildkit due to https://github.com/moby/moby/issues/37965 +endef + ## Tasks .PHONY: all @@ -74,14 +119,6 @@ test: unit validate-cyclonedx-schema integration acceptance-linux ## Run all tes help: @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}' -.PHONY: ci-bootstrap -ci-bootstrap: - DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils - -.PHONY: -ci-bootstrap-mac: - github_changelog_generator --version || sudo gem install github_changelog_generator - .PHONY: bootstrap bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir) $(call title,Bootstrapping dependencies) @@ -94,7 +131,15 @@ bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./ # install utilities [ -f "$(TEMPDIR)/golangci" ] || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ v1.26.0 [ -f "$(TEMPDIR)/bouncer" ] || curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ v0.2.0 - [ -f "$(TEMPDIR)/goreleaser" ] || curl -sfL https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh | sh -s -- -b $(TEMPDIR)/ v0.140.0 + [ -f "$(TEMPDIR)/nfpm" ] || curl -sfL curl -sfL https://install.goreleaser.com/github.com/goreleaser/nfpm.sh | sh -s -- -b $(TEMPDIR)/ v2.2.2 + [ -f "$(BREW_CMD)" ] || (mkdir -p "$(BREW_DIR)" && curl -L https://github.com/Homebrew/brew/tarball/master | tar -xz --strip 1 -C "$(BREW_DIR)") + +.PHONY: bootstrap-ci-linux +bootstrap-ci-linux: bootstrap + DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils gettext + +.PHONY: bootstrap-ci-mac +bootstrap-ci-mac: bootstrap .PHONY: static-analysis static-analysis: lint check-licenses @@ -138,10 +183,8 @@ unit: fixtures ## Run unit tests (with coverage) .PHONY: integration integration: ## Run integration tests $(call title,Running integration tests) - go test -v ./test/integration - # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted integration-fingerprint: find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint @@ -164,22 +207,32 @@ generate-json-schema: ## Generate a new json schema clear-test-cache: ## Delete all test cache (built docker image tars) find . -type f -wholename "**/test-fixtures/cache/*.tar" -delete -.PHONY: build -build: $(SNAPSHOTDIR) ## Build release snapshot binaries and packages +.PHONY: build-linux +build-linux: ## Build binaries for mac + $(call title,Building binaries for linux) + $(call build_binary,linux,amd64,$(DISTDIR)) -$(SNAPSHOTDIR): ## Build snapshot release binaries and packages - $(call title,Building snapshot artifacts) - # create a config with the dist dir overridden - echo "dist: $(SNAPSHOTDIR)" > $(TEMPDIR)/goreleaser.yaml - cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml +.PHONY: build-mac +build-mac: ## Build binaries for mac + $(call title,Building binaries for macOS) + $(call build_binary,darwin,amd64,$(DISTDIR)) - # build release snapshots - BUILD_GIT_TREE_STATE=$(GITTREESTATE) \ - $(TEMPDIR)/goreleaser release --skip-publish --rm-dist --snapshot --config $(TEMPDIR)/goreleaser.yaml +# note: mac packaging is intentionally left out (requires secrets and there is no acceptance test for packaged mac assets) +.PHONY: snapshot +snapshot: + $(call build_binary,linux,amd64,$(SNAPSHOTDIR)) + $(call build_binary,darwin,amd64,$(SNAPSHOTDIR)) + $(call build_container_image,$(SNAPSHOTDIR)) + + docker image save $(CONTAINER_IMAGE_TAG_LATEST) -o $(SNAPSHOTDIR)/image.tar + + .github/scripts/package-linux.sh \ + $(SNAPSHOTDIR) \ + $(VERSION) \ + $(TEMPDIR) -# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately .PHONY: acceptance-mac -acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac) +acceptance-mac: ## Run acceptance tests on built binaries (Mac) $(call title,Running acceptance test: Run on Mac) $(ACC_DIR)/mac.sh \ $(SNAPSHOTDIR) \ @@ -187,26 +240,11 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie $(ACC_TEST_IMAGE) \ $(RESULTSDIR) -# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately .PHONY: acceptance-linux -acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux) - -# note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests -.PHONY: compare-fingerprint -compare-fingerprint: - find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint && echo "$(COMPARE_CACHE_BUSTER)" >> test/inline-compare/inline-compare.fingerprint - -.PHONY: compare-snapshot -compare-snapshot: $(SNAPSHOTDIR) ## Compare the reports of a run of a snapshot build of syft against inline-scan - chmod 755 $(SNAPSHOT_CMD) - @cd test/inline-compare && SYFT_CMD=$(SNAPSHOT_CMD) make - -.PHONY: compare -compare: ## Compare the reports of a run of a main-branch build of syft against inline-scan - @cd test/inline-compare && make +acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on built binaries and packages (Linux) .PHONY: acceptance-test-deb-package-install -acceptance-test-deb-package-install: $(SNAPSHOTDIR) +acceptance-test-deb-package-install: $(call title,Running acceptance test: DEB install) $(ACC_DIR)/deb.sh \ $(SNAPSHOTDIR) \ @@ -215,7 +253,7 @@ acceptance-test-deb-package-install: $(SNAPSHOTDIR) $(RESULTSDIR) .PHONY: acceptance-test-rpm-package-install -acceptance-test-rpm-package-install: $(SNAPSHOTDIR) +acceptance-test-rpm-package-install: $(call title,Running acceptance test: RPM install) $(ACC_DIR)/rpm.sh \ $(SNAPSHOTDIR) \ @@ -223,11 +261,56 @@ acceptance-test-rpm-package-install: $(SNAPSHOTDIR) $(ACC_TEST_IMAGE) \ $(RESULTSDIR) +# note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests +.PHONY: compare-fingerprint +compare-fingerprint: + find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint && echo "$(COMPARE_CACHE_BUSTER)" >> test/inline-compare/inline-compare.fingerprint + +.PHONY: compare-snapshot +compare-snapshot: ## Compare the reports of a run of a snapshot build of syft against inline-scan + chmod 755 $(SNAPSHOT_CMD) + @cd test/inline-compare && SYFT_CMD=$(SNAPSHOT_CMD) make + +.PHONY: compare +compare: ## Compare the reports of a run of a main-branch build of syft against inline-scan + cd test/inline-compare && make + +.PHONY: setup-macos-signing +setup-macos-signing: ## Prepare for macOS-specific signing process + $(call title,Preparing macOS environment for code signing) + .github/scripts/mac-prepare-for-signing.sh + +.PHONY: package-mac +package-mac: setup-macos-signing bootstrap-ci-mac ## Create signed and notarized release assets for macOS + $(call title,Creating packaging for macOS -- signed and notarized) + + # Create signed and notarized assets + gon "./gon.hcl" + + # Update asset names. This won't be necessary once Gon supports variable injection. + @ORIGINAL_NAME="$(DISTDIR)/output" && NEW_NAME="$(DISTDIR)/syft_$(VERSION)_darwin_amd64" && \ + mv -v "$${ORIGINAL_NAME}.dmg" "$${NEW_NAME}.dmg" && \ + mv -v "$${ORIGINAL_NAME}.zip" "$${NEW_NAME}.zip" + +.PHONY: package-linux +package-linux: + $(call title,Creating packaging for Linux) + .github/scripts/package-linux.sh \ + $(DISTDIR) \ + $(VERSION) \ + $(TEMPDIR) + +.PHONY: package +package: package-mac package-linux + .PHONY: changlog-release +.SILIENT: changelog-release changelog-release: - @echo "Last tag: $(SECOND_TO_LAST_TAG)" - @echo "Current tag: $(VERSION)" - @github_changelog_generator \ + echo "Last tag: $(SECOND_TO_LAST_TAG)" + echo "Current tag: $(VERSION_TAG)" + docker run --rm \ + -v "$(shell pwd)":/usr/local/src/your-app \ + ferrarimarco/github-changelog-generator \ --user anchore \ --project $(BIN) \ -t ${GITHUB_TOKEN} \ @@ -236,12 +319,13 @@ changelog-release: --no-issues-wo-labels \ --since-tag $(SECOND_TO_LAST_TAG) - @printf '\n$(BOLD)$(CYAN)Release $(VERSION) Changelog$(RESET)\n\n' - @cat CHANGELOG.md + printf '\n$(BOLD)$(CYAN)Release $(VERSION_TAG) Changelog$(RESET)\n\n' + cat CHANGELOG.md .PHONY: changelog-unreleased +.SILENCE: changelog-unreleased changelog-unreleased: ## show the current changelog that will be produced on the next release (note: requires GITHUB_TOKEN set) - @docker run -it --rm \ + docker run -it --rm \ -v "$(shell pwd)":/usr/local/src/your-app \ ferrarimarco/github-changelog-generator \ --user anchore \ @@ -250,48 +334,104 @@ changelog-unreleased: ## show the current changelog that will be produced on the --exclude-labels 'duplicate,question,invalid,wontfix,size:small,size:medium,size:large,size:x-large' \ --since-tag $(LAST_TAG) - @printf '\n$(BOLD)$(CYAN)Unreleased Changes (closed PRs and issues will not be in the final changelog)$(RESET)\n' + printf '\n$(BOLD)$(CYAN)Unreleased Changes (closed PRs and issues will not be in the final changelog)$(RESET)\n' - @docker run -it --rm \ + docker run -it --rm \ -v $(shell pwd)/CHANGELOG.md:/CHANGELOG.md \ rawkode/mdv \ -t 748.5989 \ /CHANGELOG.md -.PHONY: release -release: clean-dist ci-bootstrap-mac changelog-release ## Build and publish final binaries and packages. Intended to be run only on macOS. - $(call title,Publishing release artifacts) +.PHONY: homebrew-formula-generate +.SILENT: homebrew-formula-generate +homebrew-formula-generate: + $(call title,Generating homebrew formula) + .github/scripts/homebrew-formula-generate.sh \ + "$(VERSION_TAG)" \ + "$(HOMEBREW_FORMULA_FILE)" - # Prepare for macOS-specific signing process - .github/scripts/mac-prepare-for-signing.sh +.PHONY: homebrew-formula-test +.SILENT: homebrew-formula-test +homebrew-formula-test: bootstrap + $(call title,Testing homebrew formula) - # create a config with the dist dir overridden - echo "dist: $(DISTDIR)" > $(TEMPDIR)/goreleaser.yaml - cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml + echo "Cleaning up any versions of $(BIN) previously installed by $(BREW_CMD)" + $(BREW_CMD) uninstall --force "$(HOMEBREW_FORMULA_FILE)" - # release (note the version transformation from v0.7.0 --> 0.7.0) - bash -c "\ - BUILD_GIT_TREE_STATE=$(GITTREESTATE) \ - VERSION=$(VERSION:v%=%) \ - $(TEMPDIR)/goreleaser \ - --rm-dist \ - --config $(TEMPDIR)/goreleaser.yaml \ - --release-notes <(cat CHANGELOG.md)" + echo "Testing homebrew installation using formula" + $(BREW_CMD) install --formula "$(HOMEBREW_FORMULA_FILE)" - # verify checksum signatures - .github/scripts/verify-signature.sh "$(DISTDIR)" + INSTALLED_BIN="$(BREW_BIN_DIR)/$(BIN)" && \ + echo "Now running '$${INSTALLED_BIN} version':" && \ + "$${INSTALLED_BIN}" version + +.PHONY: homebrew-formula-publish +.SILENT: homebrew-formula-publish +homebrew-formula-publish: + $(call title,Publishing homebrew formula) + + FORMULA_FILE="$$(realpath $(HOMEBREW_FORMULA_FILE))" && \ + \ + pushd "$(TEMPDIR)" && \ + rm -rfv "./homebrew-syft" && \ + gh repo clone anchore/homebrew-syft && \ + \ + pushd "homebrew-syft" && \ + cp -vf "$${FORMULA_FILE}" "./$(BIN).rb" && \ + git commit -am "Brew formula update for $(BIN) version $(VERSION_TAG)" && \ + git push && \ + popd && \ + popd + +.PHONY: version-check-update +.SILENT: version-check-update +version-check-update: + $(call title,Updating version check) # upload the version file that supports the application version update check (excluding pre-releases) - .github/scripts/update-version-file.sh "$(DISTDIR)" "$(VERSION)" + .github/scripts/update-version-file.sh "$(DISTDIR)" "$(VERSION_TAG)" + +.PHONY: stage-released-linux-artifact +stage-released-linux-artifact: + mkdir -p ./$(DISTDIR)/syft_linux_amd64 + curl -L -o ./$(DISTDIR)/syft.tar.gz https://github.com/anchore/syft/releases/download/$(VERSION_TAG)/syft_$(VERSION)_linux_amd64.tar.gz + tar -C ./$(DISTDIR)/syft_linux_amd64 -xvf ./$(DISTDIR)/syft.tar.gz syft + +.PHONY: container-image-build +.SILENT: container-image-build +container-image-build: + $(call title,Building and tagging container image for $(BIN)) + $(call build_container_image,$(DISTDIR)) + +.PHONY: container-image-test +.SILENT: container-image-test +container-image-smoke-test: + $(call title,Smoke testing container image) + docker run --pull never --rm "$(CONTAINER_IMAGE_TAG_LATEST)" version + +.PHONY: container-image-push +.SILENT: container-image-push +container-image-push: + $(call title,Pushing container image tags) + + tags=( \ + "$(CONTAINER_IMAGE_TAG_MAJOR)" \ + "$(CONTAINER_IMAGE_TAG_MINOR)" \ + "$(CONTAINER_IMAGE_TAG_PATCH)" \ + "$(CONTAINER_IMAGE_TAG_LATEST)" \ + ) && \ + for tag in $${tags[@]}; do \ + docker push "$${tag}"; \ + done .PHONY: clean clean: clean-dist clean-snapshot ## Remove previous builds and result reports rm -rf $(RESULTSDIR)/* -.PHONY: clean-snapshot -clean-snapshot: - rm -rf $(SNAPSHOTDIR) $(TEMPDIR)/goreleaser.yaml - .PHONY: clean-dist clean-dist: rm -rf $(DISTDIR) $(TEMPDIR)/goreleaser.yaml + +.PHONY: clean-snapshot +clean-snapshot: + rm -rf $(SNAPSHOTDIR) $(TEMPDIR)/goreleaser.yaml diff --git a/RELEASE.md b/RELEASE.md index 8f03c9627..eb2c68cb8 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -44,7 +44,7 @@ for each release. This project auto-generates the Changelog contents for each current release and posts the generated contents to the GitHub Release page. Leveraging the GitHub Releases feature -allows GitHub to manage the Changelog on each release outside of the git repository while +allows GitHub to manage the Changelog on each release outside the git repository while still being hosted with the released assets. The Changelog is generated from the metadata from in-repository issues and PRs, using @@ -89,11 +89,13 @@ This release process itself should be as automated as possible, and have only a milestone with a partial version, the full version should be used for the git tag (e.g. with a Milestone of `v0.1` the tag should be `v0.1.0`). -1. Push the tag, the release pipeline will generate and publish all assets as well as a - draft GitHub Release. +1. Push the tag. + +1. A release admin must approve the release on the Github Actions release pipeline run page. + Once approved, the release pipeline will generate all assets and draft a GitHub Release. -1. Promote the GitHub Release from draft to public. Note: since extra assets are made - available immediately from previous steps (i.e. the brew formula) the release should - only be in this state for a small amount of time (minutes). +1. Navigate to the Github Release draft page to review the final changelog and publish the + release. Once published, a release-follow-up pipeline will publish derivative artifacts + (docker image to DockerHub, brew formula to the external homebrew git repo, etc). 1. If there is a release Milestone, close it. diff --git a/gon.hcl b/gon.hcl index 19076ae70..6d2402be7 100644 --- a/gon.hcl +++ b/gon.hcl @@ -1,4 +1,4 @@ -source = ["./dist/syft-macos_darwin_amd64/syft"] # The 'dist' directory path should ideally reference an env var, where the source of truth is the Makefile. I wasn't able to figure out how to solve this. +source = ["./dist/syft_darwin_amd64/syft"] # The 'dist' directory path should ideally reference an env var, where the source of truth is the Makefile. I wasn't able to figure out how to solve this. bundle_id = "com.anchore.toolbox.syft" sign { diff --git a/test/acceptance/mac.sh b/test/acceptance/mac.sh index fade10d76..306781fed 100755 --- a/test/acceptance/mac.sh +++ b/test/acceptance/mac.sh @@ -7,19 +7,23 @@ ACC_DIR=$2 TEST_IMAGE=$3 RESULTSDIR=$4 -TEST_IMAGE_TAR=/tmp/image.tar TEST_TYPE=mac -WORK_DIR=`mktemp -d -t "syft-acceptance-test-${TEST_TYPE}-XXXXXX"` -NORMAL_TEST_IMAGE=$(echo ${TEST_IMAGE} | tr ':' '-' ) +WORK_DIR=$(mktemp -d -t "syft-acceptance-test-${TEST_TYPE}-XXXXXX") +TEST_IMAGE_TAR=${WORK_DIR}/image.tar +NORMAL_TEST_IMAGE=$(echo "${TEST_IMAGE}" | tr ':' '-' ) REPORT=${WORK_DIR}/acceptance-${TEST_TYPE}-${NORMAL_TEST_IMAGE}.json GOLDEN_REPORT=${ACC_DIR}/test-fixtures/acceptance-${NORMAL_TEST_IMAGE}.json +SYFT_PATH="${DISTDIR}/syft_darwin_amd64/syft" + # check if tmp dir was created if [[ ! "${WORK_DIR}" || ! -d "${WORK_DIR}" ]]; then echo "Could not create temp dir" exit 1 fi +trap "rm -f ${WORK_DIR}/*; rmdir ${WORK_DIR};" EXIT + function cleanup { # we should still preserve previous failures exit_code=$? @@ -33,20 +37,19 @@ trap cleanup EXIT skopeo --version || brew install skopeo # fetch test image -skopeo --override-os linux copy docker://docker.io/${TEST_IMAGE} docker-archive:${TEST_IMAGE_TAR} -ls -alh ${TEST_IMAGE_TAR} +skopeo --override-os linux --insecure-policy copy "docker://docker.io/${TEST_IMAGE}" "docker-archive:${TEST_IMAGE_TAR}" +ls -alh "${TEST_IMAGE_TAR}" # run syft -SYFT_PATH="${DISTDIR}/syft-macos_darwin_amd64/syft" chmod 755 "${SYFT_PATH}" "${SYFT_PATH}" version -SYFT_CHECK_FOR_APP_UPDATE=0 "${SYFT_PATH}" docker-archive://${TEST_IMAGE_TAR} -vv -o json > "${REPORT}" +SYFT_CHECK_FOR_APP_UPDATE=0 "${SYFT_PATH}" "docker-archive://${TEST_IMAGE_TAR}" -vv -o json > "${REPORT}" # keep the generated report around -mkdir -p ${RESULTSDIR} -cp ${REPORT} ${RESULTSDIR} +mkdir -p "${RESULTSDIR}" +cp "${REPORT}" "${RESULTSDIR}" # compare the results to a known good output ${ACC_DIR}/compare.py \ - ${GOLDEN_REPORT} \ - ${REPORT} | tee ${RESULTSDIR}/acceptance-${TEST_TYPE}.txt + "${GOLDEN_REPORT}" \ + "${REPORT}" | tee "${RESULTSDIR}/acceptance-${TEST_TYPE}.txt"