diff --git a/.binny.yaml b/.binny.yaml index 15e848809..9b118d2d0 100644 --- a/.binny.yaml +++ b/.binny.yaml @@ -1,67 +1,11 @@ # only pull in version updates that were released more than a week ago (low-pass filter for quickly-retracted releases) cooldown: 7d +# Most tools (binny, chronicle, cosign, golangci-lint, goreleaser, gosimports, +# bouncer, quill, syft, task, gh) are inherited from anchore/go-make's embedded +# .binny.yaml — see https://github.com/anchore/go-make. Only syft-specific tools +# or version overrides should live here. tools: - ## internal tools ############################################################################ - - # we want to use a pinned version of binny to manage the toolchain (so binny manages itself!) - - name: binny - version: - want: v0.13.0 - method: github-release - with: - repo: anchore/binny - - # used to produce SBOMs during release - - name: syft - version: - want: v1.42.3 - method: github-release - with: - repo: anchore/syft - - # used to sign mac binaries at release - - name: quill - version: - want: v0.7.1 - method: github-release - with: - repo: anchore/quill - - # used at release to generate the changelog - - name: chronicle - version: - want: v0.8.0 - method: github-release - with: - repo: anchore/chronicle - - ## external tools ############################################################################ - - # used for linting - - name: golangci-lint - version: - want: v2.11.4 - method: github-release - with: - repo: golangci/golangci-lint - - # used for showing the changelog at release - - name: glow - version: - want: v2.1.1 - method: github-release - with: - repo: charmbracelet/glow - - # used for signing the checksums file at release - - name: cosign - version: - want: v3.0.5 - method: github-release - with: - repo: sigstore/cosign - # used in integration tests to verify JSON schemas - name: yajsv version: @@ -70,46 +14,6 @@ tools: with: repo: neilpa/yajsv - # used to release all artifacts - - name: goreleaser - version: - want: v2.15.2 - method: github-release - with: - repo: goreleaser/goreleaser - - # used for organizing imports during static analysis - - name: gosimports - version: - want: v0.3.8 - method: github-release - with: - repo: rinchsan/gosimports - - # used during static analysis for license compliance - - name: bouncer - version: - want: v0.4.0 - method: github-release - with: - repo: wagoodman/go-bouncer - - # used for running all local and CI tasks - - name: task - version: - want: v3.49.1 - method: github-release - with: - repo: go-task/task - - # used for triggering a release - - name: gh - version: - want: v2.89.0 - method: github-release - with: - repo: cli/cli - # used to upload test fixture cache - name: oras version: @@ -118,7 +22,7 @@ tools: with: repo: oras-project/oras - # used to upload test fixture cache + # used to parse JSON/YAML annotations on the fixture cache image - name: yq version: want: v4.52.5 diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index 9d047394b..67717b789 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -1,55 +1,45 @@ name: "Bootstrap" -description: "Bootstrap all tools and dependencies" +description: "Bootstrap all syft tools and dependencies on top of go-make's setup action" + +# This action is a thin wrapper around anchore/go-make/.github/actions/setup which +# already handles checkout, setup-go, restore-only build/mod cache, and tool cache. +# We add the syft-specific extras here: apt packages and the test fixture cache. + inputs: go-version: - description: "Go version to install" + description: "Go version to install (passed to go-make/setup)" required: true default: "1.26.2" - go-dependencies: - description: "Download go dependencies" + cache-key-prefix: + description: "Prefix all cache keys with this value (passed to go-make/setup)" + required: true + default: "v1" + cache-enabled: + description: "Enable build/mod and tool caching (passed to go-make/setup)" required: true default: "true" - cache-key-prefix: - description: "Prefix all cache keys with this value" - required: true - default: "53ac821810" download-test-fixture-cache: description: "Download test fixture cache from OCI and github actions" required: true default: "false" - tools: - description: "whether to install tools" - default: "true" bootstrap-apt-packages: description: "Space delimited list of tools to install via apt" default: "libxml2-utils" + runs: using: "composite" steps: - # note: go mod and build is automatically cached on default with v4+ - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - if: inputs.go-version != '' + - name: Setup go + go-make tooling + uses: anchore/go-make/.github/actions/setup@88c36505984649108439f13fb35dcaea4ce61d94 # v0.4.0 with: go-version: ${{ inputs.go-version }} - check-latest: true - - name: Restore tool cache - if: inputs.tools == 'true' - id: tool-cache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ${{ github.workspace }}/.tool - key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-tool-${{ hashFiles('.binny.yaml') }} - - name: Install project tools + cache-key-prefix: ${{ inputs.cache-key-prefix }} + cache-enabled: ${{ inputs.cache-enabled }} + + - name: Install binny-managed tools shell: bash - if: inputs.tools == 'true' - run: | - make tools - .tool/binny list - .tool/binny check - - name: Install go dependencies - if: inputs.go-dependencies == 'true' - shell: bash - run: make ci-bootstrap-go + run: make binny:install + - name: Install apt packages if: inputs.bootstrap-apt-packages != '' shell: bash @@ -58,12 +48,14 @@ runs: run: | IFS=' ' read -ra packages <<< "$APT_PACKAGES" DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y "${packages[@]}" + - name: Restore ORAS cache from github actions if: inputs.download-test-fixture-cache == 'true' uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ${{ github.workspace }}/.tmp/oras-cache key: ${{ inputs.cache-key-prefix }}-oras-cache + - name: Download test fixture cache if: inputs.download-test-fixture-cache == 'true' shell: bash diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fc3b7bd6d..c7097b2b8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,7 +14,9 @@ version: 2 updates: - package-ecosystem: gomod - directory: "/" + directories: + - "/" + - "/.make" cooldown: default-days: 7 schedule: diff --git a/.github/scripts/ci-check.sh b/.github/scripts/ci-check.sh deleted file mode 100755 index 0ab83a318..000000000 --- a/.github/scripts/ci-check.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -red=$(tput setaf 1) -bold=$(tput bold) -normal=$(tput sgr0) - -# assert we are running in CI (or die!) -if [[ -z "$CI" ]]; then - echo "${bold}${red}This step should ONLY be run in CI. Exiting...${normal}" - exit 1 -fi diff --git a/.github/scripts/coverage.py b/.github/scripts/coverage.py deleted file mode 100755 index db14135cd..000000000 --- a/.github/scripts/coverage.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 -import subprocess -import sys -import shlex - - -class bcolors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKCYAN = '\033[96m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - - -if len(sys.argv) < 3: - print("Usage: coverage.py [threshold] [go-coverage-report]") - sys.exit(1) - - -threshold = float(sys.argv[1]) -report = sys.argv[2] - - -args = shlex.split(f"go tool cover -func {report}") -p = subprocess.run(args, capture_output=True, text=True) - -percent_coverage = float(p.stdout.splitlines()[-1].split()[-1].replace("%", "")) -print(f"{bcolors.BOLD}Coverage: {percent_coverage}%{bcolors.ENDC}") - -if percent_coverage < threshold: - print(f"{bcolors.BOLD}{bcolors.FAIL}Coverage below threshold of {threshold}%{bcolors.ENDC}") - sys.exit(1) diff --git a/.github/scripts/find_cache_paths.py b/.github/scripts/find_cache_paths.py index 5d66ae589..9dc25481e 100755 --- a/.github/scripts/find_cache_paths.py +++ b/.github/scripts/find_cache_paths.py @@ -1,35 +1,45 @@ #!/usr/bin/env python3 from __future__ import annotations -import os import glob -import sys -import json import hashlib - +import json +import os +import sys IGNORED_PREFIXES = [] def find_fingerprints_and_check_dirs(base_dir): - all_fingerprints = set(glob.glob(os.path.join(base_dir, '**', 'test*', '**', '*.fingerprint'), recursive=True)) + all_fingerprints = set( + glob.glob( + os.path.join(base_dir, "**", "test*", "**", "*.fingerprint"), recursive=True + ) + ) - all_fingerprints = {os.path.relpath(fp) for fp in all_fingerprints - if not any(fp.startswith(prefix) for prefix in IGNORED_PREFIXES)} + all_fingerprints = { + os.path.relpath(fp) + for fp in all_fingerprints + if not any(fp.startswith(prefix) for prefix in IGNORED_PREFIXES) + } if not all_fingerprints: show("No .fingerprint files or cache directories found.") exit(1) - missing_content = [] + orphan_fingerprints = [] + empty_content = [] valid_paths = set() fingerprint_contents = [] for fingerprint in all_fingerprints: - path = fingerprint.replace('.fingerprint', '') + path = fingerprint.replace(".fingerprint", "") if not os.path.exists(path): - missing_content.append(path) + # paired content path is entirely missing — the .fingerprint is likely + # leftover from a moved/deleted source (testdata trees are git-ignored, + # so they persist locally across rename refactors) + orphan_fingerprints.append(fingerprint) continue if not os.path.isdir(path): @@ -39,13 +49,13 @@ def find_fingerprints_and_check_dirs(base_dir): if os.listdir(path): valid_paths.add(path) else: - missing_content.append(path) + empty_content.append(path) - with open(fingerprint, 'r') as f: + with open(fingerprint, "r") as f: content = f.read().strip() fingerprint_contents.append((fingerprint, content)) - return sorted(valid_paths), missing_content, fingerprint_contents + return sorted(valid_paths), empty_content, orphan_fingerprints, fingerprint_contents def parse_fingerprint_contents(fingerprint_content): @@ -59,7 +69,9 @@ def parse_fingerprint_contents(fingerprint_content): def calculate_sha256(fingerprint_contents): sorted_fingerprint_contents = sorted(fingerprint_contents, key=lambda x: x[0]) - concatenated_contents = ''.join(content for _, content in sorted_fingerprint_contents) + concatenated_contents = "".join( + content for _, content in sorted_fingerprint_contents + ) sha256_hash = hashlib.sha256(concatenated_contents.encode()).hexdigest() @@ -68,7 +80,7 @@ def calculate_sha256(fingerprint_contents): def calculate_file_sha256(file_path): sha256_hash = hashlib.sha256() - with open(file_path, 'rb') as f: + with open(file_path, "rb") as f: for byte_block in iter(lambda: f.read(4096), b""): sha256_hash.update(byte_block) return sha256_hash.hexdigest() @@ -79,17 +91,28 @@ def show(*s: str): def main(file_path: str | None): - base_dir = '.' - valid_paths, missing_content, fingerprint_contents = find_fingerprints_and_check_dirs(base_dir) + base_dir = "." + valid_paths, empty_content, orphan_fingerprints, fingerprint_contents = ( + find_fingerprints_and_check_dirs(base_dir) + ) - if missing_content: - show("The following paths are missing or have no content, but have corresponding .fingerprint files:") - for path in sorted(missing_content): + if empty_content: + show( + "The following paths exist but are empty, and have corresponding .fingerprint files:" + ) + for path in sorted(empty_content): show(f"- {path}") # when adding new cache directories there is a time where it is not possible to have this directory without # running the tests first... but this step is a prerequisite for running the tests. We should not block on this. - # show("Please ensure these paths exist and have content if they are directories.") - # exit(1) + + if orphan_fingerprints: + show( + "The following .fingerprint files reference paths that no longer exist " + "(likely leftover from a moved/deleted cataloger — safe to delete, " + "or run `task prune-orphan-fingerprints`):" + ) + for fp in sorted(orphan_fingerprints): + show(f"- {fp}") sha256_hash = calculate_sha256(fingerprint_contents) @@ -101,30 +124,24 @@ def main(file_path: str | None): file_digest = calculate_file_sha256(fingerprint_file) # Parse the fingerprint file to get the digest/path tuples - with open(fingerprint_file, 'r') as f: + with open(fingerprint_file, "r") as f: fingerprint_content = f.read().strip() input_map = parse_fingerprint_contents(fingerprint_content) - paths_with_digests.append({ - "path": path, - "digest": file_digest, - "input": input_map - }) + paths_with_digests.append( + {"path": path, "digest": file_digest, "input": input_map} + ) except Exception as e: show(f"Error processing {fingerprint_file}: {e}") raise e - - output = { - "digest": sha256_hash, - "paths": paths_with_digests - } + output = {"digest": sha256_hash, "paths": paths_with_digests} content = json.dumps(output, indent=2, sort_keys=True) if file_path: - with open(file_path, 'w') as f: + with open(file_path, "w") as f: f.write(content) print(content) diff --git a/.github/scripts/go-mod-tidy-check.sh b/.github/scripts/go-mod-tidy-check.sh deleted file mode 100755 index 28f22fcdc..000000000 --- a/.github/scripts/go-mod-tidy-check.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -eu - -ORIGINAL_STATE_DIR=$(mktemp -d "TEMP-original-state-XXXXXXXXX") -TIDY_STATE_DIR=$(mktemp -d "TEMP-tidy-state-XXXXXXXXX") - -trap "cp -p ${ORIGINAL_STATE_DIR}/* ./ && git update-index -q --refresh && rm -fR ${ORIGINAL_STATE_DIR} ${TIDY_STATE_DIR}" EXIT - -# capturing original state of files... -cp go.mod go.sum "${ORIGINAL_STATE_DIR}" - -# capturing state of go.mod and go.sum after running go mod tidy... -go mod tidy -cp go.mod go.sum "${TIDY_STATE_DIR}" - -set +e - -# detect difference between the git HEAD state and the go mod tidy state -DIFF_MOD=$(diff -u "${ORIGINAL_STATE_DIR}/go.mod" "${TIDY_STATE_DIR}/go.mod") -DIFF_SUM=$(diff -u "${ORIGINAL_STATE_DIR}/go.sum" "${TIDY_STATE_DIR}/go.sum") - -if [[ -n "${DIFF_MOD}" || -n "${DIFF_SUM}" ]]; then - echo "go.mod diff:" - echo "${DIFF_MOD}" - echo "go.sum diff:" - echo "${DIFF_SUM}" - echo "" - printf "FAILED! go.mod and/or go.sum are NOT tidy; please run 'go mod tidy'.\n\n" - exit 1 -fi diff --git a/.github/scripts/prune_orphan_fingerprints.py b/.github/scripts/prune_orphan_fingerprints.py new file mode 100755 index 000000000..1825a2904 --- /dev/null +++ b/.github/scripts/prune_orphan_fingerprints.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +""" +Remove orphan *.fingerprint files left behind by moved/deleted catalogers. + +A fingerprint is considered orphaned when: + 1. its paired content path (the fingerprint path with `.fingerprint` stripped) + does not exist, AND + 2. the nearest ancestor `testdata/` directory has no `Makefile` claiming + responsibility for generating that path. + +The second condition is the safety check: if there is a Makefile, the +fingerprint is "live" and might just be waiting for fixtures to be built — +leave it alone. Without a Makefile, nothing in-repo will ever regenerate +the content, so the fingerprint is dead weight that triggers spurious +"missing path" warnings. + +Empty parent directories are also pruned after removing the fingerprint. + +Use --dry-run to preview without deleting. +""" +from __future__ import annotations + +import argparse +import glob +import os +import sys + + +def find_ancestor_testdata(path: str) -> str | None: + d = os.path.dirname(path) + while d and d not in (".", os.sep): + if os.path.basename(d) == "testdata": + return d + d = os.path.dirname(d) + return None + + +def is_orphan(fingerprint: str) -> bool: + paired = fingerprint[: -len(".fingerprint")] + if os.path.exists(paired): + return False + + testdata_dir = find_ancestor_testdata(fingerprint) + if testdata_dir and os.path.isfile(os.path.join(testdata_dir, "Makefile")): + # a Makefile exists that may regenerate this — not safe to prune + return False + + return True + + +def prune_empty_parents(start: str, stop_at: str = ".") -> list[str]: + removed = [] + d = os.path.dirname(start) + stop_at = os.path.abspath(stop_at) + while d and os.path.abspath(d) != stop_at: + try: + if not os.listdir(d): + os.rmdir(d) + removed.append(d) + d = os.path.dirname(d) + else: + break + except OSError: + break + return removed + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be removed without deleting anything", + ) + args = parser.parse_args() + + all_fingerprints = glob.glob("**/test*/**/*.fingerprint", recursive=True) + orphans = sorted(fp for fp in all_fingerprints if is_orphan(fp)) + + if not orphans: + print("no orphan fingerprints found") + return 0 + + verb = "would remove" if args.dry_run else "removing" + print(f"{verb} {len(orphans)} orphan fingerprint(s):") + for fp in orphans: + print(f"- {fp}") + if args.dry_run: + continue + try: + os.remove(fp) + except OSError as e: + print(f" ! failed to remove: {e}", file=sys.stderr) + continue + for d in prune_empty_parents(fp): + print(f" (also removed empty dir {d})") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/trigger-release.sh b/.github/scripts/trigger-release.sh deleted file mode 100755 index 8684585b7..000000000 --- a/.github/scripts/trigger-release.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -eu - -bold=$(tput bold) -normal=$(tput sgr0) - -GH_CLI=.tool/gh - -if ! [ -x "$(command -v $GH_CLI)" ]; then - echo "The GitHub CLI could not be found. run: make bootstrap" - exit 1 -fi - -$GH_CLI auth status - -# set the default repo in cases where multiple remotes are defined -$GH_CLI repo set-default anchore/syft - -export GITHUB_TOKEN="${GITHUB_TOKEN-"$($GH_CLI auth token)"}" - -# we need all of the git state to determine the next version. Since tagging is done by -# the release pipeline it is possible to not have all of the tags from previous releases. -git fetch --tags - -# populates the CHANGELOG.md and VERSION files -echo "${bold}Generating changelog...${normal}" -make changelog 2> /dev/null - -NEXT_VERSION=$(cat VERSION) - -if [[ "$NEXT_VERSION" == "" || "${NEXT_VERSION}" == "(Unreleased)" ]]; then - echo "Could not determine the next version to release. Exiting..." - exit 1 -fi - -while true; do - read -p "${bold}Do you want to trigger a release for version '${NEXT_VERSION}'?${normal} [y/n] " yn - case $yn in - [Yy]* ) echo; break;; - [Nn]* ) echo; echo "Cancelling release..."; exit;; - * ) echo "Please answer yes or no.";; - esac -done - -echo "${bold}Kicking off release for ${NEXT_VERSION}${normal}..." -echo -$GH_CLI workflow run release.yaml -f version=${NEXT_VERSION} - -echo -echo "${bold}Waiting for release to start...${normal}" -sleep 10 - -set +e - -echo "${bold}Head to the release workflow to monitor the release:${normal} $($GH_CLI run list --workflow=release.yaml --limit=1 --json url --jq '.[].url')" -id=$($GH_CLI run list --workflow=release.yaml --limit=1 --json databaseId --jq '.[].databaseId') -$GH_CLI run watch $id --exit-status || (echo ; echo "${bold}Logs of failed step:${normal}" && GH_PAGER="" $GH_CLI run view $id --log-failed) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2eb2738bf..db3b2ed2a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,9 +23,10 @@ on: - "install-script-only" jobs: - version-available: if: ${{ github.event.inputs.phase == 'all' }} + permissions: + contents: read # required for fetching tags uses: anchore/workflows/.github/workflows/check-version-available.yaml@8b2b1caf40e03933c6807e03b99e883e2ceb5ac8 # v0.4.0 with: version: ${{ github.event.inputs.version }} @@ -33,7 +34,7 @@ jobs: check-gate: if: ${{ github.event.inputs.phase == 'all' }} permissions: - checks: read # required for getting the status of specific check names + checks: read # required for getting the status of specific check names uses: anchore/workflows/.github/workflows/check-gate.yaml@8b2b1caf40e03933c6807e03b99e883e2ceb5ac8 # v0.4.0 with: # these are checks that should be run on pull-request and merges to main. @@ -42,8 +43,9 @@ jobs: checks: '["Acceptance tests (Linux)", "Acceptance tests (Mac)", "Build snapshot artifacts", "CLI tests (Linux)", "Integration tests", "Static analysis", "Unit tests"]' release: - needs: [ check-gate, version-available ] + needs: [check-gate, version-available] if: ${{ github.event.inputs.phase == 'all' }} + environment: release # runs-on.com: compute instances for parallel builds # spot disabled: reliability for build workflows (used for releases too) # goreleaser uses parallelism of 12, so we need more CPUs @@ -51,10 +53,9 @@ jobs: # tmpfs: faster io-intensive workflows runs-on: runs-on=${{ github.run_id }}/cpu=16+32/ram=32+128/family=c5+c6+c7+c8/spot=false/extras=s3-cache+tmpfs permissions: - contents: write - packages: write - # required for goreleaser signs section with cosign - id-token: write + contents: write # required for creating the GitHub release and pushing the version tag + packages: write # required for publishing release artifacts to GitHub packages + id-token: write # required for keyless signing (cosign/sigstore OIDC) steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 with: @@ -65,31 +66,24 @@ jobs: uses: ./.github/actions/bootstrap - name: Login to Docker Hub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 #v4.1.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 #v4.1.0 with: username: ${{ secrets.ANCHOREOSSWRITE_DH_USERNAME }} password: ${{ secrets.ANCHOREOSSWRITE_DH_PAT }} - name: Login to GitHub Container Registry - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 #v4.1.0 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 #v4.1.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Tag release - run: | - git config --global user.name "anchoreci" - git config --global user.email "anchoreci@users.noreply.github.com" - git tag -a "$VERSION" -m "Release $VERSION" - git push origin --tags - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ github.event.inputs.version }} - - name: Build & publish release artifacts run: make ci-release env: + # used for pushing tags + DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }} + RELEASE_VERSION: ${{ github.event.inputs.version }} # for mac signing and notarization... QUILL_SIGN_P12: ${{ secrets.ANCHORE_APPLE_DEVELOPER_ID_CERT_CHAIN }} QUILL_SIGN_PASSWORD: ${{ secrets.ANCHORE_APPLE_DEVELOPER_ID_CERT_PASS }} @@ -128,9 +122,11 @@ jobs: if: ${{ success() }} release-install-script: - needs: [ release ] + needs: [release] if: ${{ always() && (needs.release.result == 'success' || github.event.inputs.phase == 'install-script-only') }} - uses: "anchore/workflows/.github/workflows/release-install-script.yaml@main" + permissions: + contents: read # required for the reusable workflow to check out the repo and publish the install script + uses: anchore/workflows/.github/workflows/release-install-script.yaml@8b2b1caf40e03933c6807e03b99e883e2ceb5ac8 # v0.4.0 with: tag: ${{ github.event.inputs.version }} secrets: diff --git a/.gitignore b/.gitignore index 7fafb7faf..907f14e2d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ go.work go.work.sum .tool-versions .python-version +.mise.toml +.env # app configuration /.syft.yaml @@ -20,6 +22,8 @@ bin/ /generate /specs mise.toml +.make/.make +.conductor # changelog generation CHANGELOG.md @@ -76,5 +80,3 @@ cosign.pub __pycache__/ *.py[cod] *$py.class - - diff --git a/.make/go.mod b/.make/go.mod new file mode 100644 index 000000000..942ddc180 --- /dev/null +++ b/.make/go.mod @@ -0,0 +1,14 @@ +module github.com/anchore/syft/.make + +go 1.25.8 + +require ( + github.com/anchore/go-make v0.4.0 + github.com/goccy/go-yaml v1.19.2 +) + +require ( + github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/sys v0.44.0 // indirect +) diff --git a/.make/go.sum b/.make/go.sum new file mode 100644 index 000000000..dc48864df --- /dev/null +++ b/.make/go.sum @@ -0,0 +1,10 @@ +github.com/anchore/go-make v0.4.0 h1:poFE4PXcHwvix2E8AhdM7YHZ15bMhZb+W02i3+qrP8M= +github.com/anchore/go-make v0.4.0/go.mod h1:Nc/tkwQHW1d1Vi8+0rtS/vSrH6pxieaUQXLdrctn+8g= +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/.make/main.go b/.make/main.go new file mode 100644 index 000000000..c67dc3424 --- /dev/null +++ b/.make/main.go @@ -0,0 +1,209 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/goccy/go-yaml" + + . "github.com/anchore/go-make" + "github.com/anchore/go-make/file" + "github.com/anchore/go-make/git" + "github.com/anchore/go-make/lang" + "github.com/anchore/go-make/run" + "github.com/anchore/go-make/tasks/golint" + "github.com/anchore/go-make/tasks/goreleaser" + "github.com/anchore/go-make/tasks/gotest" +) + +// taskfileDescriptions maps Taskfile.yaml task names to their `desc:` field. +// Loaded at package init so wrap() can use Taskfile.yaml as the single source +// of truth for wrapped-task descriptions. +var taskfileDescriptions = mustReadTaskfileDescriptions() + +func main() { + Makefile( + // shared anchore tasks + golint.Tasks(), + goreleaser.Tasks(), + + // unit tests: exclude packages under any test/ directory (matches the syft + // Taskfile's prior `grep -v` against test paths). Coverage threshold of 62% + // preserves the prior coverage gate that used to live in scripts/coverage.py. + gotest.Tasks( + gotest.Name("unit"), + gotest.ExcludeGlob("**/test/**"), + gotest.CoverageThreshold(62), + ), + + // integration tests: native go-make Task. The race-detector smoke against a + // real image stays bundled here (RunsOn integration) so `make integration` + // behaves like the Taskfile version did. + gotest.Tasks( + gotest.Name("integration"), + gotest.IncludeGlob("./cmd/syft/internal/test/integration/..."), + gotest.Verbose(), + gotest.NoCoverage(), + ), + Task{ + Name: "integration:race-smoke", + Description: "exercise the CLI with the race detector", + RunsOn: lang.List("integration"), + Run: func() { + Run("go run -race cmd/syft/main.go anchore/test_images:grype-quality-dotnet-69f15d2") + }, + }, + + // cli tests: native go-make Task. Requires SYFT_BINARY_LOCATION pointing at + // an *absolute* path to the snapshot binary. Intentionally does NOT depend + // on snapshot: in CI we download a pre-built snapshot artifact and re-running + // goreleaser here would both burn ~10m and clobber the downloaded binary. + // Locally, the failure message tells you to run `make snapshot` first. + Task{ + Name: "cli", + Description: "Run CLI tests", + RunsOn: lang.List("test"), + Run: func() { + bin := snapshotBinPath() + if !file.Exists(bin) { + panic(fmt.Sprintf("snapshot binary not found at %s; run `make snapshot` first", bin)) + } + Log("testing binary: %s", bin) + Run( + "go test -count=1 -timeout=15m -v ./test/cli", + run.Env("SYFT_BINARY_LOCATION", bin), + ) + }, + }, + + // default validation pipeline (replaces Taskfile `default`/`pr-validations`/`validations`). + Task{ + Name: "default", + Description: "Run all validation tasks", + Dependencies: Deps("static-analysis", "test", "install-test"), + }, + + // --- everything below is implemented in Taskfile.yaml and surfaced here + // via wrap(). Descriptions come from Taskfile.yaml (single source of truth). + + // static analysis extras + wrap("check-json-schema-drift").RunOn("static-analysis"), + wrap("check-capability-drift"), + wrap("check-binary-fixture-size").RunOn("static-analysis"), + + // test extras + wrap("validate-cyclonedx-schema").RunOn("test"), + wrap("test-utils").RunOn("test"), + wrap("check-docker-cache").RunOn("test"), + wrap("snapshot-smoke-test"), + + // update commands + wrap("update-format-golden-files"), + + // fixture cache plumbing (heavy ORAS logic, lives in Taskfile). + // refresh-fixtures hooks into "unit" so `make unit` triggers the + // stale-cache detection + download just like `task unit` did on main + // (its `deps: [tmpdir, fixtures]` is what kept the fixture cache fresh). + wrap("fingerprints"), + wrap("refresh-fixtures").RunOn("unit"), + wrap("fixtures"), + wrap("build-fixtures"), + wrap("download-test-fixture-cache"), + wrap("upload-test-fixture-cache"), + wrap("show-test-image-cache"), + + // install-script tests (delegates to test/install/Makefile) + wrap("install-test"), + wrap("install-test-cache-save"), + wrap("install-test-cache-load"), + wrap("install-test-ci-mac"), + + // compare tests + wrap("generate-compare-file"), + wrap("compare-mac"), + wrap("compare-linux"), + wrap("compare-test-deb-package-install"), + wrap("compare-test-rpm-package-install"), + + // code/data generation (umbrella + per-target; each lives in Taskfile) + wrap("generate"), + wrap("generate-json-schema"), + wrap("generate-license-list"), + wrap("generate-cpe-dictionary-index"), + wrap("generate-capabilities"), + + // cleanup (each hooks into go-make's built-in `clean` label) + wrap("clean-snapshot").RunOn("clean"), + wrap("clean-docker-cache").RunOn("clean"), + wrap("clean-oras-cache").RunOn("clean"), + wrap("clean-cache").RunOn("clean"), + wrap("clean-test-observations").RunOn("clean"), + ) +} + +// wrap creates a go-make Task that delegates execution to `task `. The +// task's description is pulled from Taskfile.yaml's `desc:` field — descriptions +// for wrapped tasks must always live in Taskfile.yaml, never here. +func wrap(name string) Task { + desc, ok := taskfileDescriptions[name] + if !ok || desc == "" { + // loud-fail at startup so missing descs can't sneak through review. + panic(fmt.Sprintf("Taskfile.yaml task %q is missing a `desc:` field; please add one", name)) + } + return Task{ + Name: name, + Description: desc, + Run: func() { Run("task " + name) }, + } +} + +// mustReadTaskfileDescriptions parses Taskfile.yaml at the repo root and returns +// a map of task name -> desc. Runs at package init time so wrap() can use it. +func mustReadTaskfileDescriptions() map[string]string { + root := git.Root() + if root == "" { + return nil + } + path := filepath.Join(root, "Taskfile.yaml") + data, err := os.ReadFile(path) //nolint:gosec // G304: path resolved from git.Root() + if err != nil { + return nil + } + var tf struct { + Tasks map[string]struct { + Desc string `yaml:"desc"` + Aliases []string `yaml:"aliases"` + } `yaml:"tasks"` + } + lang.Throw(yaml.Unmarshal(data, &tf)) + out := make(map[string]string, len(tf.Tasks)) + for name, t := range tf.Tasks { + out[name] = t.Desc + // aliases inherit the canonical task's description so wrap() can find them. + for _, alias := range t.Aliases { + out[alias] = t.Desc + } + } + return out +} + +// snapshotBinPath replicates the SNAPSHOT_BIN computation from the prior Taskfile: +// /snapshot/-build__/syft, where arch maps amd64->amd64_v1 +// and arm64->arm64_v8.0 to match goreleaser's per-target output directory naming. +// Returns an absolute path: the cli tests' getSyftBinaryLocation contract requires +// SYFT_BINARY_LOCATION to be absolute because subtests run with cmd.Dir = t.TempDir(). +func snapshotBinPath() string { + osName := runtime.GOOS + var arch string + switch runtime.GOARCH { + case "amd64": + arch = "amd64_v1" + case "arm64": + arch = "arm64_v8.0" + default: + arch = runtime.GOARCH + } + return filepath.Join(RootDir(), "snapshot", osName+"-build_"+osName+"_"+arch, "syft") +} diff --git a/Makefile b/Makefile index 3f7916180..b8b2c6131 100644 --- a/Makefile +++ b/Makefile @@ -1,46 +1,18 @@ -OWNER = anchore -PROJECT = syft +# `test` and `snapshot` have matching directory names in this repo, so make would +# refuse to run them without an explicit .PHONY (Nothing to be done for ...). +.PHONY: test snapshot +test: + @go run -C .make . test -TOOL_DIR = .tool -BINNY = $(TOOL_DIR)/binny -TASK = $(TOOL_DIR)/task +snapshot: + @go run -C .make . snapshot -.DEFAULT_GOAL := make-default +.PHONY: * +.DEFAULT_GOAL: make-default -## Bootstrapping targets ################################# +make-default: + @go run -C .make . -# note: we need to assume that binny and task have not already been installed -$(BINNY): - @mkdir -p $(TOOL_DIR) - @curl -sSfL https://get.anchore.io/binny | sh -s -- -b $(TOOL_DIR) - -# note: we need to assume that binny and task have not already been installed -.PHONY: task -$(TASK) task: $(BINNY) - @$(BINNY) install task -q - -.PHONY: ci-bootstrap-go -ci-bootstrap-go: - go mod download - -# this is a bootstrapping catch-all, where if the target doesn't exist, we'll ensure the tools are installed and then try again +.DEFAULT: %: - @make --silent $(TASK) - @$(TASK) $@ - -## Shim targets ################################# - -.PHONY: make-default -make-default: $(TASK) - @# run the default task in the taskfile - @$(TASK) - -# for those of us that can't seem to kick the habit of typing `make ...` lets wrap the superior `task` tool -TASKS := $(shell bash -c "test -f $(TASK) && NO_COLOR=1 $(TASK) -l | grep '^\* ' | cut -d' ' -f2 | tr -d ':' | tr '\n' ' '" ) $(shell bash -c "test -f $(TASK) && NO_COLOR=1 $(TASK) -l | grep 'aliases:' | cut -d ':' -f 3 | tr '\n' ' ' | tr -d ','") - -.PHONY: $(TASKS) -$(TASKS): $(TASK) - @$(TASK) $@ - -help: $(TASK) - @$(TASK) -l + @go run -C .make . $@ diff --git a/Taskfile.yaml b/Taskfile.yaml index e16b59054..18cd532b8 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -1,6 +1,11 @@ - version: "3" +# NOTE: most generic tasks (static-analysis, format, lint, unit, snapshot, release, +# changelog, ci-release, etc.) are now provided natively by anchore/go-make and +# defined in .make/main.go. This file holds the syft-specific tasks that go-make +# wraps via `wrap("")` calls — keep descriptions (`desc:`) populated so they +# show up in `make help`. + includes: generate:cpe-index: ./task.d/generate/cpe-index.yaml @@ -25,11 +30,7 @@ vars: YQ: "{{ .TOOL_DIR }}/yq" TASK: "{{ .TOOL_DIR }}/task" - # used for changelog generation - CHANGELOG: CHANGELOG.md - NEXT_VERSION: VERSION - - # used for snapshot builds + # used for snapshot bin discovery in compare/install tasks OS: sh: uname -s | tr '[:upper:]' '[:lower:]' ARCH: @@ -42,11 +43,6 @@ vars: # e.g. when installing snapshot debs from a local path, ./ forces the deb to be installed in the current working directory instead of referencing a package name SNAPSHOT_DIR: ./snapshot SNAPSHOT_BIN: "{{ .PROJECT_ROOT }}/{{ .SNAPSHOT_DIR }}/{{ .OS }}-build_{{ .OS }}_{{ .ARCH }}/{{ .PROJECT }}" - SNAPSHOT_CMD: "{{ .TOOL_DIR }}/goreleaser release --config {{ .TMP_DIR }}/goreleaser.yaml --clean --snapshot --skip=publish --skip=sign" - BUILD_CMD: "{{ .TOOL_DIR }}/goreleaser build --config {{ .TMP_DIR }}/goreleaser.yaml --clean --snapshot --single-target" - RELEASE_CMD: "{{ .TOOL_DIR }}/goreleaser release --clean --release-notes {{ .CHANGELOG }}" - VERSION: - sh: git describe --dirty --always --tags # used for install and acceptance testing COMPARE_DIR: ./test/compare @@ -57,43 +53,10 @@ env: tasks: - ## High-level tasks ################################# - - default: - desc: Run all validation tasks - aliases: - - pr-validations - - validations - cmds: - - task: static-analysis - - task: test - - task: install-test - - static-analysis: - desc: Run all static analysis tasks - cmds: - - task: check-go-mod-tidy - - task: check-licenses - - task: lint - - task: check-json-schema-drift - - task: check-binary-fixture-size - - test: - desc: Run all levels of test - cmds: - - task: unit - - task: integration - - task: validate-cyclonedx-schema - - task: test-utils - - task: snapshot - - task: cli - - task: check-docker-cache - - ## Bootstrap tasks ################################# + ## Bootstrap (internal helpers used by other Taskfile tasks) ############### binny: internal: true - # desc: Get the binny tool generates: - "{{ .TOOL_DIR }}/binny" status: @@ -102,10 +65,8 @@ tasks: silent: true tools: - desc: Install all tools needed for CI and local development + internal: true deps: [binny] - aliases: - - bootstrap generates: - ".binny.yaml" - "{{ .TOOL_DIR }}/*" @@ -114,79 +75,14 @@ tasks: cmd: "{{ .TOOL_DIR }}/binny install -v" silent: true - update-tools: - desc: Update pinned versions of all tools to their latest available versions - deps: [binny] - generates: - - ".binny.yaml" - - "{{ .TOOL_DIR }}/*" - cmd: "{{ .TOOL_DIR }}/binny update -v" - silent: true - - list-tools: - desc: List all tools needed for CI and local development - deps: [binny] - cmd: "{{ .TOOL_DIR }}/binny list" - silent: true - - list-tool-updates: - desc: List all tools that are not up to date relative to the binny config - deps: [binny] - cmd: "{{ .TOOL_DIR }}/binny list --updates" - silent: true - tmpdir: + internal: true silent: true generates: - "{{ .TMP_DIR }}" cmd: "mkdir -p {{ .TMP_DIR }}" - ## Static analysis tasks ################################# - - format: - desc: Auto-format all source code - deps: [tools] - cmds: - - gofmt -w -s . - - "{{ .TOOL_DIR }}/gosimports -local github.com/anchore -w ." - - go mod tidy - - lint-fix: - desc: Auto-format all source code + run golangci lint fixers - deps: [tools] - cmds: - - task: format - - "{{ .TOOL_DIR }}/golangci-lint run --tests=false --fix" - - lint: - desc: Run gofmt + golangci lint checks - vars: - BAD_FMT_FILES: - sh: gofmt -l -s . - BAD_FILE_NAMES: - sh: "find . | grep -e ':' || true" - deps: [tools] - cmds: - # ensure there are no go fmt differences - - cmd: 'test -z "{{ .BAD_FMT_FILES }}" || (echo "files with gofmt issues: [{{ .BAD_FMT_FILES }}]"; exit 1)' - silent: true - # ensure there are no files with ":" in it (a known back case in the go ecosystem) - - cmd: 'test -z "{{ .BAD_FILE_NAMES }}" || (echo "files with bad names: [{{ .BAD_FILE_NAMES }}]"; exit 1)' - silent: true - # run linting - - "{{ .TOOL_DIR }}/golangci-lint run --tests=false" - - - check-licenses: - # desc: Ensure transitive dependencies are compliant with the current license policy - deps: [tools] - cmd: "{{ .TOOL_DIR }}/bouncer check ./..." - - check-go-mod-tidy: - # desc: Ensure go.mod and go.sum are up to date - cmds: - - cmd: .github/scripts/go-mod-tidy-check.sh && echo "go.mod and go.sum are tidy!" - silent: true + ## Static analysis extras ################################################# check-json-schema-drift: desc: Ensure there is no drift between the JSON schema and the code @@ -203,8 +99,8 @@ tasks: cmds: - .github/scripts/check_binary_fixture_size.sh syft/pkg/cataloger/binary/testdata/classifiers/snippets + ## Test extras ############################################################ - ## Testing tasks ################################# update-format-golden-files: desc: "Update golden (i.e. snapshot) files used by unit tests" cmds: @@ -214,59 +110,32 @@ tasks: - go test ./syft/format/cyclonedxjson -update-cyclonedx-json - go test ./syft/format/syftjson -update-json - unit: - desc: Run unit tests - deps: - - tmpdir - - fixtures - vars: - TEST_PKGS: - sh: "go list ./... | grep -v {{ .OWNER }}/{{ .PROJECT }}/test | grep -v {{ .OWNER }}/{{ .PROJECT }}/cmd/syft/internal/test | tr '\n' ' '" - - # unit test coverage threshold (in % coverage) - COVERAGE_THRESHOLD: 62 - cmds: - - task: clean-test-observations - - "go test -coverprofile {{ .TMP_DIR }}/unit-coverage-details.txt {{ .TEST_PKGS }}" - - cmd: ".github/scripts/coverage.py {{ .COVERAGE_THRESHOLD }} {{ .TMP_DIR }}/unit-coverage-details.txt" - silent: true - - integration: - desc: Run integration tests - cmds: - - "go test -v ./cmd/syft/internal/test/integration" - # exercise most of the CLI with the data race detector - # we use a larger image to ensure we're using multiple catalogers at a time - - "go run -race cmd/syft/main.go anchore/test_images:grype-quality-dotnet-69f15d2" - validate-cyclonedx-schema: desc: Validate that Syft produces valid CycloneDX documents cmds: - "cd schema/cyclonedx && make" - cli: - desc: Run CLI tests - deps: [tools] - cmds: - - cmd: "echo 'testing binary: {{ .SNAPSHOT_BIN }}'" - silent: true - - cmd: "test -f {{ .SNAPSHOT_BIN }} || (find {{ .SNAPSHOT_DIR }} && echo '\nno snapshot found for {{ .SNAPSHOT_BIN }}' && false)" - silent: true - - - "go test -count=1 -timeout=15m -v ./test/cli" - env: - SYFT_BINARY_LOCATION: "{{ .SNAPSHOT_BIN }}" - test-utils: desc: Run tests for pipeline utils cmds: - cmd: .github/scripts/labeler_test.py + snapshot-smoke-test: + desc: Run a smoke test on the snapshot builds + docker images + cmds: + - cmd: "echo 'testing snapshot binary: {{ .SNAPSHOT_BIN }}'" + silent: true + - cmd: "test -f {{ .SNAPSHOT_BIN }} || (find {{ .SNAPSHOT_DIR }} && echo '\nno snapshot found for {{ .SNAPSHOT_BIN }}' && false)" + silent: true + - "{{ .SNAPSHOT_BIN }} version" + - "{{ .SNAPSHOT_BIN }} scan alpine:latest" + - docker run --rm anchore/syft:latest version + - docker run --rm anchore/syft:latest scan alpine:latest - ## Test-fixture-related targets ################################# + ## Test-fixture-related targets ########################################### fingerprints: - desc: Generate fingerprints for all non-docker test fixture + desc: Generate fingerprints for all non-docker test fixtures silent: true # this will look for `testdata/Makefile` and invoke the `fingerprint` target to calculate all cache input fingerprint files generates: @@ -281,16 +150,10 @@ tasks: echo -e "${YELLOW}creating fingerprint files for non-docker fixtures...${RESET}" for dir in $(find . -type d -name 'testdata'); do if [ -f "$dir/Makefile" ]; then - # for debugging... - #echo -e "${YELLOW}• calculating fingerprints in $dir... ${RESET}" - (make -C "$dir" fingerprint) fi done - # for debugging... - # echo -e "generated all fixture fingerprints" - - .github/scripts/fingerprint_docker_fixtures.py - | # if DOWNLOAD_TEST_FIXTURE_CACHE is set to 'false', then we don't need to calculate the fingerprint for the cache @@ -432,6 +295,7 @@ tasks: eval $oras_command show-test-image-cache: + desc: Print the on-disk + docker daemon state of the stereoscope fixture image cache silent: true cmds: - "echo 'Docker daemon cache:'" @@ -452,28 +316,36 @@ tasks: exit 1 fi - ## install.sh testing targets ################################# + ## install.sh testing targets ############################################# install-test: + desc: Run install.sh test suite (delegates to test/install/Makefile) cmds: - "cd test/install && make" install-test-cache-save: + desc: Save the install.sh test image cache (delegates to test/install/Makefile) cmds: - "cd test/install && make save" install-test-cache-load: + desc: Load the install.sh test image cache (delegates to test/install/Makefile) cmds: - "cd test/install && make load" install-test-ci-mac: + desc: Run install.sh CI test suite on macOS (delegates to test/install/Makefile) cmds: - "cd test/install && make ci-test-mac" + ## Compare-test targets ################################################### + generate-compare-file: + desc: Generate the acceptance comparison reference JSON for the current compare image cmd: "go run ./cmd/syft {{ .COMPARE_TEST_IMAGE }} -o json > {{ .COMPARE_DIR }}/testdata/acceptance-{{ .COMPARE_TEST_IMAGE }}.json" compare-mac: + desc: Run macOS install + acceptance comparison against the snapshot build deps: [tmpdir] cmd: | {{ .COMPARE_DIR }}/mac.sh \ @@ -483,11 +355,13 @@ tasks: {{ .TMP_DIR }} compare-linux: + desc: Run Linux install + acceptance comparison (deb + rpm) against the snapshot build cmds: - task: compare-test-deb-package-install - task: compare-test-rpm-package-install compare-test-deb-package-install: + desc: Run Linux .deb install + acceptance comparison against the snapshot build deps: [tmpdir] cmd: | {{ .COMPARE_DIR }}/deb.sh \ @@ -497,6 +371,7 @@ tasks: {{ .TMP_DIR }} compare-test-rpm-package-install: + desc: Run Linux .rpm install + acceptance comparison against the snapshot build deps: [tmpdir] cmd: | {{ .COMPARE_DIR }}/rpm.sh \ @@ -506,7 +381,7 @@ tasks: {{ .TMP_DIR }} - ## Code and data generation targets ################################# + ## Code and data generation targets ###################################### generate: desc: Add data generation tasks @@ -556,104 +431,7 @@ tasks: - "SYFT_ENABLE_COMPLETENESS_TESTS=true go test -p 1 ./internal/capabilities/... -count=1" - ## Build-related targets ################################# - - build: - desc: Build the project - deps: [tools, tmpdir] - generates: - - "{{ .PROJECT }}" - cmds: - - silent: true - cmd: | - echo "dist: {{ .SNAPSHOT_DIR }}" > {{ .TMP_DIR }}/goreleaser.yaml - cat .goreleaser.yaml >> {{ .TMP_DIR }}/goreleaser.yaml - - - "{{ .BUILD_CMD }}" - - snapshot: - desc: Create a snapshot release - aliases: - - build - deps: [tools, tmpdir] - sources: - - cmd/**/*.go - - syft/**/*.go - - internal/**/*.go - method: checksum - generates: - - "{{ .SNAPSHOT_BIN }}" - cmds: - - silent: true - cmd: | - echo "dist: {{ .SNAPSHOT_DIR }}" > {{ .TMP_DIR }}/goreleaser.yaml - cat .goreleaser.yaml >> {{ .TMP_DIR }}/goreleaser.yaml - - - "{{ .SNAPSHOT_CMD }}" - - snapshot-smoke-test: - desc: Run a smoke test on the snapshot builds + docker images - cmds: - - cmd: "echo 'testing snapshot binary: {{ .SNAPSHOT_BIN }}'" - silent: true - - cmd: "test -f {{ .SNAPSHOT_BIN }} || (find {{ .SNAPSHOT_DIR }} && echo '\nno snapshot found for {{ .SNAPSHOT_BIN }}' && false)" - silent: true - - "{{ .SNAPSHOT_BIN }} version" - - "{{ .SNAPSHOT_BIN }} scan alpine:latest" - - docker run --rm anchore/syft:latest version - - docker run --rm anchore/syft:latest scan alpine:latest - - changelog: - desc: Generate a changelog - deps: [tools] - generates: - - "{{ .CHANGELOG }}" - - "{{ .NEXT_VERSION }}" - cmds: - - "{{ .TOOL_DIR }}/chronicle -vv -n --version-file {{ .NEXT_VERSION }} > {{ .CHANGELOG }}" - - "{{ .TOOL_DIR }}/glow -w 0 {{ .CHANGELOG }}" - - - ## Release targets ################################# - - release: - desc: Create a release - interactive: true - deps: [tools] - cmds: - - cmd: .github/scripts/trigger-release.sh - silent: true - - - ## CI-only targets ################################# - - ci-check: - # desc: "[CI only] Are you in CI?" - cmds: - - cmd: .github/scripts/ci-check.sh - silent: true - - ci-release: - # desc: "[CI only] Create a release" - deps: [tools] - cmds: - - task: ci-check - - "{{ .TOOL_DIR }}/chronicle -vvv > CHANGELOG.md" - - cmd: "cat CHANGELOG.md" - silent: true - - "{{ .RELEASE_CMD }}" - - - ## Cleanup targets ################################# - - clean: - desc: Remove all cache files and old builds - cmds: - - task: clean-snapshot - - task: clean-cache - - task: clean-test-observations - - task: clean-docker-cache - - task: clean-oras-cache + ## Cleanup targets ######################################################## clean-snapshot: desc: Remove any snapshot builds @@ -675,6 +453,7 @@ tasks: desc: Remove all image docker tar cache, images from the docker daemon, and ephemeral test fixtures cmds: - task: clean-docker-cache + - task: prune-orphan-fingerprints - | BOLD='\033[1m' YELLOW='\033[0;33m' @@ -690,6 +469,12 @@ tasks: echo -e "${BOLD}Deleted all ephemeral test fixtures${RESET}" - rm -f {{ .LAST_CACHE_PULL_FILE }} {{ .CACHE_PATHS_FILE }} + prune-orphan-fingerprints: + desc: Remove *.fingerprint files left behind by moved/deleted catalogers + silent: true + cmds: + - .github/scripts/prune_orphan_fingerprints.py + clean-test-observations: desc: Remove all test observations (i.e. testdata/test-observations.json) cmds: diff --git a/syft/file/cataloger/executable/testdata/elf/Makefile b/syft/file/cataloger/executable/testdata/elf/Makefile index 5d859e067..cfc6ff450 100644 --- a/syft/file/cataloger/executable/testdata/elf/Makefile +++ b/syft/file/cataloger/executable/testdata/elf/Makefile @@ -15,14 +15,18 @@ fixtures: build verify # requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted fingerprint: $(FINGERPRINT_FILE) -tools-check: - @sha256sum -c Dockerfile.sha256 || (echo "Tools Dockerfile has changed" && exit 1) - # for selfrando... # docker buildx build --platform linux/amd64 -t $(TOOL_IMAGE) . tools: - @(docker image inspect $(TOOL_IMAGE) > /dev/null 2>&1 && make tools-check) || (docker build -t $(TOOL_IMAGE) . && sha256sum Dockerfile > Dockerfile.sha256) + @if docker image inspect $(TOOL_IMAGE) > /dev/null 2>&1 \ + && test -f Dockerfile.sha256 \ + && sha256sum --quiet -c Dockerfile.sha256 2>/dev/null; then \ + : ; \ + else \ + docker build -t $(TOOL_IMAGE) . \ + && sha256sum Dockerfile > Dockerfile.sha256; \ + fi build: tools mkdir -p $(BIN) @@ -46,4 +50,4 @@ $(FINGERPRINT_FILE): clean: rm -rf $(BIN) Dockerfile.sha256 $(VERIFY_FILE) $(FINGERPRINT_FILE) -.PHONY: tools tools-check build verify debug clean \ No newline at end of file +.PHONY: tools build verify debug clean \ No newline at end of file diff --git a/syft/file/cataloger/executable/testdata/shared-info/Makefile b/syft/file/cataloger/executable/testdata/shared-info/Makefile index cc5b03a77..12ceccc68 100644 --- a/syft/file/cataloger/executable/testdata/shared-info/Makefile +++ b/syft/file/cataloger/executable/testdata/shared-info/Makefile @@ -15,11 +15,15 @@ fixtures: build # requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted fingerprint: $(FINGERPRINT_FILE) -tools-check: - @sha256sum -c Dockerfile.sha256 || (echo "Tools Dockerfile has changed" && exit 1) - tools: - @(docker image inspect $(TOOL_IMAGE) > /dev/null 2>&1 && make tools-check) || (docker build --platform linux/amd64 -t $(TOOL_IMAGE) . && sha256sum Dockerfile > Dockerfile.sha256) + @if docker image inspect $(TOOL_IMAGE) > /dev/null 2>&1 \ + && test -f Dockerfile.sha256 \ + && sha256sum --quiet -c Dockerfile.sha256 2>/dev/null; then \ + : ; \ + else \ + docker build --platform linux/amd64 -t $(TOOL_IMAGE) . \ + && sha256sum Dockerfile > Dockerfile.sha256; \ + fi build: tools @mkdir -p $(BIN) @@ -38,4 +42,4 @@ $(FINGERPRINT_FILE): clean: rm -rf $(BIN) Dockerfile.sha256 $(VERIFY_FILE) $(FINGERPRINT_FILE) -.PHONY: tools tools-check build debug clean +.PHONY: tools build debug clean