oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-10-30 14:02:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Port to go-make (#4923)

Browse Source 
* port to go-make

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refresh fixtures on running unit tests

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address refresh cache issues with old now-gitignored files

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2026-05-18 11:59:55 -04:00
committed by GitHub
parent 89cda82263
commit d61af0abab
18 changed files with 517 additions and 638 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/ci-check.sh
-11
View File
@@ -1,11 +0,0 @@
#!/usr/bin/env bash
red=$(tput setaf 1)
bold=$(tput bold)
normal=$(tput sgr0)
# assert we are running in CI (or die!)
if [[ -z "$CI" ]]; then
echo "${bold}${red}This step should ONLY be run in CI. Exiting...${normal}"
exit 1
fi
.github/scripts/coverage.py
-36
View File
@@ -1,36 +0,0 @@
#!/usr/bin/env python3
import subprocess
import sys
import shlex
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKCYAN = '\033[96m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
if len(sys.argv) < 3:
print("Usage: coverage.py [threshold] [go-coverage-report]")
sys.exit(1)
threshold = float(sys.argv[1])
report = sys.argv[2]
args = shlex.split(f"go tool cover -func {report}")
p = subprocess.run(args, capture_output=True, text=True)
percent_coverage = float(p.stdout.splitlines()[-1].split()[-1].replace("%", ""))
print(f"{bcolors.BOLD}Coverage: {percent_coverage}%{bcolors.ENDC}")
if percent_coverage < threshold:
print(f"{bcolors.BOLD}{bcolors.FAIL}Coverage below threshold of {threshold}%{bcolors.ENDC}")
sys.exit(1)
.github/scripts/find_cache_paths.py
+51 -34
View File
@@ -1,35 +1,45 @@
#!/usr/bin/env python3
from __future__ import annotations
import os
import glob
import sys
import json
import hashlib
import json
import os
import sys
IGNORED_PREFIXES = []
def find_fingerprints_and_check_dirs(base_dir):
all_fingerprints = set(glob.glob(os.path.join(base_dir, '**', 'test*', '**', '*.fingerprint'), recursive=True))
all_fingerprints = set(
glob.glob(
os.path.join(base_dir, "**", "test*", "**", "*.fingerprint"), recursive=True
)
)
all_fingerprints = {os.path.relpath(fp) for fp in all_fingerprints
if not any(fp.startswith(prefix) for prefix in IGNORED_PREFIXES)}
all_fingerprints = {
os.path.relpath(fp)
for fp in all_fingerprints
if not any(fp.startswith(prefix) for prefix in IGNORED_PREFIXES)
}
if not all_fingerprints:
show("No .fingerprint files or cache directories found.")
exit(1)
missing_content = []
orphan_fingerprints = []
empty_content = []
valid_paths = set()
fingerprint_contents = []
for fingerprint in all_fingerprints:
path = fingerprint.replace('.fingerprint', '')
path = fingerprint.replace(".fingerprint", "")
if not os.path.exists(path):
missing_content.append(path)
# paired content path is entirely missing — the .fingerprint is likely
# leftover from a moved/deleted source (testdata trees are git-ignored,
# so they persist locally across rename refactors)
orphan_fingerprints.append(fingerprint)
continue
if not os.path.isdir(path):
@@ -39,13 +49,13 @@ def find_fingerprints_and_check_dirs(base_dir):
if os.listdir(path):
valid_paths.add(path)
else:
missing_content.append(path)
empty_content.append(path)
with open(fingerprint, 'r') as f:
with open(fingerprint, "r") as f:
content = f.read().strip()
fingerprint_contents.append((fingerprint, content))
return sorted(valid_paths), missing_content, fingerprint_contents
return sorted(valid_paths), empty_content, orphan_fingerprints, fingerprint_contents
def parse_fingerprint_contents(fingerprint_content):
@@ -59,7 +69,9 @@ def parse_fingerprint_contents(fingerprint_content):
def calculate_sha256(fingerprint_contents):
sorted_fingerprint_contents = sorted(fingerprint_contents, key=lambda x: x[0])
concatenated_contents = ''.join(content for _, content in sorted_fingerprint_contents)
concatenated_contents = "".join(
content for _, content in sorted_fingerprint_contents
)
sha256_hash = hashlib.sha256(concatenated_contents.encode()).hexdigest()
@@ -68,7 +80,7 @@ def calculate_sha256(fingerprint_contents):
def calculate_file_sha256(file_path):
sha256_hash = hashlib.sha256()
with open(file_path, 'rb') as f:
with open(file_path, "rb") as f:
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
return sha256_hash.hexdigest()
@@ -79,17 +91,28 @@ def show(*s: str):
def main(file_path: str | None):
base_dir = '.'
valid_paths, missing_content, fingerprint_contents = find_fingerprints_and_check_dirs(base_dir)
base_dir = "."
valid_paths, empty_content, orphan_fingerprints, fingerprint_contents = (
find_fingerprints_and_check_dirs(base_dir)
)
if missing_content:
show("The following paths are missing or have no content, but have corresponding .fingerprint files:")
for path in sorted(missing_content):
if empty_content:
show(
"The following paths exist but are empty, and have corresponding .fingerprint files:"
)
for path in sorted(empty_content):
show(f"- {path}")
# when adding new cache directories there is a time where it is not possible to have this directory without
# running the tests first... but this step is a prerequisite for running the tests. We should not block on this.
# show("Please ensure these paths exist and have content if they are directories.")
# exit(1)
if orphan_fingerprints:
show(
"The following .fingerprint files reference paths that no longer exist "
"(likely leftover from a moved/deleted cataloger — safe to delete, "
"or run `task prune-orphan-fingerprints`):"
)
for fp in sorted(orphan_fingerprints):
show(f"- {fp}")
sha256_hash = calculate_sha256(fingerprint_contents)
@@ -101,30 +124,24 @@ def main(file_path: str | None):
file_digest = calculate_file_sha256(fingerprint_file)
# Parse the fingerprint file to get the digest/path tuples
with open(fingerprint_file, 'r') as f:
with open(fingerprint_file, "r") as f:
fingerprint_content = f.read().strip()
input_map = parse_fingerprint_contents(fingerprint_content)
paths_with_digests.append({
"path": path,
"digest": file_digest,
"input": input_map
})
paths_with_digests.append(
{"path": path, "digest": file_digest, "input": input_map}
)
except Exception as e:
show(f"Error processing {fingerprint_file}: {e}")
raise e
output = {
"digest": sha256_hash,
"paths": paths_with_digests
}
output = {"digest": sha256_hash, "paths": paths_with_digests}
content = json.dumps(output, indent=2, sort_keys=True)
if file_path:
with open(file_path, 'w') as f:
with open(file_path, "w") as f:
f.write(content)
print(content)
.github/scripts/go-mod-tidy-check.sh
-30
View File
@@ -1,30 +0,0 @@
#!/usr/bin/env bash
set -eu
ORIGINAL_STATE_DIR=$(mktemp -d "TEMP-original-state-XXXXXXXXX")
TIDY_STATE_DIR=$(mktemp -d "TEMP-tidy-state-XXXXXXXXX")
trap "cp -p ${ORIGINAL_STATE_DIR}/* ./ && git update-index -q --refresh && rm -fR ${ORIGINAL_STATE_DIR} ${TIDY_STATE_DIR}" EXIT
# capturing original state of files...
cp go.mod go.sum "${ORIGINAL_STATE_DIR}"
# capturing state of go.mod and go.sum after running go mod tidy...
go mod tidy
cp go.mod go.sum "${TIDY_STATE_DIR}"
set +e
# detect difference between the git HEAD state and the go mod tidy state
DIFF_MOD=$(diff -u "${ORIGINAL_STATE_DIR}/go.mod" "${TIDY_STATE_DIR}/go.mod")
DIFF_SUM=$(diff -u "${ORIGINAL_STATE_DIR}/go.sum" "${TIDY_STATE_DIR}/go.sum")
if [[ -n "${DIFF_MOD}" || -n "${DIFF_SUM}" ]]; then
echo "go.mod diff:"
echo "${DIFF_MOD}"
echo "go.sum diff:"
echo "${DIFF_SUM}"
echo ""
printf "FAILED! go.mod and/or go.sum are NOT tidy; please run 'go mod tidy'.\n\n"
exit 1
fi
.github/scripts/prune_orphan_fingerprints.py
Executable
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""
Remove orphan *.fingerprint files left behind by moved/deleted catalogers.
A fingerprint is considered orphaned when:
1. its paired content path (the fingerprint path with `.fingerprint` stripped)
does not exist, AND
2. the nearest ancestor `testdata/` directory has no `Makefile` claiming
responsibility for generating that path.
The second condition is the safety check: if there is a Makefile, the
fingerprint is "live" and might just be waiting for fixtures to be built —
leave it alone. Without a Makefile, nothing in-repo will ever regenerate
the content, so the fingerprint is dead weight that triggers spurious
"missing path" warnings.
Empty parent directories are also pruned after removing the fingerprint.
Use --dry-run to preview without deleting.
"""
from __future__ import annotations
import argparse
import glob
import os
import sys
def find_ancestor_testdata(path: str) -> str | None:
d = os.path.dirname(path)
while d and d not in (".", os.sep):
if os.path.basename(d) == "testdata":
return d
d = os.path.dirname(d)
return None
def is_orphan(fingerprint: str) -> bool:
paired = fingerprint[: -len(".fingerprint")]
if os.path.exists(paired):
return False
testdata_dir = find_ancestor_testdata(fingerprint)
if testdata_dir and os.path.isfile(os.path.join(testdata_dir, "Makefile")):
# a Makefile exists that may regenerate this — not safe to prune
return False
return True
def prune_empty_parents(start: str, stop_at: str = ".") -> list[str]:
removed = []
d = os.path.dirname(start)
stop_at = os.path.abspath(stop_at)
while d and os.path.abspath(d) != stop_at:
try:
if not os.listdir(d):
os.rmdir(d)
removed.append(d)
d = os.path.dirname(d)
else:
break
except OSError:
break
return removed
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--dry-run",
action="store_true",
help="Show what would be removed without deleting anything",
)
args = parser.parse_args()
all_fingerprints = glob.glob("**/test*/**/*.fingerprint", recursive=True)
orphans = sorted(fp for fp in all_fingerprints if is_orphan(fp))
if not orphans:
print("no orphan fingerprints found")
return 0
verb = "would remove" if args.dry_run else "removing"
print(f"{verb} {len(orphans)} orphan fingerprint(s):")
for fp in orphans:
print(f"- {fp}")
if args.dry_run:
continue
try:
os.remove(fp)
except OSError as e:
print(f" ! failed to remove: {e}", file=sys.stderr)
continue
for d in prune_empty_parents(fp):
print(f" (also removed empty dir {d})")
return 0
if __name__ == "__main__":
sys.exit(main())
.github/scripts/trigger-release.sh
-57
View File
@@ -1,57 +0,0 @@
#!/usr/bin/env bash
set -eu
bold=$(tput bold)
normal=$(tput sgr0)
GH_CLI=.tool/gh
if ! [ -x "$(command -v $GH_CLI)" ]; then
echo "The GitHub CLI could not be found. run: make bootstrap"
exit 1
fi
$GH_CLI auth status
# set the default repo in cases where multiple remotes are defined
$GH_CLI repo set-default anchore/syft
export GITHUB_TOKEN="${GITHUB_TOKEN-"$($GH_CLI auth token)"}"
# we need all of the git state to determine the next version. Since tagging is done by
# the release pipeline it is possible to not have all of the tags from previous releases.
git fetch --tags
# populates the CHANGELOG.md and VERSION files
echo "${bold}Generating changelog...${normal}"
make changelog 2> /dev/null
NEXT_VERSION=$(cat VERSION)
if [[ "$NEXT_VERSION" == "" || "${NEXT_VERSION}" == "(Unreleased)" ]]; then
echo "Could not determine the next version to release. Exiting..."
exit 1
fi
while true; do
read -p "${bold}Do you want to trigger a release for version '${NEXT_VERSION}'?${normal} [y/n] " yn
case $yn in
[Yy]* ) echo; break;;
[Nn]* ) echo; echo "Cancelling release..."; exit;;
* ) echo "Please answer yes or no.";;
esac
done
echo "${bold}Kicking off release for ${NEXT_VERSION}${normal}..."
echo
$GH_CLI workflow run release.yaml -f version=${NEXT_VERSION}
echo
echo "${bold}Waiting for release to start...${normal}"
sleep 10
set +e
echo "${bold}Head to the release workflow to monitor the release:${normal} $($GH_CLI run list --workflow=release.yaml --limit=1 --json url --jq '.[].url')"
id=$($GH_CLI run list --workflow=release.yaml --limit=1 --json databaseId --jq '.[].databaseId')
$GH_CLI run watch $id --exit-status || (echo ; echo "${bold}Logs of failed step:${normal}" && GH_PAGER="" $GH_CLI run view $id --log-failed)