mirror of
https://github.com/anchore/syft.git
synced 2026-10-30 22:12:28 +01:00
Slim down docker cache size (#3190)
* slim down docker cache size Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * remove old centos images Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * troubleshoot test failure Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix wget version ref Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * refactor caching mechanisms Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add cache cleanup steps Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * simplify deleting cache Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix first clone issue Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add tool dep Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
@@ -13,16 +13,15 @@ inputs:
|
||||
cache-key-prefix:
|
||||
description: "Prefix all cache keys with this value"
|
||||
required: true
|
||||
default: "1ac8281053"
|
||||
compute-fingerprints:
|
||||
description: "Compute test fixture fingerprints"
|
||||
default: "181053ac82"
|
||||
download-test-fixture-cache:
|
||||
description: "Download test fixture cache from OCI and github actions"
|
||||
required: true
|
||||
default: "true"
|
||||
default: "false"
|
||||
bootstrap-apt-packages:
|
||||
description: "Space delimited list of tools to install via apt"
|
||||
default: "libxml2-utils"
|
||||
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
@@ -54,8 +53,14 @@ runs:
|
||||
run: |
|
||||
DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y ${{ inputs.bootstrap-apt-packages }}
|
||||
|
||||
- name: Create all cache fingerprints
|
||||
if: inputs.compute-fingerprints == 'true'
|
||||
shell: bash
|
||||
run: make fingerprints
|
||||
- name: Restore ORAS cache from github actions
|
||||
if: inputs.download-test-fixture-cache == 'true'
|
||||
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
|
||||
with:
|
||||
path: ${{ github.workspace }}/.tmp/oras-cache
|
||||
key: ${{ inputs.cache-key-prefix }}-oras-cache
|
||||
|
||||
- name: Download test fixture cache
|
||||
if: inputs.download-test-fixture-cache == 'true'
|
||||
shell: bash
|
||||
run: make download-test-fixture-cache
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
red=$(tput setaf 1)
|
||||
bold=$(tput bold)
|
||||
normal=$(tput sgr0)
|
||||
|
||||
# assert we are running in CI (or die!)
|
||||
if [[ -z "$CI" ]]; then
|
||||
echo "${bold}${red}This step should ONLY be run in CI. Exiting...${normal}"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+135
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import glob
|
||||
import sys
|
||||
import json
|
||||
import hashlib
|
||||
|
||||
|
||||
IGNORED_PREFIXES = []
|
||||
|
||||
|
||||
def find_fingerprints_and_check_dirs(base_dir):
|
||||
all_fingerprints = set(glob.glob(os.path.join(base_dir, '**', 'test*', '**', '*.fingerprint'), recursive=True))
|
||||
|
||||
all_fingerprints = {os.path.relpath(fp) for fp in all_fingerprints
|
||||
if not any(fp.startswith(prefix) for prefix in IGNORED_PREFIXES)}
|
||||
|
||||
if not all_fingerprints:
|
||||
show("No .fingerprint files or cache directories found.")
|
||||
exit(1)
|
||||
|
||||
missing_content = []
|
||||
valid_paths = set()
|
||||
fingerprint_contents = []
|
||||
|
||||
for fingerprint in all_fingerprints:
|
||||
path = fingerprint.replace('.fingerprint', '')
|
||||
|
||||
if not os.path.exists(path):
|
||||
missing_content.append(path)
|
||||
continue
|
||||
|
||||
if not os.path.isdir(path):
|
||||
valid_paths.add(path)
|
||||
continue
|
||||
|
||||
if os.listdir(path):
|
||||
valid_paths.add(path)
|
||||
else:
|
||||
missing_content.append(path)
|
||||
|
||||
with open(fingerprint, 'r') as f:
|
||||
content = f.read().strip()
|
||||
fingerprint_contents.append((fingerprint, content))
|
||||
|
||||
return sorted(valid_paths), missing_content, fingerprint_contents
|
||||
|
||||
|
||||
def parse_fingerprint_contents(fingerprint_content):
|
||||
input_map = {}
|
||||
for line in fingerprint_content.splitlines():
|
||||
digest, path = line.split()
|
||||
input_map[path] = digest
|
||||
return input_map
|
||||
|
||||
|
||||
def calculate_sha256(fingerprint_contents):
|
||||
sorted_fingerprint_contents = sorted(fingerprint_contents, key=lambda x: x[0])
|
||||
|
||||
concatenated_contents = ''.join(content for _, content in sorted_fingerprint_contents)
|
||||
|
||||
sha256_hash = hashlib.sha256(concatenated_contents.encode()).hexdigest()
|
||||
|
||||
return sha256_hash
|
||||
|
||||
|
||||
def calculate_file_sha256(file_path):
|
||||
sha256_hash = hashlib.sha256()
|
||||
with open(file_path, 'rb') as f:
|
||||
for byte_block in iter(lambda: f.read(4096), b""):
|
||||
sha256_hash.update(byte_block)
|
||||
return sha256_hash.hexdigest()
|
||||
|
||||
|
||||
def show(*s: str):
|
||||
print(*s, file=sys.stderr)
|
||||
|
||||
|
||||
def main(file_path: str | None):
|
||||
base_dir = '.'
|
||||
valid_paths, missing_content, fingerprint_contents = find_fingerprints_and_check_dirs(base_dir)
|
||||
|
||||
if missing_content:
|
||||
show("The following paths are missing or have no content, but have corresponding .fingerprint files:")
|
||||
for path in sorted(missing_content):
|
||||
show(f"- {path}")
|
||||
show("Please ensure these paths exist and have content if they are directories.")
|
||||
exit(1)
|
||||
|
||||
sha256_hash = calculate_sha256(fingerprint_contents)
|
||||
|
||||
paths_with_digests = []
|
||||
for path in sorted(valid_paths):
|
||||
fingerprint_file = f"{path}.fingerprint"
|
||||
try:
|
||||
if os.path.exists(fingerprint_file):
|
||||
file_digest = calculate_file_sha256(fingerprint_file)
|
||||
|
||||
# Parse the fingerprint file to get the digest/path tuples
|
||||
with open(fingerprint_file, 'r') as f:
|
||||
fingerprint_content = f.read().strip()
|
||||
input_map = parse_fingerprint_contents(fingerprint_content)
|
||||
|
||||
paths_with_digests.append({
|
||||
"path": path,
|
||||
"digest": file_digest,
|
||||
"input": input_map
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
show(f"Error processing {fingerprint_file}: {e}")
|
||||
raise e
|
||||
|
||||
|
||||
output = {
|
||||
"digest": sha256_hash,
|
||||
"paths": paths_with_digests
|
||||
}
|
||||
|
||||
content = json.dumps(output, indent=2, sort_keys=True)
|
||||
|
||||
if file_path:
|
||||
with open(file_path, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print(content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
file_path = None
|
||||
if len(sys.argv) > 1:
|
||||
file_path = sys.argv[1]
|
||||
main(file_path)
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import hashlib
|
||||
|
||||
BOLD = '\033[1m'
|
||||
YELLOW = '\033[0;33m'
|
||||
RESET = '\033[0m'
|
||||
|
||||
|
||||
def print_message(message):
|
||||
print(f"{YELLOW}{message}{RESET}")
|
||||
|
||||
|
||||
def sha256sum(filepath):
|
||||
h = hashlib.sha256()
|
||||
with open(filepath, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def is_git_tracked_or_untracked(directory):
|
||||
"""Returns a sorted list of files in the directory that are tracked or not ignored by Git."""
|
||||
result = subprocess.run(
|
||||
["git", "ls-files", "--cached", "--others", "--exclude-standard"],
|
||||
cwd=directory,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
return sorted(result.stdout.strip().splitlines())
|
||||
|
||||
|
||||
def find_test_fixture_dirs_with_images(base_dir):
|
||||
"""Find directories that contain 'test-fixtures' and at least one 'image-*' directory."""
|
||||
for root, dirs, files in os.walk(base_dir):
|
||||
if 'test-fixtures' in root:
|
||||
image_dirs = [d for d in dirs if d.startswith('image-')]
|
||||
if image_dirs:
|
||||
yield os.path.realpath(root)
|
||||
|
||||
|
||||
def generate_fingerprints():
|
||||
print_message("creating fingerprint files for docker fixtures...")
|
||||
|
||||
for test_fixture_dir in find_test_fixture_dirs_with_images('.'):
|
||||
cache_fingerprint_path = os.path.join(test_fixture_dir, 'cache.fingerprint')
|
||||
|
||||
with open(cache_fingerprint_path, 'w') as fingerprint_file:
|
||||
for image_dir in find_image_dirs(test_fixture_dir):
|
||||
for file in is_git_tracked_or_untracked(image_dir):
|
||||
file_path = os.path.join(image_dir, file)
|
||||
checksum = sha256sum(file_path)
|
||||
path_from_fixture_dir = os.path.relpath(file_path, test_fixture_dir)
|
||||
fingerprint_file.write(f"{checksum} {path_from_fixture_dir}\n")
|
||||
|
||||
|
||||
def find_image_dirs(test_fixture_dir):
|
||||
"""Find all 'image-*' directories inside a given test-fixture directory."""
|
||||
result = []
|
||||
for root, dirs, files in os.walk(test_fixture_dir):
|
||||
for dir_name in dirs:
|
||||
if dir_name.startswith('image-'):
|
||||
result.append(os.path.join(root, dir_name))
|
||||
return sorted(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate_fingerprints()
|
||||
Regular → Executable
+2
@@ -1,3 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
Regular → Executable
+2
@@ -1,3 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import subprocess
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: "Release"
|
||||
name: "Release: version file"
|
||||
|
||||
on:
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
name: "Test fixture cache: publish"
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# run nightly at 4AM UTC
|
||||
- cron: "0 4 * * *"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
Publish:
|
||||
name: "Publish test fixture image cache"
|
||||
# we use this runner to get enough storage space for docker images and fixture cache
|
||||
runs-on: ubuntu-22.04-4core-16gb
|
||||
permissions:
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
|
||||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
with:
|
||||
# we want to rebuild the cache with no previous state
|
||||
download-test-fixture-cache: false
|
||||
|
||||
- name: Run all tests
|
||||
run: make test
|
||||
env:
|
||||
# we want to rebuild the cache with no previous state
|
||||
DOWNLOAD_TEST_FIXTURE_CACHE: "false"
|
||||
|
||||
- name: Login to GitHub Container Registry (ORAS)
|
||||
run: echo "${{ secrets.GITHUB_TOKEN }}" | .tool/oras login ghcr.io -u ${{ github.actor }} --password-stdin
|
||||
|
||||
- name: Publish test fixture cache
|
||||
run: make upload-test-fixture-cache
|
||||
@@ -19,7 +19,6 @@ jobs:
|
||||
uses: ./.github/actions/bootstrap
|
||||
with:
|
||||
bootstrap-apt-packages: ""
|
||||
compute-fingerprints: "false"
|
||||
go-dependencies: false
|
||||
|
||||
- name: "Update tool versions"
|
||||
|
||||
@@ -35,48 +35,8 @@ jobs:
|
||||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
|
||||
- name: Restore file executable test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/file/cataloger/executable/test-fixtures/elf/bin
|
||||
key: ${{ runner.os }}-unit-file-executable-elf-cache-${{ hashFiles( 'syft/file/cataloger/executable/test-fixtures/elf/cache.fingerprint' ) }}
|
||||
|
||||
- name: Restore file executable shared-info test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/file/cataloger/executable/test-fixtures/shared-info/bin
|
||||
key: ${{ runner.os }}-unit-file-executable-shared-info-cache-${{ hashFiles( 'syft/file/cataloger/executable/test-fixtures/shared-info/cache.fingerprint' ) }}
|
||||
|
||||
- name: Restore Java test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/pkg/cataloger/java/test-fixtures/java-builds/packages
|
||||
key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/pkg/cataloger/java/test-fixtures/java-builds/cache.fingerprint' ) }}
|
||||
|
||||
- name: Restore RPM test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/pkg/cataloger/redhat/test-fixtures/rpms
|
||||
key: ${{ runner.os }}-unit-rpm-cache-${{ hashFiles( 'syft/pkg/cataloger/redhat/test-fixtures/rpms.fingerprint' ) }}
|
||||
|
||||
- name: Restore go binary test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/pkg/cataloger/golang/test-fixtures/archs/binaries
|
||||
key: ${{ runner.os }}-unit-go-binaries-cache-${{ hashFiles( 'syft/pkg/cataloger/golang/test-fixtures/archs/binaries.fingerprint' ) }}
|
||||
|
||||
- name: Restore binary cataloger test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/pkg/cataloger/binary/test-fixtures/classifiers/bin
|
||||
key: ${{ runner.os }}-unit-binary-cataloger-cache-${{ hashFiles( 'syft/pkg/cataloger/binary/test-fixtures/cache.fingerprint' ) }}
|
||||
|
||||
- name: Restore Kernel test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/pkg/cataloger/kernel/test-fixtures/cache
|
||||
key: ${{ runner.os }}-unit-kernel-cache-${{ hashFiles( 'syft/pkg/cataloger/kernel/test-fixtures/cache.fingerprint' ) }}
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Run unit tests
|
||||
run: make unit
|
||||
@@ -91,16 +51,12 @@ jobs:
|
||||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
with:
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Validate syft output against the CycloneDX schema
|
||||
run: make validate-cyclonedx-schema
|
||||
|
||||
- name: Restore integration test cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: ${{ github.workspace }}/cmd/syft/internal/test/integration/test-fixtures/cache
|
||||
key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('/cmd/syft/internal/test/integration/test-fixtures/cache.fingerprint') }}
|
||||
|
||||
- name: Run integration tests
|
||||
run: make integration
|
||||
|
||||
@@ -143,6 +99,8 @@ jobs:
|
||||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
with:
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Download snapshot build
|
||||
id: snapshot-cache
|
||||
@@ -162,13 +120,6 @@ jobs:
|
||||
- name: Run comparison tests (Linux)
|
||||
run: make compare-linux
|
||||
|
||||
- name: Restore install.sh test image cache
|
||||
id: install-test-image-cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: ${{ github.workspace }}/test/install/cache
|
||||
key: ${{ runner.os }}-install-test-image-cache-${{ hashFiles('test/install/cache.fingerprint') }}
|
||||
|
||||
- name: Load test image cache
|
||||
if: steps.install-test-image-cache.outputs.cache-hit == 'true'
|
||||
run: make install-test-cache-load
|
||||
@@ -196,8 +147,8 @@ jobs:
|
||||
uses: ./.github/actions/bootstrap
|
||||
with:
|
||||
bootstrap-apt-packages: ""
|
||||
compute-fingerprints: "false"
|
||||
go-dependencies: false
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Download snapshot build
|
||||
id: snapshot-cache
|
||||
@@ -214,13 +165,6 @@ jobs:
|
||||
if: steps.snapshot-cache.outputs.cache-hit != 'true'
|
||||
run: echo "unable to download snapshots from previous job" && false
|
||||
|
||||
- name: Restore docker image cache for compare testing
|
||||
id: mac-compare-testing-cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: image.tar
|
||||
key: ${{ runner.os }}-${{ hashFiles('test/compare/mac.sh') }}
|
||||
|
||||
- name: Run comparison tests (Mac)
|
||||
run: make compare-mac
|
||||
|
||||
@@ -238,12 +182,8 @@ jobs:
|
||||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
|
||||
- name: Restore CLI test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: ${{ github.workspace }}/test/cli/test-fixtures/cache
|
||||
key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }}
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Download snapshot build
|
||||
id: snapshot-cache
|
||||
@@ -262,3 +202,22 @@ jobs:
|
||||
|
||||
- name: Run CLI Tests (Linux)
|
||||
run: make cli
|
||||
|
||||
|
||||
Cleanup-Cache:
|
||||
name: "Cleanup snapshot cache"
|
||||
if: always()
|
||||
runs-on: ubuntu-20.04
|
||||
permissions:
|
||||
actions: write
|
||||
needs:
|
||||
- Acceptance-Linux
|
||||
- Acceptance-Mac
|
||||
- Cli-Linux
|
||||
steps:
|
||||
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
|
||||
|
||||
- name: Delete snapshot cache
|
||||
run: gh cache delete "snapshot-build-${{ github.run_id }}"
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
Reference in New Issue
Block a user