Slim down docker cache size (#3190)

* slim down docker cache size

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove old centos images

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* troubleshoot test failure

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix wget version ref

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor caching mechanisms

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add cache cleanup steps

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* simplify deleting cache

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix first clone issue

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tool dep

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

.github/scripts/ci-check.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-red=$(tput setaf 1)
-bold=$(tput bold)
-normal=$(tput sgr0)
-
-# assert we are running in CI (or die!)
-if [[ -z "$CI" ]]; then
-    echo "${bold}${red}This step should ONLY be run in CI. Exiting...${normal}"
-    exit 1
-fi

.github/scripts/find_cache_paths.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import glob
+import sys
+import json
+import hashlib
+
+
+IGNORED_PREFIXES = []
+
+
+def find_fingerprints_and_check_dirs(base_dir):
+    all_fingerprints = set(glob.glob(os.path.join(base_dir, '**', 'test*', '**', '*.fingerprint'), recursive=True))
+
+    all_fingerprints = {os.path.relpath(fp) for fp in all_fingerprints
+                        if not any(fp.startswith(prefix) for prefix in IGNORED_PREFIXES)}
+
+    if not all_fingerprints:
+        show("No .fingerprint files or cache directories found.")
+        exit(1)
+
+    missing_content = []
+    valid_paths = set()
+    fingerprint_contents = []
+
+    for fingerprint in all_fingerprints:
+        path = fingerprint.replace('.fingerprint', '')
+
+        if not os.path.exists(path):
+            missing_content.append(path)
+            continue
+
+        if not os.path.isdir(path):
+            valid_paths.add(path)
+            continue
+
+        if os.listdir(path):
+            valid_paths.add(path)
+        else:
+            missing_content.append(path)
+
+        with open(fingerprint, 'r') as f:
+            content = f.read().strip()
+            fingerprint_contents.append((fingerprint, content))
+
+    return sorted(valid_paths), missing_content, fingerprint_contents
+
+
+def parse_fingerprint_contents(fingerprint_content):
+    input_map = {}
+    for line in fingerprint_content.splitlines():
+        digest, path = line.split()
+        input_map[path] = digest
+    return input_map
+
+
+def calculate_sha256(fingerprint_contents):
+    sorted_fingerprint_contents = sorted(fingerprint_contents, key=lambda x: x[0])
+
+    concatenated_contents = ''.join(content for _, content in sorted_fingerprint_contents)
+
+    sha256_hash = hashlib.sha256(concatenated_contents.encode()).hexdigest()
+
+    return sha256_hash
+
+
+def calculate_file_sha256(file_path):
+    sha256_hash = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        for byte_block in iter(lambda: f.read(4096), b""):
+            sha256_hash.update(byte_block)
+    return sha256_hash.hexdigest()
+
+
+def show(*s: str):
+    print(*s, file=sys.stderr)
+
+
+def main(file_path: str | None):
+    base_dir = '.'
+    valid_paths, missing_content, fingerprint_contents = find_fingerprints_and_check_dirs(base_dir)
+
+    if missing_content:
+        show("The following paths are missing or have no content, but have corresponding .fingerprint files:")
+        for path in sorted(missing_content):
+            show(f"- {path}")
+        show("Please ensure these paths exist and have content if they are directories.")
+        exit(1)
+
+    sha256_hash = calculate_sha256(fingerprint_contents)
+
+    paths_with_digests = []
+    for path in sorted(valid_paths):
+        fingerprint_file = f"{path}.fingerprint"
+        try:
+            if os.path.exists(fingerprint_file):
+                file_digest = calculate_file_sha256(fingerprint_file)
+
+                # Parse the fingerprint file to get the digest/path tuples
+                with open(fingerprint_file, 'r') as f:
+                    fingerprint_content = f.read().strip()
+                    input_map = parse_fingerprint_contents(fingerprint_content)
+
+                paths_with_digests.append({
+                    "path": path,
+                    "digest": file_digest,
+                    "input": input_map
+                })
+
+        except Exception as e:
+            show(f"Error processing {fingerprint_file}: {e}")
+            raise e
+
+
+    output = {
+        "digest": sha256_hash,
+        "paths": paths_with_digests
+    }
+
+    content = json.dumps(output, indent=2, sort_keys=True)
+
+    if file_path:
+        with open(file_path, 'w') as f:
+            f.write(content)
+
+    print(content)
+
+
+if __name__ == "__main__":
+    file_path = None
+    if len(sys.argv) > 1:
+        file_path = sys.argv[1]
+    main(file_path)

.github/scripts/fingerprint_docker_fixtures.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import hashlib
+
+BOLD = '\033[1m'
+YELLOW = '\033[0;33m'
+RESET = '\033[0m'
+
+
+def print_message(message):
+    print(f"{YELLOW}{message}{RESET}")
+
+
+def sha256sum(filepath):
+    h = hashlib.sha256()
+    with open(filepath, 'rb') as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def is_git_tracked_or_untracked(directory):
+    """Returns a sorted list of files in the directory that are tracked or not ignored by Git."""
+    result = subprocess.run(
+        ["git", "ls-files", "--cached", "--others", "--exclude-standard"],
+        cwd=directory,
+        stdout=subprocess.PIPE,
+        text=True
+    )
+    return sorted(result.stdout.strip().splitlines())
+
+
+def find_test_fixture_dirs_with_images(base_dir):
+    """Find directories that contain 'test-fixtures' and at least one 'image-*' directory."""
+    for root, dirs, files in os.walk(base_dir):
+        if 'test-fixtures' in root:
+            image_dirs = [d for d in dirs if d.startswith('image-')]
+            if image_dirs:
+                yield os.path.realpath(root)
+
+
+def generate_fingerprints():
+    print_message("creating fingerprint files for docker fixtures...")
+
+    for test_fixture_dir in find_test_fixture_dirs_with_images('.'):
+        cache_fingerprint_path = os.path.join(test_fixture_dir, 'cache.fingerprint')
+
+        with open(cache_fingerprint_path, 'w') as fingerprint_file:
+            for image_dir in find_image_dirs(test_fixture_dir):
+                for file in is_git_tracked_or_untracked(image_dir):
+                    file_path = os.path.join(image_dir, file)
+                    checksum = sha256sum(file_path)
+                    path_from_fixture_dir = os.path.relpath(file_path, test_fixture_dir)
+                    fingerprint_file.write(f"{checksum}  {path_from_fixture_dir}\n")
+
+
+def find_image_dirs(test_fixture_dir):
+    """Find all 'image-*' directories inside a given test-fixture directory."""
+    result = []
+    for root, dirs, files in os.walk(test_fixture_dir):
+        for dir_name in dirs:
+            if dir_name.startswith('image-'):
+                result.append(os.path.join(root, dir_name))
+    return sorted(result)
+
+
+if __name__ == "__main__":
+    generate_fingerprints()

.github/scripts/labeler.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 from __future__ import annotations
 
 import sys

.github/scripts/labeler_test.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import unittest
 from unittest.mock import patch
 import subprocess