From eb58f698bdcdb3310a2b02b8dd160f5e629c328f Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 11 Sep 2026 14:58:30 -0400 Subject: [PATCH] Fix oom when running unit tests in CI (#5284) * fix oom when testing Signed-off-by: Alex Goodman * right-size CI test runners the `r` family and `ram=32+128` were sized for the tmpfs footprint rather than the workload. with the caches back on disk the build+test peak fits in 16GB, so move to `m`/`c` at `ram=16+32`. - ~27% cheaper per instance-hour on spot, and a much deeper pool to schedule against (~20 x86 types vs 7) - `cpu=4+8` with `ram=16+32` lets price-capacity-optimized opportunistically land an 8 vCPU box when it's cheap - plain `volume=100gb` replaces the provisioned throughput. a 4 vCPU instance caps near 150MB/s of EBS and gp3's free baseline is already 125MB/s, so the override bought nothing Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- .github/actions/bootstrap/action.yaml | 12 +++++++++++- .github/workflows/validations.yaml | 7 +++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index aad6d6090..4c826b557 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -47,7 +47,17 @@ runs: APT_PACKAGES: ${{ inputs.bootstrap-apt-packages }} run: | IFS=' ' read -ra packages <<< "$APT_PACKAGES" - DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y "${packages[@]}" + # the AMI already ships most of these, and `apt update` costs ~40MB of index (minutes, + # when archive.ubuntu.com throttles us) to find that out. only touch apt if we must. + missing=() + for pkg in "${packages[@]}"; do + dpkg -s "$pkg" >/dev/null 2>&1 || missing+=("$pkg") + done + if [ ${#missing[@]} -eq 0 ]; then + echo "all apt packages already present: ${packages[*]}" + exit 0 + fi + DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y "${missing[@]}" # ORAS cache: restore-only on non-default branches / forks - name: Restore ORAS cache from github actions diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index b4bb10ec2..2656ab1f1 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -21,10 +21,13 @@ jobs: name: "Static analysis" # runs-on.com: memory & general purpose instances for testing # spot enabled: ok to interrupt non-production workloads - # tmpfs: faster io-intensive workflows + # tmpfs intentionally omitted -- it RAM-backs /home/runner, /tmp and /var/lib/docker at + # 100% of instance memory, starving `make unit` of headroom to link with -race. + # volume: the caches and docker images tmpfs used to hold. gp3 baseline throughput is + # enough; a 4 vCPU instance can't pull much more than that anyway. # note: s3-cache intentionally omitted -- PR runs are untrusted and must not write to the # shared cache backend that the trusted release workflow reads from (cache poisoning). - runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=32+128/family=r5+r6+r7+r8+m4+m5+m6+m7+m8/spot=price-capacity-optimized/extras=tmpfs" + runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=16+32/family=m5+m6+m7+m8+c5+c6+c7+c8/spot=price-capacity-optimized/volume=100gb" permissions: contents: read steps: