mirror of
https://github.com/anchore/syft.git
synced 2026-11-03 16:06:36 +01:00
Slim down docker cache size (#3190)
* slim down docker cache size Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * remove old centos images Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * troubleshoot test failure Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix wget version ref Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * refactor caching mechanisms Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add cache cleanup steps Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * simplify deleting cache Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix first clone issue Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add tool dep Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
@@ -1,6 +1,21 @@
|
||||
# change these if you want CI to not use previous stored cache
|
||||
INTEGRATION_CACHE_BUSTER := "894d8ca"
|
||||
FINGERPRINT_FILE := cache.fingerprint
|
||||
|
||||
.PHONY: cache.fingerprint
|
||||
cache.fingerprint:
|
||||
find image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | tee /dev/stderr | md5sum | tee cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> cache.fingerprint
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures:
|
||||
@echo "nothing to do"
|
||||
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: $(FINGERPRINT_FILE)
|
||||
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find image-* -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f $(FINGERPRINT_FILE)
|
||||
|
||||
@@ -1 +1,6 @@
|
||||
FROM golang:1.18.10-alpine
|
||||
FROM --platform=linux/amd64 golang:1.18.10-alpine
|
||||
|
||||
FROM scratch
|
||||
|
||||
# we don't need the entire golang toolchain, just a single binary with the stdlib baked in
|
||||
COPY --from=0 /usr/local/go/bin/gofmt bin/gofmt
|
||||
|
||||
+6
-2
@@ -1,4 +1,4 @@
|
||||
FROM jenkins/jenkins:2.346.3-slim-jdk17@sha256:028fbbd9112c60ed086f5197fcba71992317864d27644e5949cf9c52ff4b65f0
|
||||
FROM jenkins/jenkins:2.346.3-slim-jdk17@sha256:028fbbd9112c60ed086f5197fcba71992317864d27644e5949cf9c52ff4b65f0 AS base
|
||||
|
||||
USER root
|
||||
|
||||
@@ -12,7 +12,7 @@ RUN apt-get update 2>&1 > /dev/null && apt-get install -y less zip 2>&1 > /dev/n
|
||||
|
||||
RUN unzip ../jenkins.war 2>&1 > /dev/null
|
||||
|
||||
RUN rm -f ./META-INF/MANIFEST.MF
|
||||
RUN rm -rf ./META-INF/MANIFEST.MF ./WEB-INF ./jsbundles ./scripts ./css
|
||||
|
||||
WORKDIR /usr/share/jenkins
|
||||
|
||||
@@ -21,3 +21,7 @@ RUN rm -rf jenkins.war
|
||||
RUN cd ./tmp && zip -r ../jenkins.war . && cd ..
|
||||
|
||||
RUN rm -rf ./tmp
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /usr/share/jenkins/jenkins.war /jenkins.war
|
||||
|
||||
+9
-1
@@ -1,7 +1,15 @@
|
||||
FROM alpine:3.18.3@sha256:7144f7bab3d4c2648d7e59409f15ec52a18006a128c733fcff20d3a4a54ba44a
|
||||
FROM alpine:3.18.3@sha256:7144f7bab3d4c2648d7e59409f15ec52a18006a128c733fcff20d3a4a54ba44a AS base
|
||||
|
||||
RUN wget https://repo1.maven.org/maven2/org/jvnet/hudson/main/hudson-war/2.2.1/hudson-war-2.2.1.war
|
||||
|
||||
RUN mv hudson-war-2.2.1.war hudson.war
|
||||
|
||||
# let's make this image a little smaller as to not take up so much disk space
|
||||
# we'll only keep the jar metadata files (pom data + manifest) and throw away the rest
|
||||
RUN apk add --no-cache python3 py3-pip
|
||||
COPY extract.py /extract.py
|
||||
RUN python extract.py
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /slim /
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
import os
|
||||
import zipfile
|
||||
import io
|
||||
|
||||
ARCHIVE_EXTENSIONS = ('.jar', '.war', '.ear', '.hpi', '.war', '.sar', '.nar', '.par')
|
||||
METADATA_FILES = ('pom.xml', 'pom.properties', 'MANIFEST.MF')
|
||||
|
||||
|
||||
def slim_archive(archive, output_dir, base_path="", archive_name=""):
|
||||
"""
|
||||
extracts metadata files from the archive and creates a slim JAR file
|
||||
containing only these files. handles nested JARs by preserving them.
|
||||
"""
|
||||
slim_buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(archive, 'r') as zip_file:
|
||||
with zipfile.ZipFile(slim_buffer, 'w', zipfile.ZIP_DEFLATED) as slim_zip:
|
||||
for file_name in zip_file.namelist():
|
||||
# check for metadata files or nested JARs
|
||||
if file_name.endswith(METADATA_FILES):
|
||||
# add metadata files directly to the slimmed archive
|
||||
file_data = zip_file.read(file_name)
|
||||
slim_zip.writestr(file_name, file_data)
|
||||
elif file_name.endswith(ARCHIVE_EXTENSIONS):
|
||||
# if it's a nested archive, recursively slim it
|
||||
nested_archive = io.BytesIO(zip_file.read(file_name))
|
||||
nested_slim_buffer = io.BytesIO()
|
||||
slim_archive(
|
||||
nested_archive,
|
||||
nested_slim_buffer,
|
||||
base_path=os.path.join(base_path, os.path.dirname(file_name)),
|
||||
archive_name=os.path.basename(file_name)
|
||||
)
|
||||
# add the slimmed nested archive back to the parent archive
|
||||
nested_slim_buffer.seek(0)
|
||||
slim_zip.writestr(file_name, nested_slim_buffer.read())
|
||||
|
||||
# write out the slimmed JAR to the output directory if output_dir is a directory
|
||||
if isinstance(output_dir, str):
|
||||
output_path = os.path.join(output_dir, base_path, archive_name)
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
with open(output_path, 'wb') as f:
|
||||
slim_buffer.seek(0)
|
||||
f.write(slim_buffer.read())
|
||||
else:
|
||||
# if output_dir is a BytesIO buffer (for nested archives), just write to it
|
||||
output_dir.seek(0)
|
||||
output_dir.write(slim_buffer.getvalue())
|
||||
|
||||
|
||||
def walk_directory_and_slim_jars(base_dir, output_dir):
|
||||
"""
|
||||
recursively walks through a directory tree looking for .jar, .war, .ear,
|
||||
.hpi files and slims them down by keeping only metadata files.
|
||||
"""
|
||||
for dirpath, _, filenames in os.walk(base_dir):
|
||||
for filename in filenames:
|
||||
if filename.endswith(ARCHIVE_EXTENSIONS):
|
||||
archive_path = os.path.join(dirpath, filename)
|
||||
print(f"Processing {archive_path}")
|
||||
slim_archive(archive_path, output_dir, os.path.relpath(dirpath, base_dir), filename)
|
||||
|
||||
|
||||
# a helper script for slimming down JAR files by keeping only metadata files but still keeping the jar packaging,
|
||||
# including nested JARs! Useful for testing purposes.
|
||||
if __name__ == "__main__":
|
||||
BASE_DIR = "."
|
||||
OUTPUT_DIR = "./slim"
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
walk_directory_and_slim_jars(BASE_DIR, OUTPUT_DIR)
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM alpine@sha256:d9a7354e3845ea8466bb00b22224d9116b183e594527fb5b6c3d30bc01a20378
|
||||
FROM alpine@sha256:d9a7354e3845ea8466bb00b22224d9116b183e594527fb5b6c3d30bc01a20378 AS base
|
||||
|
||||
# we keep these unpinned so that if alpine
|
||||
# changes our integration tests can adapt
|
||||
@@ -6,3 +6,8 @@ RUN apk add --no-cache \
|
||||
tzdata \
|
||||
vim \
|
||||
alpine-sdk
|
||||
|
||||
# we don't need the installed bins for this test, only the APK installed metadata
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /lib/apk/db/installed /lib/apk/db/installed
|
||||
|
||||
+8
-1
@@ -1 +1,8 @@
|
||||
FROM mcr.microsoft.com/cbl-mariner/distroless/base:2.0.202205275@sha256:f550c5428df17b145851ad75983aca6d613ad4b51ca7983b2a83e67d0ac91a5d
|
||||
FROM mcr.microsoft.com/cbl-mariner/distroless/base:2.0.202205275@sha256:f550c5428df17b145851ad75983aca6d613ad4b51ca7983b2a83e67d0ac91a5d AS base
|
||||
|
||||
# let's shoot for smaller test fixtures
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /var/lib/rpmmanifest/container-manifest-2 /var/lib/rpmmanifest/container-manifest-2
|
||||
COPY --from=base /usr/bin/gencat /usr/bin/gencat
|
||||
COPY --from=base /usr/bin/openssl /usr/bin/openssl
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
FROM ubuntu:20.04@sha256:33a5cc25d22c45900796a1aca487ad7a7cb09f09ea00b779e3b2026b4fc2faba
|
||||
FROM ubuntu:20.04@sha256:33a5cc25d22c45900796a1aca487ad7a7cb09f09ea00b779e3b2026b4fc2faba AS base
|
||||
# this covers rpm-python
|
||||
RUN apt-get update && apt-get install -y python-pil=6.2.1-3
|
||||
|
||||
# let's save some space...
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /var/lib/dpkg/status /var/lib/dpkg/status
|
||||
|
||||
@@ -1 +1,5 @@
|
||||
FROM photon:5.0-20230729@sha256:4cf2a1ce0a3f4625f13a0becb6b9bccfdb014c565be6e9a2ec4c4aad1ff8a5d9
|
||||
FROM photon:5.0-20230729@sha256:4cf2a1ce0a3f4625f13a0becb6b9bccfdb014c565be6e9a2ec4c4aad1ff8a5d9 AS base
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /usr/lib/sysimage/rpm /usr/lib/sysimage/rpm
|
||||
|
||||
@@ -1 +1,6 @@
|
||||
FROM fedora:35@sha256:36af84ba69e21c9ef86a0424a090674c433b2b80c2462e57503886f1d823abe8
|
||||
FROM fedora:35@sha256:36af84ba69e21c9ef86a0424a090674c433b2b80c2462e57503886f1d823abe8 AS base
|
||||
|
||||
# lets save some space
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /var/lib/rpm /var/lib/rpm
|
||||
|
||||
@@ -1,2 +1,11 @@
|
||||
FROM registry.suse.com/suse/sle15:15.3.17.20.20@sha256:fd657ecbab5ca564d6933e887f6ae8542a9398e6a4b399f352ce10c3a24afc64
|
||||
FROM registry.suse.com/suse/sle15:15.3.17.20.20@sha256:fd657ecbab5ca564d6933e887f6ae8542a9398e6a4b399f352ce10c3a24afc64 AS base
|
||||
RUN zypper in -y wget
|
||||
|
||||
# let's save some space... we really just need an image that has an RPM DB that is linked across layers
|
||||
FROM --platform=linux/amd64 busybox:1.36.1
|
||||
|
||||
# setup a link /var/lib/rpm -> ../../usr/lib/sysimage/rpm
|
||||
RUN mkdir -p /var/lib && ln -s ../../usr/lib/sysimage/rpm /var/lib/rpm
|
||||
|
||||
# copy the RPM DB from the SUSE image
|
||||
COPY --from=base /usr/lib/sysimage/rpm/Packages.db /usr/lib/sysimage/rpm/Packages.db
|
||||
|
||||
@@ -1,3 +1,18 @@
|
||||
FROM docker.io/anchore/test_images:java-88948cc@sha256:dea0e6c24636937f53bdc997d9960c2a18966d1e38bcd8ebd0c395d4e169b806
|
||||
FROM docker.io/anchore/test_images:java-88948cc@sha256:dea0e6c24636937f53bdc997d9960c2a18966d1e38bcd8ebd0c395d4e169b806 AS base
|
||||
|
||||
RUN rm /packages/gradle-7.1.1-bin.zip
|
||||
# not covered in testing...
|
||||
RUN rm /packages/gradle-7.1.1-bin.zip
|
||||
|
||||
RUN apk add --no-cache python3 py3-pip
|
||||
|
||||
COPY extract.py /extract.py
|
||||
|
||||
WORKDIR /
|
||||
|
||||
# let's make this image a little smaller as to not take up so much disk space
|
||||
# we'll only keep the jar metadata files (pom data + manifest) and throw away the rest
|
||||
RUN python extract.py
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /slim/packages /packages
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import os
|
||||
import zipfile
|
||||
import io
|
||||
|
||||
ARCHIVE_EXTENSIONS = ('.jar', '.war', '.ear', '.hpi', '.war', '.sar', '.nar', '.par')
|
||||
METADATA_FILES = ('pom.xml', 'pom.properties', 'MANIFEST.MF')
|
||||
|
||||
|
||||
def slim_archive(archive, output_dir, base_path="", archive_name=""):
|
||||
"""
|
||||
extracts metadata files from the archive and creates a slim JAR file
|
||||
containing only these files. handles nested JARs by preserving them.
|
||||
"""
|
||||
slim_buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(archive, 'r') as zip_file:
|
||||
with zipfile.ZipFile(slim_buffer, 'w', zipfile.ZIP_DEFLATED) as slim_zip:
|
||||
for file_name in zip_file.namelist():
|
||||
# check for metadata files or nested JARs
|
||||
if file_name.endswith(METADATA_FILES):
|
||||
# add metadata files directly to the slimmed archive
|
||||
file_data = zip_file.read(file_name)
|
||||
slim_zip.writestr(file_name, file_data)
|
||||
elif file_name.endswith(ARCHIVE_EXTENSIONS):
|
||||
# if it's a nested archive, recursively slim it
|
||||
nested_archive = io.BytesIO(zip_file.read(file_name))
|
||||
nested_slim_buffer = io.BytesIO()
|
||||
slim_archive(
|
||||
nested_archive,
|
||||
nested_slim_buffer,
|
||||
base_path=os.path.join(base_path, os.path.dirname(file_name)),
|
||||
archive_name=os.path.basename(file_name)
|
||||
)
|
||||
# add the slimmed nested archive back to the parent archive
|
||||
nested_slim_buffer.seek(0)
|
||||
slim_zip.writestr(file_name, nested_slim_buffer.read())
|
||||
|
||||
# write out the slimmed JAR to the output directory if output_dir is a directory
|
||||
if isinstance(output_dir, str):
|
||||
output_path = os.path.join(output_dir, base_path, archive_name)
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
with open(output_path, 'wb') as f:
|
||||
slim_buffer.seek(0)
|
||||
f.write(slim_buffer.read())
|
||||
else:
|
||||
# if output_dir is a BytesIO buffer (for nested archives), just write to it
|
||||
output_dir.seek(0)
|
||||
output_dir.write(slim_buffer.getvalue())
|
||||
|
||||
|
||||
def walk_directory_and_slim_jars(base_dir, output_dir):
|
||||
"""
|
||||
recursively walks through a directory tree looking for .jar, .war, .ear,
|
||||
.hpi files and slims them down by keeping only metadata files.
|
||||
"""
|
||||
for dirpath, _, filenames in os.walk(base_dir):
|
||||
for filename in filenames:
|
||||
if filename.endswith(ARCHIVE_EXTENSIONS):
|
||||
archive_path = os.path.join(dirpath, filename)
|
||||
print(f"Processing {archive_path}")
|
||||
slim_archive(archive_path, output_dir, os.path.relpath(dirpath, base_dir), filename)
|
||||
|
||||
|
||||
# a helper script for slimming down JAR files by keeping only metadata files but still keeping the jar packaging,
|
||||
# including nested JARs! Useful for testing purposes.
|
||||
if __name__ == "__main__":
|
||||
BASE_DIR = "."
|
||||
OUTPUT_DIR = "./slim"
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
walk_directory_and_slim_jars(BASE_DIR, OUTPUT_DIR)
|
||||
+25
-4
@@ -1,6 +1,27 @@
|
||||
FROM centos:7.9.2009@sha256:be65f488b7764ad3638f236b7b515b3678369a5124c47b8d32916d6487418ea4
|
||||
FROM --platform=linux/amd64 rockylinux:9.3.20231119@sha256:d644d203142cd5b54ad2a83a203e1dee68af2229f8fe32f52a30c6e1d3c3a9e0 AS base
|
||||
|
||||
# modifying the RPM DB multiple times will result in duplicate packages when using all-layers (if there was no de-dup logic)
|
||||
# curl is tricky, it already exists in the image and is being upgraded
|
||||
RUN yum install -y wget-1.14-18.el7_6.1 curl-7.29.0-59.el7_9.1
|
||||
RUN yum install -y vsftpd-3.0.2-29.el7_9
|
||||
RUN yum install -y httpd-2.4.6-97.el7.centos.5
|
||||
|
||||
# but... we want to make the test image as small as possible, so we are making the changes in stages and then
|
||||
# copying the RPM DB from each stage to a final stage in separate layers. This will result in a much smaller image.
|
||||
|
||||
FROM base AS stage1
|
||||
RUN dnf install -y wget
|
||||
|
||||
FROM stage1 AS stage2
|
||||
RUN dnf update -y curl-minimal
|
||||
|
||||
FROM stage2 AS stage3
|
||||
RUN dnf install -y vsftpd
|
||||
|
||||
FROM stage3 AS stage4
|
||||
RUN dnf install -y httpd
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /var/lib/rpm /var/lib/rpm
|
||||
COPY --from=stage1 /var/lib/rpm /var/lib/rpm
|
||||
COPY --from=stage2 /var/lib/rpm /var/lib/rpm
|
||||
COPY --from=stage3 /var/lib/rpm /var/lib/rpm
|
||||
COPY --from=stage4 /var/lib/rpm /var/lib/rpm
|
||||
|
||||
Reference in New Issue
Block a user