mirror of
https://github.com/anchore/syft.git
synced 2026-11-01 23:16:30 +01:00
Slim down docker cache size (#3190)
* slim down docker cache size Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * remove old centos images Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * troubleshoot test failure Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix wget version ref Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * refactor caching mechanisms Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add cache cleanup steps Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * simplify deleting cache Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix first clone issue Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add tool dep Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
@@ -0,0 +1 @@
|
||||
results
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
)
|
||||
|
||||
func Test_AllLayersIncludesSquashed(t *testing.T) {
|
||||
// This is a verification test for issue #894 (https://github.com/anchore/syft/issues/894)
|
||||
// This is a verification test for issue grype/#894 (https://github.com/anchore/grype/issues/894)
|
||||
allLayers, _ := catalogFixtureImage(t, "image-suse-all-layers", source.AllLayersScope)
|
||||
squashed, _ := catalogFixtureImage(t, "image-suse-all-layers", source.SquashedScope)
|
||||
|
||||
|
||||
@@ -2,7 +2,9 @@ package integration
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"regexp"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
@@ -12,8 +14,6 @@ import (
|
||||
|
||||
"github.com/anchore/syft/cmd/syft/internal/options"
|
||||
"github.com/anchore/syft/syft/format"
|
||||
"github.com/anchore/syft/syft/format/cyclonedxjson"
|
||||
"github.com/anchore/syft/syft/format/cyclonedxxml"
|
||||
"github.com/anchore/syft/syft/format/syftjson"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
@@ -43,26 +43,27 @@ func TestEncodeDecodeEncodeCycleComparison(t *testing.T) {
|
||||
},
|
||||
json: true,
|
||||
},
|
||||
{
|
||||
name: cyclonedxjson.ID.String(),
|
||||
redactor: func(in []byte) []byte {
|
||||
// unstable values
|
||||
in = regexp.MustCompile(`"(timestamp|serialNumber|bom-ref|ref)":\s*"(\n|[^"])+"`).ReplaceAll(in, []byte(`"$1": "redacted"`))
|
||||
in = regexp.MustCompile(`"(dependsOn)":\s*\[(?:\s|[^]])+]`).ReplaceAll(in, []byte(`"$1": []`))
|
||||
return in
|
||||
},
|
||||
json: true,
|
||||
},
|
||||
{
|
||||
name: cyclonedxxml.ID.String(),
|
||||
redactor: func(in []byte) []byte {
|
||||
// unstable values
|
||||
in = regexp.MustCompile(`(serialNumber|bom-ref|ref)="[^"]+"`).ReplaceAll(in, []byte{})
|
||||
in = regexp.MustCompile(`<timestamp>[^<]+</timestamp>`).ReplaceAll(in, []byte{})
|
||||
|
||||
return in
|
||||
},
|
||||
},
|
||||
// TODO: ignoring the `ref` field though does create stable results to compare, but the SBOM is fundamentally gutted and not worth comparing (find a better redaction or compare method)
|
||||
//{
|
||||
// name: cyclonedxjson.ID.String(),
|
||||
// redactor: func(in []byte) []byte {
|
||||
// // unstable values
|
||||
// in = regexp.MustCompile(`"(timestamp|serialNumber|bom-ref|ref)":\s*"(\n|[^"])+"`).ReplaceAll(in, []byte(`"$1": "redacted"`))
|
||||
// in = regexp.MustCompile(`"(dependsOn)":\s*\[(?:\s|[^]])+]`).ReplaceAll(in, []byte(`"$1": []`))
|
||||
// return in
|
||||
// },
|
||||
// json: true,
|
||||
//},
|
||||
//{
|
||||
// name: cyclonedxxml.ID.String(),
|
||||
// redactor: func(in []byte) []byte {
|
||||
// // unstable values
|
||||
// in = regexp.MustCompile(`(serialNumber|bom-ref|ref)="[^"]+"`).ReplaceAll(in, []byte{})
|
||||
// in = regexp.MustCompile(`<timestamp>[^<]+</timestamp>`).ReplaceAll(in, []byte{})
|
||||
//
|
||||
// return in
|
||||
// },
|
||||
//},
|
||||
}
|
||||
|
||||
opts := options.DefaultOutput()
|
||||
@@ -112,6 +113,21 @@ func TestEncodeDecodeEncodeCycleComparison(t *testing.T) {
|
||||
diffs := dmp.DiffMain(string(by1), string(by2), true)
|
||||
t.Errorf("diff: %s", dmp.DiffPrettyText(diffs))
|
||||
}
|
||||
|
||||
// write raw IMAGE@NAME-start and IMAGE@NAME-finish to files within the results dir
|
||||
// ... this is helpful for debugging
|
||||
require.NoError(t, os.MkdirAll("results", 0700))
|
||||
|
||||
suffix := "sbom"
|
||||
switch {
|
||||
case strings.Contains(test.name, "json"):
|
||||
suffix = "json"
|
||||
case strings.Contains(test.name, "xml"):
|
||||
suffix = "xml"
|
||||
}
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join("results", image+"@"+test.name+"-start."+suffix), by1, 0600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join("results", image+"@"+test.name+"-finish."+suffix), by2, 0600))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -35,8 +35,8 @@ func TestGolangCompilerDetection(t *testing.T) {
|
||||
for _, pkg := range packages {
|
||||
foundCompilerVersions[pkg.Version] = struct{}{}
|
||||
foundPURL[pkg.PURL] = struct{}{}
|
||||
for _, cpe := range pkg.CPEs {
|
||||
foundCPE[cpe] = struct{}{}
|
||||
for _, c := range pkg.CPEs {
|
||||
foundCPE[c] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
@@ -26,13 +25,9 @@ func TestJavaPURLs(t *testing.T) {
|
||||
found[metadata.VirtualPath] = p.PURL
|
||||
}
|
||||
}
|
||||
for key, expectedPURL := range expectedPURLs {
|
||||
purl := found[key]
|
||||
assert.Equal(t, expectedPURL, purl, fmt.Sprintf("found wrong or missing PURL for %s want %s, got %s", key, expectedPURL, purl))
|
||||
}
|
||||
for key, foundPURL := range found {
|
||||
expectedPURL := expectedPURLs[key]
|
||||
assert.Equal(t, expectedPURL, foundPURL, fmt.Sprintf("found extra purl for %s want %s, got %s", key, expectedPURL, foundPURL))
|
||||
|
||||
if d := cmp.Diff(expectedPURLs, found); d != "" {
|
||||
t.Errorf("unexpected purl values:\n%s", d)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//go:build !arm64
|
||||
|
||||
package integration
|
||||
|
||||
import (
|
||||
@@ -7,7 +5,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
@@ -22,41 +19,39 @@ func TestPackageDeduplication(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
scope: source.AllLayersScope,
|
||||
packageCount: 172, // without deduplication this would be 618
|
||||
packageCount: 178, // without deduplication this would be ~600
|
||||
instanceCount: map[string]int{
|
||||
"basesystem": 1,
|
||||
"wget": 1,
|
||||
"curl": 2, // upgraded in the image
|
||||
"vsftpd": 1,
|
||||
"httpd": 1, // rpm, - we exclude binary
|
||||
"basesystem": 1,
|
||||
"wget": 1,
|
||||
"curl-minimal": 2, // upgraded in the image
|
||||
"vsftpd": 1,
|
||||
"httpd": 1, // rpm, - we exclude binary
|
||||
},
|
||||
locationCount: map[string]int{
|
||||
"basesystem-10.0-7.el7.centos": 4,
|
||||
"curl-7.29.0-59.el7": 1, // from base image
|
||||
"curl-7.29.0-59.el7_9.1": 3, // upgrade
|
||||
"wget-1.14-18.el7_6.1": 3,
|
||||
"vsftpd-3.0.2-29.el7_9": 2,
|
||||
"httpd-2.4.6-97.el7.centos.5": 1,
|
||||
// "httpd-2.4.6": 1, // binary
|
||||
"basesystem-11-13.el9": 5, // in all layers
|
||||
"curl-minimal-7.76.1-26.el9_3.2.0.1": 2, // base + wget layer
|
||||
"curl-minimal-7.76.1-29.el9_4.1": 3, // curl upgrade layer + all above layers
|
||||
"wget-1.21.1-8.el9_4": 4, // wget + all above layers
|
||||
"vsftpd-3.0.5-5.el9": 2, // vsftpd + all above layers
|
||||
"httpd-2.4.57-11.el9_4.1": 1, // last layer
|
||||
},
|
||||
},
|
||||
{
|
||||
scope: source.SquashedScope,
|
||||
packageCount: 170,
|
||||
packageCount: 172,
|
||||
instanceCount: map[string]int{
|
||||
"basesystem": 1,
|
||||
"wget": 1,
|
||||
"curl": 1, // upgraded, but the most recent
|
||||
"vsftpd": 1,
|
||||
"httpd": 1, // rpm, binary is now excluded by overlap
|
||||
"basesystem": 1,
|
||||
"wget": 1,
|
||||
"curl-minimal": 1, // upgraded, but the most recent
|
||||
"vsftpd": 1,
|
||||
"httpd": 1, // rpm, binary is now excluded by overlap
|
||||
},
|
||||
locationCount: map[string]int{
|
||||
"basesystem-10.0-7.el7.centos": 1,
|
||||
"curl-7.29.0-59.el7_9.1": 1, // upgrade
|
||||
"wget-1.14-18.el7_6.1": 1,
|
||||
"vsftpd-3.0.2-29.el7_9": 1,
|
||||
"httpd-2.4.6-97.el7.centos.5": 1,
|
||||
// "httpd-2.4.6": 1, // binary (excluded)
|
||||
"basesystem-11-13.el9": 1,
|
||||
"curl-minimal-7.76.1-29.el9_4.1": 1, // upgrade
|
||||
"wget-1.21.1-8.el9_4": 1,
|
||||
"vsftpd-3.0.5-5.el9": 1,
|
||||
"httpd-2.4.57-11.el9_4.1": 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -75,20 +70,21 @@ func TestPackageDeduplication(t *testing.T) {
|
||||
pkgs := sbom.Artifacts.Packages.PackagesByName(name)
|
||||
|
||||
// with multiple packages with the same name, something is wrong (or this is the wrong fixture)
|
||||
require.Len(t, pkgs, expectedInstanceCount)
|
||||
if assert.Len(t, pkgs, expectedInstanceCount, "unexpected package count for %s", name) {
|
||||
for _, p := range pkgs {
|
||||
nameVersion := fmt.Sprintf("%s-%s", name, p.Version)
|
||||
expectedLocationCount, ok := tt.locationCount[nameVersion]
|
||||
if !ok {
|
||||
t.Errorf("missing name-version: %s", nameVersion)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, p := range pkgs {
|
||||
nameVersion := fmt.Sprintf("%s-%s", name, p.Version)
|
||||
expectedLocationCount, ok := tt.locationCount[nameVersion]
|
||||
if !ok {
|
||||
t.Fatalf("missing name-version: %s", nameVersion)
|
||||
// we should see merged locations (assumption, there was 1 location for each package)
|
||||
assert.Len(t, p.Locations.ToSlice(), expectedLocationCount, "unexpected location count for %s", nameVersion)
|
||||
|
||||
// all paths should match
|
||||
assert.Len(t, p.Locations.CoordinateSet().Paths(), 1, "unexpected location count for %s", nameVersion)
|
||||
}
|
||||
|
||||
// we should see merged locations (assumption, there was 1 location for each package)
|
||||
assert.Len(t, p.Locations.ToSlice(), expectedLocationCount)
|
||||
|
||||
// all paths should match
|
||||
assert.Len(t, p.Locations.CoordinateSet().Paths(), 1)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,21 @@
|
||||
# change these if you want CI to not use previous stored cache
|
||||
INTEGRATION_CACHE_BUSTER := "894d8ca"
|
||||
FINGERPRINT_FILE := cache.fingerprint
|
||||
|
||||
.PHONY: cache.fingerprint
|
||||
cache.fingerprint:
|
||||
find image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | tee /dev/stderr | md5sum | tee cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> cache.fingerprint
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures:
|
||||
@echo "nothing to do"
|
||||
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: $(FINGERPRINT_FILE)
|
||||
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find image-* -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f $(FINGERPRINT_FILE)
|
||||
|
||||
@@ -1 +1,6 @@
|
||||
FROM golang:1.18.10-alpine
|
||||
FROM --platform=linux/amd64 golang:1.18.10-alpine
|
||||
|
||||
FROM scratch
|
||||
|
||||
# we don't need the entire golang toolchain, just a single binary with the stdlib baked in
|
||||
COPY --from=0 /usr/local/go/bin/gofmt bin/gofmt
|
||||
|
||||
+6
-2
@@ -1,4 +1,4 @@
|
||||
FROM jenkins/jenkins:2.346.3-slim-jdk17@sha256:028fbbd9112c60ed086f5197fcba71992317864d27644e5949cf9c52ff4b65f0
|
||||
FROM jenkins/jenkins:2.346.3-slim-jdk17@sha256:028fbbd9112c60ed086f5197fcba71992317864d27644e5949cf9c52ff4b65f0 AS base
|
||||
|
||||
USER root
|
||||
|
||||
@@ -12,7 +12,7 @@ RUN apt-get update 2>&1 > /dev/null && apt-get install -y less zip 2>&1 > /dev/n
|
||||
|
||||
RUN unzip ../jenkins.war 2>&1 > /dev/null
|
||||
|
||||
RUN rm -f ./META-INF/MANIFEST.MF
|
||||
RUN rm -rf ./META-INF/MANIFEST.MF ./WEB-INF ./jsbundles ./scripts ./css
|
||||
|
||||
WORKDIR /usr/share/jenkins
|
||||
|
||||
@@ -21,3 +21,7 @@ RUN rm -rf jenkins.war
|
||||
RUN cd ./tmp && zip -r ../jenkins.war . && cd ..
|
||||
|
||||
RUN rm -rf ./tmp
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /usr/share/jenkins/jenkins.war /jenkins.war
|
||||
|
||||
+9
-1
@@ -1,7 +1,15 @@
|
||||
FROM alpine:3.18.3@sha256:7144f7bab3d4c2648d7e59409f15ec52a18006a128c733fcff20d3a4a54ba44a
|
||||
FROM alpine:3.18.3@sha256:7144f7bab3d4c2648d7e59409f15ec52a18006a128c733fcff20d3a4a54ba44a AS base
|
||||
|
||||
RUN wget https://repo1.maven.org/maven2/org/jvnet/hudson/main/hudson-war/2.2.1/hudson-war-2.2.1.war
|
||||
|
||||
RUN mv hudson-war-2.2.1.war hudson.war
|
||||
|
||||
# let's make this image a little smaller as to not take up so much disk space
|
||||
# we'll only keep the jar metadata files (pom data + manifest) and throw away the rest
|
||||
RUN apk add --no-cache python3 py3-pip
|
||||
COPY extract.py /extract.py
|
||||
RUN python extract.py
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /slim /
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
import os
|
||||
import zipfile
|
||||
import io
|
||||
|
||||
ARCHIVE_EXTENSIONS = ('.jar', '.war', '.ear', '.hpi', '.war', '.sar', '.nar', '.par')
|
||||
METADATA_FILES = ('pom.xml', 'pom.properties', 'MANIFEST.MF')
|
||||
|
||||
|
||||
def slim_archive(archive, output_dir, base_path="", archive_name=""):
|
||||
"""
|
||||
extracts metadata files from the archive and creates a slim JAR file
|
||||
containing only these files. handles nested JARs by preserving them.
|
||||
"""
|
||||
slim_buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(archive, 'r') as zip_file:
|
||||
with zipfile.ZipFile(slim_buffer, 'w', zipfile.ZIP_DEFLATED) as slim_zip:
|
||||
for file_name in zip_file.namelist():
|
||||
# check for metadata files or nested JARs
|
||||
if file_name.endswith(METADATA_FILES):
|
||||
# add metadata files directly to the slimmed archive
|
||||
file_data = zip_file.read(file_name)
|
||||
slim_zip.writestr(file_name, file_data)
|
||||
elif file_name.endswith(ARCHIVE_EXTENSIONS):
|
||||
# if it's a nested archive, recursively slim it
|
||||
nested_archive = io.BytesIO(zip_file.read(file_name))
|
||||
nested_slim_buffer = io.BytesIO()
|
||||
slim_archive(
|
||||
nested_archive,
|
||||
nested_slim_buffer,
|
||||
base_path=os.path.join(base_path, os.path.dirname(file_name)),
|
||||
archive_name=os.path.basename(file_name)
|
||||
)
|
||||
# add the slimmed nested archive back to the parent archive
|
||||
nested_slim_buffer.seek(0)
|
||||
slim_zip.writestr(file_name, nested_slim_buffer.read())
|
||||
|
||||
# write out the slimmed JAR to the output directory if output_dir is a directory
|
||||
if isinstance(output_dir, str):
|
||||
output_path = os.path.join(output_dir, base_path, archive_name)
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
with open(output_path, 'wb') as f:
|
||||
slim_buffer.seek(0)
|
||||
f.write(slim_buffer.read())
|
||||
else:
|
||||
# if output_dir is a BytesIO buffer (for nested archives), just write to it
|
||||
output_dir.seek(0)
|
||||
output_dir.write(slim_buffer.getvalue())
|
||||
|
||||
|
||||
def walk_directory_and_slim_jars(base_dir, output_dir):
|
||||
"""
|
||||
recursively walks through a directory tree looking for .jar, .war, .ear,
|
||||
.hpi files and slims them down by keeping only metadata files.
|
||||
"""
|
||||
for dirpath, _, filenames in os.walk(base_dir):
|
||||
for filename in filenames:
|
||||
if filename.endswith(ARCHIVE_EXTENSIONS):
|
||||
archive_path = os.path.join(dirpath, filename)
|
||||
print(f"Processing {archive_path}")
|
||||
slim_archive(archive_path, output_dir, os.path.relpath(dirpath, base_dir), filename)
|
||||
|
||||
|
||||
# a helper script for slimming down JAR files by keeping only metadata files but still keeping the jar packaging,
|
||||
# including nested JARs! Useful for testing purposes.
|
||||
if __name__ == "__main__":
|
||||
BASE_DIR = "."
|
||||
OUTPUT_DIR = "./slim"
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
walk_directory_and_slim_jars(BASE_DIR, OUTPUT_DIR)
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM alpine@sha256:d9a7354e3845ea8466bb00b22224d9116b183e594527fb5b6c3d30bc01a20378
|
||||
FROM alpine@sha256:d9a7354e3845ea8466bb00b22224d9116b183e594527fb5b6c3d30bc01a20378 AS base
|
||||
|
||||
# we keep these unpinned so that if alpine
|
||||
# changes our integration tests can adapt
|
||||
@@ -6,3 +6,8 @@ RUN apk add --no-cache \
|
||||
tzdata \
|
||||
vim \
|
||||
alpine-sdk
|
||||
|
||||
# we don't need the installed bins for this test, only the APK installed metadata
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /lib/apk/db/installed /lib/apk/db/installed
|
||||
|
||||
+8
-1
@@ -1 +1,8 @@
|
||||
FROM mcr.microsoft.com/cbl-mariner/distroless/base:2.0.202205275@sha256:f550c5428df17b145851ad75983aca6d613ad4b51ca7983b2a83e67d0ac91a5d
|
||||
FROM mcr.microsoft.com/cbl-mariner/distroless/base:2.0.202205275@sha256:f550c5428df17b145851ad75983aca6d613ad4b51ca7983b2a83e67d0ac91a5d AS base
|
||||
|
||||
# let's shoot for smaller test fixtures
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /var/lib/rpmmanifest/container-manifest-2 /var/lib/rpmmanifest/container-manifest-2
|
||||
COPY --from=base /usr/bin/gencat /usr/bin/gencat
|
||||
COPY --from=base /usr/bin/openssl /usr/bin/openssl
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
FROM ubuntu:20.04@sha256:33a5cc25d22c45900796a1aca487ad7a7cb09f09ea00b779e3b2026b4fc2faba
|
||||
FROM ubuntu:20.04@sha256:33a5cc25d22c45900796a1aca487ad7a7cb09f09ea00b779e3b2026b4fc2faba AS base
|
||||
# this covers rpm-python
|
||||
RUN apt-get update && apt-get install -y python-pil=6.2.1-3
|
||||
|
||||
# let's save some space...
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /var/lib/dpkg/status /var/lib/dpkg/status
|
||||
|
||||
@@ -1 +1,5 @@
|
||||
FROM photon:5.0-20230729@sha256:4cf2a1ce0a3f4625f13a0becb6b9bccfdb014c565be6e9a2ec4c4aad1ff8a5d9
|
||||
FROM photon:5.0-20230729@sha256:4cf2a1ce0a3f4625f13a0becb6b9bccfdb014c565be6e9a2ec4c4aad1ff8a5d9 AS base
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /usr/lib/sysimage/rpm /usr/lib/sysimage/rpm
|
||||
|
||||
@@ -1 +1,6 @@
|
||||
FROM fedora:35@sha256:36af84ba69e21c9ef86a0424a090674c433b2b80c2462e57503886f1d823abe8
|
||||
FROM fedora:35@sha256:36af84ba69e21c9ef86a0424a090674c433b2b80c2462e57503886f1d823abe8 AS base
|
||||
|
||||
# lets save some space
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /var/lib/rpm /var/lib/rpm
|
||||
|
||||
@@ -1,2 +1,11 @@
|
||||
FROM registry.suse.com/suse/sle15:15.3.17.20.20@sha256:fd657ecbab5ca564d6933e887f6ae8542a9398e6a4b399f352ce10c3a24afc64
|
||||
FROM registry.suse.com/suse/sle15:15.3.17.20.20@sha256:fd657ecbab5ca564d6933e887f6ae8542a9398e6a4b399f352ce10c3a24afc64 AS base
|
||||
RUN zypper in -y wget
|
||||
|
||||
# let's save some space... we really just need an image that has an RPM DB that is linked across layers
|
||||
FROM --platform=linux/amd64 busybox:1.36.1
|
||||
|
||||
# setup a link /var/lib/rpm -> ../../usr/lib/sysimage/rpm
|
||||
RUN mkdir -p /var/lib && ln -s ../../usr/lib/sysimage/rpm /var/lib/rpm
|
||||
|
||||
# copy the RPM DB from the SUSE image
|
||||
COPY --from=base /usr/lib/sysimage/rpm/Packages.db /usr/lib/sysimage/rpm/Packages.db
|
||||
|
||||
@@ -1,3 +1,18 @@
|
||||
FROM docker.io/anchore/test_images:java-88948cc@sha256:dea0e6c24636937f53bdc997d9960c2a18966d1e38bcd8ebd0c395d4e169b806
|
||||
FROM docker.io/anchore/test_images:java-88948cc@sha256:dea0e6c24636937f53bdc997d9960c2a18966d1e38bcd8ebd0c395d4e169b806 AS base
|
||||
|
||||
RUN rm /packages/gradle-7.1.1-bin.zip
|
||||
# not covered in testing...
|
||||
RUN rm /packages/gradle-7.1.1-bin.zip
|
||||
|
||||
RUN apk add --no-cache python3 py3-pip
|
||||
|
||||
COPY extract.py /extract.py
|
||||
|
||||
WORKDIR /
|
||||
|
||||
# let's make this image a little smaller as to not take up so much disk space
|
||||
# we'll only keep the jar metadata files (pom data + manifest) and throw away the rest
|
||||
RUN python extract.py
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /slim/packages /packages
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import os
|
||||
import zipfile
|
||||
import io
|
||||
|
||||
ARCHIVE_EXTENSIONS = ('.jar', '.war', '.ear', '.hpi', '.war', '.sar', '.nar', '.par')
|
||||
METADATA_FILES = ('pom.xml', 'pom.properties', 'MANIFEST.MF')
|
||||
|
||||
|
||||
def slim_archive(archive, output_dir, base_path="", archive_name=""):
|
||||
"""
|
||||
extracts metadata files from the archive and creates a slim JAR file
|
||||
containing only these files. handles nested JARs by preserving them.
|
||||
"""
|
||||
slim_buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(archive, 'r') as zip_file:
|
||||
with zipfile.ZipFile(slim_buffer, 'w', zipfile.ZIP_DEFLATED) as slim_zip:
|
||||
for file_name in zip_file.namelist():
|
||||
# check for metadata files or nested JARs
|
||||
if file_name.endswith(METADATA_FILES):
|
||||
# add metadata files directly to the slimmed archive
|
||||
file_data = zip_file.read(file_name)
|
||||
slim_zip.writestr(file_name, file_data)
|
||||
elif file_name.endswith(ARCHIVE_EXTENSIONS):
|
||||
# if it's a nested archive, recursively slim it
|
||||
nested_archive = io.BytesIO(zip_file.read(file_name))
|
||||
nested_slim_buffer = io.BytesIO()
|
||||
slim_archive(
|
||||
nested_archive,
|
||||
nested_slim_buffer,
|
||||
base_path=os.path.join(base_path, os.path.dirname(file_name)),
|
||||
archive_name=os.path.basename(file_name)
|
||||
)
|
||||
# add the slimmed nested archive back to the parent archive
|
||||
nested_slim_buffer.seek(0)
|
||||
slim_zip.writestr(file_name, nested_slim_buffer.read())
|
||||
|
||||
# write out the slimmed JAR to the output directory if output_dir is a directory
|
||||
if isinstance(output_dir, str):
|
||||
output_path = os.path.join(output_dir, base_path, archive_name)
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
with open(output_path, 'wb') as f:
|
||||
slim_buffer.seek(0)
|
||||
f.write(slim_buffer.read())
|
||||
else:
|
||||
# if output_dir is a BytesIO buffer (for nested archives), just write to it
|
||||
output_dir.seek(0)
|
||||
output_dir.write(slim_buffer.getvalue())
|
||||
|
||||
|
||||
def walk_directory_and_slim_jars(base_dir, output_dir):
|
||||
"""
|
||||
recursively walks through a directory tree looking for .jar, .war, .ear,
|
||||
.hpi files and slims them down by keeping only metadata files.
|
||||
"""
|
||||
for dirpath, _, filenames in os.walk(base_dir):
|
||||
for filename in filenames:
|
||||
if filename.endswith(ARCHIVE_EXTENSIONS):
|
||||
archive_path = os.path.join(dirpath, filename)
|
||||
print(f"Processing {archive_path}")
|
||||
slim_archive(archive_path, output_dir, os.path.relpath(dirpath, base_dir), filename)
|
||||
|
||||
|
||||
# a helper script for slimming down JAR files by keeping only metadata files but still keeping the jar packaging,
|
||||
# including nested JARs! Useful for testing purposes.
|
||||
if __name__ == "__main__":
|
||||
BASE_DIR = "."
|
||||
OUTPUT_DIR = "./slim"
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
walk_directory_and_slim_jars(BASE_DIR, OUTPUT_DIR)
|
||||
+25
-4
@@ -1,6 +1,27 @@
|
||||
FROM centos:7.9.2009@sha256:be65f488b7764ad3638f236b7b515b3678369a5124c47b8d32916d6487418ea4
|
||||
FROM --platform=linux/amd64 rockylinux:9.3.20231119@sha256:d644d203142cd5b54ad2a83a203e1dee68af2229f8fe32f52a30c6e1d3c3a9e0 AS base
|
||||
|
||||
# modifying the RPM DB multiple times will result in duplicate packages when using all-layers (if there was no de-dup logic)
|
||||
# curl is tricky, it already exists in the image and is being upgraded
|
||||
RUN yum install -y wget-1.14-18.el7_6.1 curl-7.29.0-59.el7_9.1
|
||||
RUN yum install -y vsftpd-3.0.2-29.el7_9
|
||||
RUN yum install -y httpd-2.4.6-97.el7.centos.5
|
||||
|
||||
# but... we want to make the test image as small as possible, so we are making the changes in stages and then
|
||||
# copying the RPM DB from each stage to a final stage in separate layers. This will result in a much smaller image.
|
||||
|
||||
FROM base AS stage1
|
||||
RUN dnf install -y wget
|
||||
|
||||
FROM stage1 AS stage2
|
||||
RUN dnf update -y curl-minimal
|
||||
|
||||
FROM stage2 AS stage3
|
||||
RUN dnf install -y vsftpd
|
||||
|
||||
FROM stage3 AS stage4
|
||||
RUN dnf install -y httpd
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=base /var/lib/rpm /var/lib/rpm
|
||||
COPY --from=stage1 /var/lib/rpm /var/lib/rpm
|
||||
COPY --from=stage2 /var/lib/rpm /var/lib/rpm
|
||||
COPY --from=stage3 /var/lib/rpm /var/lib/rpm
|
||||
COPY --from=stage4 /var/lib/rpm /var/lib/rpm
|
||||
|
||||
Reference in New Issue
Block a user