oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-01 23:16:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Slim down docker cache size (#3190)

Browse Source 
* slim down docker cache size

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove old centos images

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* troubleshoot test failure

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix wget version ref

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor caching mechanisms

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add cache cleanup steps

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* simplify deleting cache

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix first clone issue

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tool dep

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2024-09-09 11:15:13 -04:00
committed by GitHub
parent deabd4115a
commit 0a3f513f92
75 changed files with 1302 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/file/cataloger/executable/test-fixtures/Makefile
+15
View File
@@ -0,0 +1,15 @@
.DEFAULT_GOAL := default
default:
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
if [ -f "$$dir/Makefile" ]; then \
$(MAKE) -C $$dir; \
fi; \
done
%:
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
if [ -f "$$dir/Makefile" ]; then \
$(MAKE) -C $$dir $@; \
fi; \
done
syft/file/cataloger/executable/test-fixtures/elf/Makefile
+20 -5
View File
@@ -1,8 +1,19 @@
BIN=./bin
TOOL_IMAGE=localhost/syft-bin-build-tools:latest
VERIFY_FILE=actual_verify
FINGERPRINT_FILE=$(BIN).fingerprint
all: build verify
ifndef BIN
$(error BIN is not set)
endif
.DEFAULT_GOAL := fixtures
# requirement 1: 'fixtures' goal to generate any and all test fixtures
fixtures: build verify
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
fingerprint: $(FINGERPRINT_FILE)
tools-check:
@sha256sum -c Dockerfile.sha256 || (echo "Tools Dockerfile has changed" && exit 1)
@@ -25,10 +36,14 @@ verify: tools
debug:
docker run -i --rm -v $(shell pwd):/mount -w /mount/project $(TOOL_IMAGE) bash
cache.fingerprint:
@find project Dockerfile Makefile -type f -exec md5sum {} + | awk '{print $1}' | sort | tee cache.fingerprint
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
.PHONY: $(FINGERPRINT_FILE)
$(FINGERPRINT_FILE):
@find project Dockerfile Makefile -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
# requirement 4: 'clean' goal to remove all generated test fixtures
clean:
rm -f $(BIN)/*
rm -rf $(BIN) Dockerfile.sha256 $(VERIFY_FILE) $(FINGERPRINT_FILE)
.PHONY: build verify debug build-image build-bins clean dockerfile-check cache.fingerprint
.PHONY: tools tools-check build verify debug clean
syft/file/cataloger/executable/test-fixtures/shared-info/Makefile
+22 -6
View File
@@ -1,8 +1,20 @@
BIN=./bin
TOOL_IMAGE=localhost/syft-shared-info-build-tools:latest
VERIFY_FILE=actual_verify
FINGERPRINT_FILE=$(BIN).fingerprint
ifndef BIN
$(error BIN is not set)
endif
.DEFAULT_GOAL := fixtures
# requirement 1: 'fixtures' goal to generate any and all test fixtures
fixtures: build
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
fingerprint: $(FINGERPRINT_FILE)
all: build
tools-check:
@sha256sum -c Dockerfile.sha256 || (echo "Tools Dockerfile has changed" && exit 1)
@@ -10,16 +22,20 @@ tools:
@(docker inspect $(TOOL_IMAGE) > /dev/null && make tools-check) || (docker build -t $(TOOL_IMAGE) . && sha256sum Dockerfile > Dockerfile.sha256)
build: tools
mkdir -p $(BIN)
@mkdir -p $(BIN)
docker run --platform linux/amd64 -i -v $(shell pwd):/mount -w /mount/project $(TOOL_IMAGE) make
debug:
docker run --platform linux/amd64 -i --rm -v $(shell pwd):/mount -w /mount/project $(TOOL_IMAGE) bash
cache.fingerprint:
@find project Dockerfile Makefile -type f -exec md5sum {} + | awk '{print $1}' | sort | tee cache.fingerprint
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
.PHONY: $(FINGERPRINT_FILE)
$(FINGERPRINT_FILE):
@find project Dockerfile Makefile -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
# requirement 4: 'clean' goal to remove all generated test fixtures
clean:
rm -f $(BIN)/*
rm -rf $(BIN) Dockerfile.sha256 $(VERIFY_FILE) $(FINGERPRINT_FILE)
.PHONY: build verify debug build-image build-bins clean dockerfile-check cache.fingerprint
.PHONY: tools tools-check build debug clean
syft/format/text/test-fixtures/image-simple/Dockerfile
-4
View File
@@ -1,4 +0,0 @@
# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one.
FROM scratch
ADD file-1.txt /somefile-1.txt
ADD file-2.txt /somefile-2.txt
syft/format/text/test-fixtures/image-simple/file-1.txt
-1
View File
@@ -1 +0,0 @@
this file has contents
syft/format/text/test-fixtures/image-simple/file-2.txt
-1
View File
@@ -1 +0,0 @@
file-2 contents!
syft/format/text/test-fixtures/snapshot/TestTextImageEncoder.golden
+2 -2
View File
@@ -1,11 +1,11 @@
[Image]
Layer: 0
Digest: sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59
Digest: sha256:100d5a55f9032faead28b7427fa3e650e4f0158f86ea89d06e1489df00cb8c6f
Size: 22
MediaType: application/vnd.docker.image.rootfs.diff.tar.gzip
Layer: 1
Digest: sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec
Digest: sha256:000fb9200890d3a19138478b20023023c0dce1c54352007c2863716780f049eb
Size: 16
MediaType: application/vnd.docker.image.rootfs.diff.tar.gzip
syft/pkg/cataloger/binary/test-fixtures/.gitignore
-1
View File
@@ -1,6 +1,5 @@
classifiers/dynamic
classifiers/bin
cache.fingerprint
# allow for lb patterns (rust, pytho, php and more)
!lib*.so
syft/pkg/cataloger/binary/test-fixtures/Makefile
+23 -8
View File
@@ -1,8 +1,14 @@
.PHONY: default list download download-all cache.fingerprint
BIN=classifiers/bin
FINGERPRINT_FILE=$(BIN).fingerprint
.DEFAULT_GOAL := default
default: download
.DEFAULT_GOAL := fixtures
# requirement 1: 'fixtures' goal to generate any and all test fixtures
fixtures: download
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
fingerprint: clean-fingerprint $(FINGERPRINT_FILE)
list: ## list all managed binaries and snippets
go run ./manager list
@@ -16,14 +22,23 @@ download-all: ## download all managed binaries
add-snippet: ## add a new snippet from an existing binary
go run ./manager add-snippet
cache.fingerprint: ## prints the sha256sum of the any input to the download command (to determine if there is a cache miss)
@cat ./config.yaml | sha256sum | awk '{print $$1}' | tee cache.fingerprint
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
.PHONY: $(FINGERPRINT_FILE)
$(FINGERPRINT_FILE): ## prints the sha256sum of the any input to the download command (to determine if there is a cache miss)
@sha256sum ./config.yaml > $(FINGERPRINT_FILE)
# requirement 4: 'clean' goal to remove all generated test fixtures
clean: ## clean up all downloaded binaries
rm -rf $(BIN)
clean-fingerprint: ## clean up all legacy fingerprint files
@find $(BIN) -name '*.fingerprint' -delete
clean: ## clean up all downloaded binaries
rm -rf ./classifiers/bin
## Halp! #################################
.PHONY: help
help:
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}'
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}'
.PHONY: default list download download-all clean clean-fingerprint add-snippet fingerprint
syft/pkg/cataloger/binary/test-fixtures/classifiers/snippets/traefik/3.0.4/linux-riscv64/traefik
BIN
View File
Binary file not shown.
syft/pkg/cataloger/binary/test-fixtures/config.yaml
+1
View File
@@ -85,6 +85,7 @@ from-images:
paths:
- /usr/local/go/bin/go
# TODO: this is no longer available from dockerhub! (the snippet is vital)
- version: 1.5.14
images:
- ref: haproxy:1.5.14@sha256:3d57e3921cc84e860f764e863ce729dd0765e3d28d444775127bc42d68f98e10
syft/pkg/cataloger/binary/test-fixtures/elf-test-fixtures/Dockerfile
+18 -1
View File
@@ -1,14 +1,31 @@
FROM rockylinux:8
FROM rockylinux:8 AS base
RUN dnf update -y; \
dnf install make automake gcc gcc-c++ kernel-devel -y; \
dnf clean all
RUN mkdir -p /usr/local/bin/elftests/elfbinwithnestedlib
RUN mkdir -p /usr/local/bin/elftests/elfbinwithsisterlib
COPY ./elfbinwithnestedlib /usr/local/bin/elftests/elfbinwithnestedlib
COPY ./elfbinwithsisterlib /usr/local/bin/elftests/elfbinwithsisterlib
ENV LD_LIBRARY_PATH=/usr/local/bin/elftests/elfbinwithnestedlib/bin/lib
WORKDIR /usr/local/bin/elftests/elfbinwithnestedlib/
RUN make
WORKDIR /usr/local/bin/elftests/elfbinwithsisterlib
RUN make
# let's make the test image smaller, since we only require the built binaries and supporting libraries
FROM busybox:1.36.1-musl
COPY --from=base /usr/local/bin/elftests /usr/local/bin/elftests
COPY --from=base /var/lib/rpm /var/lib/rpm
COPY --from=base '/usr/lib64/libstdc++.so.6.0.25' '/usr/lib64/libstdc++.so.6.0.25'
COPY --from=base '/usr/lib64/libstdc++.so.6' '/usr/lib64/libstdc++.so.6'
COPY --from=base '/usr/lib64/libc.so.6' '/usr/lib64/libc.so.6'
COPY --from=base '/usr/lib64/libc.so' '/usr/lib64/libc.so'
# prove we can operate over symlinks (/lib64 -> usr/lib64)
RUN ln -s /usr/lib64 /lib64
syft/pkg/cataloger/binary/test-fixtures/image-fedora-32bit/Dockerfile
+1 -1
View File
@@ -1,4 +1,4 @@
FROM --platform=linux/arm arm32v7/fedora:36 as build
FROM --platform=linux/arm arm32v7/fedora:36 AS build
FROM scratch
COPY --from=build /bin/sha256sum /sha256sum
syft/pkg/cataloger/binary/test-fixtures/image-fedora-64bit/Dockerfile
+1 -1
View File
@@ -1,4 +1,4 @@
FROM --platform=linux/amd64 fedora:41@sha256:c05bf79137835bf5c521c58f8252d6031780ae865a0379ab57f412e0ac6b42aa as build
FROM --platform=linux/amd64 fedora:41@sha256:c05bf79137835bf5c521c58f8252d6031780ae865a0379ab57f412e0ac6b42aa AS build
FROM scratch
syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image.go
+4 -4
View File
@@ -1,11 +1,11 @@
package config
import (
"crypto/sha256"
"fmt"
"path/filepath"
"strings"
"github.com/OneOfOne/xxhash"
"gopkg.in/yaml.v3"
)
@@ -68,13 +68,13 @@ func PlatformAsValue(platform string) string {
return strings.ReplaceAll(platform, "/", "-")
}
func (c BinaryFromImage) Fingerprint() string {
func (c BinaryFromImage) Digest() string {
by, err := yaml.Marshal(c)
if err != nil {
panic(err)
}
hasher := sha256.New()
hasher.Write(by)
hasher := xxhash.New64()
_, _ = hasher.Write(by)
return fmt.Sprintf("%x", hasher.Sum(nil))
}
syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image_test.go
+3 -3
View File
@@ -158,7 +158,7 @@ func TestPlatformAsValue(t *testing.T) {
}
}
func TestFingerprint(t *testing.T) {
func TestDigest(t *testing.T) {
tests := []struct {
name string
binary BinaryFromImage
@@ -179,13 +179,13 @@ func TestFingerprint(t *testing.T) {
"path/to/test",
},
},
expected: "54ed081c07e4eba031afed4c04315cf96047822196473971be98d0769a0e3645",
expected: "fc25c48e3d2f01e3",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.binary.Fingerprint())
assert.Equal(t, tt.expected, tt.binary.Digest())
})
}
}
syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image.go
+37 -11
View File
@@ -2,6 +2,7 @@ package internal
import (
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
@@ -14,6 +15,8 @@ import (
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/ui"
)
const digestFileSuffix = ".xxh64"
func DownloadFromImage(dest string, config config.BinaryFromImage) error {
t := ui.Title{Name: config.Name(), Version: config.Version}
t.Start()
@@ -39,22 +42,22 @@ func DownloadFromImage(dest string, config config.BinaryFromImage) error {
}
func isDownloadStale(config config.BinaryFromImage, binaryPaths []string) bool {
currentFingerprint := config.Fingerprint()
currentDigest := config.Digest()
for _, path := range binaryPaths {
fingerprintPath := path + ".fingerprint"
if _, err := os.Stat(fingerprintPath); err != nil {
digestPath := path + digestFileSuffix
if _, err := os.Stat(digestPath); err != nil {
// missing a fingerprint file means the download is stale
return true
}
writtenFingerprint, err := os.ReadFile(fingerprintPath)
writtenDigest, err := os.ReadFile(digestPath)
if err != nil {
// missing a fingerprint file means the download is stale
return true
}
if string(writtenFingerprint) != currentFingerprint {
if string(writtenDigest) != currentDigest {
// the fingerprint file does not match the current fingerprint, so the download is stale
return true
}
@@ -103,6 +106,12 @@ func pullDockerImage(imageReference, platform string) error {
cmd := exec.Command("docker", "pull", "--platform", platform, imageReference)
err := cmd.Run()
if err != nil {
// attach stderr to output message
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
err = fmt.Errorf("pull failed: %w:\n%s", err, exitErr.Stderr)
}
a.Done(err)
return err
}
@@ -152,6 +161,12 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
cmd := exec.Command("docker", "create", "--name", containerName, image.Reference)
if err = cmd.Run(); err != nil {
// attach stderr to output message
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr)
}
return err
}
@@ -162,7 +177,7 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
for i, destinationPath := range config.AllStorePathsForImage(image, destination) {
path := config.PathsInImage[i]
if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Fingerprint()); err != nil {
if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Digest()); err != nil {
return err
}
}
@@ -170,7 +185,7 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
return nil
}
func copyBinaryFromContainer(containerName, containerPath, destinationPath, fingerprint string) (err error) {
func copyBinaryFromContainer(containerName, containerPath, destinationPath, digest string) (err error) {
a := ui.Action{Msg: fmt.Sprintf("extract %s", containerPath)}
a.Start()
@@ -185,13 +200,24 @@ func copyBinaryFromContainer(containerName, containerPath, destinationPath, fing
cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:%s", containerName, containerPath), destinationPath) //nolint:gosec
// reason for gosec exception: this is for processing test fixtures only, not used in production
if err := cmd.Run(); err != nil {
// attach stderr to output message
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr)
}
return err
}
// capture fingerprint file
fingerprintPath := destinationPath + ".fingerprint"
if err := os.WriteFile(fingerprintPath, []byte(fingerprint), 0600); err != nil {
return fmt.Errorf("unable to write fingerprint file: %w", err)
// ensure permissions are 600 for destination
if err := os.Chmod(destinationPath, 0600); err != nil {
return fmt.Errorf("unable to set permissions on file %q: %w", destinationPath, err)
}
// capture digest file
digestPath := destinationPath + digestFileSuffix
if err := os.WriteFile(digestPath, []byte(digest), 0600); err != nil {
return fmt.Errorf("unable to write digest file: %w", err)
}
return nil
syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image_test.go
+15 -15
View File
@@ -14,35 +14,35 @@ import (
func TestIsDownloadStale(t *testing.T) {
cases := []struct {
name string
fingerprint string
expected bool
name string
digest string
expected bool
}{
{
name: "no fingerprint",
fingerprint: "",
expected: true,
name: "no digest",
digest: "",
expected: true,
},
{
name: "fingerprint matches",
// this is the fingerprint for config in the loop body
fingerprint: "5177d458eaca031ea16fa707841043df2e31b89be6bae7ea41290aa32f0251a6",
expected: false,
name: "digest matches",
// this is the digest for config in the loop body
digest: "c9c8007f9c55c2f1",
expected: false,
},
{
name: "fingerprint does not match",
fingerprint: "fingerprint",
expected: true,
name: "digest does not match",
digest: "bogus",
expected: true,
},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
binaryPath := filepath.Join(t.TempDir(), "binary")
fh, err := os.Create(binaryPath + ".fingerprint")
fh, err := os.Create(binaryPath + digestFileSuffix)
require.NoError(t, err)
fh.Write([]byte(tt.fingerprint))
fh.Write([]byte(tt.digest))
require.NoError(t, fh.Close())
cfg := config.BinaryFromImage{
syft/pkg/cataloger/binary/test-fixtures/manager/internal/list_entries.go
+1 -1
View File
@@ -170,7 +170,7 @@ func getLogicalKey(managedBinaryPath string) (*LogicalEntryKey, error) {
func allFilePaths(root string) ([]string, error) {
var paths []string
err := filepath.Walk(root, func(path string, info os.FileInfo, _ error) error {
if info != nil && !info.IsDir() && !strings.HasSuffix(path, ".fingerprint") {
if info != nil && !info.IsDir() && !strings.HasSuffix(path, digestFileSuffix) {
paths = append(paths, path)
}
return nil
syft/pkg/cataloger/gentoo/cataloger_test.go
+1 -1
View File
@@ -64,7 +64,7 @@ func TestPortageCataloger(t *testing.T) {
var expectedRelationships []artifact.Relationship
pkgtest.NewCatalogTester().
FromDirectory(t, "test-fixtures/image-portage").
FromDirectory(t, "test-fixtures/layout").
Expects(expectedPkgs, expectedRelationships).
TestCataloger(t, NewPortageCataloger())
syft/pkg/cataloger/gentoo/test-fixtures/image-portage/var/db/pkg/app-containers/skopeo-1.5.1/CONTENTS → syft/pkg/cataloger/gentoo/test-fixtures/layout/var/db/pkg/app-containers/skopeo-1.5.1/CONTENTS
View File
syft/pkg/cataloger/gentoo/test-fixtures/image-portage/var/db/pkg/app-containers/skopeo-1.5.1/LICENSE → syft/pkg/cataloger/gentoo/test-fixtures/layout/var/db/pkg/app-containers/skopeo-1.5.1/LICENSE
View File
syft/pkg/cataloger/gentoo/test-fixtures/image-portage/var/db/pkg/app-containers/skopeo-1.5.1/SIZE → syft/pkg/cataloger/gentoo/test-fixtures/layout/var/db/pkg/app-containers/skopeo-1.5.1/SIZE
View File
syft/pkg/cataloger/golang/test-fixtures/Makefile
+15
View File
@@ -0,0 +1,15 @@
.DEFAULT_GOAL := default
default:
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
if [ -f "$$dir/Makefile" ]; then \
$(MAKE) -C $$dir; \
fi; \
done
%:
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
if [ -f "$$dir/Makefile" ]; then \
$(MAKE) -C $$dir $@; \
fi; \
done
syft/pkg/cataloger/golang/test-fixtures/archs/Makefile
+21 -11
View File
@@ -1,29 +1,39 @@
DESTINATION=binaries
FINGERPRINT_FILE=$(DESTINATION).fingerprint
all: $(DESTINATION)/hello-mach-o-arm64 $(DESTINATION)/hello-linux-arm $(DESTINATION)/hello-linux-ppc64le $(DESTINATION)/hello-win-amd64
ifndef DESTINATION
$(error DESTINATION is not set)
endif
.DEFAULT_GOAL := fixtures
# requirement 1: 'fixtures' goal to generate any and all test fixtures
fixtures: $(DESTINATION)
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
fingerprint: $(DESTINATION).fingerprint
$(DESTINATION): $(DESTINATION)/hello-mach-o-arm64 $(DESTINATION)/hello-linux-arm $(DESTINATION)/hello-linux-ppc64le $(DESTINATION)/hello-win-amd64
$(DESTINATION)/hello-mach-o-arm64:
mkdir -p $(DESTINATION)
GOARCH=arm64 GOOS=darwin ./src/build.sh $(DESTINATION)/hello-mach-o-arm64
$(DESTINATION)/hello-linux-arm:
mkdir -p $(DESTINATION)
GOARCH=arm GOOS=linux ./src/build.sh $(DESTINATION)/hello-linux-arm
$(DESTINATION)/hello-linux-ppc64le:
mkdir -p $(DESTINATION)
GOARCH=ppc64le GOOS=linux ./src/build.sh $(DESTINATION)/hello-linux-ppc64le
$(DESTINATION)/hello-win-amd64:
mkdir -p $(DESTINATION)
GOARCH=amd64 GOOS=windows ./src/build.sh $(DESTINATION)/hello-win-amd64
# we need a way to determine if CI should bust the test cache based on the source material
$(DESTINATION).fingerprint: clean
mkdir -p $(DESTINATION)
find src -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee $(DESTINATION).fingerprint
sha256sum $(DESTINATION).fingerprint
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
.PHONY: $(FINGERPRINT_FILE)
$(FINGERPRINT_FILE):
@find src -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
# requirement 4: 'clean' goal to remove all generated test fixtures
.PHONY: clean
clean:
rm -f $(DESTINATION)/*
rm -rf $(DESTINATION)
syft/pkg/cataloger/golang/test-fixtures/archs/src/build.sh
+4 -1
View File
@@ -1,10 +1,13 @@
#!/usr/bin/env bash
set -uxe
set -ue
# note: this can be easily done in a 1-liner, however circle CI does NOT allow volume mounts from the host in docker executors (since they are on remote hosts, where the host files are inaccessible)
# note: gocache override is so we can run docker build not as root in a container without permission issues
BINARY=$1
mkdir -p "$(dirname "$BINARY")"
CTRID=$(docker create -e GOOS="${GOOS}" -e GOARCH="${GOARCH}" -u "$(id -u):$(id -g)" -e GOCACHE=/tmp -w /src golang:1.17 go build -o main main.go)
function cleanup() {
syft/pkg/cataloger/java/test-fixtures/Makefile
+15
View File
@@ -0,0 +1,15 @@
.DEFAULT_GOAL := default
default:
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
if [ -f "$$dir/Makefile" ]; then \
$(MAKE) -C $$dir; \
fi; \
done
%:
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
if [ -f "$$dir/Makefile" ]; then \
$(MAKE) -C $$dir $@; \
fi; \
done
syft/pkg/cataloger/java/test-fixtures/jar-metadata/Makefile
+40 -10
View File
@@ -1,5 +1,10 @@
CACHE_DIR = cache
CACHE_PATH = $(shell pwd)/cache
FINGERPRINT_FILE=$(CACHE_DIR).fingerprint
ifndef CACHE_DIR
$(error CACHE_DIR is not set)
endif
JACKSON_CORE = jackson-core-2.15.2
SBT_JACKSON_CORE = com.fasterxml.jackson.core.jackson-core-2.15.2
@@ -8,28 +13,53 @@ API_ALL_SOURCES = api-all-2.0.0-sources
SPRING_INSTRUMENTATION = spring-instrumentation-4.3.0-1.0
MULTIPLE_MATCHING = multiple-matching-2.11.5
$(CACHE_DIR):
mkdir -p $(CACHE_DIR)
$(CACHE_DIR)/$(JACKSON_CORE).jar: $(CACHE_DIR)
.DEFAULT_GOAL := fixtures
# requirement 1: 'fixtures' goal to generate any and all test fixtures
fixtures: $(CACHE_DIR)
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
fingerprint: $(FINGERPRINT_FILE)
$(CACHE_DIR): $(CACHE_DIR)/$(JACKSON_CORE).jar $(CACHE_DIR)/$(SBT_JACKSON_CORE).jar $(CACHE_DIR)/$(OPENSAML_CORE).jar $(CACHE_DIR)/$(API_ALL_SOURCES).jar $(CACHE_DIR)/$(SPRING_INSTRUMENTATION).jar $(CACHE_DIR)/$(MULTIPLE_MATCHING).jar
$(CACHE_DIR)/$(JACKSON_CORE).jar:
mkdir -p $(CACHE_DIR)
cd $(JACKSON_CORE) && zip -r $(CACHE_PATH)/$(JACKSON_CORE).jar .
$(CACHE_DIR)/$(SBT_JACKSON_CORE).jar: $(CACHE_DIR)
$(CACHE_DIR)/$(SBT_JACKSON_CORE).jar:
mkdir -p $(CACHE_DIR)
cd $(SBT_JACKSON_CORE) && zip -r $(CACHE_PATH)/$(SBT_JACKSON_CORE).jar .
$(CACHE_DIR)/$(OPENSAML_CORE).jar: $(CACHE_DIR)
$(CACHE_DIR)/$(OPENSAML_CORE).jar:
mkdir -p $(CACHE_DIR)
cd $(OPENSAML_CORE) && zip -r $(CACHE_PATH)/$(OPENSAML_CORE).jar .
$(CACHE_DIR)/$(API_ALL_SOURCES).jar: $(CACHE_DIR)
$(CACHE_DIR)/$(API_ALL_SOURCES).jar:
mkdir -p $(CACHE_DIR)
cd $(API_ALL_SOURCES) && zip -r $(CACHE_PATH)/$(API_ALL_SOURCES).jar .
$(CACHE_DIR)/$(SPRING_INSTRUMENTATION).jar: $(CACHE_DIR)
$(CACHE_DIR)/$(SPRING_INSTRUMENTATION).jar:
mkdir -p $(CACHE_DIR)
cd $(SPRING_INSTRUMENTATION) && zip -r $(CACHE_PATH)/$(SPRING_INSTRUMENTATION).jar .
$(CACHE_DIR)/$(MULTIPLE_MATCHING).jar: $(CACHE_DIR)
$(CACHE_DIR)/$(MULTIPLE_MATCHING).jar:
mkdir -p $(CACHE_DIR)
cd $(MULTIPLE_MATCHING) && zip -r $(CACHE_PATH)/$(MULTIPLE_MATCHING).jar .
# Jenkins plugins typically do not have the version included in the archive name,
# so it is important to not include it in the generated test fixture
$(CACHE_DIR)/gradle.hpi: $(CACHE_DIR)
cd jenkins-plugins/gradle/2.11 && zip -r $(CACHE_PATH)/gradle.hpi .
$(CACHE_DIR)/gradle.hpi:
mkdir -p $(CACHE_DIR)
cd jenkins-plugins/gradle/2.11 && zip -r $(CACHE_PATH)/gradle.hpi .
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
.PHONY: $(FINGERPRINT_FILE)
$(FINGERPRINT_FILE):
@find . ! -path '*/cache*' -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
# requirement 4: 'clean' goal to remove all generated test fixtures
clean:
rm -rf $(CACHE_DIR)/* $(FINGERPRINT_FILE)
syft/pkg/cataloger/java/test-fixtures/java-builds/Makefile
+19 -10
View File
@@ -1,17 +1,18 @@
PKGSDIR=packages
FINGERPRINT_FILE=$(PKGSDIR).fingerprint
ifndef PKGSDIR
$(error PKGSDIR is not set)
endif
all: jars archives native-image
clean: clean-examples
rm -f $(PKGSDIR)/*
.DEFAULT_GOAL := fixtures
clean-examples: clean-gradle clean-maven clean-jenkins clean-nestedjar
# requirement 1: 'fixtures' goal to generate any and all test fixtures
fixtures: jars archives native-image
.PHONY: maven gradle clean clean-gradle clean-maven clean-jenkins clean-examples clean-nestedjar jars archives
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
fingerprint: $(FINGERPRINT_FILE)
jars: $(PKGSDIR)/example-java-app-maven-0.1.0.jar $(PKGSDIR)/example-java-app-gradle-0.1.0.jar $(PKGSDIR)/example-jenkins-plugin.hpi $(PKGSDIR)/spring-boot-0.0.1-SNAPSHOT.jar
@@ -71,8 +72,16 @@ $(PKGSDIR)/example-java-app: $(PKGSDIR)/example-java-app-maven-0.1.0.jar
$(PKGSDIR)/gcc-amd64-darwin-exec-debug:
./build-example-macho-binary.sh $(PKGSDIR)
# we need a way to determine if CI should bust the test cache based on the source material
.PHONY: cache.fingerprint
cache.fingerprint:
find example* build* gradle* Makefile -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee cache.fingerprint
sha256sum cache.fingerprint
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
.PHONY: $(FINGERPRINT_FILE)
$(FINGERPRINT_FILE):
@find example-* build-* Makefile -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
# requirement 4: 'clean' goal to remove all generated test fixtures
clean: clean-examples
rm -rf $(PKGSDIR) $(FINGERPRINT_FILE)
clean-examples: clean-gradle clean-maven clean-jenkins clean-nestedjar
.PHONY: maven gradle clean clean-gradle clean-maven clean-jenkins clean-examples clean-nestedjar jars archives
syft/pkg/cataloger/kernel/test-fixtures/Makefile
+20 -6
View File
@@ -1,7 +1,21 @@
all:
FINGERPRINT_FILE=cache.fingerprint
# we need a way to determine if CI should bust the test cache based on the source material
.PHONY: cache.fingerprint
cache.fingerprint:
find Makefile **/Dockerfile -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee cache.fingerprint
sha256sum cache.fingerprint
.DEFAULT_GOAL := fixtures
# requirement 1: 'fixtures' goal to generate any and all test fixtures
fixtures:
@echo "nothing to do"
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
fingerprint: $(FINGERPRINT_FILE)
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
.PHONY: $(FINGERPRINT_FILE)
$(FINGERPRINT_FILE):
@find Makefile **/Dockerfile -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
# requirement 4: 'clean' goal to remove all generated test fixtures
clean:
rm -f $(FINGERPRINT_FILE)
syft/pkg/cataloger/python/test-fixtures/image-multi-site-package/Dockerfile
+16 -3
View File
@@ -1,9 +1,8 @@
# digest is for linux/amd64
FROM ubuntu:20.04@sha256:cc9cc8169c9517ae035cf293b15f06922cb8c6c864d625a72b7b18667f264b70
FROM ubuntu:20.04@sha256:cc9cc8169c9517ae035cf293b15f06922cb8c6c864d625a72b7b18667f264b70 AS base
# install Python 3.8 and Python 3.9
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y python3.8 python3.9 python3-pip python3-venv python3.9-venv python3.8-venv
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3.8 python3.9 python3-pip python3-venv python3.9-venv python3.8-venv
# install pip and virtualenv for both Python versions
RUN python3.8 -m pip install --upgrade pip virtualenv
@@ -35,3 +34,17 @@ RUN /app/project2/venv/bin/pip install click==8.0.3 pyyaml==6.0
RUN /app/project2/venv/bin/pip install inquirer==3.2.4 runs==1.2.2 xmod==1.8.1 six==1.16.0 wcwidth==0.2.13 blessed==1.20.0 editor==1.6.6 readchar==4.1.0
WORKDIR /app
# let's not waste disk space... we only need the above state we've setup, not all of the os-level packages
RUN rm -rf /app/project1/venv/share
RUN rm -rf /app/project2/venv/share
RUN find /app/project1/venv/lib/python3.9/site-packages/* -type d ! -name '*.dist-info' -exec rm -rf {} +
RUN find /app/project2/venv/lib/python3.8/site-packages/* -type d ! -name '*.dist-info' -exec rm -rf {} +
RUN find /usr/local/lib/python3.8/dist-packages/* -type d ! -name '*.dist-info' -exec rm -rf {} +
RUN find /usr/local/lib/python3.9/dist-packages/* -type d ! -name '*.dist-info' -exec rm -rf {} +
FROM scratch
COPY --from=base /app/ /app/
COPY --from=base /usr/local/lib/python3.8/ /usr/local/lib/python3.8/
COPY --from=base /usr/local/lib/python3.9/ /usr/local/lib/python3.9/
syft/pkg/cataloger/redhat/test-fixtures/Makefile
+27 -10
View File
@@ -1,21 +1,38 @@
RPMSDIR=rpms
FINGERPRINT_FILE=$(RPMSDIR).fingerprint
ifndef RPMSDIR
$(error RPMSDIR is not set)
endif
all: rpms
clean:
rm -rf $(RPMSDIR)
.DEFAULT_GOAL := fixtures
# requirement 1: 'fixtures' goal to generate any and all test fixtures
fixtures: rpms
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
fingerprint: $(FINGERPRINT_FILE)
rpms:
mkdir -p $(RPMSDIR)
cd $(RPMSDIR) && curl https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/a/abc-1.01-9.hg20160905.el7.x86_64.rpm -O
cd $(RPMSDIR) && curl https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/z/zork-1.0.3-1.el7.x86_64.rpm -O
@# see note from https://dl.fedoraproject.org/pub/epel/7/README
@# ATTENTION
@# ======================================
@# The contents of this directory have been moved to our archives available at:
@#
@# http://archives.fedoraproject.org/pub/archive/epel/
# we need a way to determine if CI should bust the test cache based on the source material
.PHONY: $(RPMSDIR).fingerprint
$(RPMSDIR).fingerprint:
find Makefile -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee $(RPMSDIR).fingerprint
sha256sum $(RPMSDIR).fingerprint
cd $(RPMSDIR) && curl -LO https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/a/abc-1.01-9.hg20160905.el7.x86_64.rpm
cd $(RPMSDIR) && curl -LO https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/z/zork-1.0.3-1.el7.x86_64.rpm
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
.PHONY: $(FINGERPRINT_FILE)
$(FINGERPRINT_FILE):
@find Makefile -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
# requirement 4: 'clean' goal to remove all generated test fixtures
.PHONY: clean
clean:
rm -rf $(RPMSDIR) $(FINGERPRINT_FILE)
syft/pkg/cataloger/redhat/test-fixtures/image-minimal/Dockerfile
+5 -1
View File
@@ -2,4 +2,8 @@
FROM rockylinux:9.3.20231119@sha256:45cc42828cc5ceeffa3a9b4f6363fb582fac3ab91f77bf403daa067f8f049f96
ADD remove.sh /remove.sh
RUN /remove.sh
RUN /remove.sh
# let's only keep what we need for testing (not the intermediate layers)
FROM scratch
COPY --from=0 / /