Slim down docker cache size (#3190)

* slim down docker cache size

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove old centos images

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* troubleshoot test failure

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix wget version ref

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor caching mechanisms

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add cache cleanup steps

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* simplify deleting cache

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix first clone issue

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tool dep

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

syft/pkg/cataloger/binary/test-fixtures/.gitignore

@@ -1,6 +1,5 @@
 classifiers/dynamic
 classifiers/bin
-cache.fingerprint
 
 # allow for lb patterns (rust, pytho, php and more)
 !lib*.so

syft/pkg/cataloger/binary/test-fixtures/Makefile

@@ -1,8 +1,14 @@
-.PHONY: default list download download-all cache.fingerprint
+BIN=classifiers/bin
+FINGERPRINT_FILE=$(BIN).fingerprint
 
-.DEFAULT_GOAL := default
 
-default: download
+.DEFAULT_GOAL := fixtures
+
+# requirement 1: 'fixtures' goal to generate any and all test fixtures
+fixtures: download
+
+# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
+fingerprint: clean-fingerprint $(FINGERPRINT_FILE)
 
 list: ## list all managed binaries and snippets
 	go run ./manager list
@@ -16,14 +22,23 @@ download-all: ## download all managed binaries
 add-snippet: ## add a new snippet from an existing binary
 	go run ./manager add-snippet
 
-cache.fingerprint: ## prints the sha256sum of the any input to the download command (to determine if there is a cache miss)
-	@cat ./config.yaml | sha256sum | awk '{print $$1}' | tee cache.fingerprint
+# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
+.PHONY: $(FINGERPRINT_FILE)
+$(FINGERPRINT_FILE): ## prints the sha256sum of the any input to the download command (to determine if there is a cache miss)
+	@sha256sum ./config.yaml > $(FINGERPRINT_FILE)
+
+# requirement 4: 'clean' goal to remove all generated test fixtures
+clean:  ## clean up all downloaded binaries
+	rm -rf $(BIN)
+
+clean-fingerprint: ## clean up all legacy fingerprint files
+	@find $(BIN) -name '*.fingerprint' -delete
 
-clean: ## clean up all downloaded binaries
-	rm -rf ./classifiers/bin
 
 ## Halp! #################################
 
 .PHONY: help
 help:
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}'
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}'
+
+.PHONY: default list download download-all clean clean-fingerprint add-snippet fingerprint

syft/pkg/cataloger/binary/test-fixtures/config.yaml

@@ -85,6 +85,7 @@ from-images:
     paths:
       - /usr/local/go/bin/go
 
+  # TODO: this is no longer available from dockerhub! (the snippet is vital)
   - version: 1.5.14
     images:
       - ref: haproxy:1.5.14@sha256:3d57e3921cc84e860f764e863ce729dd0765e3d28d444775127bc42d68f98e10

syft/pkg/cataloger/binary/test-fixtures/elf-test-fixtures/Dockerfile

@@ -1,14 +1,31 @@
-FROM rockylinux:8
+FROM rockylinux:8 AS base
+
 RUN dnf update -y; \
 	dnf install make automake gcc gcc-c++ kernel-devel -y; \
 	dnf clean all
 RUN mkdir -p /usr/local/bin/elftests/elfbinwithnestedlib
 RUN mkdir -p /usr/local/bin/elftests/elfbinwithsisterlib
+
 COPY ./elfbinwithnestedlib /usr/local/bin/elftests/elfbinwithnestedlib
 COPY ./elfbinwithsisterlib /usr/local/bin/elftests/elfbinwithsisterlib
+
 ENV LD_LIBRARY_PATH=/usr/local/bin/elftests/elfbinwithnestedlib/bin/lib
+
 WORKDIR /usr/local/bin/elftests/elfbinwithnestedlib/
 RUN make
+
 WORKDIR /usr/local/bin/elftests/elfbinwithsisterlib
 RUN make
 
+# let's make the test image smaller, since we only require the built binaries and supporting libraries
+FROM busybox:1.36.1-musl
+
+COPY --from=base /usr/local/bin/elftests /usr/local/bin/elftests
+COPY --from=base /var/lib/rpm /var/lib/rpm
+COPY --from=base '/usr/lib64/libstdc++.so.6.0.25' '/usr/lib64/libstdc++.so.6.0.25'
+COPY --from=base '/usr/lib64/libstdc++.so.6' '/usr/lib64/libstdc++.so.6'
+COPY --from=base '/usr/lib64/libc.so.6' '/usr/lib64/libc.so.6'
+COPY --from=base '/usr/lib64/libc.so' '/usr/lib64/libc.so'
+
+# prove we can operate over symlinks (/lib64 -> usr/lib64)
+RUN ln -s /usr/lib64 /lib64

syft/pkg/cataloger/binary/test-fixtures/image-fedora-32bit/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/arm arm32v7/fedora:36 as build
+FROM --platform=linux/arm arm32v7/fedora:36 AS build
 
 FROM scratch
 COPY --from=build /bin/sha256sum /sha256sum

syft/pkg/cataloger/binary/test-fixtures/image-fedora-64bit/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 fedora:41@sha256:c05bf79137835bf5c521c58f8252d6031780ae865a0379ab57f412e0ac6b42aa as build
+FROM --platform=linux/amd64 fedora:41@sha256:c05bf79137835bf5c521c58f8252d6031780ae865a0379ab57f412e0ac6b42aa AS build
 
 
 FROM scratch

syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image.go

@@ -1,11 +1,11 @@
 package config
 
 import (
-	"crypto/sha256"
 	"fmt"
 	"path/filepath"
 	"strings"
 
+	"github.com/OneOfOne/xxhash"
 	"gopkg.in/yaml.v3"
 )
 
@@ -68,13 +68,13 @@ func PlatformAsValue(platform string) string {
 	return strings.ReplaceAll(platform, "/", "-")
 }
 
-func (c BinaryFromImage) Fingerprint() string {
+func (c BinaryFromImage) Digest() string {
 	by, err := yaml.Marshal(c)
 	if err != nil {
 		panic(err)
 	}
 
-	hasher := sha256.New()
-	hasher.Write(by)
+	hasher := xxhash.New64()
+	_, _ = hasher.Write(by)
 	return fmt.Sprintf("%x", hasher.Sum(nil))
 }

syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image_test.go

@@ -158,7 +158,7 @@ func TestPlatformAsValue(t *testing.T) {
 	}
 }
 
-func TestFingerprint(t *testing.T) {
+func TestDigest(t *testing.T) {
 	tests := []struct {
 		name     string
 		binary   BinaryFromImage
@@ -179,13 +179,13 @@ func TestFingerprint(t *testing.T) {
 					"path/to/test",
 				},
 			},
-			expected: "54ed081c07e4eba031afed4c04315cf96047822196473971be98d0769a0e3645",
+			expected: "fc25c48e3d2f01e3",
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, tt.expected, tt.binary.Fingerprint())
+			assert.Equal(t, tt.expected, tt.binary.Digest())
 		})
 	}
 }

syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -14,6 +15,8 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/ui"
 )
 
+const digestFileSuffix = ".xxh64"
+
 func DownloadFromImage(dest string, config config.BinaryFromImage) error {
 	t := ui.Title{Name: config.Name(), Version: config.Version}
 	t.Start()
@@ -39,22 +42,22 @@ func DownloadFromImage(dest string, config config.BinaryFromImage) error {
 }
 
 func isDownloadStale(config config.BinaryFromImage, binaryPaths []string) bool {
-	currentFingerprint := config.Fingerprint()
+	currentDigest := config.Digest()
 
 	for _, path := range binaryPaths {
-		fingerprintPath := path + ".fingerprint"
-		if _, err := os.Stat(fingerprintPath); err != nil {
+		digestPath := path + digestFileSuffix
+		if _, err := os.Stat(digestPath); err != nil {
 			// missing a fingerprint file means the download is stale
 			return true
 		}
 
-		writtenFingerprint, err := os.ReadFile(fingerprintPath)
+		writtenDigest, err := os.ReadFile(digestPath)
 		if err != nil {
 			// missing a fingerprint file means the download is stale
 			return true
 		}
 
-		if string(writtenFingerprint) != currentFingerprint {
+		if string(writtenDigest) != currentDigest {
 			// the fingerprint file does not match the current fingerprint, so the download is stale
 			return true
 		}
@@ -103,6 +106,12 @@ func pullDockerImage(imageReference, platform string) error {
 	cmd := exec.Command("docker", "pull", "--platform", platform, imageReference)
 	err := cmd.Run()
 	if err != nil {
+		// attach stderr to output message
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
+			err = fmt.Errorf("pull failed: %w:\n%s", err, exitErr.Stderr)
+		}
+
 		a.Done(err)
 		return err
 	}
@@ -152,6 +161,12 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
 
 	cmd := exec.Command("docker", "create", "--name", containerName, image.Reference)
 	if err = cmd.Run(); err != nil {
+		// attach stderr to output message
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
+			err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr)
+		}
+
 		return err
 	}
 
@@ -162,7 +177,7 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
 
 	for i, destinationPath := range config.AllStorePathsForImage(image, destination) {
 		path := config.PathsInImage[i]
-		if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Fingerprint()); err != nil {
+		if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Digest()); err != nil {
 			return err
 		}
 	}
@@ -170,7 +185,7 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
 	return nil
 }
 
-func copyBinaryFromContainer(containerName, containerPath, destinationPath, fingerprint string) (err error) {
+func copyBinaryFromContainer(containerName, containerPath, destinationPath, digest string) (err error) {
 	a := ui.Action{Msg: fmt.Sprintf("extract %s", containerPath)}
 	a.Start()
 
@@ -185,13 +200,24 @@ func copyBinaryFromContainer(containerName, containerPath, destinationPath, fing
 	cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:%s", containerName, containerPath), destinationPath) //nolint:gosec
 	// reason for gosec exception: this is for processing test fixtures only, not used in production
 	if err := cmd.Run(); err != nil {
+		// attach stderr to output message
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
+			err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr)
+		}
+
 		return err
 	}
 
-	// capture fingerprint file
-	fingerprintPath := destinationPath + ".fingerprint"
-	if err := os.WriteFile(fingerprintPath, []byte(fingerprint), 0600); err != nil {
-		return fmt.Errorf("unable to write fingerprint file: %w", err)
+	// ensure permissions are 600 for destination
+	if err := os.Chmod(destinationPath, 0600); err != nil {
+		return fmt.Errorf("unable to set permissions on file %q: %w", destinationPath, err)
+	}
+
+	// capture digest file
+	digestPath := destinationPath + digestFileSuffix
+	if err := os.WriteFile(digestPath, []byte(digest), 0600); err != nil {
+		return fmt.Errorf("unable to write digest file: %w", err)
 	}
 
 	return nil

syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image_test.go

@@ -14,35 +14,35 @@ import (
 func TestIsDownloadStale(t *testing.T) {
 
 	cases := []struct {
-		name        string
-		fingerprint string
-		expected    bool
+		name     string
+		digest   string
+		expected bool
 	}{
 		{
-			name:        "no fingerprint",
-			fingerprint: "",
-			expected:    true,
+			name:     "no digest",
+			digest:   "",
+			expected: true,
 		},
 		{
-			name: "fingerprint matches",
-			// this is the fingerprint for config in the loop body
-			fingerprint: "5177d458eaca031ea16fa707841043df2e31b89be6bae7ea41290aa32f0251a6",
-			expected:    false,
+			name: "digest matches",
+			// this is the digest for config in the loop body
+			digest:   "c9c8007f9c55c2f1",
+			expected: false,
 		},
 		{
-			name:        "fingerprint does not match",
-			fingerprint: "fingerprint",
-			expected:    true,
+			name:     "digest does not match",
+			digest:   "bogus",
+			expected: true,
 		},
 	}
 
 	for _, tt := range cases {
 		t.Run(tt.name, func(t *testing.T) {
 			binaryPath := filepath.Join(t.TempDir(), "binary")
-			fh, err := os.Create(binaryPath + ".fingerprint")
+			fh, err := os.Create(binaryPath + digestFileSuffix)
 			require.NoError(t, err)
 
-			fh.Write([]byte(tt.fingerprint))
+			fh.Write([]byte(tt.digest))
 			require.NoError(t, fh.Close())
 
 			cfg := config.BinaryFromImage{

syft/pkg/cataloger/binary/test-fixtures/manager/internal/list_entries.go

@@ -170,7 +170,7 @@ func getLogicalKey(managedBinaryPath string) (*LogicalEntryKey, error) {
 func allFilePaths(root string) ([]string, error) {
 	var paths []string
 	err := filepath.Walk(root, func(path string, info os.FileInfo, _ error) error {
-		if info != nil && !info.IsDir() && !strings.HasSuffix(path, ".fingerprint") {
+		if info != nil && !info.IsDir() && !strings.HasSuffix(path, digestFileSuffix) {
 			paths = append(paths, path)
 		}
 		return nil