oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-05 00:46:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

Slim down docker cache size (#3190)

Browse Source 
* slim down docker cache size

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove old centos images

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* troubleshoot test failure

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix wget version ref

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor caching mechanisms

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add cache cleanup steps

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* simplify deleting cache

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix first clone issue

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tool dep

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2024-09-09 11:15:13 -04:00
committed by GitHub
parent deabd4115a
commit 0a3f513f92
75 changed files with 1302 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image.go
+4 -4
View File
@@ -1,11 +1,11 @@
package config
import (
"crypto/sha256"
"fmt"
"path/filepath"
"strings"
"github.com/OneOfOne/xxhash"
"gopkg.in/yaml.v3"
)
@@ -68,13 +68,13 @@ func PlatformAsValue(platform string) string {
return strings.ReplaceAll(platform, "/", "-")
}
func (c BinaryFromImage) Fingerprint() string {
func (c BinaryFromImage) Digest() string {
by, err := yaml.Marshal(c)
if err != nil {
panic(err)
}
hasher := sha256.New()
hasher.Write(by)
hasher := xxhash.New64()
_, _ = hasher.Write(by)
return fmt.Sprintf("%x", hasher.Sum(nil))
}
syft/pkg/cataloger/binary/test-fixtures/manager/internal/config/binary_from_image_test.go
+3 -3
View File
@@ -158,7 +158,7 @@ func TestPlatformAsValue(t *testing.T) {
}
}
func TestFingerprint(t *testing.T) {
func TestDigest(t *testing.T) {
tests := []struct {
name string
binary BinaryFromImage
@@ -179,13 +179,13 @@ func TestFingerprint(t *testing.T) {
"path/to/test",
},
},
expected: "54ed081c07e4eba031afed4c04315cf96047822196473971be98d0769a0e3645",
expected: "fc25c48e3d2f01e3",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.binary.Fingerprint())
assert.Equal(t, tt.expected, tt.binary.Digest())
})
}
}
syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image.go
+37 -11
View File
@@ -2,6 +2,7 @@ package internal
import (
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
@@ -14,6 +15,8 @@ import (
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/ui"
)
const digestFileSuffix = ".xxh64"
func DownloadFromImage(dest string, config config.BinaryFromImage) error {
t := ui.Title{Name: config.Name(), Version: config.Version}
t.Start()
@@ -39,22 +42,22 @@ func DownloadFromImage(dest string, config config.BinaryFromImage) error {
}
func isDownloadStale(config config.BinaryFromImage, binaryPaths []string) bool {
currentFingerprint := config.Fingerprint()
currentDigest := config.Digest()
for _, path := range binaryPaths {
fingerprintPath := path + ".fingerprint"
if _, err := os.Stat(fingerprintPath); err != nil {
digestPath := path + digestFileSuffix
if _, err := os.Stat(digestPath); err != nil {
// missing a fingerprint file means the download is stale
return true
}
writtenFingerprint, err := os.ReadFile(fingerprintPath)
writtenDigest, err := os.ReadFile(digestPath)
if err != nil {
// missing a fingerprint file means the download is stale
return true
}
if string(writtenFingerprint) != currentFingerprint {
if string(writtenDigest) != currentDigest {
// the fingerprint file does not match the current fingerprint, so the download is stale
return true
}
@@ -103,6 +106,12 @@ func pullDockerImage(imageReference, platform string) error {
cmd := exec.Command("docker", "pull", "--platform", platform, imageReference)
err := cmd.Run()
if err != nil {
// attach stderr to output message
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
err = fmt.Errorf("pull failed: %w:\n%s", err, exitErr.Stderr)
}
a.Done(err)
return err
}
@@ -152,6 +161,12 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
cmd := exec.Command("docker", "create", "--name", containerName, image.Reference)
if err = cmd.Run(); err != nil {
// attach stderr to output message
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr)
}
return err
}
@@ -162,7 +177,7 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
for i, destinationPath := range config.AllStorePathsForImage(image, destination) {
path := config.PathsInImage[i]
if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Fingerprint()); err != nil {
if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Digest()); err != nil {
return err
}
}
@@ -170,7 +185,7 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
return nil
}
func copyBinaryFromContainer(containerName, containerPath, destinationPath, fingerprint string) (err error) {
func copyBinaryFromContainer(containerName, containerPath, destinationPath, digest string) (err error) {
a := ui.Action{Msg: fmt.Sprintf("extract %s", containerPath)}
a.Start()
@@ -185,13 +200,24 @@ func copyBinaryFromContainer(containerName, containerPath, destinationPath, fing
cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:%s", containerName, containerPath), destinationPath) //nolint:gosec
// reason for gosec exception: this is for processing test fixtures only, not used in production
if err := cmd.Run(); err != nil {
// attach stderr to output message
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr)
}
return err
}
// capture fingerprint file
fingerprintPath := destinationPath + ".fingerprint"
if err := os.WriteFile(fingerprintPath, []byte(fingerprint), 0600); err != nil {
return fmt.Errorf("unable to write fingerprint file: %w", err)
// ensure permissions are 600 for destination
if err := os.Chmod(destinationPath, 0600); err != nil {
return fmt.Errorf("unable to set permissions on file %q: %w", destinationPath, err)
}
// capture digest file
digestPath := destinationPath + digestFileSuffix
if err := os.WriteFile(digestPath, []byte(digest), 0600); err != nil {
return fmt.Errorf("unable to write digest file: %w", err)
}
return nil
syft/pkg/cataloger/binary/test-fixtures/manager/internal/download_from_image_test.go
+15 -15
View File
@@ -14,35 +14,35 @@ import (
func TestIsDownloadStale(t *testing.T) {
cases := []struct {
name string
fingerprint string
expected bool
name string
digest string
expected bool
}{
{
name: "no fingerprint",
fingerprint: "",
expected: true,
name: "no digest",
digest: "",
expected: true,
},
{
name: "fingerprint matches",
// this is the fingerprint for config in the loop body
fingerprint: "5177d458eaca031ea16fa707841043df2e31b89be6bae7ea41290aa32f0251a6",
expected: false,
name: "digest matches",
// this is the digest for config in the loop body
digest: "c9c8007f9c55c2f1",
expected: false,
},
{
name: "fingerprint does not match",
fingerprint: "fingerprint",
expected: true,
name: "digest does not match",
digest: "bogus",
expected: true,
},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
binaryPath := filepath.Join(t.TempDir(), "binary")
fh, err := os.Create(binaryPath + ".fingerprint")
fh, err := os.Create(binaryPath + digestFileSuffix)
require.NoError(t, err)
fh.Write([]byte(tt.fingerprint))
fh.Write([]byte(tt.digest))
require.NoError(t, fh.Close())
cfg := config.BinaryFromImage{
syft/pkg/cataloger/binary/test-fixtures/manager/internal/list_entries.go
+1 -1
View File
@@ -170,7 +170,7 @@ func getLogicalKey(managedBinaryPath string) (*LogicalEntryKey, error) {
func allFilePaths(root string) ([]string, error) {
var paths []string
err := filepath.Walk(root, func(path string, info os.FileInfo, _ error) error {
if info != nil && !info.IsDir() && !strings.HasSuffix(path, ".fingerprint") {
if info != nil && !info.IsDir() && !strings.HasSuffix(path, digestFileSuffix) {
paths = append(paths, path)
}
return nil