mirror of
https://github.com/anchore/syft.git
synced 2026-11-05 00:46:38 +01:00
perf: build lower layer squashes only when a scan reads them (#5492)
* perf: keep only the image squash when scanning with the squashed scope - stereoscope builds a squashed tree for every layer, each a full copy of the one below, so memory grows with layers x files. The squashed scope only ever reads the top one. - new `GetSourceConfig.WithImageSquashOnly(bool)` (and `sourceproviders.Config` field) reads images with stereoscope's `image.WithImageSquashOnly`, scoped to that one read - the CLI turns it on when `--scope squashed` (the default) - an image source read this way returns an error from `FileResolver` for `all-layers` and `deep-squashed` instead of panicking on the missing layer squashes Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * chore(deps): temporarily pin stereoscope to anchore/stereoscope#731 Replaces stereoscope with the PR head on the contributor fork so this builds. Drop the replace and bump to a released version once that PR merges. Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * chore(deps): bump stereoscope to main for WithImageSquashOnly Drops the temporary replace to the contributor fork now that anchore/stereoscope#731 has merged. Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * revert: drop the squash-only image read and bump stereoscope stereoscope is replacing `image.WithImageSquashOnly` with lazy layer squashes, where every layer keeps a usable squash. That removes the need for `GetSourceConfig.WithImageSquashOnly` and the scope guard in `FileResolver`, so both go. - bumps stereoscope to `feat/lazy-layer-squashes` (anchore/stereoscope `43c8f7e`), pending that PR merging Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * perf: build lower layer squashes only when a scan reads them The image source provider now reads images with `image.WithLazyLayerSquashes`: the image squash is built during the read, and each lower layer's squash only when something reads it. A squashed scan never does, so it skips a full file tree copy per layer. - squashed scans: faster, much less memory on images with many layers - `scan`/`attest` with all-layers or deep-squashed stay eager (an internal ctx flag), since eager squashing overlaps with fetching and indexing and is faster when every layer gets read - library callers going through `syft.GetSource` get lazy for every scope, since the scope isn't known when the image is read. All-layers is still correct, just slower - no public API change Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * pull in stereoscope@main Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> Co-authored-by: Will Murphy <willmurphyscode@users.noreply.github.com>
This commit is contained in:
Alex Goodman
co-authored by
Will Murphy
Will Murphy
parent
551442b1bc
commit
fd712ee71a
@@ -260,6 +260,11 @@ func getSource(ctx context.Context, opts *options.Catalog, userInput string, sou
|
||||
cfg = cfg.WithDigestAlgorithms(hashers...)
|
||||
}
|
||||
|
||||
if source.ParseScope(opts.Scope) != source.SquashedScope {
|
||||
// every other scope reads each layer's squash, which is cheaper to build while the image is read
|
||||
ctx = internal.WithEagerLayerSquashes(ctx)
|
||||
}
|
||||
|
||||
src, err := syft.GetSource(ctx, userInput, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not determine source: %w", err)
|
||||
|
||||
@@ -39,7 +39,6 @@ require (
|
||||
github.com/dustin/go-humanize v1.1.0
|
||||
github.com/elliotchance/phpserialize v1.4.0
|
||||
github.com/facebookincubator/nvdtools v0.1.5
|
||||
github.com/github/go-spdx/v2 v2.8.0
|
||||
github.com/gkampitakis/go-snaps v0.5.24
|
||||
github.com/go-git/go-billy/v5 v5.9.2
|
||||
github.com/go-git/go-git/v5 v5.19.3
|
||||
@@ -69,6 +68,7 @@ require (
|
||||
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1
|
||||
github.com/olekukonko/tablewriter v1.1.5
|
||||
github.com/opencontainers/go-digest v1.0.0
|
||||
github.com/pb33f/ordered-map/v2 v2.3.2
|
||||
github.com/pelletier/go-toml v1.9.5
|
||||
github.com/quasilyte/go-ruleguard/dsl v0.3.23
|
||||
github.com/rust-secure-code/go-rustaudit v0.0.0-20250226111315-e20ec32e963c
|
||||
@@ -83,6 +83,7 @@ require (
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd
|
||||
github.com/ulikunitz/xz v0.5.17
|
||||
github.com/vbatts/go-mtree v0.7.0
|
||||
github.com/vifraa/gopom v1.0.0
|
||||
@@ -95,17 +96,14 @@ require (
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
golang.org/x/mod v0.42.0
|
||||
golang.org/x/net v0.61.0
|
||||
golang.org/x/sys v0.49.0
|
||||
golang.org/x/time v0.16.0
|
||||
golang.org/x/tools v0.52.0
|
||||
howett.net/plist v1.0.1
|
||||
modernc.org/sqlite v1.61.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/pb33f/ordered-map/v2 v2.3.2
|
||||
github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd
|
||||
golang.org/x/sys v0.49.0
|
||||
howett.net/plist v1.0.1
|
||||
)
|
||||
require github.com/github/go-spdx/v2 v2.8.0
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.2 // indirect
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package internal
|
||||
|
||||
import "context"
|
||||
|
||||
type eagerLayerSquashesKey struct{}
|
||||
|
||||
// WithEagerLayerSquashes has the image source provider build every layer's squashed tree while it reads the
|
||||
// image, instead of the first time each one is used. Set it when the scan will read lower layers (any scope but
|
||||
// squashed): eager squashing overlaps with fetching and indexing the layers, so those scans finish sooner.
|
||||
func WithEagerLayerSquashes(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, eagerLayerSquashesKey{}, true)
|
||||
}
|
||||
|
||||
func EagerLayerSquashes(ctx context.Context) bool {
|
||||
v, _ := ctx.Value(eagerLayerSquashesKey{}).(bool)
|
||||
return v
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/anchore/go-collections"
|
||||
"github.com/anchore/stereoscope"
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
@@ -29,6 +30,10 @@ func (l stereoscopeImageSourceProvider) Name() string {
|
||||
}
|
||||
|
||||
func (l stereoscopeImageSourceProvider) Provide(ctx context.Context) (source.Source, error) {
|
||||
if !internal.EagerLayerSquashes(ctx) {
|
||||
// lower layer squashes are only built if something reads them, so a squashed scan never pays for them
|
||||
ctx = image.WithLazyLayerSquashes(ctx)
|
||||
}
|
||||
img, err := l.stereoscopeProvider.Provide(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package stereoscopesource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
"github.com/google/go-containerregistry/pkg/v1/random"
|
||||
"github.com/google/go-containerregistry/pkg/v1/tarball"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/stereoscope"
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
func Test_stereoscopeImageSourceProvider_layerSquashes(t *testing.T) {
|
||||
img, err := random.Image(1024, 3)
|
||||
require.NoError(t, err)
|
||||
tag, err := name.NewTag("localhost/layer-squashes:latest")
|
||||
require.NoError(t, err)
|
||||
archive := filepath.Join(t.TempDir(), "image.tar")
|
||||
require.NoError(t, tarball.WriteToFile(archive, tag, img))
|
||||
|
||||
provide := func(t *testing.T, ctx context.Context) *stereoscopeImageSource {
|
||||
providers := Providers(ProviderConfig{
|
||||
StereoscopeImageProviderConfig: stereoscope.ImageProviderConfig{UserInput: archive},
|
||||
})
|
||||
for _, p := range providers {
|
||||
if p.Value.Name() != image.DockerTarballSource {
|
||||
continue
|
||||
}
|
||||
src, err := p.Value.Provide(ctx)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, src.Close()) })
|
||||
return src.(*stereoscopeImageSource)
|
||||
}
|
||||
t.Fatalf("no %s provider", image.DockerTarballSource)
|
||||
return nil
|
||||
}
|
||||
|
||||
lazy := provide(t, context.Background())
|
||||
eager := provide(t, internal.WithEagerLayerSquashes(context.Background()))
|
||||
|
||||
// lower layers are built on first use unless eager was asked for, the top is the image squash either way
|
||||
require.Len(t, lazy.image.Layers, 3)
|
||||
for idx := range lazy.image.Layers {
|
||||
_, isTree := lazy.image.Layers[idx].SquashedTree.(*filetree.FileTree)
|
||||
assert.Equalf(t, idx == 2, isTree, "lazy layer %d", idx)
|
||||
assert.IsTypef(t, &filetree.FileTree{}, eager.image.Layers[idx].SquashedTree, "eager layer %d", idx)
|
||||
}
|
||||
|
||||
// every scope sees the same files either way
|
||||
for _, scope := range []source.Scope{source.SquashedScope, source.AllLayersScope, source.DeepSquashedScope} {
|
||||
t.Run(string(scope), func(t *testing.T) {
|
||||
files := func(src *stereoscopeImageSource) []string {
|
||||
res, err := src.FileResolver(scope)
|
||||
require.NoError(t, err)
|
||||
locs, err := res.FilesByGlob("**/*")
|
||||
require.NoError(t, err)
|
||||
var out []string
|
||||
for _, l := range locs {
|
||||
out = append(out, l.RealPath+"@"+l.FileSystemID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
want := files(eager)
|
||||
require.NotEmpty(t, want)
|
||||
assert.ElementsMatch(t, want, files(lazy))
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user