From b2807f60d2ae31c58ded778f4f9849a737d7ee26 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 29 Oct 2026 12:10:12 -0400 Subject: [PATCH] perf: keep only the image squash when scanning with the squashed scope - stereoscope builds a squashed tree for every layer, each a full copy of the one below, so memory grows with layers x files. The squashed scope only ever reads the top one. - new `GetSourceConfig.WithImageSquashOnly(bool)` (and `sourceproviders.Config` field) reads images with stereoscope's `image.WithImageSquashOnly`, scoped to that one read - the CLI turns it on when `--scope squashed` (the default) - an image source read this way returns an error from `FileResolver` for `all-layers` and `deep-squashed` instead of panicking on the missing layer squashes Signed-off-by: Alex Goodman --- cmd/syft/internal/commands/scan.go | 4 +- syft/get_source_config.go | 8 +++ .../sourceproviders/source_provider_config.go | 9 +++ .../sourceproviders/source_providers.go | 5 +- syft/source/stereoscopesource/image_source.go | 15 +++++ .../image_source_provider.go | 6 ++ .../image_source_provider_test.go | 64 +++++++++++++++++++ 7 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 syft/source/stereoscopesource/image_source_provider_test.go diff --git a/cmd/syft/internal/commands/scan.go b/cmd/syft/internal/commands/scan.go index 7f92491ca..168568b86 100644 --- a/cmd/syft/internal/commands/scan.go +++ b/cmd/syft/internal/commands/scan.go @@ -239,7 +239,9 @@ func getSource(ctx context.Context, opts *options.Catalog, userInput string, sou }). WithBasePath(opts.Source.BasePath). WithSources(sources...). - WithDefaultImagePullSource(opts.Source.Image.DefaultPullSource) + WithDefaultImagePullSource(opts.Source.Image.DefaultPullSource). + // only the squashed scope reads from the image squash alone, every other scope needs each layer's squash + WithImageSquashOnly(source.ParseScope(opts.Scope) == source.SquashedScope) var err error var platform *image.Platform diff --git a/syft/get_source_config.go b/syft/get_source_config.go index 735dba516..f7528951f 100644 --- a/syft/get_source_config.go +++ b/syft/get_source_config.go @@ -57,6 +57,14 @@ func (c *GetSourceConfig) WithBasePath(basePath string) *GetSourceConfig { return c } +// WithImageSquashOnly reads container images keeping only the image squash, which cuts memory sharply on +// images with many layers. Only use it when the source will be resolved with source.SquashedScope: +// any other scope returns an error from FileResolver. +func (c *GetSourceConfig) WithImageSquashOnly(squashOnly bool) *GetSourceConfig { + c.SourceProviderConfig = c.SourceProviderConfig.WithImageSquashOnly(squashOnly) + return c +} + func (c *GetSourceConfig) WithSources(sources ...string) *GetSourceConfig { c.Sources = sources return c diff --git a/syft/source/sourceproviders/source_provider_config.go b/syft/source/sourceproviders/source_provider_config.go index dbc48bc0b..b1bac097e 100644 --- a/syft/source/sourceproviders/source_provider_config.go +++ b/syft/source/sourceproviders/source_provider_config.go @@ -15,6 +15,10 @@ type Config struct { Exclude source.ExcludeConfig DigestAlgorithms []crypto.Hash BasePath string + // ImageSquashOnly reads container images keeping only the image squash, dropping the squash every + // lower layer would otherwise hold. This cuts memory sharply on images with many layers, but the + // resulting source only supports the squashed scope. + ImageSquashOnly bool } func (c *Config) WithAlias(alias source.Alias) *Config { @@ -47,6 +51,11 @@ func (c *Config) WithBasePath(basePath string) *Config { return c } +func (c *Config) WithImageSquashOnly(squashOnly bool) *Config { + c.ImageSquashOnly = squashOnly + return c +} + func DefaultConfig() *Config { return &Config{ DigestAlgorithms: []crypto.Hash{ diff --git a/syft/source/sourceproviders/source_providers.go b/syft/source/sourceproviders/source_providers.go index fcaf7e697..f6eeca0ea 100644 --- a/syft/source/sourceproviders/source_providers.go +++ b/syft/source/sourceproviders/source_providers.go @@ -67,8 +67,9 @@ func stereoscopeSourceProviders(userInput string, cfg *Config) collections.Tagge Platform: cfg.Platform, Registry: registry, }, - Alias: cfg.Alias, - Exclude: cfg.Exclude, + Alias: cfg.Alias, + Exclude: cfg.Exclude, + ImageSquashOnly: cfg.ImageSquashOnly, }) return stereoscopeProviders } diff --git a/syft/source/stereoscopesource/image_source.go b/syft/source/stereoscopesource/image_source.go index c6dc29390..27f4b4924 100644 --- a/syft/source/stereoscopesource/image_source.go +++ b/syft/source/stereoscopesource/image_source.go @@ -99,6 +99,10 @@ func (s stereoscopeImageSource) FileResolver(scope source.Scope) (file.Resolver, var res file.Resolver var err error + if scope != source.SquashedScope && !hasLayerSquashes(s.image) { + return nil, fmt.Errorf("the %q scope needs every layer's squashed tree, but the image was read keeping only the image squash", scope) + } + switch scope { case source.SquashedScope: res, err = fileresolver.NewFromContainerImageSquash(s.image) @@ -122,6 +126,17 @@ func (s stereoscopeImageSource) FileResolver(scope source.Scope) (file.Resolver, return res, nil } +// hasLayerSquashes reports whether every layer kept its squashed tree, which is not the case for an image +// read with image.WithImageSquashOnly (only the top layer keeps one). +func hasLayerSquashes(img *image.Image) bool { + for _, l := range img.Layers { + if l.SquashedTree == nil { + return false + } + } + return true +} + func (s stereoscopeImageSource) Close() error { if s.image == nil { return nil diff --git a/syft/source/stereoscopesource/image_source_provider.go b/syft/source/stereoscopesource/image_source_provider.go index 1b1f78e4e..0cee24694 100644 --- a/syft/source/stereoscopesource/image_source_provider.go +++ b/syft/source/stereoscopesource/image_source_provider.go @@ -15,6 +15,8 @@ type ProviderConfig struct { StereoscopeImageProviderConfig stereoscope.ImageProviderConfig Exclude source.ExcludeConfig Alias source.Alias + // ImageSquashOnly reads the image keeping only the image squash, see image.WithImageSquashOnly + ImageSquashOnly bool } type stereoscopeImageSourceProvider struct { @@ -29,6 +31,10 @@ func (l stereoscopeImageSourceProvider) Name() string { } func (l stereoscopeImageSourceProvider) Provide(ctx context.Context) (source.Source, error) { + if l.cfg.ImageSquashOnly { + // scoped to this read only, the caller's ctx may be shared with reads that need every layer squash + ctx = image.WithImageSquashOnly(ctx) + } img, err := l.stereoscopeProvider.Provide(ctx) if err != nil { return nil, err diff --git a/syft/source/stereoscopesource/image_source_provider_test.go b/syft/source/stereoscopesource/image_source_provider_test.go new file mode 100644 index 000000000..b1943850d --- /dev/null +++ b/syft/source/stereoscopesource/image_source_provider_test.go @@ -0,0 +1,64 @@ +package stereoscopesource + +import ( + "context" + "path/filepath" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/stretchr/testify/require" + + "github.com/anchore/stereoscope" + "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/syft/syft/source" +) + +func Test_stereoscopeImageSourceProvider_imageSquashOnly(t *testing.T) { + img, err := random.Image(1024, 3) + require.NoError(t, err) + tag, err := name.NewTag("localhost/squash-only:latest") + require.NoError(t, err) + archive := filepath.Join(t.TempDir(), "image.tar") + require.NoError(t, tarball.WriteToFile(archive, tag, img)) + + provide := func(t *testing.T, squashOnly bool) source.Source { + providers := Providers(ProviderConfig{ + StereoscopeImageProviderConfig: stereoscope.ImageProviderConfig{UserInput: archive}, + ImageSquashOnly: squashOnly, + }) + for _, p := range providers { + if p.Value.Name() != image.DockerTarballSource { + continue + } + src, err := p.Value.Provide(context.Background()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, src.Close()) }) + return src + } + t.Fatalf("no %s provider", image.DockerTarballSource) + return nil + } + + tests := []struct { + name string + squashOnly bool + scope source.Scope + wantErr require.ErrorAssertionFunc + }{ + {name: "full read, squashed", scope: source.SquashedScope, wantErr: require.NoError}, + {name: "full read, all layers", scope: source.AllLayersScope, wantErr: require.NoError}, + {name: "full read, deep squashed", scope: source.DeepSquashedScope, wantErr: require.NoError}, + {name: "squash only, squashed", squashOnly: true, scope: source.SquashedScope, wantErr: require.NoError}, + // the per-layer resolvers would dereference the dropped layer squashes, so refuse up front + {name: "squash only, all layers", squashOnly: true, scope: source.AllLayersScope, wantErr: require.Error}, + {name: "squash only, deep squashed", squashOnly: true, scope: source.DeepSquashedScope, wantErr: require.Error}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := provide(t, tt.squashOnly).FileResolver(tt.scope) + tt.wantErr(t, err) + }) + } +}