feat: initial recursive archive cataloging impl

Signed-off-by: Keith Zantow <keith.zantow@anchore.com>

cmd/syft/internal/options/catalog.go

@@ -99,6 +99,7 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
 		WithComplianceConfig(cfg.ToComplianceConfig()).
 		WithUnknownsConfig(cfg.ToUnknownsConfig()).
 		WithSearchConfig(cfg.ToSearchConfig()).
+		WithArchiveConfig(cfg.ToArchiveConfig()).
 		WithPackagesConfig(cfg.ToPackagesConfig()).
 		WithLicenseConfig(cfg.ToLicenseConfig()).
 		WithFilesConfig(cfg.ToFilesConfig()).
@@ -115,6 +116,19 @@ func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig {
 	}
 }
 
+func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig {
+	// the exclusion patterns come from the same --exclude flag that reaches the source, narrowed to
+	// the ones whose shape reaches inside an archive. CreateSBOM fills the field from the source only
+	// when it is empty, so this value wins.
+	return cataloging.DefaultArchiveSearchConfig().
+		WithIncludeIndexedArchives(cfg.Package.SearchIndexedArchives).
+		WithIncludeUnindexedArchives(cfg.Package.SearchUnindexedArchives).
+		WithMaxDepth(cfg.Package.NestedArchiveMaxDepth).
+		WithMaxMemoryBytes(cfg.Package.NestedArchiveMaxMemoryBytes).
+		WithMaxDiskBytes(cfg.Package.NestedArchiveMaxDiskBytes).
+		WithExclusionPatterns(cataloging.ArchiveExclusionPatterns(cfg.Exclusions))
+}
+
 func (cfg Catalog) ToRelationshipsConfig() cataloging.RelationshipsConfig {
 	return cataloging.RelationshipsConfig{
 		PackageFileOwnership:        cfg.Relationships.PackageFileOwnership,

cmd/syft/internal/options/pkg.go

@@ -6,9 +6,12 @@ import (
 )
 
 type packageConfig struct {
-	SearchUnindexedArchives         bool `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
-	SearchIndexedArchives           bool `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
-	ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files
+	SearchUnindexedArchives         bool  `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
+	SearchIndexedArchives           bool  `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
+	ExcludeBinaryOverlapByOwnership bool  `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files
+	NestedArchiveMaxDepth           int   `yaml:"nested-archive-max-depth" json:"nested-archive-max-depth" mapstructure:"nested-archive-max-depth"`
+	NestedArchiveMaxMemoryBytes     int64 `yaml:"nested-archive-max-memory-bytes" json:"nested-archive-max-memory-bytes" mapstructure:"nested-archive-max-memory-bytes"`
+	NestedArchiveMaxDiskBytes       int64 `yaml:"nested-archive-max-disk-bytes" json:"nested-archive-max-disk-bytes" mapstructure:"nested-archive-max-disk-bytes"`
 }
 
 var _ interface {
@@ -23,6 +26,18 @@ note: enabling this may result in a performance impact since all discovered comp
 note: for now this only applies to the java package cataloger`)
 	descriptions.Add(&o.ExcludeBinaryOverlapByOwnership, `allows users to exclude synthetic binary packages from the sbom
 these packages are removed if an overlap with a non-synthetic package is found`)
+	descriptions.Add(&o.NestedArchiveMaxDepth, `maximum depth to recursively catalog nested archives, treating each archive as its own filesystem (0 = disabled, -1 = unlimited)
+when enabled, all archives (including java archives) are recursively cataloged by the archive cataloger
+and the java cataloger does not unarchive nested archives itself`)
+	descriptions.Add(&o.NestedArchiveMaxMemoryBytes, `maximum bytes of nested archive content held in memory at once (0 = none, always overflow to disk; negative = unbounded)
+this is an in-use limit: it falls as each archive is released, so it bounds a scan's peak rather than its total
+content is held in memory while this limit admits it and overflowed to disk instead of being refused when it does not
+there is no separate setting for the size at which content overflows; this limit alone decides it
+only applies when nested-archive-max-depth is enabled`)
+	descriptions.Add(&o.NestedArchiveMaxDiskBytes, `maximum bytes of nested archive content on disk at once, overflow content plus extracted entries (0 = none, nothing overflows; negative = unbounded)
+this is an in-use limit: it falls as each archive is released, so it bounds a scan's peak rather than its total
+an archive that would exceed it is skipped, since there is nowhere further to overflow
+only applies when nested-archive-max-depth is enabled`)
 }
 
 func defaultPackageConfig() packageConfig {
@@ -31,5 +46,8 @@ func defaultPackageConfig() packageConfig {
 		SearchIndexedArchives:           c.IncludeIndexedArchives,
 		SearchUnindexedArchives:         c.IncludeUnindexedArchives,
 		ExcludeBinaryOverlapByOwnership: true,
+		NestedArchiveMaxDepth:           c.MaxDepth,
+		NestedArchiveMaxMemoryBytes:     c.MaxMemoryBytes,
+		NestedArchiveMaxDiskBytes:       c.MaxDiskBytes,
 	}
 }

cmd/syft/internal/options/pkg_test.go

@@ -0,0 +1,91 @@
+package options
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/syft/cataloging"
+)
+
+func Test_packageConfigDefaultsMatchArchiveSearchConfig(t *testing.T) {
+	// the nested-archive settings must default to the values in syft/cataloging rather than
+	// restating them, so a config round trip with nothing set is a no-op
+	want := cataloging.DefaultArchiveSearchConfig()
+	got := defaultPackageConfig()
+
+	assert.Equal(t, want.MaxDepth, got.NestedArchiveMaxDepth)
+	assert.Equal(t, want.MaxMemoryBytes, got.NestedArchiveMaxMemoryBytes)
+	assert.Equal(t, want.MaxDiskBytes, got.NestedArchiveMaxDiskBytes)
+
+	require.Zero(t, got.NestedArchiveMaxDepth, "nested archive cataloging must be off by default")
+}
+
+func Test_packageConfigNamesNoArchiveExclusionSetting(t *testing.T) {
+	// nested archive cataloging honors the scan's own --exclude patterns, so the application config
+	// must name no archive-specific exclusion key. Asserted over the yaml tags rather than over the
+	// Go field names, because the key is what a user writes and what a config dump shows.
+	var keys []string
+	typ := reflect.TypeOf(packageConfig{})
+	for i := 0; i < typ.NumField(); i++ {
+		keys = append(keys, typ.Field(i).Tag.Get("yaml"))
+	}
+
+	assert.Equal(t, []string{
+		"search-unindexed-archives",
+		"search-indexed-archives",
+		"exclude-binary-overlap-by-ownership",
+		"nested-archive-max-depth",
+		"nested-archive-max-memory-bytes",
+		"nested-archive-max-disk-bytes",
+	}, keys)
+}
+
+func Test_ToArchiveConfig(t *testing.T) {
+	defaults := cataloging.DefaultArchiveSearchConfig()
+
+	t.Run("defaults round trip unchanged", func(t *testing.T) {
+		cfg := Catalog{Package: defaultPackageConfig()}
+		got := cfg.ToArchiveConfig()
+
+		assert.Equal(t, defaults.MaxDepth, got.MaxDepth)
+		assert.Equal(t, defaults.MaxMemoryBytes, got.MaxMemoryBytes)
+		assert.Equal(t, defaults.MaxDiskBytes, got.MaxDiskBytes)
+	})
+
+	t.Run("every setting is threaded through", func(t *testing.T) {
+		// a key that parses but is not carried into the archive config is invisible in a
+		// config dump, so assert each one arrives
+		cfg := Catalog{Package: packageConfig{
+			NestedArchiveMaxDepth:       3,
+			NestedArchiveMaxMemoryBytes: 111,
+			NestedArchiveMaxDiskBytes:   3333,
+			SearchIndexedArchives:       true,
+			SearchUnindexedArchives:     true,
+		}}
+		got := cfg.ToArchiveConfig()
+
+		assert.Equal(t, 3, got.MaxDepth)
+		assert.Equal(t, int64(111), got.MaxMemoryBytes)
+		assert.Equal(t, int64(3333), got.MaxDiskBytes)
+		assert.True(t, got.IncludeIndexedArchives)
+		assert.True(t, got.IncludeUnindexedArchives)
+	})
+
+	t.Run("--exclude patterns arrive already filtered to the in-scope subset, with no source involved", func(t *testing.T) {
+		// every-boundary-populates-the-patterns: the CLI is one of the two boundaries that fills
+		// ExclusionPatterns, from Catalog.Exclusions, applying the same shape rule the library
+		// entry point applies to what the source publishes - only a pattern that reaches inside an
+		// archive (any-depth, "**/") is in scope. A root-anchored or one-level pattern is not, and
+		// no source is consulted to produce this: Catalog carries no source at all.
+		cfg := Catalog{Exclusions: []string{"./root-anchored", "**/*.rpm", "*/one-level", "**/vendor"}}
+		assert.Equal(t, []string{"**/*.rpm", "**/vendor"}, cfg.ToArchiveConfig().ExclusionPatterns)
+	})
+
+	t.Run("no --exclude patterns leaves the field empty", func(t *testing.T) {
+		cfg := Catalog{}
+		assert.Empty(t, cfg.ToArchiveConfig().ExclusionPatterns)
+	})
+}

cmd/syft/internal/test/integration/package_catalogers_represented_test.go

@@ -50,7 +50,13 @@ func TestAllPackageCatalogersReachableInTasks(t *testing.T) {
 		constructorCount += constructors.Size()
 	}
 
-	assert.Equal(t, len(taskTagsByName), constructorCount, "mismatch in number of cataloger constructors and task names")
+	// java has two archive catalogers - one that reads an archive file and one that reads an archive
+	// already extracted to a filesystem - and the java archive slot in the task list yields whichever
+	// the config selects, never both. So exactly one constructor has no task under the default config.
+	const configSelectedAlternates = 1
+
+	assert.Equal(t, len(taskTagsByName)+configSelectedAlternates, constructorCount,
+		"mismatch in number of cataloger constructors and task names")
 
 	exceptions := strset.New(
 		// not reachable since they are deprecated

cmd/syft/internal/ui/ui.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -184,7 +185,15 @@ func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 }
 
 func (m UI) View() string {
-	return m.frame.View()
+	v := m.frame.View()
+	// always end on an empty line: bubbletea erases the line the cursor rests on when the program stops, so
+	// a frame whose last line carries content loses it as the run finishes. Until now a log line in
+	// the footer happened to supply that newline, which is why this only showed on sources that log
+	// nothing at all.
+	if !strings.HasSuffix(v, "\n") {
+		return v + "\n"
+	}
+	return v
 }
 
 func runWithTimeout(timeout time.Duration, fn func() error) (err error) {

cmd/syft/internal/ui/ui_test.go

@@ -0,0 +1,31 @@
+package ui
+
+import (
+	"io"
+	"testing"
+
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/bubbly/bubbles/frame"
+)
+
+var _ tea.Model = (*staticModel)(nil)
+
+type staticModel string
+
+func (m staticModel) Init() tea.Cmd { return nil }
+
+func (m staticModel) Update(tea.Msg) (tea.Model, tea.Cmd) { return m, nil }
+
+func (m staticModel) View() string { return string(m) }
+
+func Test_UI_viewEndsOnAnEmptyLine(t *testing.T) {
+	// bubbletea erases the line the cursor rests on when the program stops, so a view whose last line
+	// carries content loses that line as the run finishes. Until a log line happened to be in the
+	// footer, that is what dropped the last row of the cataloging tree on any source that logs nothing.
+	subject := New(io.Discard, false)
+	subject.frame.(*frame.Frame).AppendModel(staticModel("first row\nlast row"))
+
+	require.Equal(t, "first row\nlast row\n", subject.View())
+}