Add nested archive cataloging (#5260)

Signed-off-by: Kudryavcev Nikolay <kydry.nikolau@gmail.com>
Signed-off-by: Thomas Bechtold <thomasbechtold@jpberlin.de>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Signed-off-by: Keith Zantow <kzantow@gmail.com>
Co-authored-by: Kudryavcev Nikolay <kydry.nikolau@gmail.com>
Co-authored-by: Thomas Bechtold <thomasbechtold@jpberlin.de>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>

cmd/syft/internal/options/catalog.go

@@ -102,6 +102,7 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
 		WithComplianceConfig(cfg.ToComplianceConfig()).
 		WithUnknownsConfig(cfg.ToUnknownsConfig()).
 		WithSearchConfig(cfg.ToSearchConfig()).
+		WithArchiveConfig(cfg.ToArchiveConfig()).
 		WithPackagesConfig(cfg.ToPackagesConfig()).
 		WithLicenseConfig(cfg.ToLicenseConfig()).
 		WithFilesConfig(cfg.ToFilesConfig()).
@@ -118,6 +119,15 @@ func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig {
 	}
 }
 
+func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig {
+	return cataloging.DefaultArchiveSearchConfig().
+		WithIncludeIndexedArchives(cfg.Package.SearchIndexedArchives).
+		WithIncludeUnindexedArchives(cfg.Package.SearchUnindexedArchives).
+		WithMaxDepth(cfg.Package.NestedArchiveMaxDepth).
+		WithMaxMemoryBytes(cfg.Package.NestedArchiveMaxMemoryBytes).
+		WithMaxDiskBytes(cfg.Package.NestedArchiveMaxDiskBytes)
+}
+
 func (cfg Catalog) ToRelationshipsConfig() cataloging.RelationshipsConfig {
 	return cataloging.RelationshipsConfig{
 		PackageFileOwnership:        cfg.Relationships.PackageFileOwnership,
@@ -136,6 +146,7 @@ func (cfg Catalog) ToComplianceConfig() cataloging.ComplianceConfig {
 
 func (cfg Catalog) ToUnknownsConfig() cataloging.UnknownsConfig {
 	return cataloging.UnknownsConfig{
+		RemoveWhenPackagesDefined:         cfg.Unknowns.RemoveWhenPackagesDefined,
 		IncludeExecutablesWithoutPackages: cfg.Unknowns.ExecutablesWithoutPackages,
 		IncludeUnexpandedArchives:         cfg.Unknowns.UnexpandedArchives,
 	}
@@ -169,10 +180,6 @@ func (cfg Catalog) ToLicenseConfig() cataloging.LicenseConfig {
 }
 
 func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
-	archiveSearch := cataloging.ArchiveSearchConfig{
-		IncludeIndexedArchives:   cfg.Package.SearchIndexedArchives,
-		IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
-	}
 	return pkgcataloging.Config{
 		Binary: binary.DefaultClassifierCatalogerConfig(),
 		Cpp: cpp.DefaultCatalogerConfig().
@@ -219,7 +226,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
 			WithMavenLocalRepositoryDir(cfg.Java.MavenLocalRepositoryDir).
 			WithUseNetwork(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Java, task.Maven), cfg.Java.UseNetwork)).
 			WithMavenBaseURL(cfg.Java.MavenURL).
-			WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth).
+			WithMaxParentRecursiveDepth(cfg.Java.MaxParentRecursiveDepth).
 			WithResolveTransitiveDependencies(cfg.Java.ResolveTransitiveDependencies),
 	}
 }

cmd/syft/internal/options/pkg.go

@@ -6,9 +6,12 @@ import (
 )
 
 type packageConfig struct {
-	SearchUnindexedArchives         bool `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
-	SearchIndexedArchives           bool `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
-	ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files
+	SearchUnindexedArchives         bool  `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
+	SearchIndexedArchives           bool  `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
+	ExcludeBinaryOverlapByOwnership bool  `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files
+	NestedArchiveMaxDepth           int   `yaml:"nested-archive-max-depth" json:"nested-archive-max-depth" mapstructure:"nested-archive-max-depth"`
+	NestedArchiveMaxMemoryBytes     int64 `yaml:"nested-archive-max-memory-bytes" json:"nested-archive-max-memory-bytes" mapstructure:"nested-archive-max-memory-bytes"`
+	NestedArchiveMaxDiskBytes       int64 `yaml:"nested-archive-max-disk-bytes" json:"nested-archive-max-disk-bytes" mapstructure:"nested-archive-max-disk-bytes"`
 }
 
 var _ interface {
@@ -17,12 +20,17 @@ var _ interface {
 
 func (o *packageConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
 	descriptions.Add(&o.SearchIndexedArchives, `search within archives that do contain a file index to search against (zip)
-note: for now this only applies to the java package cataloger`)
+note: this only applies to the java package cataloger, and is ignored when nested-archive-max-depth is not 0 (every archive is searched)`)
 	descriptions.Add(&o.SearchUnindexedArchives, `search within archives that do not contain a file index to search against (tar, tar.gz, tar.bz2, etc)
 note: enabling this may result in a performance impact since all discovered compressed tars will be decompressed
-note: for now this only applies to the java package cataloger`)
+note: this only applies to the java package cataloger, and is ignored when nested-archive-max-depth is not 0 (every archive is searched)`)
 	descriptions.Add(&o.ExcludeBinaryOverlapByOwnership, `allows users to exclude synthetic binary packages from the sbom
 these packages are removed if an overlap with a non-synthetic package is found`)
+	descriptions.Add(&o.NestedArchiveMaxDepth, `maximum depth to recursively catalog nested archives (0 = disabled, -1 = as deep as 16 levels, the most allowed)
+note: when enabled this replaces the java cataloger's own unbounded jar-in-jar recursion, so archives deeper than this are reported as unknowns rather than cataloged`)
+	descriptions.Add(&o.NestedArchiveMaxMemoryBytes, `maximum bytes of nested archive content held in memory at once (0 = none, -1 = unbounded)`)
+	descriptions.Add(&o.NestedArchiveMaxDiskBytes, `maximum bytes of nested archive content on disk at once (0 = none, -1 = unbounded)
+note: this and nested-archive-max-memory-bytes cannot both be 0 when nested-archive-max-depth is not 0`)
 }
 
 func defaultPackageConfig() packageConfig {
@@ -31,5 +39,8 @@ func defaultPackageConfig() packageConfig {
 		SearchIndexedArchives:           c.IncludeIndexedArchives,
 		SearchUnindexedArchives:         c.IncludeUnindexedArchives,
 		ExcludeBinaryOverlapByOwnership: true,
+		NestedArchiveMaxDepth:           c.MaxDepth,
+		NestedArchiveMaxMemoryBytes:     c.MaxMemoryBytes,
+		NestedArchiveMaxDiskBytes:       c.MaxDiskBytes,
 	}
 }

cmd/syft/internal/options/pkg_test.go

@@ -0,0 +1,53 @@
+package options
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/syft/cataloging"
+)
+
+func Test_packageConfigDefaultsMatchArchiveSearchConfig(t *testing.T) {
+	want := cataloging.DefaultArchiveSearchConfig()
+	got := defaultPackageConfig()
+
+	assert.Equal(t, want.MaxDepth, got.NestedArchiveMaxDepth)
+	assert.Equal(t, want.MaxMemoryBytes, got.NestedArchiveMaxMemoryBytes)
+	assert.Equal(t, want.MaxDiskBytes, got.NestedArchiveMaxDiskBytes)
+
+	require.Zero(t, got.NestedArchiveMaxDepth, "nested archive cataloging must be off by default")
+}
+
+func Test_ToArchiveConfig(t *testing.T) {
+	defaults := cataloging.DefaultArchiveSearchConfig()
+
+	t.Run("defaults round trip unchanged", func(t *testing.T) {
+		cfg := Catalog{Package: defaultPackageConfig()}
+		got := cfg.ToArchiveConfig()
+
+		assert.Equal(t, defaults.MaxDepth, got.MaxDepth)
+		assert.Equal(t, defaults.MaxMemoryBytes, got.MaxMemoryBytes)
+		assert.Equal(t, defaults.MaxDiskBytes, got.MaxDiskBytes)
+	})
+
+	t.Run("every setting is threaded through", func(t *testing.T) {
+		// a key that parses but is not carried into the archive config is invisible in a config
+		// dump, so assert each one arrives
+		cfg := Catalog{Package: packageConfig{
+			NestedArchiveMaxDepth:       3,
+			NestedArchiveMaxMemoryBytes: 111,
+			NestedArchiveMaxDiskBytes:   3333,
+			SearchIndexedArchives:       true,
+			SearchUnindexedArchives:     true,
+		}}
+		got := cfg.ToArchiveConfig()
+
+		assert.Equal(t, 3, got.MaxDepth)
+		assert.Equal(t, int64(111), got.MaxMemoryBytes)
+		assert.Equal(t, int64(3333), got.MaxDiskBytes)
+		assert.True(t, got.IncludeIndexedArchives)
+		assert.True(t, got.IncludeUnindexedArchives)
+	})
+}

cmd/syft/internal/ui/ui.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -186,7 +187,12 @@ func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 }
 
 func (m UI) View() string {
-	return m.frame.View()
+	v := m.frame.View()
+	// bubbletea erases the line the cursor rests on at exit, which would drop the frame's last line
+	if !strings.HasSuffix(v, "\n") {
+		return v + "\n"
+	}
+	return v
 }
 
 var (

cmd/syft/internal/ui/ui_test.go

@@ -0,0 +1,31 @@
+package ui
+
+import (
+	"io"
+	"testing"
+
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/bubbly/bubbles/frame"
+)
+
+var _ tea.Model = (*staticModel)(nil)
+
+type staticModel string
+
+func Test_UI_viewEndsOnAnEmptyLine(t *testing.T) {
+	// bubbletea erases the line the cursor rests on when the program stops, so a view whose last line
+	// carries content loses it. Unless a log line happened to be in the footer, that dropped the last row
+	// of the cataloging tree on any source that logs nothing.
+	subject := New(io.Discard, false)
+	subject.frame.(*frame.Frame).AppendModel(staticModel("first row\nlast row"))
+
+	require.Equal(t, "first row\nlast row\n", subject.View())
+}
+
+func (m staticModel) Init() tea.Cmd { return nil }
+
+func (m staticModel) Update(tea.Msg) (tea.Model, tea.Cmd) { return m, nil }
+
+func (m staticModel) View() string { return string(m) }