feat(cataloging): extend ArchiveSearchConfig with recursive extraction settings

Add configuration fields for generic recursive archive extraction:
- MaxDepth: maximum nesting depth for archive extraction (default 0 = disabled)
- MaxExtractionSizeBytes: per-archive extraction size limit (default 500MB)
- MaxFileCount: per-archive file count limit (default 10,000)
- MaxTotalExtractionBytes: total extraction size limit (default 2GB)
- ExcludeExtensions: archive extensions to skip during extraction

These fields lay the groundwork for cross-cataloger recursive archive
cataloging, where archives discovered during scanning can be extracted
and their contents made visible to all catalogers.

Signed-off-by: Thomas Bechtold <thomasbechtold@jpberlin.de>
This commit is contained in:
Thomas Bechtold
2026-04-27 08:46:09 +02:00
parent 3562dab445
commit 05b4cf1c84
2 changed files with 95 additions and 0 deletions
+51
View File
@@ -1,17 +1,43 @@
package cataloging
const (
DefaultArchiveMaxDepth = 0 // disabled by default for backwards compatibility
DefaultArchiveMaxExtractionSizeBytes = 500 * 1024 * 1024 // 500 MB per archive
DefaultArchiveMaxFileCount = 10000
DefaultArchiveMaxTotalExtractionBytes = 2 * 1024 * 1024 * 1024 // 2 GB total
)
type ArchiveSearchConfig struct {
// IncludeIndexedArchives indicates whether to search within indexed archive files (e.g., .zip).
IncludeIndexedArchives bool `yaml:"include-indexed-archives" json:"include-indexed-archives" mapstructure:"include-indexed-archives"`
// IncludeUnindexedArchives indicates whether to search within unindexed archive files (e.g., .tar*).
IncludeUnindexedArchives bool `yaml:"include-unindexed-archives" json:"include-unindexed-archives" mapstructure:"include-unindexed-archives"`
// MaxDepth is the maximum depth of recursive archive extraction (0 = disabled, no recursive extraction).
MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"`
// MaxExtractionSizeBytes is the maximum total bytes to extract from a single archive before stopping.
MaxExtractionSizeBytes int64 `yaml:"max-extraction-size-bytes" json:"max-extraction-size-bytes" mapstructure:"max-extraction-size-bytes"`
// MaxFileCount is the maximum number of files to extract from a single archive before stopping.
MaxFileCount int `yaml:"max-file-count" json:"max-file-count" mapstructure:"max-file-count"`
// MaxTotalExtractionBytes is the maximum total bytes to extract across all archives before stopping.
MaxTotalExtractionBytes int64 `yaml:"max-total-extraction-bytes" json:"max-total-extraction-bytes" mapstructure:"max-total-extraction-bytes"`
// ExcludeExtensions is a list of archive extensions to skip during recursive extraction (e.g., [".rpm"]).
ExcludeExtensions []string `yaml:"exclude-extensions" json:"exclude-extensions" mapstructure:"exclude-extensions"`
}
func DefaultArchiveSearchConfig() ArchiveSearchConfig {
return ArchiveSearchConfig{
IncludeIndexedArchives: true,
IncludeUnindexedArchives: false,
MaxDepth: DefaultArchiveMaxDepth,
MaxExtractionSizeBytes: DefaultArchiveMaxExtractionSizeBytes,
MaxFileCount: DefaultArchiveMaxFileCount,
MaxTotalExtractionBytes: DefaultArchiveMaxTotalExtractionBytes,
}
}
@@ -24,3 +50,28 @@ func (c ArchiveSearchConfig) WithIncludeUnindexedArchives(include bool) ArchiveS
c.IncludeUnindexedArchives = include
return c
}
func (c ArchiveSearchConfig) WithMaxDepth(depth int) ArchiveSearchConfig {
c.MaxDepth = depth
return c
}
func (c ArchiveSearchConfig) WithMaxExtractionSizeBytes(size int64) ArchiveSearchConfig {
c.MaxExtractionSizeBytes = size
return c
}
func (c ArchiveSearchConfig) WithMaxFileCount(count int) ArchiveSearchConfig {
c.MaxFileCount = count
return c
}
func (c ArchiveSearchConfig) WithMaxTotalExtractionBytes(size int64) ArchiveSearchConfig {
c.MaxTotalExtractionBytes = size
return c
}
func (c ArchiveSearchConfig) WithExcludeExtensions(extensions []string) ArchiveSearchConfig {
c.ExcludeExtensions = extensions
return c
}
+44
View File
@@ -0,0 +1,44 @@
package cataloging
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestDefaultArchiveSearchConfig(t *testing.T) {
cfg := DefaultArchiveSearchConfig()
assert.True(t, cfg.IncludeIndexedArchives)
assert.False(t, cfg.IncludeUnindexedArchives)
assert.Equal(t, DefaultArchiveMaxDepth, cfg.MaxDepth)
assert.Equal(t, int64(DefaultArchiveMaxExtractionSizeBytes), cfg.MaxExtractionSizeBytes)
assert.Equal(t, DefaultArchiveMaxFileCount, cfg.MaxFileCount)
assert.Equal(t, int64(DefaultArchiveMaxTotalExtractionBytes), cfg.MaxTotalExtractionBytes)
assert.Nil(t, cfg.ExcludeExtensions)
}
func TestArchiveSearchConfig_WithMethods(t *testing.T) {
cfg := DefaultArchiveSearchConfig()
cfg = cfg.WithMaxDepth(5)
assert.Equal(t, 5, cfg.MaxDepth)
cfg = cfg.WithMaxExtractionSizeBytes(1024)
assert.Equal(t, int64(1024), cfg.MaxExtractionSizeBytes)
cfg = cfg.WithMaxFileCount(100)
assert.Equal(t, 100, cfg.MaxFileCount)
cfg = cfg.WithMaxTotalExtractionBytes(2048)
assert.Equal(t, int64(2048), cfg.MaxTotalExtractionBytes)
cfg = cfg.WithExcludeExtensions([]string{".rpm", ".deb"})
assert.Equal(t, []string{".rpm", ".deb"}, cfg.ExcludeExtensions)
cfg = cfg.WithIncludeIndexedArchives(false)
assert.False(t, cfg.IncludeIndexedArchives)
cfg = cfg.WithIncludeUnindexedArchives(true)
assert.True(t, cfg.IncludeUnindexedArchives)
}