External sources configuration (#1158)

cmd/syft/cli/options/packages.go

@@ -26,6 +26,7 @@ type PackagesOptions struct {
 	OverwriteExistingImage bool
 	ImportTimeout          uint
 	Catalogers             []string
+	ExternalSourcesEnabled bool
 }
 
 var _ Interface = (*PackagesOptions)(nil)
@@ -70,9 +71,13 @@ func (o *PackagesOptions) AddFlags(cmd *cobra.Command, v *viper.Viper) error {
 	cmd.Flags().UintVarP(&o.ImportTimeout, "import-timeout", "", 30,
 		"set a timeout duration (in seconds) for the upload to Anchore Enterprise")
 
+	cmd.Flags().BoolVarP(&o.ExternalSourcesEnabled, "external-sources-enabled", "", false,
+		"shut off any use of external sources during sbom generation (default false")
+
 	return bindPackageConfigOptions(cmd.Flags(), v)
 }
 
+//nolint:funlen
 func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
 	// Formatting & Input options //////////////////////////////////////////////
 
@@ -104,6 +109,10 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
 		return err
 	}
 
+	if err := v.BindPFlag("external_sources.external-sources-enabled", flags.Lookup("external-sources-enabled")); err != nil {
+		return err
+	}
+
 	// Upload options //////////////////////////////////////////////////////////
 
 	if err := v.BindPFlag("anchore.host", flags.Lookup("host")); err != nil {

internal/config/application.go

@@ -57,6 +57,7 @@ type Application struct {
 	Exclusions         []string           `yaml:"exclude" json:"exclude" mapstructure:"exclude"`
 	Attest             attest             `yaml:"attest" json:"attest" mapstructure:"attest"`
 	Platform           string             `yaml:"platform" json:"platform" mapstructure:"platform"`
+	ExternalSources    ExternalSources    `yaml:"external_sources" json:"external_sources" mapstructure:"external_sources"`
 }
 
 func (cfg Application) ToCatalogerConfig() cataloger.Config {
@@ -66,7 +67,8 @@ func (cfg Application) ToCatalogerConfig() cataloger.Config {
 			IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
 			Scope:                    cfg.Package.Cataloger.ScopeOpt,
 		},
-		Catalogers: cfg.Catalogers,
+		Catalogers:             cfg.Catalogers,
+		ExternalSourcesEnabled: cfg.ExternalSources.ExternalSourcesEnabled,
 	}
 }
 

internal/config/datasources.go

@@ -0,0 +1,11 @@
+package config
+
+import "github.com/spf13/viper"
+
+type ExternalSources struct {
+	ExternalSourcesEnabled bool `yaml:"external-sources-enabled" json:"external-sources-enabled" mapstructure:"external-sources-enabled"`
+}
+
+func (e ExternalSources) loadDefaultValues(v *viper.Viper) {
+	v.SetDefault("external-sources-enabled", false)
+}

syft/pkg/cataloger/alpm/cataloger.go

@@ -23,6 +23,11 @@ func (c *Cataloger) Name() string {
 	return catalogerName
 }
 
+// UsesExternalSources indicates that the alpmdb cataloger does not use external sources
+func (c *Cataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	fileMatches, err := resolver.FilesByGlob(pkg.AlpmDBGlob)

syft/pkg/cataloger/cataloger.go

@@ -41,6 +41,8 @@ type Cataloger interface {
 	Name() string
 	// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
 	Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error)
+	// UsesExternalSources returns if the cataloger uses external sources, such as querying a database
+	UsesExternalSources() bool
 }
 
 // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
@@ -58,7 +60,7 @@ func ImageCatalogers(cfg Config) []Cataloger {
 		golang.NewGoModuleBinaryCataloger(),
 		dotnet.NewDotnetDepsCataloger(),
 		portage.NewPortageCataloger(),
-	}, cfg.Catalogers)
+	}, cfg)
 }
 
 // DirectoryCatalogers returns a slice of locally implemented catalogers that are fit for detecting packages from index files (and select installations)
@@ -84,7 +86,7 @@ func DirectoryCatalogers(cfg Config) []Cataloger {
 		cpp.NewConanfileCataloger(),
 		portage.NewPortageCataloger(),
 		haskell.NewHackageCataloger(),
-	}, cfg.Catalogers)
+	}, cfg)
 }
 
 // AllCatalogers returns all implemented catalogers
@@ -114,10 +116,20 @@ func AllCatalogers(cfg Config) []Cataloger {
 		cpp.NewConanfileCataloger(),
 		portage.NewPortageCataloger(),
 		haskell.NewHackageCataloger(),
-	}, cfg.Catalogers)
+	}, cfg)
 }
 
+// RequestedAllCatalogers returns true if all Catalogers have been requested. Takes into account cfg.ExternalSourcesEnabled
 func RequestedAllCatalogers(cfg Config) bool {
+	// if external sources are disabled, only return false if there actually are any catalogers that use external sources
+	if !cfg.ExternalSourcesEnabled {
+		for _, cat := range AllCatalogers(Config{Catalogers: []string{"all"}, ExternalSourcesEnabled: true}) {
+			if cat.UsesExternalSources() {
+				return false
+			}
+		}
+	}
+
 	for _, enableCatalogerPattern := range cfg.Catalogers {
 		if enableCatalogerPattern == AllCatalogersPattern {
 			return true
@@ -126,14 +138,33 @@ func RequestedAllCatalogers(cfg Config) bool {
 	return false
 }
 
-func filterCatalogers(catalogers []Cataloger, enabledCatalogerPatterns []string) []Cataloger {
+func filterForExternalSources(catalogers []Cataloger, cfg Config) []Cataloger {
+	if cfg.ExternalSourcesEnabled {
+		return catalogers
+	}
+
+	var enabledCatalogers []Cataloger
+	for _, cataloger := range catalogers {
+		if !cataloger.UsesExternalSources() {
+			enabledCatalogers = append(enabledCatalogers, cataloger)
+		} else {
+			log.Infof("cataloger %v will not be used because external sources are disabled", cataloger.Name())
+		}
+	}
+
+	return enabledCatalogers
+}
+
+func filterCatalogers(catalogers []Cataloger, cfg Config) []Cataloger {
+	enabledCatalogerPatterns := cfg.Catalogers
+
 	// if cataloger is not set, all applicable catalogers are enabled by default
 	if len(enabledCatalogerPatterns) == 0 {
-		return catalogers
+		return filterForExternalSources(catalogers, cfg)
 	}
 	for _, enableCatalogerPattern := range enabledCatalogerPatterns {
 		if enableCatalogerPattern == AllCatalogersPattern {
-			return catalogers
+			return filterForExternalSources(catalogers, cfg)
 		}
 	}
 	var keepCatalogers []Cataloger
@@ -144,7 +175,7 @@ func filterCatalogers(catalogers []Cataloger, enabledCatalogerPatterns []string)
 		}
 		log.Infof("skipping cataloger %q", cataloger.Name())
 	}
-	return keepCatalogers
+	return filterForExternalSources(keepCatalogers, cfg)
 }
 
 func contains(enabledPartial []string, catalogerName string) bool {

syft/pkg/cataloger/cataloger_test.go

@@ -1,11 +1,12 @@
 package cataloger
 
 import (
+	"testing"
+
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
 	"github.com/stretchr/testify/assert"
-	"testing"
 )
 
 var _ Cataloger = (*dummy)(nil)
@@ -22,12 +23,17 @@ func (d dummy) Catalog(_ source.FileResolver) ([]pkg.Package, []artifact.Relatio
 	panic("not implemented")
 }
 
+func (d dummy) UsesExternalSources() bool {
+	return false
+}
+
 func Test_filterCatalogers(t *testing.T) {
 	tests := []struct {
-		name       string
+		name                   string
-		patterns   []string
+		patterns               []string
-		catalogers []string
+		ExternalSourcesEnabled bool
-		want       []string
+		catalogers             []string
+		want                   []string
 	}{
 		{
 			name:     "no filtering",
@@ -142,6 +148,21 @@ func Test_filterCatalogers(t *testing.T) {
 				"go-module-binary-cataloger",
 			},
 		},
+		{ // Note: no catalogers with external sources are currently implemented
+			name:                   "external sources enabled",
+			patterns:               []string{"all"},
+			ExternalSourcesEnabled: true,
+			catalogers: []string{
+				"ruby-gemspec-cataloger",
+				"python-package-cataloger",
+				"rekor-cataloger",
+			},
+			want: []string{
+				"ruby-gemspec-cataloger",
+				"python-package-cataloger",
+				"rekor-cataloger",
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -149,7 +170,8 @@ func Test_filterCatalogers(t *testing.T) {
 			for _, n := range tt.catalogers {
 				catalogers = append(catalogers, dummy{name: n})
 			}
-			got := filterCatalogers(catalogers, tt.patterns)
+			cfg := Config{Catalogers: tt.patterns, ExternalSourcesEnabled: tt.ExternalSourcesEnabled}
+			got := filterCatalogers(catalogers, cfg)
 			var gotNames []string
 			for _, g := range got {
 				gotNames = append(gotNames, g.Name())

syft/pkg/cataloger/common/generic_cataloger.go

@@ -39,6 +39,11 @@ func (c *GenericCataloger) Name() string {
 	return c.upstreamCataloger
 }
 
+// UsesExternalSources indicates that any GenericCatalogor does not use external sources
+func (c *GenericCataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
 func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	var packages []pkg.Package

syft/pkg/cataloger/config.go

@@ -5,8 +5,9 @@ import (
 )
 
 type Config struct {
-	Search     SearchConfig
+	Search                 SearchConfig
-	Catalogers []string
+	Catalogers             []string
+	ExternalSourcesEnabled bool
 }
 
 func DefaultConfig() Config {

syft/pkg/cataloger/deb/cataloger.go

@@ -36,6 +36,11 @@ func (c *Cataloger) Name() string {
 	return "dpkgdb-cataloger"
 }
 
+// UsesExternalSources indicates that the dpkgdb cataloger does not use external sources
+func (c *Cataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	dbFileMatches, err := resolver.FilesByGlob(pkg.DpkgDBGlob)

syft/pkg/cataloger/golang/binary_cataloger.go

@@ -28,6 +28,11 @@ func (c *Cataloger) Name() string {
 	return catalogerName
 }
 
+// UsesExternalSources indicates that the golang binary cataloger does not use external sources
+func (c *Cataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	var pkgs []pkg.Package

syft/pkg/cataloger/portage/cataloger.go

@@ -37,6 +37,11 @@ func (c *Cataloger) Name() string {
 	return "portage-cataloger"
 }
 
+// UsesExternalSources indicates that the portage cataloger does not use external sources
+func (c *Cataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing portage support files.
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	dbFileMatches, err := resolver.FilesByGlob(pkg.PortageDBGlob)

syft/pkg/cataloger/python/package_cataloger.go

@@ -33,6 +33,11 @@ func (c *PackageCataloger) Name() string {
 	return "python-package-cataloger"
 }
 
+// UsesExternalSources indicates that the python package cataloger does not use external sources
+func (c *PackageCataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
 func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	var fileMatches []source.Location

syft/pkg/cataloger/rpmdb/cataloger.go

@@ -27,6 +27,11 @@ func (c *Cataloger) Name() string {
 	return catalogerName
 }
 
+// UsesExternalSources indicates that the rpmdb cataloger does not use external sources
+func (c *Cataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	fileMatches, err := resolver.FilesByGlob(pkg.RpmDBGlob)

syft/pkg/cataloger/rust/audit_binary_cataloger.go

@@ -27,6 +27,11 @@ func (c *Cataloger) Name() string {
 	return catalogerName
 }
 
+// UsesExternalSources indicates that the audit binary cataloger does not use external sources
+func (c *Cataloger) UsesExternalSources() bool {
+	return false
+}
+
 // Catalog identifies executables then attempts to read Rust dependency information from them
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	var pkgs []pkg.Package