Revert "External sources configuration (#1158)" (#1191)

reverted as functionality is to be merged with dev branch of kubecon draft

cmd/syft/cli/options/packages.go

@@ -26,7 +26,6 @@ type PackagesOptions struct {
 	OverwriteExistingImage bool
 	ImportTimeout          uint
 	Catalogers             []string
-	ExternalSourcesEnabled bool
 }
 
 var _ Interface = (*PackagesOptions)(nil)
@@ -71,13 +70,9 @@ func (o *PackagesOptions) AddFlags(cmd *cobra.Command, v *viper.Viper) error {
 	cmd.Flags().UintVarP(&o.ImportTimeout, "import-timeout", "", 30,
 		"set a timeout duration (in seconds) for the upload to Anchore Enterprise")
 
-	cmd.Flags().BoolVarP(&o.ExternalSourcesEnabled, "external-sources-enabled", "", false,
-		"shut off any use of external sources during sbom generation (default false")
-
 	return bindPackageConfigOptions(cmd.Flags(), v)
 }
 
-//nolint:funlen
 func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
 	// Formatting & Input options //////////////////////////////////////////////
 
@@ -109,10 +104,6 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
 		return err
 	}
 
-	if err := v.BindPFlag("external_sources.external-sources-enabled", flags.Lookup("external-sources-enabled")); err != nil {
-		return err
-	}
-
 	// Upload options //////////////////////////////////////////////////////////
 
 	if err := v.BindPFlag("anchore.host", flags.Lookup("host")); err != nil {

internal/config/application.go

@@ -57,7 +57,6 @@ type Application struct {
 	Exclusions         []string           `yaml:"exclude" json:"exclude" mapstructure:"exclude"`
 	Attest             attest             `yaml:"attest" json:"attest" mapstructure:"attest"`
 	Platform           string             `yaml:"platform" json:"platform" mapstructure:"platform"`
-	ExternalSources    ExternalSources    `yaml:"external_sources" json:"external_sources" mapstructure:"external_sources"`
 }
 
 func (cfg Application) ToCatalogerConfig() cataloger.Config {
@@ -67,8 +66,7 @@ func (cfg Application) ToCatalogerConfig() cataloger.Config {
 			IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
 			Scope:                    cfg.Package.Cataloger.ScopeOpt,
 		},
-		Catalogers:             cfg.Catalogers,
+		Catalogers: cfg.Catalogers,
-		ExternalSourcesEnabled: cfg.ExternalSources.ExternalSourcesEnabled,
 	}
 }
 

internal/config/datasources.go

@@ -1,11 +0,0 @@
-package config
-
-import "github.com/spf13/viper"
-
-type ExternalSources struct {
-	ExternalSourcesEnabled bool `yaml:"external-sources-enabled" json:"external-sources-enabled" mapstructure:"external-sources-enabled"`
-}
-
-func (e ExternalSources) loadDefaultValues(v *viper.Viper) {
-	v.SetDefault("external-sources-enabled", false)
-}

syft/pkg/cataloger/alpm/cataloger.go

@@ -23,11 +23,6 @@ func (c *Cataloger) Name() string {
 	return catalogerName
 }
 
-// UsesExternalSources indicates that the alpmdb cataloger does not use external sources
-func (c *Cataloger) UsesExternalSources() bool {
-	return false
-}
-
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	fileMatches, err := resolver.FilesByGlob(pkg.AlpmDBGlob)

syft/pkg/cataloger/cataloger.go

@@ -41,8 +41,6 @@ type Cataloger interface {
 	Name() string
 	// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
 	Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error)
-	// UsesExternalSources returns if the cataloger uses external sources, such as querying a database
-	UsesExternalSources() bool
 }
 
 // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
@@ -60,7 +58,7 @@ func ImageCatalogers(cfg Config) []Cataloger {
 		golang.NewGoModuleBinaryCataloger(),
 		dotnet.NewDotnetDepsCataloger(),
 		portage.NewPortageCataloger(),
-	}, cfg)
+	}, cfg.Catalogers)
 }
 
 // DirectoryCatalogers returns a slice of locally implemented catalogers that are fit for detecting packages from index files (and select installations)
@@ -86,7 +84,7 @@ func DirectoryCatalogers(cfg Config) []Cataloger {
 		cpp.NewConanfileCataloger(),
 		portage.NewPortageCataloger(),
 		haskell.NewHackageCataloger(),
-	}, cfg)
+	}, cfg.Catalogers)
 }
 
 // AllCatalogers returns all implemented catalogers
@@ -116,20 +114,10 @@ func AllCatalogers(cfg Config) []Cataloger {
 		cpp.NewConanfileCataloger(),
 		portage.NewPortageCataloger(),
 		haskell.NewHackageCataloger(),
-	}, cfg)
+	}, cfg.Catalogers)
 }
 
-// RequestedAllCatalogers returns true if all Catalogers have been requested. Takes into account cfg.ExternalSourcesEnabled
 func RequestedAllCatalogers(cfg Config) bool {
-	// if external sources are disabled, only return false if there actually are any catalogers that use external sources
-	if !cfg.ExternalSourcesEnabled {
-		for _, cat := range AllCatalogers(Config{Catalogers: []string{"all"}, ExternalSourcesEnabled: true}) {
-			if cat.UsesExternalSources() {
-				return false
-			}
-		}
-	}
-
 	for _, enableCatalogerPattern := range cfg.Catalogers {
 		if enableCatalogerPattern == AllCatalogersPattern {
 			return true
@@ -138,33 +126,14 @@ func RequestedAllCatalogers(cfg Config) bool {
 	return false
 }
 
-func filterForExternalSources(catalogers []Cataloger, cfg Config) []Cataloger {
+func filterCatalogers(catalogers []Cataloger, enabledCatalogerPatterns []string) []Cataloger {
-	if cfg.ExternalSourcesEnabled {
-		return catalogers
-	}
-
-	var enabledCatalogers []Cataloger
-	for _, cataloger := range catalogers {
-		if !cataloger.UsesExternalSources() {
-			enabledCatalogers = append(enabledCatalogers, cataloger)
-		} else {
-			log.Infof("cataloger %v will not be used because external sources are disabled", cataloger.Name())
-		}
-	}
-
-	return enabledCatalogers
-}
-
-func filterCatalogers(catalogers []Cataloger, cfg Config) []Cataloger {
-	enabledCatalogerPatterns := cfg.Catalogers
-
 	// if cataloger is not set, all applicable catalogers are enabled by default
 	if len(enabledCatalogerPatterns) == 0 {
-		return filterForExternalSources(catalogers, cfg)
+		return catalogers
 	}
 	for _, enableCatalogerPattern := range enabledCatalogerPatterns {
 		if enableCatalogerPattern == AllCatalogersPattern {
-			return filterForExternalSources(catalogers, cfg)
+			return catalogers
 		}
 	}
 	var keepCatalogers []Cataloger
@@ -175,7 +144,7 @@ func filterCatalogers(catalogers []Cataloger, cfg Config) []Cataloger {
 		}
 		log.Infof("skipping cataloger %q", cataloger.Name())
 	}
-	return filterForExternalSources(keepCatalogers, cfg)
+	return keepCatalogers
 }
 
 func contains(enabledPartial []string, catalogerName string) bool {

syft/pkg/cataloger/cataloger_test.go

@@ -1,12 +1,11 @@
 package cataloger
 
 import (
-	"testing"
-
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
 	"github.com/stretchr/testify/assert"
+	"testing"
 )
 
 var _ Cataloger = (*dummy)(nil)
@@ -23,17 +22,12 @@ func (d dummy) Catalog(_ source.FileResolver) ([]pkg.Package, []artifact.Relatio
 	panic("not implemented")
 }
 
-func (d dummy) UsesExternalSources() bool {
-	return false
-}
-
 func Test_filterCatalogers(t *testing.T) {
 	tests := []struct {
-		name                   string
+		name       string
-		patterns               []string
+		patterns   []string
-		ExternalSourcesEnabled bool
+		catalogers []string
-		catalogers             []string
+		want       []string
-		want                   []string
 	}{
 		{
 			name:     "no filtering",
@@ -148,21 +142,6 @@ func Test_filterCatalogers(t *testing.T) {
 				"go-module-binary-cataloger",
 			},
 		},
-		{ // Note: no catalogers with external sources are currently implemented
-			name:                   "external sources enabled",
-			patterns:               []string{"all"},
-			ExternalSourcesEnabled: true,
-			catalogers: []string{
-				"ruby-gemspec-cataloger",
-				"python-package-cataloger",
-				"rekor-cataloger",
-			},
-			want: []string{
-				"ruby-gemspec-cataloger",
-				"python-package-cataloger",
-				"rekor-cataloger",
-			},
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -170,8 +149,7 @@ func Test_filterCatalogers(t *testing.T) {
 			for _, n := range tt.catalogers {
 				catalogers = append(catalogers, dummy{name: n})
 			}
-			cfg := Config{Catalogers: tt.patterns, ExternalSourcesEnabled: tt.ExternalSourcesEnabled}
+			got := filterCatalogers(catalogers, tt.patterns)
-			got := filterCatalogers(catalogers, cfg)
 			var gotNames []string
 			for _, g := range got {
 				gotNames = append(gotNames, g.Name())

syft/pkg/cataloger/common/generic_cataloger.go

@@ -39,11 +39,6 @@ func (c *GenericCataloger) Name() string {
 	return c.upstreamCataloger
 }
 
-// UsesExternalSources indicates that any GenericCatalogor does not use external sources
-func (c *GenericCataloger) UsesExternalSources() bool {
-	return false
-}
-
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
 func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	var packages []pkg.Package

syft/pkg/cataloger/config.go

@@ -5,9 +5,8 @@ import (
 )
 
 type Config struct {
-	Search                 SearchConfig
+	Search     SearchConfig
-	Catalogers             []string
+	Catalogers []string
-	ExternalSourcesEnabled bool
 }
 
 func DefaultConfig() Config {

syft/pkg/cataloger/deb/cataloger.go

@@ -36,11 +36,6 @@ func (c *Cataloger) Name() string {
 	return "dpkgdb-cataloger"
 }
 
-// UsesExternalSources indicates that the dpkgdb cataloger does not use external sources
-func (c *Cataloger) UsesExternalSources() bool {
-	return false
-}
-
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	dbFileMatches, err := resolver.FilesByGlob(pkg.DpkgDBGlob)

syft/pkg/cataloger/golang/binary_cataloger.go

@@ -28,11 +28,6 @@ func (c *Cataloger) Name() string {
 	return catalogerName
 }
 
-// UsesExternalSources indicates that the golang binary cataloger does not use external sources
-func (c *Cataloger) UsesExternalSources() bool {
-	return false
-}
-
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	var pkgs []pkg.Package

syft/pkg/cataloger/portage/cataloger.go

@@ -37,11 +37,6 @@ func (c *Cataloger) Name() string {
 	return "portage-cataloger"
 }
 
-// UsesExternalSources indicates that the portage cataloger does not use external sources
-func (c *Cataloger) UsesExternalSources() bool {
-	return false
-}
-
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing portage support files.
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	dbFileMatches, err := resolver.FilesByGlob(pkg.PortageDBGlob)

syft/pkg/cataloger/python/package_cataloger.go

@@ -33,11 +33,6 @@ func (c *PackageCataloger) Name() string {
 	return "python-package-cataloger"
 }
 
-// UsesExternalSources indicates that the python package cataloger does not use external sources
-func (c *PackageCataloger) UsesExternalSources() bool {
-	return false
-}
-
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
 func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	var fileMatches []source.Location

syft/pkg/cataloger/rpmdb/cataloger.go

@@ -27,11 +27,6 @@ func (c *Cataloger) Name() string {
 	return catalogerName
 }
 
-// UsesExternalSources indicates that the rpmdb cataloger does not use external sources
-func (c *Cataloger) UsesExternalSources() bool {
-	return false
-}
-
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	fileMatches, err := resolver.FilesByGlob(pkg.RpmDBGlob)

syft/pkg/cataloger/rust/audit_binary_cataloger.go

@@ -27,11 +27,6 @@ func (c *Cataloger) Name() string {
 	return catalogerName
 }
 
-// UsesExternalSources indicates that the audit binary cataloger does not use external sources
-func (c *Cataloger) UsesExternalSources() bool {
-	return false
-}
-
 // Catalog identifies executables then attempts to read Rust dependency information from them
 func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
 	var pkgs []pkg.Package