Add catalogers configuration (#1038)

* Option to enable specific language or ecosystem cataloger

Signed-off-by: ramanan-ravi <ramanan@deepfence.io>
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* Disable dotnet cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* Option to enable specific language or ecosystem cataloger

Signed-off-by: Ramanan Ravikumar <ramanan@deepfence.io>
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* rename "enable-cataloger" option to "catalogers"

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add cli test for --catalogers option

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update readme with latest cataloger names

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* enable dotnet cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix cataloger imports

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update readme with alpmdb cataloger config example

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Co-authored-by: ramanan-ravi <ramanan@deepfence.io>

README.md

@@ -394,6 +394,29 @@ exclude: []
 # same as --platform; SYFT_PLATFORM env var
 platform: ""
 
+# set the list of package catalogers to use when generating the SBOM
+# default = empty (cataloger set determined automatically by the source type [image or file/directory])
+# catalogers:
+#   - ruby-gemfile
+#   - ruby-gemspec
+#   - python-index
+#   - python-package
+#   - javascript-lock
+#   - javascript-package
+#   - php-composer-installed
+#   - php-composer-lock
+#   - alpmdb
+#   - dpkgdb
+#   - rpmdb
+#   - java
+#   - apkdb
+#   - go-module-binary
+#   - go-mod-file
+#   - dartlang-lock
+#   - rust
+#   - dotnet-deps
+catalogers:
+
 # cataloging packages is exposed through the packages and power-user subcommands
 package:
 

cmd/syft/cli/eventloop/tasks.go

@@ -46,7 +46,7 @@ func generateCatalogPackagesTask(app *config.Application) (Task, error) {
 	}
 
 	task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
-		packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, app.Package.ToConfig())
+		packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, app.ToCatalogerConfig())
 		if err != nil {
 			return nil, err
 		}

cmd/syft/cli/options/packages.go

@@ -25,48 +25,52 @@ type PackagesOptions struct {
 	Exclude                []string
 	OverwriteExistingImage bool
 	ImportTimeout          uint
+	Catalogers             []string
 }
 
 var _ Interface = (*PackagesOptions)(nil)
 
 func (o *PackagesOptions) AddFlags(cmd *cobra.Command, v *viper.Viper) error {
-	cmd.PersistentFlags().StringVarP(&o.Scope, "scope", "s", cataloger.DefaultSearchConfig().Scope.String(),
+	cmd.Flags().StringVarP(&o.Scope, "scope", "s", cataloger.DefaultSearchConfig().Scope.String(),
 		fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes))
 
-	cmd.PersistentFlags().StringArrayVarP(&o.Output, "output", "o", FormatAliases(table.ID),
+	cmd.Flags().StringArrayVarP(&o.Output, "output", "o", FormatAliases(table.ID),
 		fmt.Sprintf("report output format, options=%v", FormatAliases(syft.FormatIDs()...)))
 
-	cmd.PersistentFlags().StringVarP(&o.OutputTemplatePath, "template", "t", "",
+	cmd.Flags().StringVarP(&o.File, "file", "", "",
-		"specify the path to a Go template file")
-
-	cmd.PersistentFlags().StringVarP(&o.File, "file", "", "",
 		"file to write the default report output to (default is STDOUT)")
 
-	cmd.PersistentFlags().StringVarP(&o.Platform, "platform", "", "",
+	cmd.Flags().StringVarP(&o.OutputTemplatePath, "template", "t", "",
+		"specify the path to a Go template file")
+
+	cmd.Flags().StringVarP(&o.Platform, "platform", "", "",
 		"an optional platform specifier for container image sources (e.g. 'linux/arm64', 'linux/arm64/v8', 'arm64', 'linux')")
 
-	cmd.PersistentFlags().StringVarP(&o.Host, "host", "H", "",
+	cmd.Flags().StringVarP(&o.Host, "host", "H", "",
 		"the hostname or URL of the Anchore Enterprise instance to upload to")
 
-	cmd.PersistentFlags().StringVarP(&o.Username, "username", "u", "",
+	cmd.Flags().StringVarP(&o.Username, "username", "u", "",
 		"the username to authenticate against Anchore Enterprise")
 
-	cmd.PersistentFlags().StringVarP(&o.Password, "password", "p", "",
+	cmd.Flags().StringVarP(&o.Password, "password", "p", "",
 		"the password to authenticate against Anchore Enterprise")
 
-	cmd.PersistentFlags().StringVarP(&o.Dockerfile, "dockerfile", "d", "",
+	cmd.Flags().StringVarP(&o.Dockerfile, "dockerfile", "d", "",
 		"include dockerfile for upload to Anchore Enterprise")
 
-	cmd.PersistentFlags().StringArrayVarP(&o.Exclude, "exclude", "", nil,
+	cmd.Flags().StringArrayVarP(&o.Exclude, "exclude", "", nil,
 		"exclude paths from being scanned using a glob expression")
 
-	cmd.PersistentFlags().BoolVarP(&o.OverwriteExistingImage, "overwrite-existing-image", "", false,
+	cmd.Flags().StringArrayVarP(&o.Catalogers, "catalogers", "", nil,
+		"enable one or more package catalogers")
+
+	cmd.Flags().BoolVarP(&o.OverwriteExistingImage, "overwrite-existing-image", "", false,
 		"overwrite an existing image during the upload to Anchore Enterprise")
 
-	cmd.PersistentFlags().UintVarP(&o.ImportTimeout, "import-timeout", "", 30,
+	cmd.Flags().UintVarP(&o.ImportTimeout, "import-timeout", "", 30,
 		"set a timeout duration (in seconds) for the upload to Anchore Enterprise")
 
-	return bindPackageConfigOptions(cmd.PersistentFlags(), v)
+	return bindPackageConfigOptions(cmd.Flags(), v)
 }
 
 func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
@@ -84,6 +88,10 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
 		return err
 	}
 
+	if err := v.BindPFlag("catalogers", flags.Lookup("catalogers")); err != nil {
+		return err
+	}
+
 	if err := v.BindPFlag("output", flags.Lookup("output")); err != nil {
 		return err
 	}

internal/config/application.go

@@ -5,8 +5,11 @@ import (
 	"fmt"
 	"path"
 	"reflect"
+	"sort"
 	"strings"
 
+	"github.com/anchore/syft/syft/pkg/cataloger"
+
 	"github.com/sirupsen/logrus"
 
 	"github.com/adrg/xdg"
@@ -44,6 +47,7 @@ type Application struct {
 	Anchore            anchore            `yaml:"anchore" json:"anchore" mapstructure:"anchore"`                                        // options for interacting with Anchore Engine/Enterprise
 	Dev                development        `yaml:"dev" json:"dev" mapstructure:"dev"`
 	Log                logging            `yaml:"log" json:"log" mapstructure:"log"` // all logging-related options
+	Catalogers         []string           `yaml:"catalogers" json:"catalogers" mapstructure:"catalogers"`
 	Package            pkg                `yaml:"package" json:"package" mapstructure:"package"`
 	FileMetadata       FileMetadata       `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"`
 	FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"`
@@ -55,6 +59,17 @@ type Application struct {
 	Platform           string             `yaml:"platform" json:"platform" mapstructure:"platform"`
 }
 
+func (cfg Application) ToCatalogerConfig() cataloger.Config {
+	return cataloger.Config{
+		Search: cataloger.SearchConfig{
+			IncludeIndexedArchives:   cfg.Package.SearchIndexedArchives,
+			IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
+			Scope:                    cfg.Package.Cataloger.ScopeOpt,
+		},
+		Catalogers: cfg.Catalogers,
+	}
+}
+
 func (cfg *Application) LoadAllValues(v *viper.Viper, configPath string) error {
 	// priority order: viper.Set, flag, env, config, kv, defaults
 	// flags have already been loaded into viper by command construction
@@ -86,6 +101,16 @@ func (cfg *Application) LoadAllValues(v *viper.Viper, configPath string) error {
 }
 
 func (cfg *Application) parseConfigValues() error {
+	// parse options on this struct
+	var catalogers []string
+	for _, c := range cfg.Catalogers {
+		for _, f := range strings.Split(c, ",") {
+			catalogers = append(catalogers, strings.TrimSpace(f))
+		}
+	}
+	sort.Strings(catalogers)
+	cfg.Catalogers = catalogers
+
 	// parse application config options
 	for _, optionFn := range []func() error{
 		cfg.parseUploadOptions,
@@ -173,6 +198,7 @@ func loadDefaultValues(v *viper.Viper) {
 	// set the default values for primitive fields in this struct
 	v.SetDefault("quiet", false)
 	v.SetDefault("check-for-app-update", true)
+	v.SetDefault("catalogers", nil)
 
 	// for each field in the configuration struct, see if the field implements the defaultValueLoader interface and invoke it if it does
 	value := reflect.ValueOf(Application{})

internal/config/pkg.go

@@ -21,13 +21,3 @@ func (cfg pkg) loadDefaultValues(v *viper.Viper) {
 func (cfg *pkg) parseConfigValues() error {
 	return cfg.Cataloger.parseConfigValues()
 }
-
-func (cfg pkg) ToConfig() cataloger.Config {
-	return cataloger.Config{
-		Search: cataloger.SearchConfig{
-			IncludeIndexedArchives:   cfg.SearchIndexedArchives,
-			IncludeUnindexedArchives: cfg.SearchUnindexedArchives,
-			Scope:                    cfg.Cataloger.ScopeOpt,
-		},
-	}
-}

syft/pkg/cataloger/cataloger.go

@@ -6,6 +6,9 @@ catalogers defined in child packages as well as the interface definition to impl
 package cataloger
 
 import (
+	"strings"
+
+	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/alpm"
@@ -36,7 +39,7 @@ type Cataloger interface {
 
 // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
 func ImageCatalogers(cfg Config) []Cataloger {
-	return []Cataloger{
+	return filterCatalogers([]Cataloger{
 		alpm.NewAlpmdbCataloger(),
 		ruby.NewGemSpecCataloger(),
 		python.NewPythonPackageCataloger(),
@@ -48,12 +51,12 @@ func ImageCatalogers(cfg Config) []Cataloger {
 		apkdb.NewApkdbCataloger(),
 		golang.NewGoModuleBinaryCataloger(),
 		dotnet.NewDotnetDepsCataloger(),
-	}
+	}, cfg.Catalogers)
 }
 
 // DirectoryCatalogers returns a slice of locally implemented catalogers that are fit for detecting packages from index files (and select installations)
 func DirectoryCatalogers(cfg Config) []Cataloger {
-	return []Cataloger{
+	return filterCatalogers([]Cataloger{
 		alpm.NewAlpmdbCataloger(),
 		ruby.NewGemFileLockCataloger(),
 		python.NewPythonIndexCataloger(),
@@ -69,12 +72,12 @@ func DirectoryCatalogers(cfg Config) []Cataloger {
 		rust.NewCargoLockCataloger(),
 		dart.NewPubspecLockCataloger(),
 		dotnet.NewDotnetDepsCataloger(),
-	}
+	}, cfg.Catalogers)
 }
 
 // AllCatalogers returns all implemented catalogers
 func AllCatalogers(cfg Config) []Cataloger {
-	return []Cataloger{
+	return filterCatalogers([]Cataloger{
 		alpm.NewAlpmdbCataloger(),
 		ruby.NewGemFileLockCataloger(),
 		ruby.NewGemSpecCataloger(),
@@ -91,5 +94,35 @@ func AllCatalogers(cfg Config) []Cataloger {
 		rust.NewCargoLockCataloger(),
 		dart.NewPubspecLockCataloger(),
 		dotnet.NewDotnetDepsCataloger(),
-	}
+	}, cfg.Catalogers)
+}
+
+func filterCatalogers(catalogers []Cataloger, enabledCatalogerPatterns []string) []Cataloger {
+	// if cataloger is not set, all applicable catalogers are enabled by default
+	if len(enabledCatalogerPatterns) == 0 {
+		return catalogers
+	}
+	var keepCatalogers []Cataloger
+	for _, cataloger := range catalogers {
+		if contains(enabledCatalogerPatterns, cataloger.Name()) {
+			keepCatalogers = append(keepCatalogers, cataloger)
+			continue
+		}
+		log.Infof("skipping cataloger %q", cataloger.Name())
+	}
+	return keepCatalogers
+}
+
+func contains(enabledPartial []string, catalogerName string) bool {
+	catalogerName = strings.TrimSuffix(catalogerName, "-cataloger")
+	for _, partial := range enabledPartial {
+		partial = strings.TrimSuffix(partial, "-cataloger")
+		if partial == "" {
+			continue
+		}
+		if strings.Contains(catalogerName, partial) {
+			return true
+		}
+	}
+	return false
 }

syft/pkg/cataloger/cataloger_test.go

@@ -0,0 +1,201 @@
+package cataloger
+
+import (
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+var _ Cataloger = (*dummy)(nil)
+
+type dummy struct {
+	name string
+}
+
+func (d dummy) Name() string {
+	return d.name
+}
+
+func (d dummy) Catalog(_ source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
+	panic("not implemented")
+}
+
+func Test_filterCatalogers(t *testing.T) {
+	tests := []struct {
+		name       string
+		patterns   []string
+		catalogers []string
+		want       []string
+	}{
+		{
+			name:     "no filtering",
+			patterns: nil,
+			catalogers: []string{
+				"ruby-gemspec-cataloger",
+				"python-package-cataloger",
+				"php-composer-installed-cataloger",
+				"javascript-package-cataloger",
+				"dpkgdb-cataloger",
+				"rpmdb-cataloger",
+				"java-cataloger",
+				"apkdb-cataloger",
+				"go-module-binary-cataloger",
+			},
+			want: []string{
+				"ruby-gemspec-cataloger",
+				"python-package-cataloger",
+				"php-composer-installed-cataloger",
+				"javascript-package-cataloger",
+				"dpkgdb-cataloger",
+				"rpmdb-cataloger",
+				"java-cataloger",
+				"apkdb-cataloger",
+				"go-module-binary-cataloger",
+			},
+		},
+		{
+			name: "exact name match",
+			patterns: []string{
+				"rpmdb-cataloger",
+				"javascript-package-cataloger",
+			},
+			catalogers: []string{
+				"ruby-gemspec-cataloger",
+				"python-package-cataloger",
+				"php-composer-installed-cataloger",
+				"javascript-package-cataloger",
+				"dpkgdb-cataloger",
+				"rpmdb-cataloger",
+				"java-cataloger",
+				"apkdb-cataloger",
+				"go-module-binary-cataloger",
+			},
+			want: []string{
+				"javascript-package-cataloger",
+				"rpmdb-cataloger",
+			},
+		},
+		{
+			name: "partial name match",
+			patterns: []string{
+				"ruby",
+				"installed",
+			},
+			catalogers: []string{
+				"ruby-gemspec-cataloger",
+				"ruby-gemfile-cataloger",
+				"python-package-cataloger",
+				"php-composer-installed-cataloger",
+				"javascript-package-cataloger",
+				"dpkgdb-cataloger",
+				"rpmdb-cataloger",
+				"java-cataloger",
+				"apkdb-cataloger",
+				"go-module-binary-cataloger",
+			},
+			want: []string{
+				"php-composer-installed-cataloger",
+				"ruby-gemspec-cataloger",
+				"ruby-gemfile-cataloger",
+			},
+		},
+		{
+			name: "ignore 'cataloger' keyword",
+			patterns: []string{
+				"cataloger",
+			},
+			catalogers: []string{
+				"ruby-gemspec-cataloger",
+				"ruby-gemfile-cataloger",
+				"python-package-cataloger",
+				"php-composer-installed-cataloger",
+				"javascript-package-cataloger",
+				"dpkgdb-cataloger",
+				"rpmdb-cataloger",
+				"java-cataloger",
+				"apkdb-cataloger",
+				"go-module-binary-cataloger",
+			},
+			want: []string{},
+		},
+		{
+			name: "only some patterns match",
+			patterns: []string{
+				"cataloger",
+				"go-module",
+			},
+			catalogers: []string{
+				"ruby-gemspec-cataloger",
+				"ruby-gemfile-cataloger",
+				"python-package-cataloger",
+				"php-composer-installed-cataloger",
+				"javascript-package-cataloger",
+				"dpkgdb-cataloger",
+				"rpmdb-cataloger",
+				"java-cataloger",
+				"apkdb-cataloger",
+				"go-module-binary-cataloger",
+			},
+			want: []string{
+				"go-module-binary-cataloger",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var catalogers []Cataloger
+			for _, n := range tt.catalogers {
+				catalogers = append(catalogers, dummy{name: n})
+			}
+			got := filterCatalogers(catalogers, tt.patterns)
+			var gotNames []string
+			for _, g := range got {
+				gotNames = append(gotNames, g.Name())
+			}
+			assert.ElementsMatch(t, tt.want, gotNames)
+		})
+	}
+}
+
+func Test_contains(t *testing.T) {
+	type args struct {
+	}
+	tests := []struct {
+		name              string
+		enabledCatalogers []string
+		catalogerName     string
+		want              bool
+	}{
+		{
+			name: "keep exact match",
+			enabledCatalogers: []string{
+				"php-composer-installed-cataloger",
+			},
+			catalogerName: "php-composer-installed-cataloger",
+			want:          true,
+		},
+		{
+			name: "match substring",
+			enabledCatalogers: []string{
+				"python",
+			},
+			catalogerName: "python-package-cataloger",
+			want:          true,
+		},
+		{
+			name: "dont match on 'cataloger'",
+			enabledCatalogers: []string{
+				"cataloger",
+			},
+			catalogerName: "python-package-cataloger",
+			want:          false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, contains(tt.enabledCatalogers, tt.catalogerName))
+		})
+	}
+}

syft/pkg/cataloger/config.go

@@ -5,7 +5,8 @@ import (
 )
 
 type Config struct {
-	Search SearchConfig
+	Search     SearchConfig
+	Catalogers []string
 }
 
 func DefaultConfig() Config {

test/cli/packages_cmd_test.go

@@ -227,6 +227,14 @@ func TestPackagesCmdFlags(t *testing.T) {
 				),
 			},
 		},
+		{
+			name: "catalogers-option",
+			args: []string{"packages", "-o", "json", "--catalogers", "python,ruby-gemspec", coverageImage},
+			assertions: []traitAssertion{
+				assertPackageCount(6),
+				assertSuccessfulReturnCode,
+			},
+		},
 	}
 
 	for _, test := range tests {