diff --git a/README.md b/README.md index d87e3d43b..818939b70 100644 --- a/README.md +++ b/README.md @@ -394,6 +394,29 @@ exclude: [] # same as --platform; SYFT_PLATFORM env var platform: "" +# set the list of package catalogers to use when generating the SBOM +# default = empty (cataloger set determined automatically by the source type [image or file/directory]) +# catalogers: +# - ruby-gemfile +# - ruby-gemspec +# - python-index +# - python-package +# - javascript-lock +# - javascript-package +# - php-composer-installed +# - php-composer-lock +# - alpmdb +# - dpkgdb +# - rpmdb +# - java +# - apkdb +# - go-module-binary +# - go-mod-file +# - dartlang-lock +# - rust +# - dotnet-deps +catalogers: + # cataloging packages is exposed through the packages and power-user subcommands package: diff --git a/cmd/syft/cli/eventloop/tasks.go b/cmd/syft/cli/eventloop/tasks.go index c98bcc15f..4d16f64f6 100644 --- a/cmd/syft/cli/eventloop/tasks.go +++ b/cmd/syft/cli/eventloop/tasks.go @@ -46,7 +46,7 @@ func generateCatalogPackagesTask(app *config.Application) (Task, error) { } task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { - packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, app.Package.ToConfig()) + packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, app.ToCatalogerConfig()) if err != nil { return nil, err } diff --git a/cmd/syft/cli/options/packages.go b/cmd/syft/cli/options/packages.go index 45bfea00d..8891f814e 100644 --- a/cmd/syft/cli/options/packages.go +++ b/cmd/syft/cli/options/packages.go @@ -25,48 +25,52 @@ type PackagesOptions struct { Exclude []string OverwriteExistingImage bool ImportTimeout uint + Catalogers []string } var _ Interface = (*PackagesOptions)(nil) func (o *PackagesOptions) AddFlags(cmd *cobra.Command, v *viper.Viper) error { - cmd.PersistentFlags().StringVarP(&o.Scope, "scope", "s", cataloger.DefaultSearchConfig().Scope.String(), + cmd.Flags().StringVarP(&o.Scope, "scope", "s", cataloger.DefaultSearchConfig().Scope.String(), fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes)) - cmd.PersistentFlags().StringArrayVarP(&o.Output, "output", "o", FormatAliases(table.ID), + cmd.Flags().StringArrayVarP(&o.Output, "output", "o", FormatAliases(table.ID), fmt.Sprintf("report output format, options=%v", FormatAliases(syft.FormatIDs()...))) - cmd.PersistentFlags().StringVarP(&o.OutputTemplatePath, "template", "t", "", - "specify the path to a Go template file") - - cmd.PersistentFlags().StringVarP(&o.File, "file", "", "", + cmd.Flags().StringVarP(&o.File, "file", "", "", "file to write the default report output to (default is STDOUT)") - cmd.PersistentFlags().StringVarP(&o.Platform, "platform", "", "", + cmd.Flags().StringVarP(&o.OutputTemplatePath, "template", "t", "", + "specify the path to a Go template file") + + cmd.Flags().StringVarP(&o.Platform, "platform", "", "", "an optional platform specifier for container image sources (e.g. 'linux/arm64', 'linux/arm64/v8', 'arm64', 'linux')") - cmd.PersistentFlags().StringVarP(&o.Host, "host", "H", "", + cmd.Flags().StringVarP(&o.Host, "host", "H", "", "the hostname or URL of the Anchore Enterprise instance to upload to") - cmd.PersistentFlags().StringVarP(&o.Username, "username", "u", "", + cmd.Flags().StringVarP(&o.Username, "username", "u", "", "the username to authenticate against Anchore Enterprise") - cmd.PersistentFlags().StringVarP(&o.Password, "password", "p", "", + cmd.Flags().StringVarP(&o.Password, "password", "p", "", "the password to authenticate against Anchore Enterprise") - cmd.PersistentFlags().StringVarP(&o.Dockerfile, "dockerfile", "d", "", + cmd.Flags().StringVarP(&o.Dockerfile, "dockerfile", "d", "", "include dockerfile for upload to Anchore Enterprise") - cmd.PersistentFlags().StringArrayVarP(&o.Exclude, "exclude", "", nil, + cmd.Flags().StringArrayVarP(&o.Exclude, "exclude", "", nil, "exclude paths from being scanned using a glob expression") - cmd.PersistentFlags().BoolVarP(&o.OverwriteExistingImage, "overwrite-existing-image", "", false, + cmd.Flags().StringArrayVarP(&o.Catalogers, "catalogers", "", nil, + "enable one or more package catalogers") + + cmd.Flags().BoolVarP(&o.OverwriteExistingImage, "overwrite-existing-image", "", false, "overwrite an existing image during the upload to Anchore Enterprise") - cmd.PersistentFlags().UintVarP(&o.ImportTimeout, "import-timeout", "", 30, + cmd.Flags().UintVarP(&o.ImportTimeout, "import-timeout", "", 30, "set a timeout duration (in seconds) for the upload to Anchore Enterprise") - return bindPackageConfigOptions(cmd.PersistentFlags(), v) + return bindPackageConfigOptions(cmd.Flags(), v) } func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error { @@ -84,6 +88,10 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error { return err } + if err := v.BindPFlag("catalogers", flags.Lookup("catalogers")); err != nil { + return err + } + if err := v.BindPFlag("output", flags.Lookup("output")); err != nil { return err } diff --git a/internal/config/application.go b/internal/config/application.go index ecb869c47..d5898b0d1 100644 --- a/internal/config/application.go +++ b/internal/config/application.go @@ -5,8 +5,11 @@ import ( "fmt" "path" "reflect" + "sort" "strings" + "github.com/anchore/syft/syft/pkg/cataloger" + "github.com/sirupsen/logrus" "github.com/adrg/xdg" @@ -44,6 +47,7 @@ type Application struct { Anchore anchore `yaml:"anchore" json:"anchore" mapstructure:"anchore"` // options for interacting with Anchore Engine/Enterprise Dev development `yaml:"dev" json:"dev" mapstructure:"dev"` Log logging `yaml:"log" json:"log" mapstructure:"log"` // all logging-related options + Catalogers []string `yaml:"catalogers" json:"catalogers" mapstructure:"catalogers"` Package pkg `yaml:"package" json:"package" mapstructure:"package"` FileMetadata FileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"` FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"` @@ -55,6 +59,17 @@ type Application struct { Platform string `yaml:"platform" json:"platform" mapstructure:"platform"` } +func (cfg Application) ToCatalogerConfig() cataloger.Config { + return cataloger.Config{ + Search: cataloger.SearchConfig{ + IncludeIndexedArchives: cfg.Package.SearchIndexedArchives, + IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives, + Scope: cfg.Package.Cataloger.ScopeOpt, + }, + Catalogers: cfg.Catalogers, + } +} + func (cfg *Application) LoadAllValues(v *viper.Viper, configPath string) error { // priority order: viper.Set, flag, env, config, kv, defaults // flags have already been loaded into viper by command construction @@ -86,6 +101,16 @@ func (cfg *Application) LoadAllValues(v *viper.Viper, configPath string) error { } func (cfg *Application) parseConfigValues() error { + // parse options on this struct + var catalogers []string + for _, c := range cfg.Catalogers { + for _, f := range strings.Split(c, ",") { + catalogers = append(catalogers, strings.TrimSpace(f)) + } + } + sort.Strings(catalogers) + cfg.Catalogers = catalogers + // parse application config options for _, optionFn := range []func() error{ cfg.parseUploadOptions, @@ -173,6 +198,7 @@ func loadDefaultValues(v *viper.Viper) { // set the default values for primitive fields in this struct v.SetDefault("quiet", false) v.SetDefault("check-for-app-update", true) + v.SetDefault("catalogers", nil) // for each field in the configuration struct, see if the field implements the defaultValueLoader interface and invoke it if it does value := reflect.ValueOf(Application{}) diff --git a/internal/config/pkg.go b/internal/config/pkg.go index 2e695a995..88d622541 100644 --- a/internal/config/pkg.go +++ b/internal/config/pkg.go @@ -21,13 +21,3 @@ func (cfg pkg) loadDefaultValues(v *viper.Viper) { func (cfg *pkg) parseConfigValues() error { return cfg.Cataloger.parseConfigValues() } - -func (cfg pkg) ToConfig() cataloger.Config { - return cataloger.Config{ - Search: cataloger.SearchConfig{ - IncludeIndexedArchives: cfg.SearchIndexedArchives, - IncludeUnindexedArchives: cfg.SearchUnindexedArchives, - Scope: cfg.Cataloger.ScopeOpt, - }, - } -} diff --git a/syft/pkg/cataloger/cataloger.go b/syft/pkg/cataloger/cataloger.go index de36fa4bf..1aded4adf 100644 --- a/syft/pkg/cataloger/cataloger.go +++ b/syft/pkg/cataloger/cataloger.go @@ -6,6 +6,9 @@ catalogers defined in child packages as well as the interface definition to impl package cataloger import ( + "strings" + + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/alpm" @@ -36,7 +39,7 @@ type Cataloger interface { // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages. func ImageCatalogers(cfg Config) []Cataloger { - return []Cataloger{ + return filterCatalogers([]Cataloger{ alpm.NewAlpmdbCataloger(), ruby.NewGemSpecCataloger(), python.NewPythonPackageCataloger(), @@ -48,12 +51,12 @@ func ImageCatalogers(cfg Config) []Cataloger { apkdb.NewApkdbCataloger(), golang.NewGoModuleBinaryCataloger(), dotnet.NewDotnetDepsCataloger(), - } + }, cfg.Catalogers) } // DirectoryCatalogers returns a slice of locally implemented catalogers that are fit for detecting packages from index files (and select installations) func DirectoryCatalogers(cfg Config) []Cataloger { - return []Cataloger{ + return filterCatalogers([]Cataloger{ alpm.NewAlpmdbCataloger(), ruby.NewGemFileLockCataloger(), python.NewPythonIndexCataloger(), @@ -69,12 +72,12 @@ func DirectoryCatalogers(cfg Config) []Cataloger { rust.NewCargoLockCataloger(), dart.NewPubspecLockCataloger(), dotnet.NewDotnetDepsCataloger(), - } + }, cfg.Catalogers) } // AllCatalogers returns all implemented catalogers func AllCatalogers(cfg Config) []Cataloger { - return []Cataloger{ + return filterCatalogers([]Cataloger{ alpm.NewAlpmdbCataloger(), ruby.NewGemFileLockCataloger(), ruby.NewGemSpecCataloger(), @@ -91,5 +94,35 @@ func AllCatalogers(cfg Config) []Cataloger { rust.NewCargoLockCataloger(), dart.NewPubspecLockCataloger(), dotnet.NewDotnetDepsCataloger(), - } + }, cfg.Catalogers) +} + +func filterCatalogers(catalogers []Cataloger, enabledCatalogerPatterns []string) []Cataloger { + // if cataloger is not set, all applicable catalogers are enabled by default + if len(enabledCatalogerPatterns) == 0 { + return catalogers + } + var keepCatalogers []Cataloger + for _, cataloger := range catalogers { + if contains(enabledCatalogerPatterns, cataloger.Name()) { + keepCatalogers = append(keepCatalogers, cataloger) + continue + } + log.Infof("skipping cataloger %q", cataloger.Name()) + } + return keepCatalogers +} + +func contains(enabledPartial []string, catalogerName string) bool { + catalogerName = strings.TrimSuffix(catalogerName, "-cataloger") + for _, partial := range enabledPartial { + partial = strings.TrimSuffix(partial, "-cataloger") + if partial == "" { + continue + } + if strings.Contains(catalogerName, partial) { + return true + } + } + return false } diff --git a/syft/pkg/cataloger/cataloger_test.go b/syft/pkg/cataloger/cataloger_test.go new file mode 100644 index 000000000..e47944dab --- /dev/null +++ b/syft/pkg/cataloger/cataloger_test.go @@ -0,0 +1,201 @@ +package cataloger + +import ( + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" + "github.com/stretchr/testify/assert" + "testing" +) + +var _ Cataloger = (*dummy)(nil) + +type dummy struct { + name string +} + +func (d dummy) Name() string { + return d.name +} + +func (d dummy) Catalog(_ source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { + panic("not implemented") +} + +func Test_filterCatalogers(t *testing.T) { + tests := []struct { + name string + patterns []string + catalogers []string + want []string + }{ + { + name: "no filtering", + patterns: nil, + catalogers: []string{ + "ruby-gemspec-cataloger", + "python-package-cataloger", + "php-composer-installed-cataloger", + "javascript-package-cataloger", + "dpkgdb-cataloger", + "rpmdb-cataloger", + "java-cataloger", + "apkdb-cataloger", + "go-module-binary-cataloger", + }, + want: []string{ + "ruby-gemspec-cataloger", + "python-package-cataloger", + "php-composer-installed-cataloger", + "javascript-package-cataloger", + "dpkgdb-cataloger", + "rpmdb-cataloger", + "java-cataloger", + "apkdb-cataloger", + "go-module-binary-cataloger", + }, + }, + { + name: "exact name match", + patterns: []string{ + "rpmdb-cataloger", + "javascript-package-cataloger", + }, + catalogers: []string{ + "ruby-gemspec-cataloger", + "python-package-cataloger", + "php-composer-installed-cataloger", + "javascript-package-cataloger", + "dpkgdb-cataloger", + "rpmdb-cataloger", + "java-cataloger", + "apkdb-cataloger", + "go-module-binary-cataloger", + }, + want: []string{ + "javascript-package-cataloger", + "rpmdb-cataloger", + }, + }, + { + name: "partial name match", + patterns: []string{ + "ruby", + "installed", + }, + catalogers: []string{ + "ruby-gemspec-cataloger", + "ruby-gemfile-cataloger", + "python-package-cataloger", + "php-composer-installed-cataloger", + "javascript-package-cataloger", + "dpkgdb-cataloger", + "rpmdb-cataloger", + "java-cataloger", + "apkdb-cataloger", + "go-module-binary-cataloger", + }, + want: []string{ + "php-composer-installed-cataloger", + "ruby-gemspec-cataloger", + "ruby-gemfile-cataloger", + }, + }, + { + name: "ignore 'cataloger' keyword", + patterns: []string{ + "cataloger", + }, + catalogers: []string{ + "ruby-gemspec-cataloger", + "ruby-gemfile-cataloger", + "python-package-cataloger", + "php-composer-installed-cataloger", + "javascript-package-cataloger", + "dpkgdb-cataloger", + "rpmdb-cataloger", + "java-cataloger", + "apkdb-cataloger", + "go-module-binary-cataloger", + }, + want: []string{}, + }, + { + name: "only some patterns match", + patterns: []string{ + "cataloger", + "go-module", + }, + catalogers: []string{ + "ruby-gemspec-cataloger", + "ruby-gemfile-cataloger", + "python-package-cataloger", + "php-composer-installed-cataloger", + "javascript-package-cataloger", + "dpkgdb-cataloger", + "rpmdb-cataloger", + "java-cataloger", + "apkdb-cataloger", + "go-module-binary-cataloger", + }, + want: []string{ + "go-module-binary-cataloger", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var catalogers []Cataloger + for _, n := range tt.catalogers { + catalogers = append(catalogers, dummy{name: n}) + } + got := filterCatalogers(catalogers, tt.patterns) + var gotNames []string + for _, g := range got { + gotNames = append(gotNames, g.Name()) + } + assert.ElementsMatch(t, tt.want, gotNames) + }) + } +} + +func Test_contains(t *testing.T) { + type args struct { + } + tests := []struct { + name string + enabledCatalogers []string + catalogerName string + want bool + }{ + { + name: "keep exact match", + enabledCatalogers: []string{ + "php-composer-installed-cataloger", + }, + catalogerName: "php-composer-installed-cataloger", + want: true, + }, + { + name: "match substring", + enabledCatalogers: []string{ + "python", + }, + catalogerName: "python-package-cataloger", + want: true, + }, + { + name: "dont match on 'cataloger'", + enabledCatalogers: []string{ + "cataloger", + }, + catalogerName: "python-package-cataloger", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, contains(tt.enabledCatalogers, tt.catalogerName)) + }) + } +} diff --git a/syft/pkg/cataloger/config.go b/syft/pkg/cataloger/config.go index 4e82957c0..478fc292d 100644 --- a/syft/pkg/cataloger/config.go +++ b/syft/pkg/cataloger/config.go @@ -5,7 +5,8 @@ import ( ) type Config struct { - Search SearchConfig + Search SearchConfig + Catalogers []string } func DefaultConfig() Config { diff --git a/test/cli/packages_cmd_test.go b/test/cli/packages_cmd_test.go index d49e74245..ccb64c708 100644 --- a/test/cli/packages_cmd_test.go +++ b/test/cli/packages_cmd_test.go @@ -227,6 +227,14 @@ func TestPackagesCmdFlags(t *testing.T) { ), }, }, + { + name: "catalogers-option", + args: []string{"packages", "-o", "json", "--catalogers", "python,ruby-gemspec", coverageImage}, + assertions: []traitAssertion{ + assertPackageCount(6), + assertSuccessfulReturnCode, + }, + }, } for _, test := range tests {