oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-02-12 02:26:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add catalogers configuration (#1038)

Browse Source 
* Option to enable specific language or ecosystem cataloger

Signed-off-by: ramanan-ravi <ramanan@deepfence.io>
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* Disable dotnet cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* Option to enable specific language or ecosystem cataloger

Signed-off-by: Ramanan Ravikumar <ramanan@deepfence.io>
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* rename "enable-cataloger" option to "catalogers"

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add cli test for --catalogers option

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update readme with latest cataloger names

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* enable dotnet cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix cataloger imports

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update readme with alpmdb cataloger config example

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Co-authored-by: ramanan-ravi <ramanan@deepfence.io>
This commit is contained in:
Alex Goodman 2022-06-21 09:06:25 -04:00 committed by GitHub
parent aed1599c4d
commit ea611dab5f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 323 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
README.md
View File

@ -394,6 +394,29 @@ exclude: []
# same as --platform; SYFT_PLATFORM env var
platform: ""
# set the list of package catalogers to use when generating the SBOM
# default = empty (cataloger set determined automatically by the source type [image or file/directory])
# catalogers:
# - ruby-gemfile
# - ruby-gemspec
# - python-index
# - python-package
# - javascript-lock
# - javascript-package
# - php-composer-installed
# - php-composer-lock
# - alpmdb
# - dpkgdb
# - rpmdb
# - java
# - apkdb
# - go-module-binary
# - go-mod-file
# - dartlang-lock
# - rust
# - dotnet-deps
catalogers:
# cataloging packages is exposed through the packages and power-user subcommands
package:

2
cmd/syft/cli/eventloop/tasks.go
View File

@ -46,7 +46,7 @@ func generateCatalogPackagesTask(app *config.Application) (Task, error) {
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, app.Package.ToConfig())
packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, app.ToCatalogerConfig())
if err != nil {
return nil, err
}

38
cmd/syft/cli/options/packages.go
View File

@ -25,48 +25,52 @@ type PackagesOptions struct {
Exclude []string
OverwriteExistingImage bool
ImportTimeout uint
Catalogers []string
}
var _ Interface = (*PackagesOptions)(nil)
func (o *PackagesOptions) AddFlags(cmd *cobra.Command, v *viper.Viper) error {
cmd.PersistentFlags().StringVarP(&o.Scope, "scope", "s", cataloger.DefaultSearchConfig().Scope.String(),
cmd.Flags().StringVarP(&o.Scope, "scope", "s", cataloger.DefaultSearchConfig().Scope.String(),
fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes))
cmd.PersistentFlags().StringArrayVarP(&o.Output, "output", "o", FormatAliases(table.ID),
cmd.Flags().StringArrayVarP(&o.Output, "output", "o", FormatAliases(table.ID),
fmt.Sprintf("report output format, options=%v", FormatAliases(syft.FormatIDs()...)))
cmd.PersistentFlags().StringVarP(&o.OutputTemplatePath, "template", "t", "",
"specify the path to a Go template file")
cmd.PersistentFlags().StringVarP(&o.File, "file", "", "",
cmd.Flags().StringVarP(&o.File, "file", "", "",
"file to write the default report output to (default is STDOUT)")
cmd.PersistentFlags().StringVarP(&o.Platform, "platform", "", "",
cmd.Flags().StringVarP(&o.OutputTemplatePath, "template", "t", "",
"specify the path to a Go template file")
cmd.Flags().StringVarP(&o.Platform, "platform", "", "",
"an optional platform specifier for container image sources (e.g. 'linux/arm64', 'linux/arm64/v8', 'arm64', 'linux')")
cmd.PersistentFlags().StringVarP(&o.Host, "host", "H", "",
cmd.Flags().StringVarP(&o.Host, "host", "H", "",
"the hostname or URL of the Anchore Enterprise instance to upload to")
cmd.PersistentFlags().StringVarP(&o.Username, "username", "u", "",
cmd.Flags().StringVarP(&o.Username, "username", "u", "",
"the username to authenticate against Anchore Enterprise")
cmd.PersistentFlags().StringVarP(&o.Password, "password", "p", "",
cmd.Flags().StringVarP(&o.Password, "password", "p", "",
"the password to authenticate against Anchore Enterprise")
cmd.PersistentFlags().StringVarP(&o.Dockerfile, "dockerfile", "d", "",
cmd.Flags().StringVarP(&o.Dockerfile, "dockerfile", "d", "",
"include dockerfile for upload to Anchore Enterprise")
cmd.PersistentFlags().StringArrayVarP(&o.Exclude, "exclude", "", nil,
cmd.Flags().StringArrayVarP(&o.Exclude, "exclude", "", nil,
"exclude paths from being scanned using a glob expression")
cmd.PersistentFlags().BoolVarP(&o.OverwriteExistingImage, "overwrite-existing-image", "", false,
cmd.Flags().StringArrayVarP(&o.Catalogers, "catalogers", "", nil,
"enable one or more package catalogers")
cmd.Flags().BoolVarP(&o.OverwriteExistingImage, "overwrite-existing-image", "", false,
"overwrite an existing image during the upload to Anchore Enterprise")
cmd.PersistentFlags().UintVarP(&o.ImportTimeout, "import-timeout", "", 30,
cmd.Flags().UintVarP(&o.ImportTimeout, "import-timeout", "", 30,
"set a timeout duration (in seconds) for the upload to Anchore Enterprise")
return bindPackageConfigOptions(cmd.PersistentFlags(), v)
return bindPackageConfigOptions(cmd.Flags(), v)
}
func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
@ -84,6 +88,10 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
return err
}
if err := v.BindPFlag("catalogers", flags.Lookup("catalogers")); err != nil {
return err
}
if err := v.BindPFlag("output", flags.Lookup("output")); err != nil {
return err
}

26
internal/config/application.go
View File

@ -5,8 +5,11 @@ import (
"fmt"
"path"
"reflect"
"sort"
"strings"
"github.com/anchore/syft/syft/pkg/cataloger"
"github.com/sirupsen/logrus"
"github.com/adrg/xdg"
@ -44,6 +47,7 @@ type Application struct {
Anchore anchore `yaml:"anchore" json:"anchore" mapstructure:"anchore"` // options for interacting with Anchore Engine/Enterprise
Dev development `yaml:"dev" json:"dev" mapstructure:"dev"`
Log logging `yaml:"log" json:"log" mapstructure:"log"` // all logging-related options
Catalogers []string `yaml:"catalogers" json:"catalogers" mapstructure:"catalogers"`
Package pkg `yaml:"package" json:"package" mapstructure:"package"`
FileMetadata FileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"`
FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"`
@ -55,6 +59,17 @@ type Application struct {
Platform string `yaml:"platform" json:"platform" mapstructure:"platform"`
}
func (cfg Application) ToCatalogerConfig() cataloger.Config {
return cataloger.Config{
Search: cataloger.SearchConfig{
IncludeIndexedArchives: cfg.Package.SearchIndexedArchives,
IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
Scope: cfg.Package.Cataloger.ScopeOpt,
},
Catalogers: cfg.Catalogers,
}
}
func (cfg *Application) LoadAllValues(v *viper.Viper, configPath string) error {
// priority order: viper.Set, flag, env, config, kv, defaults
// flags have already been loaded into viper by command construction
@ -86,6 +101,16 @@ func (cfg *Application) LoadAllValues(v *viper.Viper, configPath string) error {
}
func (cfg *Application) parseConfigValues() error {
// parse options on this struct
var catalogers []string
for _, c := range cfg.Catalogers {
for _, f := range strings.Split(c, ",") {
catalogers = append(catalogers, strings.TrimSpace(f))
}
}
sort.Strings(catalogers)
cfg.Catalogers = catalogers
// parse application config options
for _, optionFn := range []func() error{
cfg.parseUploadOptions,
@ -173,6 +198,7 @@ func loadDefaultValues(v *viper.Viper) {
// set the default values for primitive fields in this struct
v.SetDefault("quiet", false)
v.SetDefault("check-for-app-update", true)
v.SetDefault("catalogers", nil)
// for each field in the configuration struct, see if the field implements the defaultValueLoader interface and invoke it if it does
value := reflect.ValueOf(Application{})

10
internal/config/pkg.go
View File

@ -21,13 +21,3 @@ func (cfg pkg) loadDefaultValues(v *viper.Viper) {
func (cfg *pkg) parseConfigValues() error {
return cfg.Cataloger.parseConfigValues()
}
func (cfg pkg) ToConfig() cataloger.Config {
return cataloger.Config{
Search: cataloger.SearchConfig{
IncludeIndexedArchives: cfg.SearchIndexedArchives,
IncludeUnindexedArchives: cfg.SearchUnindexedArchives,
Scope: cfg.Cataloger.ScopeOpt,
},
}
}

45
syft/pkg/cataloger/cataloger.go
View File

@ -6,6 +6,9 @@ catalogers defined in child packages as well as the interface definition to impl
package cataloger
import (
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/alpm"
@ -36,7 +39,7 @@ type Cataloger interface {
// ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
func ImageCatalogers(cfg Config) []Cataloger {
return []Cataloger{
return filterCatalogers([]Cataloger{
alpm.NewAlpmdbCataloger(),
ruby.NewGemSpecCataloger(),
python.NewPythonPackageCataloger(),
@ -48,12 +51,12 @@ func ImageCatalogers(cfg Config) []Cataloger {
apkdb.NewApkdbCataloger(),
golang.NewGoModuleBinaryCataloger(),
dotnet.NewDotnetDepsCataloger(),
}
}, cfg.Catalogers)
}
// DirectoryCatalogers returns a slice of locally implemented catalogers that are fit for detecting packages from index files (and select installations)
func DirectoryCatalogers(cfg Config) []Cataloger {
return []Cataloger{
return filterCatalogers([]Cataloger{
alpm.NewAlpmdbCataloger(),
ruby.NewGemFileLockCataloger(),
python.NewPythonIndexCataloger(),
@ -69,12 +72,12 @@ func DirectoryCatalogers(cfg Config) []Cataloger {
rust.NewCargoLockCataloger(),
dart.NewPubspecLockCataloger(),
dotnet.NewDotnetDepsCataloger(),
}
}, cfg.Catalogers)
}
// AllCatalogers returns all implemented catalogers
func AllCatalogers(cfg Config) []Cataloger {
return []Cataloger{
return filterCatalogers([]Cataloger{
alpm.NewAlpmdbCataloger(),
ruby.NewGemFileLockCataloger(),
ruby.NewGemSpecCataloger(),
@ -91,5 +94,35 @@ func AllCatalogers(cfg Config) []Cataloger {
rust.NewCargoLockCataloger(),
dart.NewPubspecLockCataloger(),
dotnet.NewDotnetDepsCataloger(),
}
}, cfg.Catalogers)
}
func filterCatalogers(catalogers []Cataloger, enabledCatalogerPatterns []string) []Cataloger {
// if cataloger is not set, all applicable catalogers are enabled by default
if len(enabledCatalogerPatterns) == 0 {
return catalogers
}
var keepCatalogers []Cataloger
for _, cataloger := range catalogers {
if contains(enabledCatalogerPatterns, cataloger.Name()) {
keepCatalogers = append(keepCatalogers, cataloger)
continue
}
log.Infof("skipping cataloger %q", cataloger.Name())
}
return keepCatalogers
}
func contains(enabledPartial []string, catalogerName string) bool {
catalogerName = strings.TrimSuffix(catalogerName, "-cataloger")
for _, partial := range enabledPartial {
partial = strings.TrimSuffix(partial, "-cataloger")
if partial == "" {
continue
}
if strings.Contains(catalogerName, partial) {
return true
}
}
return false
}

201
syft/pkg/cataloger/cataloger_test.go Normal file
View File

@ -0,0 +1,201 @@
package cataloger
import (
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
"github.com/stretchr/testify/assert"
"testing"
)
var _ Cataloger = (*dummy)(nil)
type dummy struct {
name string
}
func (d dummy) Name() string {
return d.name
}
func (d dummy) Catalog(_ source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
panic("not implemented")
}
func Test_filterCatalogers(t *testing.T) {
tests := []struct {
name string
patterns []string
catalogers []string
want []string
}{
{
name: "no filtering",
patterns: nil,
catalogers: []string{
"ruby-gemspec-cataloger",
"python-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkgdb-cataloger",
"rpmdb-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
want: []string{
"ruby-gemspec-cataloger",
"python-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkgdb-cataloger",
"rpmdb-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
},
{
name: "exact name match",
patterns: []string{
"rpmdb-cataloger",
"javascript-package-cataloger",
},
catalogers: []string{
"ruby-gemspec-cataloger",
"python-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkgdb-cataloger",
"rpmdb-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
want: []string{
"javascript-package-cataloger",
"rpmdb-cataloger",
},
},
{
name: "partial name match",
patterns: []string{
"ruby",
"installed",
},
catalogers: []string{
"ruby-gemspec-cataloger",
"ruby-gemfile-cataloger",
"python-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkgdb-cataloger",
"rpmdb-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
want: []string{
"php-composer-installed-cataloger",
"ruby-gemspec-cataloger",
"ruby-gemfile-cataloger",
},
},
{
name: "ignore 'cataloger' keyword",
patterns: []string{
"cataloger",
},
catalogers: []string{
"ruby-gemspec-cataloger",
"ruby-gemfile-cataloger",
"python-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkgdb-cataloger",
"rpmdb-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
want: []string{},
},
{
name: "only some patterns match",
patterns: []string{
"cataloger",
"go-module",
},
catalogers: []string{
"ruby-gemspec-cataloger",
"ruby-gemfile-cataloger",
"python-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkgdb-cataloger",
"rpmdb-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
want: []string{
"go-module-binary-cataloger",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var catalogers []Cataloger
for _, n := range tt.catalogers {
catalogers = append(catalogers, dummy{name: n})
}
got := filterCatalogers(catalogers, tt.patterns)
var gotNames []string
for _, g := range got {
gotNames = append(gotNames, g.Name())
}
assert.ElementsMatch(t, tt.want, gotNames)
})
}
}
func Test_contains(t *testing.T) {
type args struct {
}
tests := []struct {
name string
enabledCatalogers []string
catalogerName string
want bool
}{
{
name: "keep exact match",
enabledCatalogers: []string{
"php-composer-installed-cataloger",
},
catalogerName: "php-composer-installed-cataloger",
want: true,
},
{
name: "match substring",
enabledCatalogers: []string{
"python",
},
catalogerName: "python-package-cataloger",
want: true,
},
{
name: "dont match on 'cataloger'",
enabledCatalogers: []string{
"cataloger",
},
catalogerName: "python-package-cataloger",
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, contains(tt.enabledCatalogers, tt.catalogerName))
})
}
}

3
syft/pkg/cataloger/config.go
View File

@ -5,7 +5,8 @@ import (
)
type Config struct {
Search SearchConfig
Search SearchConfig
Catalogers []string
}
func DefaultConfig() Config {

8
test/cli/packages_cmd_test.go
View File

@ -227,6 +227,14 @@ func TestPackagesCmdFlags(t *testing.T) {
),
},
},
{
name: "catalogers-option",
args: []string{"packages", "-o", "json", "--catalogers", "python,ruby-gemspec", coverageImage},
assertions: []traitAssertion{
assertPackageCount(6),
assertSuccessfulReturnCode,
},
},
}
for _, test := range tests {