oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2025-11-17 08:23:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

add task id to constructor

Browse Source 
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman 2022-06-07 17:44:24 -04:00
parent a5dd485672
commit b06a2d4f27
No known key found for this signature in database
GPG Key ID: 5CB45AE22BAB7EA7
23 changed files with 300 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/attest.go
View File

@ -193,7 +193,7 @@ func attestationExecWorker(si source.Input, format sbom.Format, predicateType st
go func() {
defer close(errs)
catalogingConfig, err := appConfig.ToCatalogingConfig()
catalogingOpts, err := appConfig.ToCatalogingOptions()
if err != nil {
errs <- err
return
@ -208,7 +208,7 @@ func attestationExecWorker(si source.Input, format sbom.Format, predicateType st
return
}
s, err := generateSBOM(src, catalogingConfig)
s, err := syft.Catalog(src, catalogingOpts...)
if err != nil {
errs <- err
return

20
cmd/packages.go
View File

@ -112,6 +112,12 @@ func setPackageFlags(flags *pflag.FlagSet) {
"an optional platform specifier for container image sources (e.g. 'linux/arm64', 'linux/arm64/v8', 'arm64', 'linux')",
)
flags.StringArrayP(
"cataloger", "", nil,
// TODO: have a nice way for a user to get the set of catalogers that exist
"set which catalogers to use when generating an SBOM",
)
// Upload options //////////////////////////////////////////////////////////
flags.StringP(
"host", "H", "",
@ -169,6 +175,10 @@ func bindExclusivePackagesConfigOptions(flags *pflag.FlagSet) error {
return err
}
if err := viper.BindPFlag("catalogers", flags.Lookup("cataloger")); err != nil {
return err
}
if err := viper.BindPFlag("file", flags.Lookup("file")); err != nil {
return err
}
@ -257,18 +267,12 @@ func isVerbose() (result bool) {
return appConfig.CliOptions.Verbosity > 0 || isPipedInput
}
func generateSBOM(src *source.Source, config *syft.CatalogingConfig) (*sbom.SBOM, error) {
return syft.Catalog(src,
syft.WithConfig(*config),
)
}
func packagesExecWorker(si source.Input, writer sbom.Writer) <-chan error {
errs := make(chan error)
go func() {
defer close(errs)
catalogingConfig, err := appConfig.ToCatalogingConfig()
catalogingOpts, err := appConfig.ToCatalogingOptions()
if err != nil {
errs <- err
return
@ -283,7 +287,7 @@ func packagesExecWorker(si source.Input, writer sbom.Writer) <-chan error {
return
}
s, err := generateSBOM(src, catalogingConfig)
s, err := syft.Catalog(src, catalogingOpts...)
if err != nil {
errs <- err
return

5
cmd/power_user.go
View File

@ -2,6 +2,7 @@ package cmd
import (
"fmt"
"github.com/anchore/syft/syft"
"os"
"github.com/anchore/stereoscope"
@ -110,7 +111,7 @@ func powerUserExecWorker(userInput string, writer sbom.Writer) <-chan error {
//appConfig.FileContents.Cataloger.Enabled = true
//appConfig.FileClassification.Cataloger.Enabled = true
catalogingConfig, err := appConfig.ToCatalogingConfig()
catalogingOpts, err := appConfig.ToCatalogingOptions()
if err != nil {
errs <- err
return
@ -131,7 +132,7 @@ func powerUserExecWorker(userInput string, writer sbom.Writer) <-chan error {
defer cleanup()
}
s, err := generateSBOM(src, catalogingConfig)
s, err := syft.Catalog(src, catalogingOpts...)
if err != nil {
errs <- err
return

44
internal/config/application.go
View File

@ -3,7 +3,6 @@ package config
import (
"errors"
"fmt"
"github.com/anchore/syft/syft/cataloger/files/fileclassifier"
"github.com/anchore/syft/syft/source"
"path"
"reflect"
@ -42,6 +41,7 @@ type Application struct {
CheckForAppUpdate bool `yaml:"check-for-app-update" json:"check-for-app-update" mapstructure:"check-for-app-update"` // whether to check for an application update on start up or not
Anchore anchore `yaml:"anchore" json:"anchore" mapstructure:"anchore"` // options for interacting with Anchore Engine/Enterprise
CliOptions CliOnlyOptions `yaml:"-" json:"-"` // all options only available through the CLI (not via env vars or config)
Catalogers []string `yaml:"catalogers" json:"catalogers" mapstructure:"catalogers"`
Scope string `yaml:"scope" json:"scope" mapstructure:"scope"`
Dev development `yaml:"dev" json:"dev" mapstructure:"dev"`
Log logging `yaml:"log" json:"log" mapstructure:"log"` // all logging-related options
@ -55,7 +55,8 @@ type Application struct {
Platform string `yaml:"platform" json:"platform" mapstructure:"platform"`
}
func (cfg Application) ToCatalogingConfig() (*syft.CatalogingConfig, error) {
func (cfg Application) ToCatalogingOptions() ([]syft.CatalogingOption, error) {
digests, err := file.DigestHashesByName(cfg.FileMetadata.Digests...)
if err != nil {
return nil, fmt.Errorf("unable to parse config item 'file-metadata.digests': %w", err)
@ -76,19 +77,23 @@ func (cfg Application) ToCatalogingConfig() (*syft.CatalogingConfig, error) {
return nil, fmt.Errorf("bad scope value %q", cfg.Secrets.Scope)
}
return &syft.CatalogingConfig{
// note: package catalogers cannot be determined until runtime
ToolName: internal.ApplicationName,
ToolVersion: version.FromBuild().Version,
ToolConfiguration: cfg,
DefaultScope: scopeOption,
ProcessTasksInSerial: false,
DigestHashes: digests,
SecretsSearch: *secretsConfig,
SecretsScope: secretsScopeOption,
FileClassifiers: fileclassifier.DefaultClassifiers(),
ContentsSearch: cfg.FileContents.ToConfig(),
}, nil
c := syft.DefaultCatalogingConfig()
c.ToolVersion = version.FromBuild().Version
c.ToolConfiguration = &c
c.DefaultScope = scopeOption
c.ProcessTasksInSerial = false
c.DigestHashes = digests
c.SecretsSearch = *secretsConfig
c.SecretsScope = secretsScopeOption
c.ContentsSearch = cfg.FileContents.ToConfig()
opts := []syft.CatalogingOption{
syft.WithConfig(c),
syft.WithCatalogers(cfg.Catalogers...),
}
return opts, nil
}
// PowerUserCatalogerEnabledDefault switches all catalogers to be enabled when running power-user command
@ -142,6 +147,15 @@ func (cfg Application) loadDefaultValues(v *viper.Viper) {
}
func (cfg *Application) parseConfigValues() error {
// parse fields on application directyy
var catalogers []string
for _, c := range cfg.Catalogers {
for _, f := range strings.Split(c, ",") {
catalogers = append(catalogers, strings.TrimSpace(f))
}
}
cfg.Catalogers = catalogers
// parse application config options
for _, optionFn := range []func() error{
cfg.parseUploadOptions,

34
syft/catalog.go
View File

@ -33,32 +33,26 @@ func Catalog(src *source.Source, options ...CatalogingOption) (*sbom.SBOM, error
}
if config.availableTasks == nil {
config.availableTasks = newTaskCollection()
}
tc := config.availableTasks
if err := tc.addAllCatalogers(config); err != nil {
return nil, fmt.Errorf("unable to register catalogers: %w", err)
}
var catalogingTasks []task
if len(config.EnabledCatalogers) == 0 {
switch src.Metadata.Scheme {
case source.ImageType:
catalogingTasks = tc.tasks(tc.withLabels(packageTaskLabel, installedTaskLabel)...)
case source.FileType:
catalogingTasks = tc.tasks(tc.all()...)
case source.DirectoryType:
// TODO: it looks like gemspec was left out on main, is this intentional? if so it's not accounted for here...
catalogingTasks = tc.tasks(tc.withLabels(packageTaskLabel)...)
var err error
config.availableTasks, err = newTaskCollection()
if err != nil {
return nil, err
}
}
if len(catalogingTasks) == 0 {
if len(config.EnabledCatalogers) == 0 {
return nil, fmt.Errorf("no cataloging tasks configured to run")
}
catalogingTasks, err := config.availableTasks.tasks(config, config.EnabledCatalogers...)
if err != nil {
return nil, err
}
if len(catalogingTasks) == 0 {
return nil, fmt.Errorf("no cataloging tasks found to run")
}
// special case: we need to identify the linux distro for downstream processing
identifyLinuxDistroTask, err := newIdentifyDistroTask(config)
if err != nil {

2
syft/cataloger/packages/apkdb/cataloger.go
View File

@ -14,5 +14,5 @@ func NewApkdbCataloger() *generic.Cataloger {
pkg.ApkDBGlob: parseApkDB,
}
return generic.NewCataloger(nil, globParsers, "apkdb-cataloger")
return generic.NewCataloger(nil, globParsers)
}

6
syft/cataloger/packages/deb/cataloger.go
View File

@ -32,11 +32,6 @@ func NewDpkgdbCataloger() *Cataloger {
return &Cataloger{}
}
// Name returns a string that uniquely describes a cataloger
func (c *Cataloger) Name() string {
return "dpkgdb-cataloger"
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
func (c *Cataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
dbFileMatches, err := resolver.FilesByGlob(pkg.DpkgDBGlob)
@ -60,7 +55,6 @@ func (c *Cataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.R
for i := range pkgs {
p := &pkgs[i]
p.FoundBy = c.Name()
p.Locations = []file.Location{dbLocation}
// the current entry only has what may have been listed in the status file, however, there are additional

20
syft/cataloger/packages/generic/cataloger.go
View File

@ -18,25 +18,18 @@ import (
// Cataloger implements the Catalog interface and is responsible for dispatching the proper parser function for
// a given path or glob pattern. This is intended to be reusable across many package cataloger types.
type Cataloger struct {
globParsers map[string]Parser
pathParsers map[string]Parser
upstreamCataloger string
globParsers map[string]Parser
pathParsers map[string]Parser
}
// NewCataloger if provided path-to-parser-function and glob-to-parser-function lookups creates a Cataloger
func NewCataloger(pathParsers map[string]Parser, globParsers map[string]Parser, upstreamCataloger string) *Cataloger {
func NewCataloger(pathParsers map[string]Parser, globParsers map[string]Parser) *Cataloger {
return &Cataloger{
globParsers: globParsers,
pathParsers: pathParsers,
upstreamCataloger: upstreamCataloger,
globParsers: globParsers,
pathParsers: pathParsers,
}
}
// Name returns a string that uniquely describes the upstream cataloger that this Generic Cataloger represents.
func (c *Cataloger) Name() string {
return c.upstreamCataloger
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
func (c *Cataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
var packages []pkg.Package
@ -53,12 +46,11 @@ func (c *Cataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.R
internal.CloseAndLogError(contentReader, location.AccessPath)
if err != nil {
// TODO: should we fail? or only log?
log.Warnf("cataloger '%s' failed to parse entries at location=%+v: %+v", c.upstreamCataloger, location, err)
log.Warnf("cataloger failed to parse entries at location=%+v: %+v", location, err)
continue
}
for _, p := range discoveredPackages {
p.FoundBy = c.upstreamCataloger
p.Locations = append(p.Locations, location)
p.SetID()

7
syft/cataloger/packages/golang/binary_cataloger.go
View File

@ -14,8 +14,6 @@ import (
"github.com/anchore/syft/syft/pkg"
)
const catalogerName = "go-module-binary-cataloger"
type Cataloger struct{}
// NewGoModuleBinaryCataloger returns a new Golang cataloger object.
@ -23,11 +21,6 @@ func NewGoModuleBinaryCataloger() *Cataloger {
return &Cataloger{}
}
// Name returns a string that uniquely describes a cataloger
func (c *Cataloger) Name() string {
return catalogerName
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
func (c *Cataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
var pkgs []pkg.Package

2
syft/cataloger/packages/golang/mod_cataloger.go
View File

@ -13,5 +13,5 @@ func NewGoModFileCataloger() *generic.Cataloger {
"**/go.mod": parseGoMod,
}
return generic.NewCataloger(nil, globParsers, "go-mod-file-cataloger")
return generic.NewCataloger(nil, globParsers)
}

2
syft/cataloger/packages/java/cataloger.go
View File

@ -30,5 +30,5 @@ func NewJavaCataloger(cfg CatalogerConfig) *generic.Cataloger {
}
}
return generic.NewCataloger(nil, globParsers, "java-cataloger")
return generic.NewCataloger(nil, globParsers)
}

6
syft/cataloger/packages/javascript/cataloger.go
View File

@ -13,7 +13,7 @@ func NewJavascriptPackageCataloger() *generic.Cataloger {
"**/package.json": parsePackageJSON,
}
return generic.NewCataloger(nil, globParsers, "javascript-package-json-cataloger")
return generic.NewCataloger(nil, globParsers)
}
// NewJavascriptLockCataloger returns a new Javascript cataloger object base on package lock files.
@ -22,7 +22,7 @@ func NewJavascriptPackageLockCataloger() *generic.Cataloger {
"**/package-lock.json": parsePackageLock,
}
return generic.NewCataloger(nil, globParsers, "javascript-package-lock-cataloger")
return generic.NewCataloger(nil, globParsers)
}
func NewJavascriptYarnLockCataloger() *generic.Cataloger {
@ -30,5 +30,5 @@ func NewJavascriptYarnLockCataloger() *generic.Cataloger {
"**/yarn.lock": parseYarnLock,
}
return generic.NewCataloger(nil, globParsers, "javascript-yarn-lock-cataloger")
return generic.NewCataloger(nil, globParsers)
}

4
syft/cataloger/packages/php/cataloger.go
View File

@ -13,7 +13,7 @@ func NewPHPComposerInstalledCataloger() *generic.Cataloger {
"**/installed.json": parseInstalledJSON,
}
return generic.NewCataloger(nil, globParsers, "php-composer-installed-cataloger")
return generic.NewCataloger(nil, globParsers)
}
// NewPHPComposerLockCataloger returns a new cataloger for PHP composer.lock files.
@ -22,5 +22,5 @@ func NewPHPComposerLockCataloger() *generic.Cataloger {
"**/composer.lock": parseComposerLock,
}
return generic.NewCataloger(nil, globParsers, "php-composer-lock-cataloger")
return generic.NewCataloger(nil, globParsers)
}

8
syft/cataloger/packages/python/index_cataloger.go
View File

@ -13,7 +13,7 @@ func NewPythonRequirementsCataloger() *generic.Cataloger {
"**/*requirements*.txt": parseRequirementsTxt,
}
return generic.NewCataloger(nil, globParsers, "python-requirements-cataloger")
return generic.NewCataloger(nil, globParsers)
}
func NewPythonPoetryCataloger() *generic.Cataloger {
@ -21,7 +21,7 @@ func NewPythonPoetryCataloger() *generic.Cataloger {
"**/poetry.lock": parsePoetryLock,
}
return generic.NewCataloger(nil, globParsers, "python-poetry-cataloger")
return generic.NewCataloger(nil, globParsers)
}
func NewPythonPipfileCataloger() *generic.Cataloger {
@ -29,7 +29,7 @@ func NewPythonPipfileCataloger() *generic.Cataloger {
"**/Pipfile.lock": parsePipfileLock,
}
return generic.NewCataloger(nil, globParsers, "python-pipfile-cataloger")
return generic.NewCataloger(nil, globParsers)
}
func NewPythonSetupCataloger() *generic.Cataloger {
@ -37,5 +37,5 @@ func NewPythonSetupCataloger() *generic.Cataloger {
"**/setup.py": parseSetup,
}
return generic.NewCataloger(nil, globParsers, "python-setup-cataloger")
return generic.NewCataloger(nil, globParsers)
}

6
syft/cataloger/packages/python/package_cataloger.go
View File

@ -28,11 +28,6 @@ func NewPythonPackageCataloger() *PackageCataloger {
return &PackageCataloger{}
}
// Name returns a string that uniquely describes a cataloger
func (c *PackageCataloger) Name() string {
return "python-package-cataloger"
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
func (c *PackageCataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
var fileMatches []file.Location
@ -79,7 +74,6 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver file.Resolver, metadataLoc
p := &pkg.Package{
Name: metadata.Name,
Version: metadata.Version,
FoundBy: c.Name(),
Locations: sources,
Licenses: licenses,
Language: pkg.Python,

7
syft/cataloger/packages/rpmdb/cataloger.go
View File

@ -13,8 +13,6 @@ import (
"github.com/anchore/syft/syft/pkg"
)
const catalogerName = "rpmdb-cataloger"
type Cataloger struct{}
// NewRpmdbCataloger returns a new RPM DB cataloger object.
@ -22,11 +20,6 @@ func NewRpmdbCataloger() *Cataloger {
return &Cataloger{}
}
// Name returns a string that uniquely describes a cataloger
func (c *Cataloger) Name() string {
return catalogerName
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
func (c *Cataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
fileMatches, err := resolver.FilesByGlob(pkg.RpmDBGlob)

1
syft/cataloger/packages/rpmdb/parse_rpmdb.go
View File

@ -63,7 +63,6 @@ func parseRpmDB(resolver file.PathResolver, dbLocation file.Location, reader io.
Name: entry.Name,
Version: toELVersion(metadata),
Locations: []file.Location{dbLocation},
FoundBy: catalogerName,
Type: pkg.RpmPkg,
MetadataType: pkg.RpmdbMetadataType,
Metadata: metadata,

4
syft/cataloger/packages/ruby/catalogers.go
View File

@ -13,7 +13,7 @@ func NewGemFileLockCataloger() *generic.Cataloger {
"**/Gemfile.lock": parseGemFileLockEntries,
}
return generic.NewCataloger(nil, globParsers, "ruby-gemfile-cataloger")
return generic.NewCataloger(nil, globParsers)
}
// NewGemSpecCataloger returns a new Bundler cataloger object tailored for detecting installations of gems (e.g. Gemspec).
@ -22,5 +22,5 @@ func NewGemSpecCataloger() *generic.Cataloger {
"**/specifications/**/*.gemspec": parseGemSpecEntries,
}
return generic.NewCataloger(nil, globParsers, "ruby-gemspec-cataloger")
return generic.NewCataloger(nil, globParsers)
}

2
syft/cataloger/packages/rust/cataloger.go
View File

@ -13,5 +13,5 @@ func NewCargoLockCataloger() *generic.Cataloger {
"**/Cargo.lock": parseCargoLock,
}
return generic.NewCataloger(nil, globParsers, "rust-cataloger")
return generic.NewCataloger(nil, globParsers)
}

83
syft/cataloging_option.go
View File

@ -2,11 +2,13 @@ package syft
import (
"crypto"
"fmt"
"github.com/anchore/syft/syft/cataloger"
"github.com/anchore/syft/syft/cataloger/files/fileclassifier"
"github.com/anchore/syft/syft/cataloger/files/secrets"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
"strings"
)
type CatalogingOption func(*source.Source, *CatalogingConfig) error
@ -47,30 +49,87 @@ func WithToolConfiguration(c interface{}) CatalogingOption {
}
}
func WithCataloger(id cataloger.ID, c pkg.Cataloger) CatalogingOption {
func WithPackageCataloger(id cataloger.ID, c pkg.Cataloger) CatalogingOption {
return func(_ *source.Source, config *CatalogingConfig) error {
if config.availableTasks == nil {
config.availableTasks = newTaskCollection()
var err error
config.availableTasks, err = newTaskCollection()
if err != nil {
return err
}
}
var cfg CatalogingConfig
if config != nil {
cfg = *config
gen := func(id cataloger.ID, cfg CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, cfg, c), nil
}
config.EnabledCatalogers = append(config.EnabledCatalogers, id)
return config.availableTasks.add(pkgCatalogerTask{
id: id,
cataloger: c,
config: cfg,
})
return config.availableTasks.add(string(id), gen)
}
}
func WithDefaultCatalogers() CatalogingOption {
return func(src *source.Source, config *CatalogingConfig) error {
if config.availableTasks == nil {
var err error
config.availableTasks, err = newTaskCollection()
if err != nil {
return err
}
}
// override any previously added catalogers
config.availableTasks = newTaskCollection()
config.EnabledCatalogers = nil
tc := config.availableTasks
if len(config.EnabledCatalogers) == 0 {
switch src.Metadata.Scheme {
case source.ImageType:
config.EnabledCatalogers = tc.withLabels(packageTaskLabel, installedTaskLabel)
case source.FileType:
config.EnabledCatalogers = tc.all()
case source.DirectoryType:
// TODO: it looks like gemspec was left out on main, is this intentional? if so it's not accounted for here...
config.EnabledCatalogers = tc.withLabels(packageTaskLabel)
}
}
return nil
}
}
func WithCatalogers(catalogers ...string) CatalogingOption {
return func(src *source.Source, config *CatalogingConfig) error {
if config.availableTasks == nil {
var err error
config.availableTasks, err = newTaskCollection()
if err != nil {
return err
}
}
// override any previously added catalogers
for _, q := range catalogers {
var ids []cataloger.ID
isAdditive := strings.HasPrefix(q, "+")
if isAdditive {
ids = config.availableTasks.query(strings.TrimPrefix(q, "+"))
} else {
ids = config.availableTasks.query(q)
}
if len(ids) == 0 {
return fmt.Errorf("cataloger selection invalid: %q", q)
}
if isAdditive && len(config.EnabledCatalogers) == 0 {
// stick in the default set of catalogers first
if err := WithDefaultCatalogers()(src, config); err != nil {
return fmt.Errorf("unable to set default catalogers: %w", err)
}
}
config.EnabledCatalogers = append(config.EnabledCatalogers, ids...)
}
return nil
}
}

2
syft/pkg/cataloger.go
View File

@ -9,8 +9,6 @@ import (
// contents for the purpose of discovering Packages. Each concrete implementation should focus on discovering Packages
// for a specific Package Type or ecosystem.
type Cataloger interface {
// Name returns a string that uniquely describes a cataloger
Name() string
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
Catalog(resolver file.Resolver) ([]Package, []artifact.Relationship, error)
}

141
syft/task_collection.go
View File

@ -10,7 +10,9 @@ import (
const (
packageTaskLabel = "package"
packagesTaskLabel = "packages"
fileTaskLabel = "file"
filesTaskLabel = "files"
osTaskLabel = "os"
languageTaskLabel = "language"
installedTaskLabel = "installed"
@ -18,36 +20,24 @@ const (
)
type taskCollection struct {
taskByName map[string]task // name -> generator
namesByLabel map[string][]string // label -> names
taskConstructorByName map[string]taskGenerator // name -> generator
namesByLabel map[string][]string // label -> names
}
func newTaskCollection() *taskCollection {
return &taskCollection{
taskByName: make(map[string]task),
namesByLabel: make(map[string][]string),
func newTaskCollection() (*taskCollection, error) {
c := &taskCollection{
taskConstructorByName: make(map[string]taskGenerator),
namesByLabel: make(map[string][]string),
}
return c, c.addAllCatalogers()
}
func (c *taskCollection) add(t task, labels ...string) error {
var name string
switch v := t.(type) {
case pkgCatalogerTask:
name = string(v.id)
case catalogerTask:
name = string(v.id)
default:
if len(labels) == 0 {
return fmt.Errorf("no ID found for generic task")
}
name = labels[0]
func (c *taskCollection) add(name string, g taskGenerator, labels ...string) error {
if _, exists := c.taskConstructorByName[name]; exists {
return fmt.Errorf("task constructor already exists: %q", name)
}
if _, exists := c.taskByName[name]; exists {
return fmt.Errorf("task already exists: %q", name)
}
c.taskByName[name] = t
c.taskConstructorByName[name] = g
labelSet := strset.New(labels...)
labelSet.Add(name)
@ -57,118 +47,134 @@ func (c *taskCollection) add(t task, labels ...string) error {
return nil
}
func (c *taskCollection) addAllCatalogers(config CatalogingConfig) error {
func (c *taskCollection) addAllCatalogers() error {
for _, d := range []struct {
id cataloger.ID
generator taskGenerator
labels []string
}{
{
id: cataloger.ApkDBID,
generator: newAPKDBCatalogingTask,
labels: []string{packageTaskLabel, osTaskLabel, installedTaskLabel, "alpine", "apk", "apkdb"},
labels: []string{packageTaskLabel, packagesTaskLabel, osTaskLabel, installedTaskLabel, "alpine", "apk", "apkdb"},
},
{
id: cataloger.DpkgID,
generator: newDPKGCatalogingTask,
labels: []string{packageTaskLabel, osTaskLabel, installedTaskLabel, "debian", "dpkg", "deb", "dpkgdb"},
labels: []string{packageTaskLabel, packagesTaskLabel, osTaskLabel, installedTaskLabel, "debian", "dpkg", "deb", "dpkgdb"},
},
{
id: cataloger.RpmDBID,
generator: newRPMDBCatalogingTask,
labels: []string{packageTaskLabel, osTaskLabel, installedTaskLabel, "redhat", "rhel", "centos", "rpm", "rpmdb"},
labels: []string{packageTaskLabel, packagesTaskLabel, osTaskLabel, installedTaskLabel, "redhat", "rhel", "centos", "rpm", "rpmdb"},
},
{
id: cataloger.RubyGemspecID,
generator: newRubyGemSpecCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, installedTaskLabel, "ruby", "gemspec", "gem"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, installedTaskLabel, "ruby", "gemspec", "gem"},
},
{
id: cataloger.RubyGemfileLockID,
generator: newRubyGemFileLockCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, declaredTaskLabel, "ruby", "gemfile", "gem", "gemfile.lock"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, declaredTaskLabel, "ruby", "gemfile", "gem", "gemfile.lock"},
},
{
id: cataloger.PythonPackageID,
generator: newPythonPackageCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, installedTaskLabel, "python", "egg", "wheel"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, installedTaskLabel, "python", "egg", "wheel"},
},
{
id: cataloger.PythonRequirementsID,
generator: newPythonRequirementsCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, declaredTaskLabel, "python", "requirements", "requirements.txt"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, declaredTaskLabel, "python", "requirements", "requirements.txt"},
},
{
id: cataloger.PythonPoetryID,
generator: newPythonPoetryCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, declaredTaskLabel, "python", "poetry", "poetry.lock"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, declaredTaskLabel, "python", "poetry", "poetry.lock"},
},
{
id: cataloger.PythonSetupID,
generator: newPythonSetupCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, declaredTaskLabel, "python", "setup", "setup.py"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, declaredTaskLabel, "python", "setup", "setup.py"},
},
{
id: cataloger.PythonPipFileID,
generator: newPythonPipfileCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, declaredTaskLabel, "python", "pip", "pipfile"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, declaredTaskLabel, "python", "pip", "pipfile"},
},
{
id: cataloger.JavascriptPackageJSONID,
generator: newJavascriptPackageJSONCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, installedTaskLabel, "javascript", "node", "package.json"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, installedTaskLabel, "javascript", "node", "package.json"},
},
{
id: cataloger.JavascriptPackageLockID,
generator: newJavascriptPackageLockCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, declaredTaskLabel, "javascript", "node", "package-lock.json"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, declaredTaskLabel, "javascript", "node", "package-lock.json"},
},
{
id: cataloger.JavaScriptYarnLockID,
generator: newJavascriptYarnLockCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, declaredTaskLabel, "javascript", "node", "yarn", "yarn.lock"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, declaredTaskLabel, "javascript", "node", "yarn", "yarn.lock"},
},
{
id: cataloger.JavaArchiveID,
generator: newJavaCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, installedTaskLabel, "java", "maven", "jar", "war", "ear", "jenkins", "hudson", "hpi", "jpi", "par", "sar", "lpkg"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, installedTaskLabel, "java", "maven", "jar", "war", "ear", "jenkins", "hudson", "hpi", "jpi", "par", "sar", "lpkg"},
},
{
id: cataloger.GoModID,
generator: newGolangModuleCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, declaredTaskLabel, "go", "golang", "go-module", "go.mod"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, declaredTaskLabel, "go", "golang", "go-module", "go.mod"},
},
{
id: cataloger.GoBinaryID,
generator: newGolangBinaryCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, installedTaskLabel, "go", "golang", "go-module", "binary"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, installedTaskLabel, "go", "golang", "go-module", "binary"},
},
{
id: cataloger.RustCargoLockID,
generator: newRustCargoLockCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, declaredTaskLabel, "rust", "cargo", "cargo.lock"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, declaredTaskLabel, "rust", "cargo", "cargo.lock"},
},
{
id: cataloger.PHPInstalledJSONID,
generator: newPHPInstalledCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, installedTaskLabel, "php", "composer", "installed.json"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, installedTaskLabel, "php", "composer", "installed.json"},
},
{
id: cataloger.PHPComposerLockID,
generator: newPHPComposerLockCatalogingTask,
labels: []string{packageTaskLabel, languageTaskLabel, declaredTaskLabel, "php", "composer", "composer.lock"},
labels: []string{packageTaskLabel, packagesTaskLabel, languageTaskLabel, declaredTaskLabel, "php", "composer", "composer.lock"},
},
{
id: cataloger.FileMetadataID,
generator: newFileMetadataCatalogingTask,
labels: []string{fileTaskLabel},
labels: []string{fileTaskLabel, filesTaskLabel},
},
{
id: cataloger.FileDigestsID,
generator: newFileDigestsCatalogingTask,
labels: []string{fileTaskLabel, "digests", "digest", "file-digests"},
labels: []string{fileTaskLabel, filesTaskLabel, "digests", "digest", "file-digests"},
},
{
id: cataloger.SecretsID,
generator: newSecretsCatalogingTask,
labels: []string{"secrets"},
},
{
id: cataloger.FileClassifierID,
generator: newFileClassifierTask,
labels: []string{fileTaskLabel, "classifier"},
labels: []string{fileTaskLabel, filesTaskLabel, "classifier"},
},
{
id: cataloger.FileContentsID,
generator: newFileContentsCatalogingTask,
labels: []string{fileTaskLabel, "contents", "content", "file-contents"},
labels: []string{fileTaskLabel, filesTaskLabel, "contents", "content", "file-contents"},
},
} {
t, err := d.generator(config)
if err != nil {
return err
}
if t == nil {
continue
}
if err := c.add(t, d.labels...); err != nil {
if err := c.add(string(d.id), d.generator, d.labels...); err != nil {
return err
}
}
@ -178,7 +184,7 @@ func (c *taskCollection) addAllCatalogers(config CatalogingConfig) error {
func (c taskCollection) query(q string) []cataloger.ID {
fields := strings.FieldsFunc(q, func(r rune) bool {
switch r {
case '+', ',', '&':
case '+', '&':
return true
}
return false
@ -189,7 +195,7 @@ func (c taskCollection) query(q string) []cataloger.ID {
func (c taskCollection) all() []cataloger.ID {
var ret []cataloger.ID
for k := range c.taskByName {
for k := range c.taskConstructorByName {
ret = append(ret, cataloger.ID(k))
}
@ -221,13 +227,24 @@ func (c taskCollection) withLabels(q ...string) []cataloger.ID {
return ret
}
func (c taskCollection) tasks(ids ...cataloger.ID) (ts []task) {
func (c taskCollection) tasks(config CatalogingConfig, ids ...cataloger.ID) ([]task, error) {
var ts []task
for _, id := range ids {
t, exists := c.taskByName[string(id)]
g, exists := c.taskConstructorByName[string(id)]
if !exists {
continue
}
t, err := g(id, config)
if err != nil {
return nil, fmt.Errorf("unable to construct task %q: %w", id, err)
}
if t == nil {
continue
}
ts = append(ts, t)
}
return ts
return ts, nil
}

200
syft/tasks.go
View File

@ -2,6 +2,7 @@ package syft
import (
"fmt"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cataloger"
"github.com/anchore/syft/syft/cataloger/files/fileclassifier"
"github.com/anchore/syft/syft/cataloger/files/filecontents"
@ -28,7 +29,7 @@ import (
"github.com/anchore/syft/syft/source"
)
type taskGenerator func(CatalogingConfig) (task, error)
type taskGenerator func(cataloger.ID, CatalogingConfig) (task, error)
type task interface {
Run(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error)
@ -53,6 +54,14 @@ type pkgCatalogerTask struct {
config CatalogingConfig
}
func newPkgCatalogerTask(id cataloger.ID, config CatalogingConfig, c pkg.Cataloger) pkgCatalogerTask {
return pkgCatalogerTask{
id: id,
cataloger: c,
config: config,
}
}
func (t pkgCatalogerTask) Run(artifacts *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(t.config.DefaultScope)
if err != nil {
@ -90,166 +99,96 @@ func newIdentifyDistroTask(config CatalogingConfig) (task, error) {
}, nil
}
func newAPKDBCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.ApkDBID,
cataloger: apkdb.NewApkdbCataloger(),
config: config,
}, nil
func newAPKDBCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, apkdb.NewApkdbCataloger()), nil
}
func newDPKGCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.DpkgID,
cataloger: deb.NewDpkgdbCataloger(),
config: config,
}, nil
func newDPKGCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, deb.NewDpkgdbCataloger()), nil
}
func newGolangBinaryCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.GoBinaryID,
cataloger: golang.NewGoModuleBinaryCataloger(),
config: config,
}, nil
func newGolangBinaryCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, golang.NewGoModuleBinaryCataloger()), nil
}
func newGolangModuleCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.GoModID,
cataloger: golang.NewGoModFileCataloger(),
config: config,
}, nil
func newGolangModuleCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, golang.NewGoModFileCataloger()), nil
}
func newJavaCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.JavaArchiveID,
cataloger: java.NewJavaCataloger(java.CatalogerConfig{
SearchUnindexedArchives: config.PackageSearch.IncludeUnindexedArchives,
SearchIndexedArchives: config.PackageSearch.IncludeIndexedArchives,
}),
config: config,
}, nil
func newJavaCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(
id,
config,
java.NewJavaCataloger(
java.CatalogerConfig{
SearchUnindexedArchives: config.PackageSearch.IncludeUnindexedArchives,
SearchIndexedArchives: config.PackageSearch.IncludeIndexedArchives,
},
),
), nil
}
func newJavascriptPackageJSONCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.JavascriptPackageJSONID,
cataloger: javascript.NewJavascriptPackageCataloger(),
config: config,
}, nil
func newJavascriptPackageJSONCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, javascript.NewJavascriptPackageCataloger()), nil
}
func newJavascriptPackageLockCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.JavascriptPackageLockID,
cataloger: javascript.NewJavascriptPackageLockCataloger(),
config: config,
}, nil
func newJavascriptPackageLockCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, javascript.NewJavascriptPackageLockCataloger()), nil
}
func newJavascriptYarnLockCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.JavaScriptYarnLockID,
cataloger: javascript.NewJavascriptYarnLockCataloger(),
config: config,
}, nil
func newJavascriptYarnLockCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, javascript.NewJavascriptYarnLockCataloger()), nil
}
func newPHPComposerLockCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.PHPComposerLockID,
cataloger: php.NewPHPComposerLockCataloger(),
config: config,
}, nil
func newPHPComposerLockCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, php.NewPHPComposerLockCataloger()), nil
}
func newPHPInstalledCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.PHPInstalledJSONID,
cataloger: php.NewPHPComposerInstalledCataloger(),
config: config,
}, nil
func newPHPInstalledCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, php.NewPHPComposerInstalledCataloger()), nil
}
func newPythonPackageCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.PythonPackageID,
cataloger: python.NewPythonPackageCataloger(),
config: config,
}, nil
func newPythonPackageCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, python.NewPythonPackageCataloger()), nil
}
func newPythonRequirementsCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.PythonRequirementsID,
cataloger: python.NewPythonRequirementsCataloger(),
config: config,
}, nil
func newPythonRequirementsCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, python.NewPythonRequirementsCataloger()), nil
}
func newPythonPoetryCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.PythonPoetryID,
cataloger: python.NewPythonPoetryCataloger(),
config: config,
}, nil
func newPythonPoetryCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, python.NewPythonPoetryCataloger()), nil
}
func newPythonPipfileCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.PythonPipFileID,
cataloger: python.NewPythonPipfileCataloger(),
config: config,
}, nil
func newPythonPipfileCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, python.NewPythonPipfileCataloger()), nil
}
func newPythonSetupCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.PythonSetupID,
cataloger: python.NewPythonSetupCataloger(),
config: config,
}, nil
func newPythonSetupCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, python.NewPythonSetupCataloger()), nil
}
func newRPMDBCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.RpmDBID,
cataloger: rpmdb.NewRpmdbCataloger(),
config: config,
}, nil
func newRPMDBCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, rpmdb.NewRpmdbCataloger()), nil
}
func newRubyGemFileLockCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.RubyGemfileLockID,
cataloger: ruby.NewGemFileLockCataloger(),
config: config,
}, nil
func newRubyGemFileLockCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, ruby.NewGemFileLockCataloger()), nil
}
func newRubyGemSpecCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.RubyGemspecID,
cataloger: ruby.NewGemSpecCataloger(),
config: config,
}, nil
func newRubyGemSpecCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, ruby.NewGemSpecCataloger()), nil
}
func newRustCargoLockCatalogingTask(config CatalogingConfig) (task, error) {
return pkgCatalogerTask{
id: cataloger.RustCargoLockID,
cataloger: rust.NewCargoLockCataloger(),
config: config,
}, nil
func newRustCargoLockCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
return newPkgCatalogerTask(id, config, rust.NewCargoLockCataloger()), nil
}
func newFileMetadataCatalogingTask(config CatalogingConfig) (task, error) {
func newFileMetadataCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
c := filemetadata.NewCataloger()
return catalogerTask{
id: cataloger.FileMetadataID,
id: id,
genericTask: genericTask{
run: func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(config.DefaultScope)
@ -268,8 +207,9 @@ func newFileMetadataCatalogingTask(config CatalogingConfig) (task, error) {
}, nil
}
func newFileDigestsCatalogingTask(config CatalogingConfig) (task, error) {
func newFileDigestsCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
if len(config.DigestHashes) == 0 {
log.Warn("using file-digest cataloger with no file digest algorithms configured")
return nil, nil
}
@ -279,7 +219,7 @@ func newFileDigestsCatalogingTask(config CatalogingConfig) (task, error) {
}
return catalogerTask{
id: cataloger.FileDigestsID,
id: id,
genericTask: genericTask{
run: func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(config.DefaultScope)
@ -298,8 +238,9 @@ func newFileDigestsCatalogingTask(config CatalogingConfig) (task, error) {
}, nil
}
func newFileContentsCatalogingTask(config CatalogingConfig) (task, error) {
func newFileContentsCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
if len(config.ContentsSearch.Globs) == 0 {
log.Warn("using file-content cataloger with no content file paths/globs configured")
return nil, nil
}
@ -309,7 +250,7 @@ func newFileContentsCatalogingTask(config CatalogingConfig) (task, error) {
}
return catalogerTask{
id: cataloger.FileContentsID,
id: id,
genericTask: genericTask{
run: func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(config.DefaultScope)
@ -328,15 +269,14 @@ func newFileContentsCatalogingTask(config CatalogingConfig) (task, error) {
}, nil
}
func newSecretsCatalogingTask(config CatalogingConfig) (task, error) {
func newSecretsCatalogingTask(id cataloger.ID, config CatalogingConfig) (task, error) {
c, err := secrets.NewCataloger(config.SecretsSearch)
if err != nil {
return nil, err
}
return catalogerTask{
id: cataloger.SecretsID,
id: id,
genericTask: genericTask{
run: func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(config.SecretsScope)
@ -355,15 +295,14 @@ func newSecretsCatalogingTask(config CatalogingConfig) (task, error) {
}, nil
}
func newFileClassifierTask(config CatalogingConfig) (task, error) {
func newFileClassifierTask(id cataloger.ID, config CatalogingConfig) (task, error) {
c, err := fileclassifier.NewCataloger(config.FileClassifiers)
if err != nil {
return nil, err
}
return catalogerTask{
id: cataloger.FileClassifierID,
id: id,
genericTask: genericTask{
run: func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(config.DefaultScope)
@ -383,7 +322,6 @@ func newFileClassifierTask(config CatalogingConfig) (task, error) {
}
func newSynthesizePackageRelationshipsTasks(config CatalogingConfig) (task, error) {
return genericTask{
run: func(artifacts *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(config.DefaultScope)