oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-10-31 06:26:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: --enrich flag for data enrichment feature enablement (#3182)

Browse Source 
Signed-off-by: Keith Zantow <kzantow@gmail.com>
This commit is contained in:
Keith Zantow
2024-09-12 10:45:18 -04:00
committed by GitHub
parent fcd5ec951d
commit 1b863268df
8 changed files with 217 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/syft/internal/options/catalog.go
+81 -18
View File
@@ -11,6 +11,7 @@ import (
"github.com/anchore/fangs"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/task"
"github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/filecataloging"
@@ -36,6 +37,7 @@ type Catalog struct {
Scope string `yaml:"scope" json:"scope" mapstructure:"scope"`
Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel
Relationships relationshipsConfig `yaml:"relationships" json:"relationships" mapstructure:"relationships"`
Enrich []string `yaml:"enrich" json:"enrich" mapstructure:"enrich"`
// ecosystem-specific cataloger configuration
Golang golangConfig `yaml:"golang" json:"golang" mapstructure:"golang"`
@@ -55,7 +57,7 @@ type Catalog struct {
var _ interface {
clio.FlagAdder
clio.PostLoader
fangs.FieldDescriber
clio.FieldDescriber
} = (*Catalog)(nil)
func DefaultCatalog() Catalog {
@@ -130,9 +132,9 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
return pkgcataloging.Config{
Binary: binary.DefaultClassifierCatalogerConfig(),
Golang: golang.DefaultCatalogerConfig().
WithSearchLocalModCacheLicenses(cfg.Golang.SearchLocalModCacheLicenses).
WithSearchLocalModCacheLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Go, task.Golang), cfg.Golang.SearchLocalModCacheLicenses)).
WithLocalModCacheDir(cfg.Golang.LocalModCacheDir).
WithSearchRemoteLicenses(cfg.Golang.SearchRemoteLicenses).
WithSearchRemoteLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Go, task.Golang), cfg.Golang.SearchRemoteLicenses)).
WithProxy(cfg.Golang.Proxy).
WithNoProxy(cfg.Golang.NoProxy).
WithMainModuleVersion(
@@ -142,7 +144,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
WithFromLDFlags(cfg.Golang.MainModuleVersion.FromLDFlags),
),
JavaScript: javascript.DefaultCatalogerConfig().
WithSearchRemoteLicenses(cfg.JavaScript.SearchRemoteLicenses).
WithSearchRemoteLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.JavaScript, task.Node, task.NPM), cfg.JavaScript.SearchRemoteLicenses)).
WithNpmBaseURL(cfg.JavaScript.NpmBaseURL),
LinuxKernel: kernel.LinuxKernelCatalogerConfig{
CatalogModules: cfg.LinuxKernel.CatalogModules,
@@ -151,9 +153,9 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
GuessUnpinnedRequirements: cfg.Python.GuessUnpinnedRequirements,
},
JavaArchive: java.DefaultArchiveCatalogerConfig().
WithUseMavenLocalRepository(cfg.Java.UseMavenLocalRepository).
WithUseMavenLocalRepository(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Java, task.Maven), cfg.Java.UseMavenLocalRepository)).
WithMavenLocalRepositoryDir(cfg.Java.MavenLocalRepositoryDir).
WithUseNetwork(cfg.Java.UseNetwork).
WithUseNetwork(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Java, task.Maven), cfg.Java.UseNetwork)).
WithMavenBaseURL(cfg.Java.MavenURL).
WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth),
}
@@ -193,6 +195,9 @@ func (cfg *Catalog) AddFlags(flags clio.FlagSet) {
flags.StringArrayVarP(&cfg.SelectCatalogers, "select-catalogers", "",
"add, remove, and filter the catalogers to be used")
flags.StringArrayVarP(&cfg.Enrich, "enrich", "",
fmt.Sprintf("enable package data enrichment from local and online sources (options: %s)", strings.Join(publicisedEnrichmentOptions, ", ")))
flags.StringVarP(&cfg.Source.Name, "source-name", "",
"set the name of the target being analyzed")
@@ -205,6 +210,10 @@ func (cfg *Catalog) AddFlags(flags clio.FlagSet) {
func (cfg *Catalog) DescribeFields(descriptions fangs.FieldDescriptionSet) {
descriptions.Add(&cfg.Parallelism, "number of cataloger workers to run in parallel")
descriptions.Add(&cfg.Enrich, fmt.Sprintf(`Enable data enrichment operations, which can utilize services such as Maven Central and NPM.
By default all enrichment is disabled, use: all to enable everything.
Available options are: %s`, strings.Join(publicisedEnrichmentOptions, ", ")))
}
func (cfg *Catalog) PostLoad() error {
@@ -215,23 +224,12 @@ func (cfg *Catalog) PostLoad() error {
return fmt.Errorf("cannot use both 'catalogers' and 'select-catalogers'/'default-catalogers' flags")
}
flatten := func(l []string) []string {
var out []string
for _, v := range l {
for _, s := range strings.Split(v, ",") {
out = append(out, strings.TrimSpace(s))
}
}
sort.Strings(out)
return out
}
cfg.From = flatten(cfg.From)
cfg.Catalogers = flatten(cfg.Catalogers)
cfg.DefaultCatalogers = flatten(cfg.DefaultCatalogers)
cfg.SelectCatalogers = flatten(cfg.SelectCatalogers)
cfg.Enrich = flatten(cfg.Enrich)
// for backwards compatibility
cfg.DefaultCatalogers = append(cfg.DefaultCatalogers, cfg.Catalogers...)
@@ -243,3 +241,68 @@ func (cfg *Catalog) PostLoad() error {
return nil
}
func flatten(commaSeparatedEntries []string) []string {
var out []string
for _, v := range commaSeparatedEntries {
for _, s := range strings.Split(v, ",") {
out = append(out, strings.TrimSpace(s))
}
}
sort.Strings(out)
return out
}
var publicisedEnrichmentOptions = []string{
"all",
task.Golang,
task.Java,
task.JavaScript,
}
func enrichmentEnabled(enrichDirectives []string, features ...string) *bool {
if len(enrichDirectives) == 0 {
return nil
}
enabled := func(features ...string) *bool {
for _, directive := range enrichDirectives {
enable := true
directive = strings.TrimPrefix(directive, "+") // +java and java are equivalent
if strings.HasPrefix(directive, "-") {
directive = directive[1:]
enable = false
}
for _, feature := range features {
if directive == feature {
return &enable
}
}
}
return nil
}
enableAll := enabled("all")
disableAll := enabled("none")
if disableAll != nil && *disableAll {
if enableAll != nil {
log.Warn("you have specified to both enable and disable all enrichment functionality, defaulting to disabled")
}
enableAll = ptr(false)
}
// check for explicit enable/disable of feature names
for _, feat := range features {
enableFeature := enabled(feat)
if enableFeature != nil {
return enableFeature
}
}
return enableAll
}
func ptr[T any](val T) *T {
return &val
}
cmd/syft/internal/options/catalog_test.go
+66
View File
@@ -70,3 +70,69 @@ func TestCatalog_PostLoad(t *testing.T) {
})
}
}
func Test_enrichmentEnabled(t *testing.T) {
tests := []struct {
directives string
test string
expected *bool
}{
{
directives: "",
test: "java",
expected: nil,
},
{
directives: "none",
test: "java",
expected: ptr(false),
},
{
directives: "none,+java",
test: "java",
expected: ptr(true),
},
{
directives: "all,none",
test: "java",
expected: ptr(false),
},
{
directives: "all",
test: "java",
expected: ptr(true),
},
{
directives: "golang,js",
test: "java",
expected: nil,
},
{
directives: "golang,-js,java",
test: "java",
expected: ptr(true),
},
{
directives: "golang,js,-java",
test: "java",
expected: ptr(false),
},
{
directives: "all",
test: "java",
expected: ptr(true),
},
{
directives: "all,-java",
test: "java",
expected: ptr(false),
},
}
for _, test := range tests {
t.Run(test.directives, func(t *testing.T) {
got := enrichmentEnabled(flatten([]string{test.directives}), test.test)
assert.Equal(t, test.expected, got)
})
}
}
cmd/syft/internal/options/golang.go
+4 -4
View File
@@ -8,9 +8,9 @@ import (
)
type golangConfig struct {
SearchLocalModCacheLicenses bool `json:"search-local-mod-cache-licenses" yaml:"search-local-mod-cache-licenses" mapstructure:"search-local-mod-cache-licenses"`
SearchLocalModCacheLicenses *bool `json:"search-local-mod-cache-licenses" yaml:"search-local-mod-cache-licenses" mapstructure:"search-local-mod-cache-licenses"`
LocalModCacheDir string `json:"local-mod-cache-dir" yaml:"local-mod-cache-dir" mapstructure:"local-mod-cache-dir"`
SearchRemoteLicenses bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
SearchRemoteLicenses *bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
Proxy string `json:"proxy" yaml:"proxy" mapstructure:"proxy"`
NoProxy string `json:"no-proxy" yaml:"no-proxy" mapstructure:"no-proxy"`
MainModuleVersion golangMainModuleVersionConfig `json:"main-module-version" yaml:"main-module-version" mapstructure:"main-module-version"`
@@ -47,9 +47,9 @@ type golangMainModuleVersionConfig struct {
func defaultGolangConfig() golangConfig {
def := golang.DefaultCatalogerConfig()
return golangConfig{
SearchLocalModCacheLicenses: def.SearchLocalModCacheLicenses,
SearchLocalModCacheLicenses: nil, // this defaults to false, which is the API default
LocalModCacheDir: def.LocalModCacheDir,
SearchRemoteLicenses: def.SearchRemoteLicenses,
SearchRemoteLicenses: nil, // this defaults to false, which is the API default
Proxy: strings.Join(def.Proxies, ","),
NoProxy: strings.Join(def.NoProxy, ","),
MainModuleVersion: golangMainModuleVersionConfig{
cmd/syft/internal/options/java.go
+4 -4
View File
@@ -6,8 +6,8 @@ import (
)
type javaConfig struct {
UseNetwork bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"`
UseMavenLocalRepository bool `yaml:"use-maven-local-repository" json:"use-maven-local-repository" mapstructure:"use-maven-local-repository"`
UseNetwork *bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"`
UseMavenLocalRepository *bool `yaml:"use-maven-local-repository" json:"use-maven-local-repository" mapstructure:"use-maven-local-repository"`
MavenLocalRepositoryDir string `yaml:"maven-local-repository-dir" json:"maven-local-repository-dir" mapstructure:"maven-local-repository-dir"`
MavenURL string `yaml:"maven-url" json:"maven-url" mapstructure:"maven-url"`
MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"`
@@ -17,9 +17,9 @@ func defaultJavaConfig() javaConfig {
def := java.DefaultArchiveCatalogerConfig()
return javaConfig{
UseNetwork: def.UseNetwork,
UseNetwork: nil, // this defaults to false, which is the API default
MaxParentRecursiveDepth: def.MaxParentRecursiveDepth,
UseMavenLocalRepository: def.UseMavenLocalRepository,
UseMavenLocalRepository: nil, // this defaults to false, which is the API default
MavenLocalRepositoryDir: def.MavenLocalRepositoryDir,
MavenURL: def.MavenBaseURL,
}
cmd/syft/internal/options/javascript.go
+1 -1
View File
@@ -3,7 +3,7 @@ package options
import "github.com/anchore/clio"
type javaScriptConfig struct {
SearchRemoteLicenses bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
SearchRemoteLicenses *bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
NpmBaseURL string `json:"npm-base-url" yaml:"npm-base-url" mapstructure:"npm-base-url"`
}