Replace core SBOM-creation API with builder pattern (#1383)

* remove existing cataloging API

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add file cataloging config

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add package cataloging config

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add configs for cross-cutting concerns

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename CLI option configs to not require import aliases later

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update all nested structs for the Catalog struct

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update Catalog cli options

- add new cataloger selection options (selection and default)
- remove the excludeBinaryOverlapByOwnership
- deprecate "catalogers" flag
- add new javascript configuration

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* migrate relationship capabilities to separate internal package

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor golang cataloger to use configuration options when creating packages

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* create internal object to facilitate reading from and writing to an SBOM

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* create a command-like object (task) to facilitate partial SBOM creation

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add cataloger selection capability

- be able to parse string expressions into a set of resolved actions against sets
- be able to use expressions to select/add/remove tasks to/from the final set of tasks to run

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add package, file, and environment related tasks

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update existing file catalogers to use nested UI elements

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add CreateSBOMConfig that drives the SBOM creation process

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* capture SBOM creation info as a struct

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add CreateSBOM() function

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix tests

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update docs with SBOM selection help + breaking changes

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix multiple override default inputs

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix deprecation flag printing to stdout

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor cataloger selection description to separate object

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* keep expression errors and show specific suggestions only

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address additional review feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address more review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* addressed additional PR review feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix file selection references

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove guess language data generation option

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tests for coordinatesForSelection

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename relationship attributes

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add descriptions to relationships config fields

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* improve documentation around configuration options

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add explicit errors around legacy config entries

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2024-01-12 17:39:13 -05:00
committed by GitHub
parent 1bffc6ef0c
commit b0ab75fd89
126 changed files with 5740 additions and 1841 deletions
+6 -5
View File
@@ -1,6 +1,7 @@
package commands
import (
"context"
"fmt"
"io"
"os"
@@ -66,7 +67,7 @@ func Attest(app clio.Application) *cobra.Command {
restoreStdout := ui.CaptureStdoutToTraceLog()
defer restoreStdout()
return runAttest(id, &opts, args[0])
return runAttest(cmd.Context(), id, &opts, args[0])
},
}, &opts)
}
@@ -98,7 +99,7 @@ func defaultAttestOutputOptions() options.Output {
}
//nolint:funlen
func runAttest(id clio.Identification, opts *attestOptions, userInput string) error {
func runAttest(ctx context.Context, id clio.Identification, opts *attestOptions, userInput string) error {
// TODO: what other validation here besides binary name?
if !commandExists(cosignBinName) {
return fmt.Errorf("'syft attest' requires cosign to be installed, however it does not appear to be on PATH")
@@ -111,7 +112,7 @@ func runAttest(id clio.Identification, opts *attestOptions, userInput string) er
}
defer os.Remove(f.Name())
s, err := generateSBOMForAttestation(id, &opts.Catalog, userInput)
s, err := generateSBOMForAttestation(ctx, id, &opts.Catalog, userInput)
if err != nil {
return fmt.Errorf("unable to build SBOM: %w", err)
}
@@ -245,7 +246,7 @@ func predicateType(outputName string) string {
}
}
func generateSBOMForAttestation(id clio.Identification, opts *options.Catalog, userInput string) (*sbom.SBOM, error) {
func generateSBOMForAttestation(ctx context.Context, id clio.Identification, opts *options.Catalog, userInput string) (*sbom.SBOM, error) {
src, err := getSource(opts, userInput, onlyContainerImages)
if err != nil {
@@ -260,7 +261,7 @@ func generateSBOMForAttestation(id clio.Identification, opts *options.Catalog, u
}
}()
s, err := generateSBOM(id, src, opts)
s, err := generateSBOM(ctx, id, src, opts)
if err != nil {
return nil, err
}
+2 -1
View File
@@ -2,6 +2,7 @@ package commands
import (
"bytes"
"context"
"fmt"
"os/exec"
"regexp"
@@ -258,7 +259,7 @@ func Test_buildSBOMForAttestation(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = require.NoError
}
_, err := generateSBOMForAttestation(tt.args.id, tt.args.opts, tt.args.userInput)
_, err := generateSBOMForAttestation(context.Background(), tt.args.id, tt.args.opts, tt.args.userInput)
tt.wantErr(t, err)
if err != nil {
return
+1 -1
View File
@@ -23,7 +23,7 @@ func Packages(app clio.Application, scanCmd *cobra.Command) *cobra.Command {
restoreStdout := ui.CaptureStdoutToTraceLog()
defer restoreStdout()
return runScan(id, opts, args[0])
return runScan(cmd.Context(), id, opts, args[0])
},
}, opts)
+300
View File
@@ -0,0 +1,300 @@
package commands
import (
"errors"
"fmt"
"testing"
"github.com/hashicorp/go-multierror"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/internal/task"
)
func Test_filterExpressionErrors_expressionErrorsHelp(t *testing.T) {
tests := []struct {
name string
err error
wantExpErrs []task.ErrInvalidExpression
wantErr assert.ErrorAssertionFunc
wantHelp string
}{
{
name: "no errors",
err: nil,
wantExpErrs: nil,
wantErr: assert.NoError,
wantHelp: "",
},
{
name: "single non-expression error is retained",
err: errors.New("foo"),
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
return assert.Equal(t, "foo", err.Error())
},
wantHelp: "",
},
{
name: "multiple non-expression sibling errors are retained",
err: func() error {
var err error
err = multierror.Append(err, errors.New("foo"))
err = multierror.Append(err, errors.New("bar"))
return err
}(),
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
// note: this is the default formatting from the hashicorp multierror object
expected := `2 errors occurred:
* foo
* bar
`
return assert.Equal(t, expected, err.Error())
},
wantHelp: "",
},
{
name: "has multiple expression errors (with sibling errors)",
err: func() error {
var err error
err = multierror.Append(err, errors.New("foo"))
err = multierror.Append(err, task.ErrInvalidExpression{Expression: "foo", Operation: task.AddOperation, Err: task.ErrTagsNotAllowed})
err = multierror.Append(err, errors.New("bar"))
err = multierror.Append(err, task.ErrInvalidExpression{Expression: "bar", Operation: task.SubSelectOperation, Err: task.ErrNamesNotAllowed})
err = multierror.Append(err, errors.New("last"))
return err
}(),
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
expected := `5 errors occurred:
* foo
* invalid expression: "foo": tags are not allowed with this operation (must use exact names)
* bar
* invalid expression: "bar": names are not allowed with this operation (must use tags)
* last
`
return assert.Equal(t, expected, err.Error())
},
wantExpErrs: []task.ErrInvalidExpression{
{Expression: "foo", Operation: task.AddOperation, Err: task.ErrTagsNotAllowed},
{Expression: "bar", Operation: task.SubSelectOperation, Err: task.ErrNamesNotAllowed},
},
wantHelp: `Suggestions:
❖ Given expression "--select-catalogers foo"
However, tags are not allowed with this operation (must use exact names).
Adding groups of catalogers may result in surprising behavior (create inaccurate SBOMs).
If you are certain this is what you want to do, use "--override-default-catalogers foo" instead.
❖ Given expression "--select-catalogers bar"
However, names are not allowed with this operation (must use tags).
It seems like you are intending to add a cataloger in addition to the default set.
... Did you mean "--select-catalogers +bar" instead?
`,
},
{
name: "has multiple expression errors (with error chains and sibling errors)",
err: func() error {
var err error
err = multierror.Append(err, fmt.Errorf("foo: %w", fmt.Errorf("bar: %w", errors.New("last"))))
err = multierror.Append(err, task.ErrInvalidExpression{Expression: "foo", Operation: task.AddOperation, Err: task.ErrTagsNotAllowed})
err = multierror.Append(err, task.ErrInvalidExpression{Expression: "bar", Operation: task.SubSelectOperation, Err: task.ErrNamesNotAllowed})
err = multierror.Append(err, errors.New("bottom"))
return fmt.Errorf("top: %w", fmt.Errorf("middle: %w", err))
}(),
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
expected := `top: middle: 4 errors occurred:
* foo: bar: last
* invalid expression: "foo": tags are not allowed with this operation (must use exact names)
* invalid expression: "bar": names are not allowed with this operation (must use tags)
* bottom
`
return assert.Equal(t, expected, err.Error())
},
wantExpErrs: []task.ErrInvalidExpression{
{Expression: "foo", Operation: task.AddOperation, Err: task.ErrTagsNotAllowed},
{Expression: "bar", Operation: task.SubSelectOperation, Err: task.ErrNamesNotAllowed},
},
wantHelp: `Suggestions:
❖ Given expression "--select-catalogers foo"
However, tags are not allowed with this operation (must use exact names).
Adding groups of catalogers may result in surprising behavior (create inaccurate SBOMs).
If you are certain this is what you want to do, use "--override-default-catalogers foo" instead.
❖ Given expression "--select-catalogers bar"
However, names are not allowed with this operation (must use tags).
It seems like you are intending to add a cataloger in addition to the default set.
... Did you mean "--select-catalogers +bar" instead?
`,
},
{
name: "has multiple expression errors (with error chains and sibling errors)",
err: func() error {
var err error
err = multierror.Append(err, fmt.Errorf("foo: %w", fmt.Errorf("bar: %w", errors.New("last"))))
err = multierror.Append(err, task.ErrInvalidExpression{Expression: "foo", Operation: task.AddOperation, Err: task.ErrTagsNotAllowed})
err = multierror.Append(err, task.ErrInvalidExpression{Expression: "bar", Operation: task.SubSelectOperation, Err: task.ErrNamesNotAllowed})
err = multierror.Append(err, errors.New("bottom"))
// note we wrap the top error in a chain
return fmt.Errorf("top: %w", fmt.Errorf("middle: %w", err))
}(),
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
expected := `top: middle: 4 errors occurred:
* foo: bar: last
* invalid expression: "foo": tags are not allowed with this operation (must use exact names)
* invalid expression: "bar": names are not allowed with this operation (must use tags)
* bottom
`
return assert.Equal(t, expected, err.Error())
},
wantExpErrs: []task.ErrInvalidExpression{
{Expression: "foo", Operation: task.AddOperation, Err: task.ErrTagsNotAllowed},
{Expression: "bar", Operation: task.SubSelectOperation, Err: task.ErrNamesNotAllowed},
},
wantHelp: `Suggestions:
❖ Given expression "--select-catalogers foo"
However, tags are not allowed with this operation (must use exact names).
Adding groups of catalogers may result in surprising behavior (create inaccurate SBOMs).
If you are certain this is what you want to do, use "--override-default-catalogers foo" instead.
❖ Given expression "--select-catalogers bar"
However, names are not allowed with this operation (must use tags).
It seems like you are intending to add a cataloger in addition to the default set.
... Did you mean "--select-catalogers +bar" instead?
`,
},
{
name: "preserve for any errors within ErrInvalidExpression types",
err: func() error {
var err error
err = multierror.Append(err, task.ErrInvalidExpression{Expression: "foo", Operation: task.AddOperation, Err: task.ErrTagsNotAllowed})
err = multierror.Append(err, task.ErrInvalidExpression{Expression: "bar", Operation: task.SubSelectOperation, Err: errors.New("explanation")}) // this is what makes this test different...
return err
}(),
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
// note: the errors are removed and the help text shows the enriched error help
expected := `2 errors occurred:
* invalid expression: "foo": tags are not allowed with this operation (must use exact names)
* invalid expression: "bar": explanation
`
return assert.Equal(t, expected, err.Error())
},
wantExpErrs: []task.ErrInvalidExpression{
{Expression: "foo", Operation: task.AddOperation, Err: task.ErrTagsNotAllowed},
{Expression: "bar", Operation: task.SubSelectOperation, Err: errors.New("explanation")},
},
wantHelp: `Suggestions:
❖ Given expression "--select-catalogers foo"
However, tags are not allowed with this operation (must use exact names).
Adding groups of catalogers may result in surprising behavior (create inaccurate SBOMs).
If you are certain this is what you want to do, use "--override-default-catalogers foo" instead.
`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotExpErrs := filterExpressionErrors(tt.err)
tt.wantErr(t, tt.err) // ensure the error still remains
assert.Equal(t, tt.wantExpErrs, gotExpErrs)
gotHelp := expressionErrorsHelp(gotExpErrs)
assert.Equal(t, tt.wantHelp, gotHelp)
})
}
}
func Test_expressionSuggestions(t *testing.T) {
tests := []struct {
name string
expErr task.ErrInvalidExpression
want string
}{
{
name: "no embedded error",
expErr: task.ErrInvalidExpression{
Expression: "example",
},
want: ``,
},
{
name: "general error",
expErr: task.ErrInvalidExpression{
Err: errors.New("general error message"),
Expression: "example",
},
want: ``,
},
{
name: "ErrUnknownNameOrTag with add operation",
expErr: task.ErrInvalidExpression{
Err: task.ErrUnknownNameOrTag,
Operation: task.AddOperation,
Expression: "+example",
},
want: ``,
},
{
name: "ErrUnknownNameOrTag with subselect operation",
expErr: task.ErrInvalidExpression{
Err: task.ErrUnknownNameOrTag,
Operation: task.SubSelectOperation,
Expression: "example",
},
want: ``,
},
{
name: "ErrNamesNotAllowed with subselect operator",
expErr: task.ErrInvalidExpression{
Err: task.ErrNamesNotAllowed,
Operation: task.SubSelectOperation,
Expression: "example",
},
want: ` ❖ Given expression "--select-catalogers example"
However, names are not allowed with this operation (must use tags).
It seems like you are intending to add a cataloger in addition to the default set.
... Did you mean "--select-catalogers +example" instead?
`,
},
{
name: "ErrTagsNotAllowed with add operation",
expErr: task.ErrInvalidExpression{
Err: task.ErrTagsNotAllowed,
Operation: task.AddOperation,
Expression: "+example",
},
want: ` ❖ Given expression "--select-catalogers +example"
However, tags are not allowed with this operation (must use exact names).
Adding groups of catalogers may result in surprising behavior (create inaccurate SBOMs).
If you are certain this is what you want to do, use "--override-default-catalogers example" instead.
`,
},
{
name: "ErrAllNotAllowed with subselect operation",
expErr: task.ErrInvalidExpression{
Err: task.ErrAllNotAllowed,
Operation: task.SubSelectOperation,
Expression: "example",
},
want: ` ❖ Given expression "--select-catalogers example"
However, you cannot use the 'all' operand in this context.
It seems like you are intending to use all catalogers (which is not recommended).
... Did you mean "--override-default-catalogers example" instead?
`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, expressionSuggetions(tt.expErr))
})
}
}
+1 -1
View File
@@ -25,7 +25,7 @@ func Root(app clio.Application, packagesCmd *cobra.Command) *cobra.Command {
restoreStdout := ui.CaptureStdoutToTraceLog()
defer restoreStdout()
return runScan(id, opts, args[0])
return runScan(cmd.Context(), id, opts, args[0])
},
}, opts)
}
+238 -46
View File
@@ -1,21 +1,27 @@
package commands
import (
"context"
"errors"
"fmt"
"os"
"reflect"
"strings"
"github.com/hashicorp/go-multierror"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/anchore/clio"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/cmd/syft/cli/eventloop"
"github.com/anchore/syft/cmd/syft/cli/options"
"github.com/anchore/syft/cmd/syft/internal/ui"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/internal/task"
"github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
@@ -89,11 +95,57 @@ func Scan(app clio.Application) *cobra.Command {
restoreStdout := ui.CaptureStdoutToTraceLog()
defer restoreStdout()
return runScan(id, opts, args[0])
return runScan(cmd.Context(), id, opts, args[0])
},
}, opts)
}
func (o *scanOptions) PostLoad() error {
return o.validateLegacyOptionsNotUsed()
}
func (o *scanOptions) validateLegacyOptionsNotUsed() error {
if o.Config.ConfigFile == "" {
return nil
}
// check for legacy config file shapes that are no longer valid
type legacyConfig struct {
BasePath *string `yaml:"base-path" json:"base-path" mapstructure:"base-path"`
DefaultImagePullSource *string `yaml:"default-image-pull-source" json:"default-image-pull-source" mapstructure:"default-image-pull-source"`
ExcludeBinaryOverlapByOwnership *bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"`
File any `yaml:"file" json:"file" mapstructure:"file"`
}
by, err := os.ReadFile(o.Config.ConfigFile)
if err != nil {
return fmt.Errorf("unable to read config file during validations %q: %w", o.Config.ConfigFile, err)
}
var legacy legacyConfig
if err := yaml.Unmarshal(by, &legacy); err != nil {
return fmt.Errorf("unable to parse config file during validations %q: %w", o.Config.ConfigFile, err)
}
if legacy.DefaultImagePullSource != nil {
return fmt.Errorf("the config file option 'default-image-pull-source' has been removed, please use 'source.image.default-pull-source' instead")
}
if legacy.ExcludeBinaryOverlapByOwnership != nil {
return fmt.Errorf("the config file option 'exclude-binary-overlap-by-ownership' has been removed, please use 'relationships.exclude-binary-packages-with-file-ownership-overlap' instead")
}
if legacy.BasePath != nil {
return fmt.Errorf("the config file option 'base-path' has been removed, please use 'source.base-path' instead")
}
if legacy.File != nil && reflect.TypeOf(legacy.File).Kind() == reflect.String {
return fmt.Errorf("the config file option 'file' has been removed, please use 'outputs' instead")
}
return nil
}
func validateScanArgs(cmd *cobra.Command, args []string) error {
return validateArgs(cmd, args, "an image/directory argument is required")
}
@@ -111,7 +163,7 @@ func validateArgs(cmd *cobra.Command, args []string, error string) error {
}
// nolint:funlen
func runScan(id clio.Identification, opts *scanOptions, userInput string) error {
func runScan(ctx context.Context, id clio.Identification, opts *scanOptions, userInput string) error {
writer, err := opts.SBOMWriter()
if err != nil {
return err
@@ -131,7 +183,7 @@ func runScan(id clio.Identification, opts *scanOptions, userInput string) error
}
}()
s, err := generateSBOM(id, src, &opts.Catalog)
s, err := generateSBOM(ctx, id, src, &opts.Catalog)
if err != nil {
return err
}
@@ -151,7 +203,7 @@ func getSource(opts *options.Catalog, userInput string, filters ...func(*source.
detection, err := source.Detect(
userInput,
source.DetectConfig{
DefaultImageSource: opts.DefaultImagePullSource,
DefaultImageSource: opts.Source.Image.DefaultPullSource,
},
)
if err != nil {
@@ -175,7 +227,7 @@ func getSource(opts *options.Catalog, userInput string, filters ...func(*source.
hashers, err := file.Hashers(opts.Source.File.Digests...)
if err != nil {
return nil, fmt.Errorf("invalid hash: %w", err)
return nil, fmt.Errorf("invalid hash algorithm: %w", err)
}
src, err := detection.NewSource(
@@ -190,7 +242,7 @@ func getSource(opts *options.Catalog, userInput string, filters ...func(*source.
Paths: opts.Exclusions,
},
DigestAlgorithms: hashers,
BasePath: opts.BasePath,
BasePath: opts.Source.BasePath,
},
)
@@ -204,52 +256,192 @@ func getSource(opts *options.Catalog, userInput string, filters ...func(*source.
return src, nil
}
func generateSBOM(id clio.Identification, src source.Source, opts *options.Catalog) (*sbom.SBOM, error) {
tasks, err := eventloop.Tasks(opts)
func generateSBOM(ctx context.Context, id clio.Identification, src source.Source, opts *options.Catalog) (*sbom.SBOM, error) {
s, err := syft.CreateSBOM(ctx, src, opts.ToSBOMConfig(id))
if err != nil {
expErrs := filterExpressionErrors(err)
notifyExpressionErrors(expErrs)
return nil, err
}
s := sbom.SBOM{
Source: src.Describe(),
Descriptor: sbom.Descriptor{
Name: id.Name,
Version: id.Version,
Configuration: opts,
},
}
err = buildRelationships(&s, src, tasks)
return &s, err
return s, nil
}
func buildRelationships(s *sbom.SBOM, src source.Source, tasks []eventloop.Task) error {
var errs error
func filterExpressionErrors(err error) []task.ErrInvalidExpression {
if err == nil {
return nil
}
var relationships []<-chan artifact.Relationship
for _, task := range tasks {
c := make(chan artifact.Relationship)
relationships = append(relationships, c)
go func(task eventloop.Task) {
err := eventloop.RunTask(task, &s.Artifacts, src, c)
if err != nil {
errs = multierror.Append(errs, err)
expErrs := processErrors(err)
return expErrs
}
// processErrors traverses error chains and multierror lists and returns all ErrInvalidExpression errors found
func processErrors(err error) []task.ErrInvalidExpression {
var result []task.ErrInvalidExpression
var processError func(...error)
processError = func(errs ...error) {
for _, e := range errs {
// note: using errors.As will result in surprising behavior (since that will traverse the error chain,
// potentially skipping over nodes in a list of errors)
if cerr, ok := e.(task.ErrInvalidExpression); ok {
result = append(result, cerr)
continue
}
var multiErr *multierror.Error
if errors.As(e, &multiErr) {
processError(multiErr.Errors...)
}
}(task)
}
s.Relationships = append(s.Relationships, mergeRelationships(relationships...)...)
return errs
}
func mergeRelationships(cs ...<-chan artifact.Relationship) (relationships []artifact.Relationship) {
for _, c := range cs {
for n := range c {
relationships = append(relationships, n)
}
}
return relationships
processError(err)
return result
}
func notifyExpressionErrors(expErrs []task.ErrInvalidExpression) {
helpText := expressionErrorsHelp(expErrs)
if helpText == "" {
return
}
bus.Notify(helpText)
}
func expressionErrorsHelp(expErrs []task.ErrInvalidExpression) string {
// enrich all errors found with CLI hints
if len(expErrs) == 0 {
return ""
}
sb := strings.Builder{}
sb.WriteString("Suggestions:\n\n")
found := false
for i, expErr := range expErrs {
help := expressionSuggetions(expErr)
if help == "" {
continue
}
found = true
sb.WriteString(help)
if i != len(expErrs)-1 {
sb.WriteString("\n")
}
}
if !found {
return ""
}
return sb.String()
}
const expressionHelpTemplate = " ❖ Given expression %q\n%s%s"
func expressionSuggetions(expErr task.ErrInvalidExpression) string {
if expErr.Err == nil {
return ""
}
hint := getHintPhrase(expErr)
if hint == "" {
return ""
}
return fmt.Sprintf(expressionHelpTemplate,
getExpression(expErr),
indentMsg(getExplanation(expErr)),
indentMsg(hint),
)
}
func indentMsg(msg string) string {
if msg == "" {
return ""
}
lines := strings.Split(msg, "\n")
for i, line := range lines {
lines[i] = " " + line
}
return strings.Join(lines, "\n") + "\n"
}
func getExpression(expErr task.ErrInvalidExpression) string {
flag := "--select-catalogers"
if expErr.Operation == task.SetOperation {
flag = "--override-default-catalogers"
}
return fmt.Sprintf("%s %s", flag, expErr.Expression)
}
func getExplanation(expErr task.ErrInvalidExpression) string {
err := expErr.Err
if errors.Is(err, task.ErrUnknownNameOrTag) {
noun := ""
switch expErr.Operation {
case task.AddOperation:
noun = "name"
case task.SubSelectOperation:
noun = "tag"
default:
noun = "name or tag"
}
return fmt.Sprintf("However, %q is not a recognized cataloger %s.", trimOperation(expErr.Expression), noun)
}
if errors.Is(err, task.ErrNamesNotAllowed) {
if expErr.Operation == task.SubSelectOperation {
return "However, " + err.Error() + ".\nIt seems like you are intending to add a cataloger in addition to the default set." // nolint:goconst
}
return "However, " + err.Error() + "." // nolint:goconst
}
if errors.Is(err, task.ErrTagsNotAllowed) {
return "However, " + err.Error() + ".\nAdding groups of catalogers may result in surprising behavior (create inaccurate SBOMs)." // nolint:goconst
}
if errors.Is(err, task.ErrAllNotAllowed) {
return "However, you " + err.Error() + ".\nIt seems like you are intending to use all catalogers (which is not recommended)."
}
if err != nil {
return "However, this is not valid: " + err.Error()
}
return ""
}
func getHintPhrase(expErr task.ErrInvalidExpression) string {
if errors.Is(expErr.Err, task.ErrUnknownNameOrTag) {
return ""
}
switch expErr.Operation {
case task.AddOperation:
if errors.Is(expErr.Err, task.ErrTagsNotAllowed) {
return fmt.Sprintf("If you are certain this is what you want to do, use %q instead.", "--override-default-catalogers "+trimOperation(expErr.Expression))
}
case task.SubSelectOperation:
didYouMean := "... Did you mean %q instead?"
if errors.Is(expErr.Err, task.ErrNamesNotAllowed) {
return fmt.Sprintf(didYouMean, "--select-catalogers +"+expErr.Expression)
}
if errors.Is(expErr.Err, task.ErrAllNotAllowed) {
return fmt.Sprintf(didYouMean, "--override-default-catalogers "+expErr.Expression)
}
}
return ""
}
func trimOperation(x string) string {
return strings.TrimLeft(x, "+-")
}
+66
View File
@@ -0,0 +1,66 @@
package commands
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/cmd/syft/cli/options"
)
func Test_scanOptions_validateLegacyOptionsNotUsed(t *testing.T) {
tests := []struct {
name string
cfg string
wantErr assert.ErrorAssertionFunc
}{
{
name: "no config file",
},
{
name: "config file with no legacy options",
cfg: "test-fixtures/scan-configs/no-legacy-options.yaml",
},
{
name: "config file with default image pull source legacy option",
cfg: "test-fixtures/scan-configs/with-default-pull-source.yaml",
wantErr: assertErrorContains("source.image.default-pull-source"),
},
{
name: "config file with exclude-binary-overlap-by-ownership legacy option",
cfg: "test-fixtures/scan-configs/with-exclude-binary-overlap-by-ownership.yaml",
wantErr: assertErrorContains("relationships.exclude-binary-packages-with-file-ownership-overlap"),
},
{
name: "config file with file string legacy option",
cfg: "test-fixtures/scan-configs/with-file-string.yaml",
wantErr: assertErrorContains("outputs"),
},
{
name: "config file with file section",
cfg: "test-fixtures/scan-configs/with-file-section.yaml",
},
{
name: "config file with base-path legacy option",
cfg: "test-fixtures/scan-configs/with-base-path.yaml",
wantErr: assertErrorContains("source.base-path"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = assert.NoError
}
o := &scanOptions{
Config: options.Config{ConfigFile: tt.cfg},
}
tt.wantErr(t, o.validateLegacyOptionsNotUsed())
})
}
}
func assertErrorContains(contains string) assert.ErrorAssertionFunc {
return func(t assert.TestingT, err error, i ...interface{}) bool {
return assert.ErrorContains(t, err, contains, i...)
}
}
@@ -0,0 +1,2 @@
# simple, valid syft config with no legacy fields used
parallelism: 5
@@ -0,0 +1 @@
base-path: "something"
@@ -0,0 +1 @@
default-image-pull-source: "something"
@@ -0,0 +1 @@
exclude-binary-overlap-by-ownership: true
@@ -0,0 +1,2 @@
file:
selection: all
@@ -0,0 +1 @@
file: path
-151
View File
@@ -1,151 +0,0 @@
package eventloop
import (
"github.com/anchore/syft/cmd/syft/cli/options"
"github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file/cataloger/filecontent"
"github.com/anchore/syft/syft/file/cataloger/filedigest"
"github.com/anchore/syft/syft/file/cataloger/filemetadata"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
type Task func(*sbom.Artifacts, source.Source) ([]artifact.Relationship, error)
func Tasks(opts *options.Catalog) ([]Task, error) {
var tasks []Task
generators := []func(opts *options.Catalog) (Task, error){
generateCatalogPackagesTask,
generateCatalogFileMetadataTask,
generateCatalogFileDigestsTask,
generateCatalogContentsTask,
}
for _, generator := range generators {
task, err := generator(opts)
if err != nil {
return nil, err
}
if task != nil {
tasks = append(tasks, task)
}
}
return tasks, nil
}
func generateCatalogPackagesTask(opts *options.Catalog) (Task, error) {
if !opts.Package.Cataloger.Enabled {
return nil, nil
}
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, opts.ToCatalogerConfig())
results.Packages = packageCatalog
results.LinuxDistribution = theDistro
return relationships, err
}
return task, nil
}
func generateCatalogFileMetadataTask(opts *options.Catalog) (Task, error) {
if !opts.FileMetadata.Cataloger.Enabled {
return nil, nil
}
metadataCataloger := filemetadata.NewCataloger()
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(opts.FileMetadata.Cataloger.GetScope())
if err != nil {
return nil, err
}
result, err := metadataCataloger.Catalog(resolver)
if err != nil {
return nil, err
}
results.FileMetadata = result
return nil, nil
}
return task, nil
}
func generateCatalogFileDigestsTask(opts *options.Catalog) (Task, error) {
if !opts.FileMetadata.Cataloger.Enabled {
return nil, nil
}
hashes, err := file.Hashers(opts.FileMetadata.Digests...)
if err != nil {
return nil, err
}
digestsCataloger := filedigest.NewCataloger(hashes)
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(opts.FileMetadata.Cataloger.GetScope())
if err != nil {
return nil, err
}
result, err := digestsCataloger.Catalog(resolver)
if err != nil {
return nil, err
}
results.FileDigests = result
return nil, nil
}
return task, nil
}
func generateCatalogContentsTask(opts *options.Catalog) (Task, error) {
if !opts.FileContents.Cataloger.Enabled {
return nil, nil
}
contentsCataloger, err := filecontent.NewCataloger(opts.FileContents.Globs, opts.FileContents.SkipFilesAboveSize) //nolint:staticcheck
if err != nil {
return nil, err
}
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(opts.FileContents.Cataloger.GetScope())
if err != nil {
return nil, err
}
result, err := contentsCataloger.Catalog(resolver)
if err != nil {
return nil, err
}
results.FileContents = result
return nil, nil
}
return task, nil
}
func RunTask(t Task, a *sbom.Artifacts, src source.Source, c chan<- artifact.Relationship) error {
defer close(c)
relationships, err := t(a, src)
if err != nil {
return err
}
for _, relationship := range relationships {
c <- relationship
}
return nil
}
+163 -106
View File
@@ -6,58 +6,141 @@ import (
"strings"
"github.com/iancoleman/strcase"
"github.com/mitchellh/go-homedir"
"github.com/scylladb/go-set/strset"
"github.com/anchore/clio"
"github.com/anchore/fangs"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/pkg/cataloger"
binaryCataloger "github.com/anchore/syft/syft/pkg/cataloger/binary"
golangCataloger "github.com/anchore/syft/syft/pkg/cataloger/golang"
javaCataloger "github.com/anchore/syft/syft/pkg/cataloger/java"
javascriptCataloger "github.com/anchore/syft/syft/pkg/cataloger/javascript"
"github.com/anchore/syft/syft/cataloging/filecataloging"
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
"github.com/anchore/syft/syft/file/cataloger/filecontent"
"github.com/anchore/syft/syft/pkg/cataloger/binary"
"github.com/anchore/syft/syft/pkg/cataloger/golang"
"github.com/anchore/syft/syft/pkg/cataloger/java"
"github.com/anchore/syft/syft/pkg/cataloger/javascript"
"github.com/anchore/syft/syft/pkg/cataloger/kernel"
pythonCataloger "github.com/anchore/syft/syft/pkg/cataloger/python"
"github.com/anchore/syft/syft/pkg/cataloger/python"
"github.com/anchore/syft/syft/source"
)
type Catalog struct {
Catalogers []string `yaml:"catalogers" json:"catalogers" mapstructure:"catalogers"`
Package pkg `yaml:"package" json:"package" mapstructure:"package"`
Golang golang `yaml:"golang" json:"golang" mapstructure:"golang"`
Java java `yaml:"java" json:"java" mapstructure:"java"`
Javascript javascript `yaml:"javascript" json:"javascript" mapstructure:"javascript"`
LinuxKernel linuxKernel `yaml:"linux-kernel" json:"linux-kernel" mapstructure:"linux-kernel"`
Python python `yaml:"python" json:"python" mapstructure:"python"`
FileMetadata fileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"`
FileContents fileContents `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"`
Registry registry `yaml:"registry" json:"registry" mapstructure:"registry"`
Exclusions []string `yaml:"exclude" json:"exclude" mapstructure:"exclude"`
Platform string `yaml:"platform" json:"platform" mapstructure:"platform"`
Name string `yaml:"name" json:"name" mapstructure:"name"`
Source sourceCfg `yaml:"source" json:"source" mapstructure:"source"`
Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel
DefaultImagePullSource string `yaml:"default-image-pull-source" json:"default-image-pull-source" mapstructure:"default-image-pull-source"` // specify default image pull source
BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths
ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files
// high-level cataloger configuration
Catalogers []string `yaml:"-" json:"catalogers" mapstructure:"catalogers"` // deprecated and not shown in yaml output
DefaultCatalogers []string `yaml:"default-catalogers" json:"default-catalogers" mapstructure:"default-catalogers"`
SelectCatalogers []string `yaml:"select-catalogers" json:"select-catalogers" mapstructure:"select-catalogers"`
Package packageConfig `yaml:"package" json:"package" mapstructure:"package"`
File fileConfig `yaml:"file" json:"file" mapstructure:"file"`
Scope string `yaml:"scope" json:"scope" mapstructure:"scope"`
Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel
Relationships relationshipsConfig `yaml:"relationships" json:"relationships" mapstructure:"relationships"`
// ecosystem-specific cataloger configuration
Golang golangConfig `yaml:"golang" json:"golang" mapstructure:"golang"`
Java javaConfig `yaml:"java" json:"java" mapstructure:"java"`
JavaScript javaScriptConfig `yaml:"javascript" json:"javascript" mapstructure:"javascript"`
LinuxKernel linuxKernelConfig `yaml:"linux-kernel" json:"linux-kernel" mapstructure:"linux-kernel"`
Python pythonConfig `yaml:"python" json:"python" mapstructure:"python"`
// configuration for the source (the subject being analyzed)
Registry registryConfig `yaml:"registry" json:"registry" mapstructure:"registry"`
Platform string `yaml:"platform" json:"platform" mapstructure:"platform"`
Name string `yaml:"name" json:"name" mapstructure:"name"` // deprecated
Source sourceConfig `yaml:"source" json:"source" mapstructure:"source"`
Exclusions []string `yaml:"exclude" json:"exclude" mapstructure:"exclude"`
}
var _ interface {
clio.FlagAdder
clio.PostLoader
fangs.FieldDescriber
} = (*Catalog)(nil)
func DefaultCatalog() Catalog {
return Catalog{
Package: defaultPkg(),
LinuxKernel: defaultLinuxKernel(),
FileMetadata: defaultFileMetadata(),
FileContents: defaultFileContents(),
Source: defaultSourceCfg(),
Parallelism: 1,
ExcludeBinaryOverlapByOwnership: true,
Scope: source.SquashedScope.String(),
Package: defaultPackageConfig(),
LinuxKernel: defaultLinuxKernelConfig(),
File: defaultFileConfig(),
Relationships: defaultRelationshipsConfig(),
Source: defaultSourceConfig(),
Parallelism: 1,
}
}
func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
return syft.DefaultCreateSBOMConfig().
WithTool(id.Name, id.Version).
WithParallelism(cfg.Parallelism).
WithRelationshipsConfig(cfg.ToRelationshipsConfig()).
WithSearchConfig(cfg.ToSearchConfig()).
WithPackagesConfig(cfg.ToPackagesConfig()).
WithFilesConfig(cfg.ToFilesConfig()).
WithCatalogerSelection(
pkgcataloging.NewSelectionRequest().
WithDefaults(cfg.DefaultCatalogers...).
WithExpression(cfg.SelectCatalogers...),
)
}
func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig {
return cataloging.SearchConfig{
Scope: source.ParseScope(cfg.Scope),
}
}
func (cfg Catalog) ToRelationshipsConfig() cataloging.RelationshipsConfig {
return cataloging.RelationshipsConfig{
PackageFileOwnership: cfg.Relationships.PackageFileOwnership,
PackageFileOwnershipOverlap: cfg.Relationships.PackageFileOwnershipOverlap,
// note: this option was surfaced in the syft application configuration before this relationships section was added
ExcludeBinaryPackagesWithFileOwnershipOverlap: cfg.Package.ExcludeBinaryOverlapByOwnership,
}
}
func (cfg Catalog) ToFilesConfig() filecataloging.Config {
hashers, err := intFile.Hashers(cfg.File.Metadata.Digests...)
if err != nil {
log.WithFields("error", err).Warn("unable to configure file hashers")
}
return filecataloging.Config{
Selection: cfg.File.Metadata.Selection,
Hashers: hashers,
Content: filecontent.Config{
Globs: cfg.File.Content.Globs,
SkipFilesAboveSize: cfg.File.Content.SkipFilesAboveSize,
},
}
}
func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
archiveSearch := cataloging.ArchiveSearchConfig{
IncludeIndexedArchives: cfg.Package.SearchIndexedArchives,
IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
}
return pkgcataloging.Config{
Binary: binary.DefaultCatalogerConfig(),
Golang: golang.DefaultCatalogerConfig().
WithSearchLocalModCacheLicenses(cfg.Golang.SearchLocalModCacheLicenses).
WithLocalModCacheDir(cfg.Golang.LocalModCacheDir).
WithSearchRemoteLicenses(cfg.Golang.SearchRemoteLicenses).
WithProxy(cfg.Golang.Proxy).
WithNoProxy(cfg.Golang.NoProxy),
JavaScript: javascript.DefaultCatalogerConfig().
WithSearchRemoteLicenses(cfg.JavaScript.SearchRemoteLicenses).
WithNpmBaseURL(cfg.JavaScript.NpmBaseURL),
LinuxKernel: kernel.LinuxKernelCatalogerConfig{
CatalogModules: cfg.LinuxKernel.CatalogModules,
},
Python: python.CatalogerConfig{
GuessUnpinnedRequirements: cfg.Python.GuessUnpinnedRequirements,
},
JavaArchive: java.DefaultArchiveCatalogerConfig().
WithUseNetwork(cfg.Java.UseNetwork).
WithMavenBaseURL(cfg.Java.MavenURL).
WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth),
}
}
@@ -66,7 +149,7 @@ func (cfg *Catalog) AddFlags(flags clio.FlagSet) {
for _, scope := range source.AllScopes {
validScopeValues = append(validScopeValues, strcase.ToDelimited(string(scope), '-'))
}
flags.StringVarP(&cfg.Package.Cataloger.Scope, "scope", "s",
flags.StringVarP(&cfg.Scope, "scope", "s",
fmt.Sprintf("selection of layers to catalog, options=%v", validScopeValues))
flags.StringVarP(&cfg.Platform, "platform", "",
@@ -78,6 +161,20 @@ func (cfg *Catalog) AddFlags(flags clio.FlagSet) {
flags.StringArrayVarP(&cfg.Catalogers, "catalogers", "",
"enable one or more package catalogers")
if pfp, ok := flags.(fangs.PFlagSetProvider); ok {
if err := pfp.PFlagSet().MarkDeprecated("catalogers", "use: override-default-catalogers and select-catalogers"); err != nil {
panic(err)
}
} else {
panic("unable to mark flags as deprecated")
}
flags.StringArrayVarP(&cfg.DefaultCatalogers, "override-default-catalogers", "",
"set the base set of catalogers to use (defaults to 'image' or 'directory' depending on the scan source)")
flags.StringArrayVarP(&cfg.SelectCatalogers, "select-catalogers", "",
"add, remove, and filter the catalogers to be used")
flags.StringVarP(&cfg.Source.Name, "name", "",
"set the name of the target being analyzed")
@@ -92,25 +189,15 @@ func (cfg *Catalog) AddFlags(flags clio.FlagSet) {
flags.StringVarP(&cfg.Source.Version, "source-version", "",
"set the version of the target being analyzed")
flags.StringVarP(&cfg.BasePath, "base-path", "",
flags.StringVarP(&cfg.Source.BasePath, "base-path", "",
"base directory for scanning, no links will be followed above this directory, and all paths will be reported relative to this directory")
}
func (cfg *Catalog) DescribeFields(descriptions fangs.FieldDescriptionSet) {
descriptions.Add(&cfg.Parallelism, "number of cataloger workers to run in parallel")
}
func (cfg *Catalog) PostLoad() error {
// parse options on this struct
var catalogers []string
for _, c := range cfg.Catalogers {
for _, f := range strings.Split(c, ",") {
catalogers = append(catalogers, strings.TrimSpace(f))
}
}
sort.Strings(catalogers)
cfg.Catalogers = catalogers
if err := checkDefaultSourceValues(cfg.DefaultImagePullSource); err != nil {
return err
}
if cfg.Name != "" {
log.Warnf("name parameter is deprecated. please use: source-name. name will be removed in a future version")
if cfg.Source.Name == "" {
@@ -118,66 +205,36 @@ func (cfg *Catalog) PostLoad() error {
}
}
return nil
}
usingLegacyCatalogers := len(cfg.Catalogers) > 0
usingNewCatalogers := len(cfg.DefaultCatalogers) > 0 || len(cfg.SelectCatalogers) > 0
func (cfg Catalog) ToCatalogerConfig() cataloger.Config {
return cataloger.Config{
Search: cataloger.SearchConfig{
IncludeIndexedArchives: cfg.Package.SearchIndexedArchives,
IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
Scope: cfg.Package.Cataloger.GetScope(),
},
Catalogers: cfg.Catalogers,
Parallelism: cfg.Parallelism,
Golang: golangCataloger.DefaultCatalogerConfig().
WithSearchLocalModCacheLicenses(cfg.Golang.SearchLocalModCacheLicenses).
WithLocalModCacheDir(cfg.Golang.LocalModCacheDir).
WithSearchRemoteLicenses(cfg.Golang.SearchRemoteLicenses).
WithProxy(cfg.Golang.Proxy).
WithNoProxy(cfg.Golang.NoProxy),
LinuxKernel: kernel.LinuxKernelCatalogerConfig{
CatalogModules: cfg.LinuxKernel.CatalogModules,
},
Java: javaCataloger.DefaultArchiveCatalogerConfig().
WithUseNetwork(cfg.Java.UseNetwork).
WithMavenBaseURL(cfg.Java.MavenURL).
WithArchiveTraversal(
cataloging.ArchiveSearchConfig{
IncludeIndexedArchives: cfg.Package.SearchIndexedArchives,
IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
},
cfg.Java.MaxParentRecursiveDepth),
Javascript: javascriptCataloger.DefaultCatalogerConfig().
WithSearchRemoteLicenses(cfg.Javascript.SearchRemoteLicenses).
WithNpmBaseURL(cfg.Javascript.NpmBaseURL),
Binary: binaryCataloger.DefaultCatalogerConfig(),
Python: pythonCataloger.CatalogerConfig{
GuessUnpinnedRequirements: cfg.Python.GuessUnpinnedRequirements,
},
ExcludeBinaryOverlapByOwnership: cfg.ExcludeBinaryOverlapByOwnership,
}
}
var validDefaultSourceValues = []string{"registry", "docker", "podman", ""}
func checkDefaultSourceValues(source string) error {
validValues := strset.New(validDefaultSourceValues...)
if !validValues.Has(source) {
validValuesString := strings.Join(validDefaultSourceValues, ", ")
return fmt.Errorf("%s is not a valid default source; please use one of the following: %s''", source, validValuesString)
if usingLegacyCatalogers && usingNewCatalogers {
return fmt.Errorf("cannot use both 'catalogers' and 'select-catalogers'/'default-catalogers' flags")
}
return nil
}
func expandFilePath(file string) (string, error) {
if file != "" {
expandedPath, err := homedir.Expand(file)
if err != nil {
return "", fmt.Errorf("unable to expand file path=%q: %w", file, err)
flatten := func(l []string) []string {
var out []string
for _, v := range l {
for _, s := range strings.Split(v, ",") {
out = append(out, strings.TrimSpace(s))
}
}
file = expandedPath
sort.Strings(out)
return out
}
return file, nil
cfg.Catalogers = flatten(cfg.Catalogers)
cfg.DefaultCatalogers = flatten(cfg.DefaultCatalogers)
cfg.SelectCatalogers = flatten(cfg.SelectCatalogers)
// for backwards compatibility
cfg.DefaultCatalogers = append(cfg.DefaultCatalogers, cfg.Catalogers...)
s := source.ParseScope(cfg.Scope)
if s == source.UnknownScope {
return fmt.Errorf("bad scope value %q", cfg.Scope)
}
return nil
}
+72
View File
@@ -0,0 +1,72 @@
package options
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCatalog_PostLoad(t *testing.T) {
tests := []struct {
name string
options Catalog
assert func(t *testing.T, options Catalog)
wantErr assert.ErrorAssertionFunc
}{
{
name: "mutually exclusive cataloger flags (cat / def-cat)",
options: Catalog{
Catalogers: []string{"foo,bar", "42"},
DefaultCatalogers: []string{"some,thing"},
Scope: "squashed",
},
wantErr: assert.Error,
},
{
name: "mutually exclusive cataloger flags (cat / sel-cat)",
options: Catalog{
Catalogers: []string{"foo,bar", "42"},
SelectCatalogers: []string{"some,thing"},
Scope: "squashed",
},
wantErr: assert.Error,
},
{
name: "allow old cataloger flags",
options: Catalog{
Catalogers: []string{"foo,bar"},
Scope: "squashed",
},
assert: func(t *testing.T, options Catalog) {
assert.Equal(t, []string{"bar", "foo"}, options.DefaultCatalogers) // note: sorted order
assert.Equal(t, []string{"bar", "foo"}, options.Catalogers) // note: sorted order
},
},
{
name: "allow new cataloger flags",
options: Catalog{
SelectCatalogers: []string{"foo,bar", "42"},
DefaultCatalogers: []string{"some,thing"},
Scope: "squashed",
},
assert: func(t *testing.T, options Catalog) {
assert.Equal(t, []string{"42", "bar", "foo"}, options.SelectCatalogers) // note: sorted order
assert.Equal(t, []string{"some", "thing"}, options.DefaultCatalogers) // note: sorted order
assert.Empty(t, options.Catalogers)
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = assert.NoError
}
tt.wantErr(t, tt.options.PostLoad(), fmt.Sprintf("PostLoad()"))
if tt.assert != nil {
tt.assert(t, tt.options)
}
})
}
}
+6
View File
@@ -1,6 +1,12 @@
package options
import "github.com/anchore/fangs"
// Config holds a reference to the specific config file that was used to load application configuration
type Config struct {
ConfigFile string `yaml:"config" json:"config" mapstructure:"config"`
}
func (cfg *Config) DescribeFields(descriptions fangs.FieldDescriptionSet) {
descriptions.Add(&cfg.ConfigFile, "the configuration file that was used to load application configuration")
}
+43
View File
@@ -0,0 +1,43 @@
package options
import (
"fmt"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft/file"
)
type fileConfig struct {
Metadata fileMetadata `yaml:"metadata" json:"metadata" mapstructure:"metadata"`
Content fileContent `yaml:"content" json:"content" mapstructure:"content"`
}
type fileMetadata struct {
Selection file.Selection `yaml:"selection" json:"selection" mapstructure:"selection"`
Digests []string `yaml:"digests" json:"digests" mapstructure:"digests"`
}
type fileContent struct {
SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"`
Globs []string `yaml:"globs" json:"globs" mapstructure:"globs"`
}
func defaultFileConfig() fileConfig {
return fileConfig{
Metadata: fileMetadata{
Selection: file.FilesOwnedByPackageSelection,
Digests: []string{"sha1", "sha256"},
},
Content: fileContent{
SkipFilesAboveSize: 250 * intFile.KB,
},
}
}
func (c *fileConfig) PostLoad() error {
switch c.Metadata.Selection {
case file.NoFilesSelection, file.FilesOwnedByPackageSelection, file.AllFilesSelection:
return nil
}
return fmt.Errorf("invalid file metadata selection: %q", c.Metadata.Selection)
}
-21
View File
@@ -1,21 +0,0 @@
package options
import (
"github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft/source"
)
type fileContents struct {
Cataloger scope `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"`
Globs []string `yaml:"globs" json:"globs" mapstructure:"globs"`
}
func defaultFileContents() fileContents {
return fileContents{
Cataloger: scope{
Scope: source.SquashedScope.String(),
},
SkipFilesAboveSize: 1 * file.MB,
}
}
-19
View File
@@ -1,19 +0,0 @@
package options
import (
"github.com/anchore/syft/syft/source"
)
type fileMetadata struct {
Cataloger scope `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
Digests []string `yaml:"digests" json:"digests" mapstructure:"digests"`
}
func defaultFileMetadata() fileMetadata {
return fileMetadata{
Cataloger: scope{
Scope: source.SquashedScope.String(),
},
Digests: []string{"sha256"},
}
}
+1 -1
View File
@@ -1,6 +1,6 @@
package options
type golang struct {
type golangConfig struct {
SearchLocalModCacheLicenses bool `json:"search-local-mod-cache-licenses" yaml:"search-local-mod-cache-licenses" mapstructure:"search-local-mod-cache-licenses"`
LocalModCacheDir string `json:"local-mod-cache-dir" yaml:"local-mod-cache-dir" mapstructure:"local-mod-cache-dir"`
SearchRemoteLicenses bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
+1 -1
View File
@@ -1,6 +1,6 @@
package options
type java struct {
type javaConfig struct {
UseNetwork bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"`
MavenURL string `yaml:"maven-url" json:"maven-url" mapstructure:"maven-url"`
MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"`
+1 -1
View File
@@ -1,6 +1,6 @@
package options
type javascript struct {
type javaScriptConfig struct {
SearchRemoteLicenses bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"`
NpmBaseURL string `json:"npm-base-url" yaml:"npm-base-url" mapstructure:"npm-base-url"`
}
+3 -3
View File
@@ -1,11 +1,11 @@
package options
type linuxKernel struct {
type linuxKernelConfig struct {
CatalogModules bool `json:"catalog-modules" yaml:"catalog-modules" mapstructure:"catalog-modules"`
}
func defaultLinuxKernel() linuxKernel {
return linuxKernel{
func defaultLinuxKernelConfig() linuxKernelConfig {
return linuxKernelConfig{
CatalogModules: true,
}
}
+1 -1
View File
@@ -96,7 +96,7 @@ func (o Output) SBOMWriter() (sbom.Writer, error) {
}
}
return makeSBOMWriter(o.Outputs, o.File, encoders)
return makeSBOMWriter(o.Outputs, o.LegacyFile, encoders)
}
func (o Output) OutputNameSet() *strset.Set {
+22 -7
View File
@@ -1,6 +1,10 @@
package options
import (
"fmt"
"github.com/mitchellh/go-homedir"
"github.com/anchore/clio"
"github.com/anchore/fangs"
"github.com/anchore/syft/syft/sbom"
@@ -13,13 +17,13 @@ var _ interface {
// Deprecated: OutputFile supports the --file to write the SBOM output to
type OutputFile struct {
Enabled bool `yaml:"-" json:"-" mapstructure:"-"`
File string `yaml:"file" json:"file" mapstructure:"file"`
Enabled bool `yaml:"-" json:"-" mapstructure:"-"`
LegacyFile string `yaml:"-" json:"-" mapstructure:"legacyFile"`
}
func (o *OutputFile) AddFlags(flags clio.FlagSet) {
if o.Enabled {
flags.StringVarP(&o.File, "file", "",
flags.StringVarP(&o.LegacyFile, "file", "",
"file to write the default report output to (default is STDOUT)")
if pfp, ok := flags.(fangs.PFlagSetProvider); ok {
@@ -33,12 +37,12 @@ func (o *OutputFile) PostLoad() error {
if !o.Enabled {
return nil
}
if o.File != "" {
file, err := expandFilePath(o.File)
if o.LegacyFile != "" {
file, err := expandFilePath(o.LegacyFile)
if err != nil {
return err
}
o.File = file
o.LegacyFile = file
}
return nil
}
@@ -47,10 +51,21 @@ func (o *OutputFile) SBOMWriter(f sbom.FormatEncoder) (sbom.Writer, error) {
if !o.Enabled {
return nil, nil
}
writer, err := newSBOMMultiWriter(newSBOMWriterDescription(f, o.File))
writer, err := newSBOMMultiWriter(newSBOMWriterDescription(f, o.LegacyFile))
if err != nil {
return nil, err
}
return writer, nil
}
func expandFilePath(file string) (string, error) {
if file != "" {
expandedPath, err := homedir.Expand(file)
if err != nil {
return "", fmt.Errorf("unable to expand file path=%q: %w", file, err)
}
file = expandedPath
}
return file, nil
}
+11 -16
View File
@@ -1,23 +1,18 @@
package options
import (
"github.com/anchore/syft/syft/pkg/cataloger"
)
import "github.com/anchore/syft/syft/cataloging"
type pkg struct {
Cataloger scope `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
SearchUnindexedArchives bool `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
SearchIndexedArchives bool `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
type packageConfig struct {
SearchUnindexedArchives bool `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
SearchIndexedArchives bool `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files
}
func defaultPkg() pkg {
c := cataloger.DefaultSearchConfig()
return pkg{
SearchIndexedArchives: c.IncludeIndexedArchives,
SearchUnindexedArchives: c.IncludeUnindexedArchives,
Cataloger: scope{
Enabled: true,
Scope: c.Scope.String(),
},
func defaultPackageConfig() packageConfig {
c := cataloging.DefaultArchiveSearchConfig()
return packageConfig{
SearchIndexedArchives: c.IncludeIndexedArchives,
SearchUnindexedArchives: c.IncludeUnindexedArchives,
ExcludeBinaryOverlapByOwnership: true,
}
}
+1 -1
View File
@@ -1,5 +1,5 @@
package options
type python struct {
type pythonConfig struct {
GuessUnpinnedRequirements bool `json:"guess-unpinned-requirements" yaml:"guess-unpinned-requirements" mapstructure:"guess-unpinned-requirements"`
}
+4 -4
View File
@@ -18,16 +18,16 @@ type RegistryCredentials struct {
TLSKey string `yaml:"tls-key,omitempty" json:"tls-key,omitempty" mapstructure:"tls-key"`
}
type registry struct {
type registryConfig struct {
InsecureSkipTLSVerify bool `yaml:"insecure-skip-tls-verify" json:"insecure-skip-tls-verify" mapstructure:"insecure-skip-tls-verify"`
InsecureUseHTTP bool `yaml:"insecure-use-http" json:"insecure-use-http" mapstructure:"insecure-use-http"`
Auth []RegistryCredentials `yaml:"auth" json:"auth" mapstructure:"auth"`
CACert string `yaml:"ca-cert" json:"ca-cert" mapstructure:"ca-cert"`
}
var _ clio.PostLoader = (*registry)(nil)
var _ clio.PostLoader = (*registryConfig)(nil)
func (cfg *registry) PostLoad() error {
func (cfg *registryConfig) PostLoad() error {
// there may be additional credentials provided by env var that should be appended to the set of credentials
authority, username, password, token, tlsCert, tlsKey :=
os.Getenv("SYFT_REGISTRY_AUTH_AUTHORITY"),
@@ -62,7 +62,7 @@ func hasNonEmptyCredentials(username, password, token, tlsCert, tlsKey string) b
return hasUserPass || hasToken || hasTLSMaterial
}
func (cfg *registry) ToOptions() *image.RegistryOptions {
func (cfg *registryConfig) ToOptions() *image.RegistryOptions {
var auth = make([]image.RegistryCredentials, len(cfg.Auth))
for i, a := range cfg.Auth {
auth[i] = image.RegistryCredentials{
+6 -6
View File
@@ -72,19 +72,19 @@ func TestHasNonEmptyCredentials(t *testing.T) {
func Test_registry_ToOptions(t *testing.T) {
tests := []struct {
name string
input registry
input registryConfig
expected image.RegistryOptions
}{
{
name: "no registry options",
input: registry{},
input: registryConfig{},
expected: image.RegistryOptions{
Credentials: []image.RegistryCredentials{},
},
},
{
name: "set InsecureSkipTLSVerify",
input: registry{
input: registryConfig{
InsecureSkipTLSVerify: true,
},
expected: image.RegistryOptions{
@@ -94,7 +94,7 @@ func Test_registry_ToOptions(t *testing.T) {
},
{
name: "set InsecureUseHTTP",
input: registry{
input: registryConfig{
InsecureUseHTTP: true,
},
expected: image.RegistryOptions{
@@ -104,7 +104,7 @@ func Test_registry_ToOptions(t *testing.T) {
},
{
name: "set all bool options",
input: registry{
input: registryConfig{
InsecureSkipTLSVerify: true,
InsecureUseHTTP: true,
},
@@ -116,7 +116,7 @@ func Test_registry_ToOptions(t *testing.T) {
},
{
name: "provide all tls configuration",
input: registry{
input: registryConfig{
CACert: "ca.crt",
InsecureSkipTLSVerify: true,
Auth: []RegistryCredentials{
+22
View File
@@ -0,0 +1,22 @@
package options
import "github.com/anchore/fangs"
var _ fangs.FieldDescriber = (*relationshipsConfig)(nil)
type relationshipsConfig struct {
PackageFileOwnership bool `mapstructure:"package-file-ownership" json:"package-file-ownership" yaml:"package-file-ownership"`
PackageFileOwnershipOverlap bool `mapstructure:"package-file-ownership-overlap" json:"package-file-ownership-overlap" yaml:"package-file-ownership-overlap"`
}
func defaultRelationshipsConfig() relationshipsConfig {
return relationshipsConfig{
PackageFileOwnership: true,
PackageFileOwnershipOverlap: true,
}
}
func (r *relationshipsConfig) DescribeFields(descriptions fangs.FieldDescriptionSet) {
descriptions.Add(&r.PackageFileOwnership, "include package-to-file relationships that indicate which files are owned by which packages.")
descriptions.Add(&r.PackageFileOwnershipOverlap, "include package-to-package relationships that indicate one package is owned by another due to files claimed to be owned by one package are also evidence of another package's existence.")
}
-27
View File
@@ -1,27 +0,0 @@
package options
import (
"fmt"
"github.com/anchore/clio"
"github.com/anchore/syft/syft/source"
)
type scope struct {
Enabled bool `yaml:"enabled" json:"enabled" mapstructure:"enabled"`
Scope string `yaml:"scope" json:"scope" mapstructure:"scope"`
}
var _ clio.PostLoader = (*scope)(nil)
func (opt *scope) PostLoad() error {
s := opt.GetScope()
if s == source.UnknownScope {
return fmt.Errorf("bad scope value %v", opt.Scope)
}
return nil
}
func (opt scope) GetScope() source.Scope {
return source.ParseScope(opt.Scope)
}
+38 -6
View File
@@ -1,19 +1,51 @@
package options
type sourceCfg struct {
Name string `json:"name" yaml:"name" mapstructure:"name"`
Version string `json:"version" yaml:"version" mapstructure:"version"`
File fileSource `json:"file" yaml:"file" mapstructure:"file"`
import (
"fmt"
"strings"
"github.com/scylladb/go-set/strset"
)
type sourceConfig struct {
Name string `json:"name" yaml:"name" mapstructure:"name"`
Version string `json:"version" yaml:"version" mapstructure:"version"`
BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths
File fileSource `json:"file" yaml:"file" mapstructure:"file"`
Image imageSource `json:"image" yaml:"image" mapstructure:"image"`
}
type fileSource struct {
Digests []string `json:"digests" yaml:"digests" mapstructure:"digests"`
}
func defaultSourceCfg() sourceCfg {
return sourceCfg{
type imageSource struct {
DefaultPullSource string `json:"default-pull-source" yaml:"default-pull-source" mapstructure:"default-pull-source"`
}
func defaultSourceConfig() sourceConfig {
return sourceConfig{
File: fileSource{
Digests: []string{"sha256"},
},
Image: imageSource{
DefaultPullSource: "",
},
}
}
func (c imageSource) PostLoad() error {
return checkDefaultSourceValues(c.DefaultPullSource)
}
var validDefaultSourceValues = []string{"registry", "docker", "podman", ""}
func checkDefaultSourceValues(source string) error {
validValues := strset.New(validDefaultSourceValues...)
if !validValues.Has(source) {
validValuesString := strings.Join(validDefaultSourceValues, ", ")
return fmt.Errorf("%s is not a valid default source; please use one of the following: %s''", source, validValuesString)
}
return nil
}
+6
View File
@@ -1,5 +1,7 @@
package options
import "github.com/anchore/fangs"
type UpdateCheck struct {
CheckForAppUpdate bool `yaml:"check-for-app-update" json:"check-for-app-update" mapstructure:"check-for-app-update"` // whether to check for an application update on start up or not
}
@@ -9,3 +11,7 @@ func DefaultUpdateCheck() UpdateCheck {
CheckForAppUpdate: true,
}
}
func (cfg *UpdateCheck) DescribeFields(descriptions fangs.FieldDescriptionSet) {
descriptions.Add(&cfg.CheckForAppUpdate, "whether to check for an application update on start up or not")
}
@@ -1,8 +0,0 @@
[TestHandler_handleFileDigestsCatalogerStarted/cataloging_in_progress - 1]
⠋ Cataloging file digests ━━━━━━━━━━━━━━━━━━━━ [current]
---
[TestHandler_handleFileDigestsCatalogerStarted/cataloging_complete - 1]
✔ Cataloged file digests
---
@@ -1,8 +0,0 @@
[TestHandler_handleFileMetadataCatalogerStarted/cataloging_in_progress - 1]
⠋ Cataloging file metadata ━━━━━━━━━━━━━━━━━━━━ [current]
---
[TestHandler_handleFileMetadataCatalogerStarted/cataloging_complete - 1]
✔ Cataloged file metadata
---
@@ -1,16 +0,0 @@
[TestHandler_handlePackageCatalogerStarted/cataloging_in_progress - 1]
⠋ Cataloging packages [50 packages]
---
[TestHandler_handlePackageCatalogerStarted/cataloging_only_files_complete - 1]
⠋ Cataloging packages [50 packages]
---
[TestHandler_handlePackageCatalogerStarted/cataloging_only_packages_complete - 1]
⠋ Cataloging packages [100 packages]
---
[TestHandler_handlePackageCatalogerStarted/cataloging_complete - 1]
✔ Cataloged packages [100 packages]
---
@@ -1,8 +0,0 @@
[TestHandler_handleSecretsCatalogerStarted/cataloging_in_progress - 1]
⠋ Cataloging secrets ━━━━━━━━━━━━━━━━━━━━ [64 secrets]
---
[TestHandler_handleSecretsCatalogerStarted/cataloging_complete - 1]
✔ Cataloged secrets [64 secrets]
---