Replace core SBOM-creation API with builder pattern (#1383)

* remove existing cataloging API

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add file cataloging config

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add package cataloging config

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add configs for cross-cutting concerns

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename CLI option configs to not require import aliases later

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update all nested structs for the Catalog struct

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update Catalog cli options

- add new cataloger selection options (selection and default)
- remove the excludeBinaryOverlapByOwnership
- deprecate "catalogers" flag
- add new javascript configuration

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* migrate relationship capabilities to separate internal package

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor golang cataloger to use configuration options when creating packages

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* create internal object to facilitate reading from and writing to an SBOM

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* create a command-like object (task) to facilitate partial SBOM creation

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add cataloger selection capability

- be able to parse string expressions into a set of resolved actions against sets
- be able to use expressions to select/add/remove tasks to/from the final set of tasks to run

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add package, file, and environment related tasks

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update existing file catalogers to use nested UI elements

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add CreateSBOMConfig that drives the SBOM creation process

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* capture SBOM creation info as a struct

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add CreateSBOM() function

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix tests

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update docs with SBOM selection help + breaking changes

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix multiple override default inputs

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix deprecation flag printing to stdout

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor cataloger selection description to separate object

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* keep expression errors and show specific suggestions only

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address additional review feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address more review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* addressed additional PR review feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix file selection references

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove guess language data generation option

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tests for coordinatesForSelection

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename relationship attributes

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add descriptions to relationships config fields

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* improve documentation around configuration options

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add explicit errors around legacy config entries

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2024-01-12 17:39:13 -05:00
committed by GitHub
parent 1bffc6ef0c
commit b0ab75fd89
126 changed files with 5740 additions and 1841 deletions
+24
View File
@@ -0,0 +1,24 @@
package task
import (
"context"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
)
// TODO: add tui element here?
func NewEnvironmentTask() Task {
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
release := linux.IdentifyRelease(resolver)
if release != nil {
builder.SetLinuxDistribution(*release)
}
return nil
}
return NewTask("environment-cataloger", fn)
}
+72
View File
@@ -0,0 +1,72 @@
package task
import (
"context"
"fmt"
"runtime/debug"
"sync"
"github.com/hashicorp/go-multierror"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
)
type Executor struct {
numWorkers int
tasks chan Task
}
func NewTaskExecutor(tasks []Task, numWorkers int) *Executor {
p := &Executor{
numWorkers: numWorkers,
tasks: make(chan Task, len(tasks)),
}
for i := range tasks {
p.tasks <- tasks[i]
}
close(p.tasks)
return p
}
func (p *Executor) Execute(ctx context.Context, resolver file.Resolver, s sbomsync.Builder, prog *monitor.CatalogerTaskProgress) error {
var errs error
wg := &sync.WaitGroup{}
for i := 0; i < p.numWorkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
tsk, ok := <-p.tasks
if !ok {
return
}
if err := runTaskSafely(ctx, tsk, resolver, s); err != nil {
errs = multierror.Append(errs, fmt.Errorf("failed to run task: %w", err))
prog.SetError(err)
}
prog.Increment()
}
}()
}
wg.Wait()
return errs
}
func runTaskSafely(ctx context.Context, t Task, resolver file.Resolver, s sbomsync.Builder) (err error) {
// handle individual cataloger panics
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("%v at:\n%s", e, string(debug.Stack()))
}
}()
return t.Execute(ctx, resolver, s)
}
+27
View File
@@ -0,0 +1,27 @@
package task
import (
"context"
"testing"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
)
func Test_TaskExecutor_PanicHandling(t *testing.T) {
tsk := NewTask("panicking-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
panic("something bad happened")
})
ex := NewTaskExecutor([]Task{tsk}, 1)
err := ex.Execute(context.Background(), nil, nil, &monitor.CatalogerTaskProgress{
Manual: progress.NewManual(-1),
})
require.Error(t, err)
require.Contains(t, err.Error(), "executor_test.go")
}
+238
View File
@@ -0,0 +1,238 @@
package task
import (
"fmt"
"regexp"
"sort"
"github.com/hashicorp/go-multierror"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
)
var expressionNodePattern = regexp.MustCompile(`^([a-zA-Z0-9][a-zA-Z0-9-+]*)+$`)
const (
SetOperation Operation = "set"
AddOperation Operation = "add"
SubSelectOperation Operation = "sub-select"
RemoveOperation Operation = "remove"
)
var (
ErrEmptyToken = fmt.Errorf("no value given")
ErrInvalidToken = fmt.Errorf("invalid token given: only alphanumeric characters and hyphens are allowed")
ErrInvalidOperator = fmt.Errorf("invalid operator given")
ErrUnknownNameOrTag = fmt.Errorf("unknown name or tag given")
ErrTagsNotAllowed = fmt.Errorf("tags are not allowed with this operation (must use exact names)")
ErrNamesNotAllowed = fmt.Errorf("names are not allowed with this operation (must use tags)")
ErrAllNotAllowed = fmt.Errorf("cannot use the 'all' operand in this context")
)
// ErrInvalidExpression represents an expression that cannot be parsed or can be parsed but is logically invalid.
type ErrInvalidExpression struct {
Expression string
Operation Operation
Err error
}
func (e ErrInvalidExpression) Error() string {
return fmt.Sprintf("invalid expression: %q: %s", e.Expression, e.Err.Error())
}
func newErrInvalidExpression(exp string, op Operation, err error) ErrInvalidExpression {
return ErrInvalidExpression{
Expression: exp,
Operation: op,
Err: err,
}
}
// Expression represents a single operation-operand pair with (all validation errors).
// E.g. "+foo", "-bar", or "something" are all expressions. Some validations are relevant to not only the
// syntax (operation and operator) but other are sensitive to the context of the operand (e.g. if a given operand
// is a tag or a name, validated against the operation).
type Expression struct {
Operation Operation
Operand string
Errors []error
}
// Operation represents the type of operation to perform on the operand (set, add, remove, sub-select).
type Operation string
// Expressions represents a list of expressions.
type Expressions []Expression
// expressionContext represents all information needed to validate an expression (e.g. the set of all tasks and their tags).
type expressionContext struct {
Names *strset.Set
Tags *strset.Set
}
func newExpressionContext(ts []Task) *expressionContext {
ec := &expressionContext{
Names: strset.New(tasks(ts).Names()...),
Tags: strset.New(tasks(ts).Tags()...),
}
ec.Tags.Add("all")
return ec
}
// newExpression creates a new validated Expression object relative to the task names and tags.
func (ec expressionContext) newExpression(exp string, operation Operation, token string) Expression {
if token == "" {
return Expression{
Operation: operation,
Operand: token,
Errors: []error{newErrInvalidExpression(exp, operation, ErrEmptyToken)},
}
}
if !isValidNode(token) {
return Expression{
Operation: operation,
Operand: token,
Errors: []error{newErrInvalidExpression(exp, operation, ErrInvalidToken)},
}
}
var err error
switch operation {
case SetOperation, RemoveOperation:
// names and tags allowed
if !ec.Tags.Has(token) && !ec.Names.Has(token) {
err = newErrInvalidExpression(exp, operation, ErrUnknownNameOrTag)
}
case AddOperation:
// only names are allowed
if !ec.Names.Has(token) {
if ec.Tags.Has(token) {
err = newErrInvalidExpression(exp, operation, ErrTagsNotAllowed)
} else {
err = newErrInvalidExpression(exp, operation, ErrUnknownNameOrTag)
}
}
case SubSelectOperation:
if token == "all" {
// special case: we cannot sub-select all (this is most likely a misconfiguration and the user intended to use the set operation)
err = newErrInvalidExpression(exp, operation, ErrAllNotAllowed)
} else if !ec.Tags.Has(token) {
// only tags are allowed...
if ec.Names.Has(token) {
err = newErrInvalidExpression(exp, operation, ErrNamesNotAllowed)
} else {
err = newErrInvalidExpression(exp, operation, ErrUnknownNameOrTag)
}
}
}
var errs []error
if err != nil {
errs = append(errs, err)
}
return Expression{
Operation: operation,
Operand: token,
Errors: errs,
}
}
func newExpressionsFromSelectionRequest(nc *expressionContext, selectionRequest pkgcataloging.SelectionRequest) Expressions {
var all Expressions
for _, exp := range selectionRequest.DefaultNamesOrTags {
all = append(all, nc.newExpression(exp, SetOperation, exp))
}
for _, exp := range selectionRequest.SubSelectTags {
all = append(all, nc.newExpression(exp, SubSelectOperation, exp))
}
for _, exp := range selectionRequest.AddNames {
all = append(all, nc.newExpression(exp, AddOperation, exp))
}
for _, exp := range selectionRequest.RemoveNamesOrTags {
all = append(all, nc.newExpression(exp, RemoveOperation, exp))
}
sort.Sort(all)
return all
}
func isValidNode(s string) bool {
return expressionNodePattern.Match([]byte(s))
}
func (e Expressions) Clone() Expressions {
clone := make(Expressions, len(e))
copy(clone, e)
return clone
}
func (e Expression) String() string {
var op string
switch e.Operation {
case AddOperation:
op = "+"
case RemoveOperation:
op = "-"
case SubSelectOperation:
op = ""
case SetOperation:
op = ""
default:
op = "?"
}
return op + e.Operand
}
func (e Expressions) Len() int {
return len(e)
}
func (e Expressions) Swap(i, j int) {
e[i], e[j] = e[j], e[i]
}
// order of operations
var orderOfOps = map[Operation]int{
SetOperation: 1,
SubSelectOperation: 2,
RemoveOperation: 3,
AddOperation: 4,
}
func (e Expressions) Less(i, j int) bool {
ooi := orderOfOps[e[i].Operation]
ooj := orderOfOps[e[j].Operation]
if ooi != ooj {
return ooi < ooj
}
return i < j
}
func (e Expressions) Errors() (errs []error) {
for _, n := range e {
if len(n.Errors) > 0 {
errs = append(errs, n.Errors...)
}
}
return errs
}
func (e Expressions) Validate() error {
errs := e.Errors()
if len(errs) == 0 {
return nil
}
var err error
return multierror.Append(err, e.Errors()...)
}
+242
View File
@@ -0,0 +1,242 @@
package task
import (
"sort"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
)
func Test_newExpressionsFromSelectionRequest(t *testing.T) {
ts := []Task{
dummyTask("1", "t1"),
dummyTask("2", "t2"),
dummyTask("3", "t3"),
dummyTask("4", "t4"),
dummyTask("5"),
dummyTask("6"),
}
nc := newExpressionContext(ts)
var tests = []struct {
name string
basis []string
expressions []string
expected Expressions
expectedErrors []error
}{
{
name: "empty input",
basis: []string{},
expressions: []string{},
expected: nil,
},
{
name: "valid single set operation",
basis: []string{"1"},
expressions: []string{},
expected: []Expression{
{Operation: SetOperation, Operand: "1"},
},
},
{
name: "add operation",
basis: []string{},
expressions: []string{"+4"},
expected: []Expression{
{Operation: AddOperation, Operand: "4"},
},
},
{
name: "remove operation",
basis: []string{},
expressions: []string{"-3"},
expected: []Expression{
{Operation: RemoveOperation, Operand: "3"},
},
},
{
name: "select operation",
basis: []string{},
expressions: []string{"t2"},
expected: []Expression{
{Operation: SubSelectOperation, Operand: "t2"},
},
},
{
name: "mixed operations order",
basis: []string{"1"},
expressions: []string{"+4", "-3", "t2"},
expected: []Expression{
// note they are sorted by operation
{Operation: SetOperation, Operand: "1"},
{Operation: SubSelectOperation, Operand: "t2"},
{Operation: RemoveOperation, Operand: "3"},
{Operation: AddOperation, Operand: "4"},
},
},
{
name: "invalid token",
basis: []string{"!1"},
expressions: []string{},
expected: nil,
expectedErrors: []error{ErrInvalidToken},
},
{
name: "use + operator in basis",
basis: []string{"+1"},
expressions: []string{},
expected: nil,
expectedErrors: []error{ErrInvalidToken},
},
{
name: "use - operator in basis",
basis: []string{"-1"},
expressions: []string{},
expected: nil,
expectedErrors: []error{ErrInvalidToken},
},
{
name: "invalid name",
basis: []string{},
expressions: []string{"+t1"},
expected: nil,
expectedErrors: []error{ErrTagsNotAllowed},
},
{
name: "invalid tag",
basis: []string{},
expressions: []string{"1"},
expected: nil,
expectedErrors: []error{ErrNamesNotAllowed},
},
{
name: "invalid use of all",
basis: []string{},
expressions: []string{"all"},
expected: nil,
expectedErrors: []error{ErrAllNotAllowed},
},
{
name: "allow all operand",
basis: []string{"all"},
expressions: []string{},
expected: []Expression{
// note they are sorted by operation
{Operation: SetOperation, Operand: "all"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := pkgcataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...)
result := newExpressionsFromSelectionRequest(nc, req)
if tt.expectedErrors != nil {
errs := result.Errors()
require.Len(t, errs, len(tt.expectedErrors))
for i, err := range tt.expectedErrors {
var target ErrInvalidExpression
require.ErrorAs(t, errs[i], &target)
assert.Equal(t, err, target.Err)
}
} else {
assert.Empty(t, result.Errors())
assert.Equal(t, tt.expected, result)
}
})
}
}
func Test_expressionNodes_sort(t *testing.T) {
tests := []struct {
name string
subject Expressions
want Expressions
}{
{
name: "sort operations but keep token order",
subject: []Expression{
{
Operation: AddOperation,
Operand: "8",
},
{
Operation: AddOperation,
Operand: "7",
},
{
Operation: RemoveOperation,
Operand: "6",
},
{
Operation: RemoveOperation,
Operand: "5",
},
{
Operation: SetOperation,
Operand: "2",
},
{
Operation: SetOperation,
Operand: "1",
},
{
Operation: SubSelectOperation,
Operand: "4",
},
{
Operation: SubSelectOperation,
Operand: "3",
},
},
want: []Expression{
{
Operation: SetOperation,
Operand: "2",
},
{
Operation: SetOperation,
Operand: "1",
},
{
Operation: SubSelectOperation,
Operand: "4",
},
{
Operation: SubSelectOperation,
Operand: "3",
},
{
Operation: RemoveOperation,
Operand: "6",
},
{
Operation: RemoveOperation,
Operand: "5",
},
{
Operation: AddOperation,
Operand: "8",
},
{
Operation: AddOperation,
Operand: "7",
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := tt.subject.Clone()
sort.Sort(s)
assert.Equal(t, tt.want, s)
})
}
}
+135
View File
@@ -0,0 +1,135 @@
package task
import (
"context"
"crypto"
"fmt"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/file/cataloger/filecontent"
"github.com/anchore/syft/syft/file/cataloger/filedigest"
"github.com/anchore/syft/syft/file/cataloger/filemetadata"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
)
func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash) Task {
if selection == file.NoFilesSelection || len(hashers) == 0 {
return nil
}
digestsCataloger := filedigest.NewCataloger(hashers)
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
accessor := builder.(sbomsync.Accessor)
coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor))
if !ok {
return nil
}
result, err := digestsCataloger.Catalog(resolver, coordinates...)
if err != nil {
return fmt.Errorf("unable to catalog file digests: %w", err)
}
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileDigests = result
})
return nil
}
return NewTask("file-digest-cataloger", fn)
}
func NewFileMetadataCatalogerTask(selection file.Selection) Task {
if selection == file.NoFilesSelection {
return nil
}
metadataCataloger := filemetadata.NewCataloger()
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
accessor := builder.(sbomsync.Accessor)
coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor))
if !ok {
return nil
}
result, err := metadataCataloger.Catalog(resolver, coordinates...)
if err != nil {
return err
}
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileMetadata = result
})
return nil
}
return NewTask("file-metadata-cataloger", fn)
}
func NewFileContentCatalogerTask(cfg filecontent.Config) Task {
if len(cfg.Globs) == 0 {
return nil
}
cat := filecontent.NewCataloger(cfg)
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
accessor := builder.(sbomsync.Accessor)
result, err := cat.Catalog(resolver)
if err != nil {
return err
}
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileContents = result
})
return nil
}
return NewTask("file-content-cataloger", fn)
}
// TODO: this should be replaced with a fix that allows passing a coordinate or location iterator to the cataloger
// Today internal to both cataloger this functions differently: a slice of coordinates vs a channel of locations
func coordinatesForSelection(selection file.Selection, accessor sbomsync.Accessor) ([]file.Coordinates, bool) {
if selection == file.AllFilesSelection {
return nil, true
}
if selection == file.FilesOwnedByPackageSelection {
var coordinates []file.Coordinates
accessor.ReadFromSBOM(func(sbom *sbom.SBOM) {
for _, r := range sbom.Relationships {
if r.Type != artifact.ContainsRelationship {
continue
}
if _, ok := r.From.(pkg.Package); !ok {
continue
}
if c, ok := r.To.(file.Coordinates); ok {
coordinates = append(coordinates, c)
}
}
})
if len(coordinates) == 0 {
return nil, false
}
return coordinates, true
}
return nil, false
}
+104
View File
@@ -0,0 +1,104 @@
package task
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
)
func Test_coordinatesForSelection(t *testing.T) {
tests := []struct {
name string
selection file.Selection
sbom *sbom.SBOM
files []file.Coordinates
ok bool
}{
{
name: "all files",
selection: file.AllFilesSelection,
files: nil,
ok: true,
},
{
name: "no files",
selection: file.NoFilesSelection,
files: nil,
ok: false,
},
{
name: "specific files with hits",
selection: file.FilesOwnedByPackageSelection,
sbom: &sbom.SBOM{
Relationships: []artifact.Relationship{
{
From: pkg.Package{},
To: file.Coordinates{
RealPath: "path",
FileSystemID: "fs",
},
Type: artifact.ContainsRelationship,
},
},
},
files: []file.Coordinates{
{
RealPath: "path",
FileSystemID: "fs",
},
},
ok: true,
},
{
name: "specific files no hits (by wrong type)",
selection: file.FilesOwnedByPackageSelection,
sbom: &sbom.SBOM{
Relationships: []artifact.Relationship{
{
From: pkg.Package{},
To: file.Coordinates{
RealPath: "path",
FileSystemID: "fs",
},
// wrong type
Type: artifact.DependencyOfRelationship,
},
},
},
files: nil,
ok: false,
},
{
name: "specific files no hits (by wrong node types)",
selection: file.FilesOwnedByPackageSelection,
sbom: &sbom.SBOM{
Relationships: []artifact.Relationship{
{
From: file.Coordinates{}, // wrong type
To: file.Coordinates{
RealPath: "path",
FileSystemID: "fs",
},
Type: artifact.ContainsRelationship,
},
},
},
files: nil,
ok: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
files, ok := coordinatesForSelection(tt.selection, sbomsync.NewBuilder(tt.sbom).(sbomsync.Accessor))
assert.Equal(t, tt.files, files)
assert.Equal(t, tt.ok, ok)
})
}
}
+207
View File
@@ -0,0 +1,207 @@
package task
import (
"context"
"fmt"
"sort"
"strings"
"unicode"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common/cpe"
)
type packageTaskFactory func(cfg CatalogingFactoryConfig) Task
type PackageTaskFactories []packageTaskFactory
type CatalogingFactoryConfig struct {
SearchConfig cataloging.SearchConfig
RelationshipsConfig cataloging.RelationshipsConfig
DataGenerationConfig cataloging.DataGenerationConfig
PackagesConfig pkgcataloging.Config
}
func DefaultCatalogingFactoryConfig() CatalogingFactoryConfig {
return CatalogingFactoryConfig{
SearchConfig: cataloging.DefaultSearchConfig(),
RelationshipsConfig: cataloging.DefaultRelationshipsConfig(),
DataGenerationConfig: cataloging.DefaultDataGenerationConfig(),
PackagesConfig: pkgcataloging.DefaultConfig(),
}
}
func newPackageTaskFactory(catalogerFactory func(CatalogingFactoryConfig) pkg.Cataloger, tags ...string) packageTaskFactory {
return func(cfg CatalogingFactoryConfig) Task {
return NewPackageTask(cfg, catalogerFactory(cfg), tags...)
}
}
func newSimplePackageTaskFactory(catalogerFactory func() pkg.Cataloger, tags ...string) packageTaskFactory {
return func(cfg CatalogingFactoryConfig) Task {
return NewPackageTask(cfg, catalogerFactory(), tags...)
}
}
func (f PackageTaskFactories) Tasks(cfg CatalogingFactoryConfig) ([]Task, error) {
var allTasks []Task
taskNames := strset.New()
duplicateTaskNames := strset.New()
var err error
for _, factory := range f {
tsk := factory(cfg)
if tsk == nil {
continue
}
tskName := tsk.Name()
if taskNames.Has(tskName) {
duplicateTaskNames.Add(tskName)
}
allTasks = append(allTasks, tsk)
taskNames.Add(tskName)
}
if duplicateTaskNames.Size() > 0 {
names := duplicateTaskNames.List()
sort.Strings(names)
err = fmt.Errorf("duplicate cataloger task names: %v", strings.Join(names, ", "))
}
return allTasks, err
}
// NewPackageTask creates a Task function for a generic pkg.Cataloger, honoring the common configuration options.
//
//nolint:funlen
func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string) Task {
fn := func(ctx context.Context, resolver file.Resolver, sbom sbomsync.Builder) error {
catalogerName := c.Name()
log.WithFields("name", catalogerName).Trace("starting package cataloger")
info := monitor.GenericTask{
Title: monitor.Title{
Default: prettyName(catalogerName),
},
ID: catalogerName,
ParentID: monitor.PackageCatalogingTaskID,
Context: "",
HideOnSuccess: true,
}
t := bus.StartCatalogerTask(info, -1, "")
pkgs, relationships, err := c.Catalog(resolver)
if err != nil {
return fmt.Errorf("unable to catalog packages with %q: %w", c.Name(), err)
}
log.WithFields("cataloger", c.Name()).Debugf("discovered %d packages", len(pkgs))
for i, p := range pkgs {
if cfg.DataGenerationConfig.GenerateCPEs {
// generate CPEs (note: this is excluded from package ID, so is safe to mutate)
// we might have binary classified CPE already with the package so we want to append here
dictionaryCPE, ok := cpe.DictionaryFind(p)
if ok {
log.Tracef("used CPE dictionary to find CPE for %s package %q: %s", p.Type, p.Name, dictionaryCPE.BindToFmtString())
p.CPEs = append(p.CPEs, dictionaryCPE)
} else {
p.CPEs = append(p.CPEs, cpe.Generate(p)...)
}
}
// if we were not able to identify the language we have an opportunity
// to try and get this value from the PURL. Worst case we assert that
// we could not identify the language at either stage and set UnknownLanguage
if p.Language == "" {
p.Language = pkg.LanguageFromPURL(p.PURL)
}
if cfg.RelationshipsConfig.PackageFileOwnership {
// create file-to-package relationships for files owned by the package
owningRelationships, err := packageFileOwnershipRelationships(p, resolver)
if err != nil {
log.Warnf("unable to create any package-file relationships for package name=%q type=%q: %w", p.Name, p.Type, err)
} else {
relationships = append(relationships, owningRelationships...)
}
}
pkgs[i] = p
}
sbom.AddPackages(pkgs...)
sbom.AddRelationships(relationships...)
t.Add(int64(len(pkgs)))
t.SetCompleted()
log.WithFields("name", c.Name()).Trace("package cataloger completed")
return nil
}
tags = append(tags, pkgcataloging.PackageTag)
return NewTask(c.Name(), fn, tags...)
}
func prettyName(s string) string {
if s == "" {
return ""
}
// Convert first character to uppercase
r := []rune(s)
r[0] = unicode.ToUpper(r[0])
return strings.ReplaceAll(string(r), "-", " ")
}
func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver) ([]artifact.Relationship, error) {
fileOwner, ok := p.Metadata.(pkg.FileOwner)
if !ok {
return nil, nil
}
locations := map[artifact.ID]file.Location{}
for _, path := range fileOwner.OwnedFiles() {
pathRefs, err := resolver.FilesByPath(path)
if err != nil {
return nil, fmt.Errorf("unable to find path for path=%q: %w", path, err)
}
if len(pathRefs) == 0 {
// ideally we want to warn users about missing files from a package, however, it is very common for
// container image authors to delete files that are not needed in order to keep image sizes small. Adding
// a warning here would be needlessly noisy (even for popular base images).
continue
}
for _, ref := range pathRefs {
if oldRef, ok := locations[ref.Coordinates.ID()]; ok {
log.Debugf("found path duplicate of %s", oldRef.RealPath)
}
locations[ref.Coordinates.ID()] = ref
}
}
var relationships []artifact.Relationship
for _, location := range locations {
relationships = append(relationships, artifact.Relationship{
From: p,
To: location.Coordinates,
Type: artifact.ContainsRelationship,
})
}
return relationships, nil
}
+120
View File
@@ -0,0 +1,120 @@
package task
import (
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/alpine"
"github.com/anchore/syft/syft/pkg/cataloger/arch"
"github.com/anchore/syft/syft/pkg/cataloger/binary"
"github.com/anchore/syft/syft/pkg/cataloger/cpp"
"github.com/anchore/syft/syft/pkg/cataloger/dart"
"github.com/anchore/syft/syft/pkg/cataloger/debian"
"github.com/anchore/syft/syft/pkg/cataloger/dotnet"
"github.com/anchore/syft/syft/pkg/cataloger/elixir"
"github.com/anchore/syft/syft/pkg/cataloger/erlang"
"github.com/anchore/syft/syft/pkg/cataloger/gentoo"
"github.com/anchore/syft/syft/pkg/cataloger/githubactions"
"github.com/anchore/syft/syft/pkg/cataloger/golang"
"github.com/anchore/syft/syft/pkg/cataloger/haskell"
"github.com/anchore/syft/syft/pkg/cataloger/java"
"github.com/anchore/syft/syft/pkg/cataloger/javascript"
"github.com/anchore/syft/syft/pkg/cataloger/nix"
"github.com/anchore/syft/syft/pkg/cataloger/php"
"github.com/anchore/syft/syft/pkg/cataloger/python"
"github.com/anchore/syft/syft/pkg/cataloger/r"
"github.com/anchore/syft/syft/pkg/cataloger/redhat"
"github.com/anchore/syft/syft/pkg/cataloger/ruby"
"github.com/anchore/syft/syft/pkg/cataloger/rust"
sbomCataloger "github.com/anchore/syft/syft/pkg/cataloger/sbom"
"github.com/anchore/syft/syft/pkg/cataloger/swift"
)
//nolint:funlen
func DefaultPackageTaskFactories() PackageTaskFactories {
return []packageTaskFactory{
// OS package installed catalogers ///////////////////////////////////////////////////////////////////////////
newSimplePackageTaskFactory(arch.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "alpm", "archlinux"),
newSimplePackageTaskFactory(alpine.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "apk", "alpine"),
newSimplePackageTaskFactory(debian.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "dpkg", "debian"),
newSimplePackageTaskFactory(gentoo.NewPortageCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "portage", "gentoo"),
newSimplePackageTaskFactory(redhat.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "rpm", "redhat"),
// OS package declared catalogers ///////////////////////////////////////////////////////////////////////////
newSimplePackageTaskFactory(redhat.NewArchiveCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.OSTag, "linux", "rpm", "redhat"),
// language-specific package installed catalogers ///////////////////////////////////////////////////////////////////////////
newSimplePackageTaskFactory(cpp.NewConanInfoCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "cpp", "conan"),
newSimplePackageTaskFactory(javascript.NewPackageCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "javascript", "node"),
newSimplePackageTaskFactory(php.NewComposerInstalledCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "php", "composer"),
newSimplePackageTaskFactory(r.NewPackageCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "r"),
newSimplePackageTaskFactory(ruby.NewInstalledGemSpecCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "ruby", "gem", "gemspec"),
newSimplePackageTaskFactory(rust.NewAuditBinaryCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "rust", "binary"),
// language-specific package declared catalogers ///////////////////////////////////////////////////////////////////////////
newSimplePackageTaskFactory(cpp.NewConanCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "cpp", "conan"),
newSimplePackageTaskFactory(dart.NewPubspecLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "dart"),
newSimplePackageTaskFactory(dotnet.NewDotnetDepsCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "dotnet", "c#"),
newSimplePackageTaskFactory(elixir.NewMixLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "elixir"),
newSimplePackageTaskFactory(erlang.NewRebarLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "erlang"),
newSimplePackageTaskFactory(haskell.NewHackageCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "haskell", "hackage", "cabal"),
newPackageTaskFactory(
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
return golang.NewGoModuleFileCataloger(cfg.PackagesConfig.Golang)
},
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "go", "golang", "gomod",
),
newSimplePackageTaskFactory(java.NewGradleLockfileCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "java", "gradle"),
newPackageTaskFactory(
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
return java.NewPomCataloger(cfg.PackagesConfig.JavaArchive)
},
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "java", "maven",
),
newPackageTaskFactory(
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
return javascript.NewLockCataloger(cfg.PackagesConfig.JavaScript)
},
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "javascript", "node", "npm",
),
newSimplePackageTaskFactory(php.NewComposerLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "php", "composer"),
newPackageTaskFactory(
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
return python.NewPackageCataloger(cfg.PackagesConfig.Python)
},
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "python",
),
newSimplePackageTaskFactory(ruby.NewGemFileLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "ruby", "gem"),
newSimplePackageTaskFactory(ruby.NewGemSpecCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "ruby", "gem", "gemspec"),
newSimplePackageTaskFactory(rust.NewCargoLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "rust", "cargo"),
newSimplePackageTaskFactory(swift.NewCocoapodsCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "swift", "cocoapods"),
newSimplePackageTaskFactory(swift.NewSwiftPackageManagerCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "swift", "spm"),
// language-specific package for both image and directory scans (but not necessarily declared) ////////////////////////////////////////
newSimplePackageTaskFactory(dotnet.NewDotnetPortableExecutableCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "dotnet", "c#", "binary"),
newSimplePackageTaskFactory(python.NewInstalledPackageCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "python"),
newPackageTaskFactory(
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
return golang.NewGoModuleBinaryCataloger(cfg.PackagesConfig.Golang)
},
pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "go", "golang", "gomod", "binary",
),
newPackageTaskFactory(
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
return java.NewArchiveCataloger(cfg.PackagesConfig.JavaArchive)
},
pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "java", "maven",
),
newSimplePackageTaskFactory(java.NewNativeImageCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "java"),
newSimplePackageTaskFactory(nix.NewStoreCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "nix"),
// other package catalogers ///////////////////////////////////////////////////////////////////////////
newPackageTaskFactory(
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
return binary.NewCataloger(cfg.PackagesConfig.Binary)
},
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, "binary"),
newSimplePackageTaskFactory(githubactions.NewActionUsageCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, "github", "github-actions"),
newSimplePackageTaskFactory(githubactions.NewWorkflowUsageCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, "github", "github-actions"),
newSimplePackageTaskFactory(sbomCataloger.NewCataloger, pkgcataloging.ImageTag, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.ImageTag, "sbom"), // note: not evidence of installed packages
}
}
+35
View File
@@ -0,0 +1,35 @@
package task
import (
"context"
"github.com/anchore/syft/internal/relationship"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source"
)
var _ artifact.Identifiable = (*sourceIdentifierAdapter)(nil)
type sourceIdentifierAdapter struct {
desc source.Description
}
func (s sourceIdentifierAdapter) ID() artifact.ID {
return artifact.ID(s.desc.ID)
}
func NewRelationshipsTask(cfg cataloging.RelationshipsConfig, src source.Description) Task {
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
relationship.Finalize(
builder,
cfg,
&sourceIdentifierAdapter{desc: src})
return nil
}
return NewTask("relationships-cataloger", fn)
}
+176
View File
@@ -0,0 +1,176 @@
package task
import (
"fmt"
"sort"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
)
// Selection represents the users request for a subset of tasks to run and the resulting set of task names that were
// selected. Additionally, all tokens that were matched on to reach the returned conclusion are also provided.
type Selection struct {
Request pkgcataloging.SelectionRequest
Result *strset.Set
TokensByTask map[string]TokenSelection
}
// TokenSelection represents the tokens that were matched on to either include or exclude a given task (based on expression evaluation).
type TokenSelection struct {
SelectedOn *strset.Set
DeselectedOn *strset.Set
}
func newTokenSelection(selected, deselected []string) TokenSelection {
return TokenSelection{
SelectedOn: strset.New(selected...),
DeselectedOn: strset.New(deselected...),
}
}
func (ts *TokenSelection) merge(other ...TokenSelection) {
for _, o := range other {
if ts.SelectedOn != nil {
ts.SelectedOn.Add(o.SelectedOn.List()...)
}
if ts.DeselectedOn != nil {
ts.DeselectedOn.Add(o.DeselectedOn.List()...)
}
}
}
func newSelection() Selection {
return Selection{
Result: strset.New(),
TokensByTask: make(map[string]TokenSelection),
}
}
// Select parses the given expressions as two sets: expressions that represent a "set" operation, and expressions that
// represent all other operations. The parsed expressions are then evaluated against the given tasks to return
// a subset (or the same) set of tasks.
func Select(allTasks []Task, selectionRequest pkgcataloging.SelectionRequest) ([]Task, Selection, error) {
nodes := newExpressionsFromSelectionRequest(newExpressionContext(allTasks), selectionRequest)
finalTasks, selection := selectByExpressions(allTasks, nodes)
selection.Request = selectionRequest
return finalTasks, selection, nodes.Validate()
}
// selectByExpressions the set of tasks to run based on the given expression(s).
func selectByExpressions(ts tasks, nodes Expressions) (tasks, Selection) {
if len(nodes) == 0 {
return ts, newSelection()
}
finalSet := newSet()
selectionSet := newSet()
addSet := newSet()
removeSet := newSet()
allSelections := make(map[string]TokenSelection)
nodes = nodes.Clone()
sort.Sort(nodes)
for i, node := range nodes {
if len(node.Errors) > 0 {
continue
}
selectedTasks, selections := evaluateExpression(ts, node)
for name, ss := range selections {
if selection, exists := allSelections[name]; exists {
ss.merge(selection)
}
allSelections[name] = ss
}
if len(selectedTasks) == 0 {
log.WithFields("selection", fmt.Sprintf("%q", node.String())).Warn("no cataloger tasks selected found for given selection (this might be a misconfiguration)")
}
switch node.Operation {
case SetOperation:
finalSet.Add(selectedTasks...)
case AddOperation, "":
addSet.Add(selectedTasks...)
case RemoveOperation:
removeSet.Add(selectedTasks...)
case SubSelectOperation:
selectionSet.Add(selectedTasks...)
default:
nodes[i].Errors = append(nodes[i].Errors, ErrInvalidOperator)
}
}
if len(selectionSet.tasks) > 0 {
finalSet.Intersect(selectionSet.Tasks()...)
}
finalSet.Remove(removeSet.Tasks()...)
finalSet.Add(addSet.Tasks()...)
finalTasks := finalSet.Tasks()
return finalTasks, Selection{
Result: strset.New(finalTasks.Names()...),
TokensByTask: allSelections,
}
}
// evaluateExpression returns the set of tasks that match the given expression (as well as all tokens that were matched
// on to reach the returned conclusion).
func evaluateExpression(ts tasks, node Expression) ([]Task, map[string]TokenSelection) {
selection := make(map[string]TokenSelection)
var finalTasks []Task
for _, t := range ts {
if !isSelected(t, node.Operand) {
continue
}
s := newTokenSelection(nil, nil)
switch node.Operation {
case SetOperation, SubSelectOperation, AddOperation:
s.SelectedOn.Add(node.Operand)
case RemoveOperation:
s.DeselectedOn.Add(node.Operand)
}
finalTasks = append(finalTasks, t)
if og, exists := selection[t.Name()]; exists {
s.merge(og)
}
selection[t.Name()] = s
}
return finalTasks, selection
}
// isSelected returns true if the given task matches the given token. If the token is "all" then the task is always selected.
func isSelected(td Task, token string) bool {
if token == "all" {
return true
}
if ts, ok := td.(Selector); ok {
// use the selector to verify all tags
if ts.HasAllSelectors(token) {
return true
}
}
// only do exact name matching
if td.Name() == token {
return true
}
return false
}
+369
View File
@@ -0,0 +1,369 @@
package task
import (
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
"github.com/anchore/syft/syft/file"
)
func dummyTask(name string, tags ...string) Task {
return NewTask(name, func(ctx context.Context, resolver file.Resolver, sbom sbomsync.Builder) error {
panic("not implemented")
}, tags...)
}
// note: this test fixture does not need to be kept up to date here, but makes a great test subject
func createDummyTasks() tasks {
return []Task{
// OS package installed catalogers
dummyTask("alpm-db-cataloger", "directory", "installed", "image", "os", "alpm", "archlinux"),
dummyTask("apk-db-cataloger", "directory", "installed", "image", "os", "apk", "alpine"),
dummyTask("dpkg-db-cataloger", "directory", "installed", "image", "os", "dpkg", "debian"),
dummyTask("portage-cataloger", "directory", "installed", "image", "os", "portage", "gentoo"),
dummyTask("rpm-db-cataloger", "directory", "installed", "image", "os", "rpm", "redhat"),
// OS package declared catalogers
dummyTask("rpm-archive-cataloger", "declared", "directory", "os", "rpm", "redhat"),
// language-specific package installed catalogers
dummyTask("conan-info-cataloger", "installed", "image", "language", "cpp", "conan"),
dummyTask("javascript-package-cataloger", "installed", "image", "language", "javascript", "node"),
dummyTask("php-composer-installed-cataloger", "installed", "image", "language", "php", "composer"),
dummyTask("ruby-installed-gemspec-cataloger", "installed", "image", "language", "ruby", "gem", "gemspec"),
dummyTask("rust-cargo-lock-cataloger", "installed", "image", "language", "rust", "binary"),
// language-specific package declared catalogers
dummyTask("conan-cataloger", "declared", "directory", "language", "cpp", "conan"),
dummyTask("dart-pubspec-lock-cataloger", "declared", "directory", "language", "dart"),
dummyTask("dotnet-deps-cataloger", "declared", "directory", "language", "dotnet", "c#"),
dummyTask("elixir-mix-lock-cataloger", "declared", "directory", "language", "elixir"),
dummyTask("erlang-rebar-lock-cataloger", "declared", "directory", "language", "erlang"),
dummyTask("javascript-lock-cataloger", "declared", "directory", "language", "javascript", "node", "npm"),
// language-specific package for both image and directory scans (but not necessarily declared)
dummyTask("dotnet-portable-executable-cataloger", "directory", "installed", "image", "language", "dotnet", "c#"),
dummyTask("python-installed-package-cataloger", "directory", "installed", "image", "language", "python"),
dummyTask("go-module-binary-cataloger", "directory", "installed", "image", "language", "go", "golang", "gomod", "binary"),
dummyTask("java-archive-cataloger", "directory", "installed", "image", "language", "java", "maven"),
dummyTask("graalvm-native-image-cataloger", "directory", "installed", "image", "language", "java"),
// other package catalogers
dummyTask("binary-cataloger", "declared", "directory", "image", "binary"),
dummyTask("github-actions-usage-cataloger", "declared", "directory", "github", "github-actions"),
dummyTask("github-action-workflow-usage-cataloger", "declared", "directory", "github", "github-actions"),
dummyTask("sbom-cataloger", "declared", "directory", "image", "sbom"),
}
}
func TestSelect(t *testing.T) {
tests := []struct {
name string
allTasks []Task
basis []string
expressions []string
wantNames []string
wantTokens map[string]TokenSelection
wantRequest pkgcataloging.SelectionRequest
wantErr assert.ErrorAssertionFunc
}{
{
name: "empty input",
allTasks: []Task{},
basis: []string{},
expressions: []string{},
wantNames: []string{},
wantTokens: map[string]TokenSelection{},
wantRequest: pkgcataloging.SelectionRequest{},
},
{
name: "use default tasks",
allTasks: createDummyTasks(),
basis: []string{
"image",
},
expressions: []string{},
wantNames: []string{
"alpm-db-cataloger",
"apk-db-cataloger",
"dpkg-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
"conan-info-cataloger",
"javascript-package-cataloger",
"php-composer-installed-cataloger",
"ruby-installed-gemspec-cataloger",
"rust-cargo-lock-cataloger",
"dotnet-portable-executable-cataloger",
"python-installed-package-cataloger",
"go-module-binary-cataloger",
"java-archive-cataloger",
"graalvm-native-image-cataloger",
"binary-cataloger",
"sbom-cataloger",
},
wantTokens: map[string]TokenSelection{
"alpm-db-cataloger": newTokenSelection([]string{"image"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"image"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"image"}, nil),
"portage-cataloger": newTokenSelection([]string{"image"}, nil),
"rpm-db-cataloger": newTokenSelection([]string{"image"}, nil),
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil),
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
},
wantRequest: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"},
},
},
{
name: "select, add, and remove tasks",
allTasks: createDummyTasks(),
basis: []string{
"image",
},
expressions: []string{
"+github-actions-usage-cataloger",
"-dpkg",
"os",
},
wantNames: []string{
"alpm-db-cataloger",
"apk-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
"github-actions-usage-cataloger",
},
wantTokens: map[string]TokenSelection{
// selected
"alpm-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"image", "os"}, []string{"dpkg"}),
"portage-cataloger": newTokenSelection([]string{"image", "os"}, nil),
"rpm-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
"github-actions-usage-cataloger": newTokenSelection([]string{"github-actions-usage-cataloger"}, nil),
// ultimately not selected
"rpm-archive-cataloger": newTokenSelection([]string{"os"}, nil),
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil),
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
},
wantRequest: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"},
SubSelectTags: []string{"os"},
RemoveNamesOrTags: []string{"dpkg"},
AddNames: []string{"github-actions-usage-cataloger"},
},
},
{
name: "allow for partial selections",
allTasks: createDummyTasks(),
basis: []string{
"image",
},
expressions: []string{
// valid...
"+github-actions-usage-cataloger",
"-dpkg",
"os",
// invalid...
"+python",
"rust-cargo-lock-cataloger",
},
wantNames: []string{
"alpm-db-cataloger",
"apk-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
"github-actions-usage-cataloger",
},
wantTokens: map[string]TokenSelection{
// selected
"alpm-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"image", "os"}, []string{"dpkg"}),
"portage-cataloger": newTokenSelection([]string{"image", "os"}, nil),
"rpm-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
"github-actions-usage-cataloger": newTokenSelection([]string{"github-actions-usage-cataloger"}, nil),
// ultimately not selected
"rpm-archive-cataloger": newTokenSelection([]string{"os"}, nil),
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil), // note: there is no python token used for selection
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
},
wantRequest: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"},
SubSelectTags: []string{"os", "rust-cargo-lock-cataloger"},
RemoveNamesOrTags: []string{"dpkg"},
AddNames: []string{"github-actions-usage-cataloger", "python"},
},
wantErr: assert.Error, // !important!
},
{
name: "select all tasks",
allTasks: createDummyTasks(),
basis: []string{
"all",
},
expressions: []string{},
wantNames: []string{
"alpm-db-cataloger",
"apk-db-cataloger",
"dpkg-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
"rpm-archive-cataloger",
"conan-info-cataloger",
"javascript-package-cataloger",
"php-composer-installed-cataloger",
"ruby-installed-gemspec-cataloger",
"rust-cargo-lock-cataloger",
"conan-cataloger",
"dart-pubspec-lock-cataloger",
"dotnet-deps-cataloger",
"elixir-mix-lock-cataloger",
"erlang-rebar-lock-cataloger",
"javascript-lock-cataloger",
"dotnet-portable-executable-cataloger",
"python-installed-package-cataloger",
"go-module-binary-cataloger",
"java-archive-cataloger",
"graalvm-native-image-cataloger",
"binary-cataloger",
"github-actions-usage-cataloger",
"github-action-workflow-usage-cataloger",
"sbom-cataloger",
},
wantTokens: map[string]TokenSelection{
"alpm-db-cataloger": newTokenSelection([]string{"all"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"all"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"all"}, nil),
"portage-cataloger": newTokenSelection([]string{"all"}, nil),
"rpm-db-cataloger": newTokenSelection([]string{"all"}, nil),
"rpm-archive-cataloger": newTokenSelection([]string{"all"}, nil),
"conan-info-cataloger": newTokenSelection([]string{"all"}, nil),
"javascript-package-cataloger": newTokenSelection([]string{"all"}, nil),
"php-composer-installed-cataloger": newTokenSelection([]string{"all"}, nil),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"all"}, nil),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"all"}, nil),
"conan-cataloger": newTokenSelection([]string{"all"}, nil),
"dart-pubspec-lock-cataloger": newTokenSelection([]string{"all"}, nil),
"dotnet-deps-cataloger": newTokenSelection([]string{"all"}, nil),
"elixir-mix-lock-cataloger": newTokenSelection([]string{"all"}, nil),
"erlang-rebar-lock-cataloger": newTokenSelection([]string{"all"}, nil),
"javascript-lock-cataloger": newTokenSelection([]string{"all"}, nil),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"all"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"all"}, nil),
"go-module-binary-cataloger": newTokenSelection([]string{"all"}, nil),
"java-archive-cataloger": newTokenSelection([]string{"all"}, nil),
"graalvm-native-image-cataloger": newTokenSelection([]string{"all"}, nil),
"binary-cataloger": newTokenSelection([]string{"all"}, nil),
"github-actions-usage-cataloger": newTokenSelection([]string{"all"}, nil),
"github-action-workflow-usage-cataloger": newTokenSelection([]string{"all"}, nil),
"sbom-cataloger": newTokenSelection([]string{"all"}, nil),
},
wantRequest: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"all"},
},
},
{
name: "set default with multiple tags",
allTasks: createDummyTasks(),
basis: []string{
"gemspec",
"python",
},
expressions: []string{},
wantNames: []string{
"ruby-installed-gemspec-cataloger",
"python-installed-package-cataloger",
},
wantTokens: map[string]TokenSelection{
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"gemspec"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"python"}, nil),
},
wantRequest: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"gemspec", "python"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = assert.NoError
}
req := pkgcataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...)
got, gotEvidence, err := Select(tt.allTasks, req)
tt.wantErr(t, err)
if err != nil {
// dev note: this is useful for debugging when needed...
//for _, e := range gotEvidence.Request.Expressions {
// t.Logf("expression (errors %q): %#v", e.Errors, e)
//}
// note: we DON'T bail early in validations... this is because we should always return the full set of
// of selected tasks and surrounding evidence.
}
gotNames := make([]string, 0)
for _, g := range got {
gotNames = append(gotNames, g.Name())
}
assert.Equal(t, tt.wantNames, gotNames)
// names in selection should match all tasks returned
require.Len(t, tt.wantNames, gotEvidence.Result.Size(), "selected tasks should match all tasks returned (but does not)")
assert.ElementsMatch(t, tt.wantNames, gotEvidence.Result.List(), "selected tasks should match all tasks returned (but does not)")
setCompare := cmp.Comparer(func(x, y *strset.Set) bool {
return x.IsEqual(y)
})
if d := cmp.Diff(tt.wantTokens, gotEvidence.TokensByTask, setCompare); d != "" {
t.Errorf("unexpected tokens by task (-want +got):\n%s", d)
}
assert.Equal(t, tt.wantRequest, gotEvidence.Request)
})
}
}
+70
View File
@@ -0,0 +1,70 @@
package task
type set struct {
order []string
tasks map[string]Task
}
func newSet(tasks ...Task) *set {
s := &set{
order: []string{},
tasks: make(map[string]Task),
}
s.Add(tasks...)
return s
}
func (ts *set) Len() int {
return len(ts.tasks)
}
func (ts *set) Add(tasks ...Task) {
for _, t := range tasks {
taskName := t.Name()
if _, exists := ts.tasks[taskName]; exists {
continue
}
ts.tasks[taskName] = t
ts.order = append(ts.order, taskName)
}
}
func (ts *set) Remove(tasks ...Task) {
for _, t := range tasks {
taskName := t.Name()
if _, exists := ts.tasks[taskName]; !exists {
continue
}
delete(ts.tasks, taskName)
for i, t := range ts.order {
if t == taskName {
ts.order = append(ts.order[:i], ts.order[i+1:]...)
break
}
}
}
}
func (ts *set) Intersect(tasks ...Task) {
other := newSet(tasks...)
result := newSet()
for _, taskName := range ts.order {
// we make a new set to prevent the original set from being modified while we are iterating over "order"
if _, exists := other.tasks[taskName]; exists {
// note: keep the original task and ordering
result.Add(ts.tasks[taskName])
}
}
*ts = *result
}
func (ts set) Tasks() tasks {
var result []Task
for _, name := range ts.order {
result = append(result, ts.tasks[name])
}
return result
}
+154
View File
@@ -0,0 +1,154 @@
package task
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/file"
)
var _ Task = (*mockTask)(nil)
type mockTask struct {
name string
}
func (m mockTask) Execute(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
panic("implement me")
}
func (m mockTask) Name() string {
return m.name
}
func Test_set_Add(t *testing.T) {
tests := []struct {
name string
initialTasks []Task
newTasks []Task
expected []string
}{
{
name: "add unique tasks",
initialTasks: []Task{mockTask{"task2"}, mockTask{"task1"}},
newTasks: []Task{mockTask{"task3"}},
expected: []string{
"task2", // note order is honored
"task1",
"task3",
},
},
{
name: "add duplicate tasks",
initialTasks: []Task{mockTask{"task1"}, mockTask{"task2"}},
newTasks: []Task{mockTask{"task1"}},
expected: []string{
"task1",
"task2",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := newSet(tt.initialTasks...)
s.Add(tt.newTasks...)
got := s.Tasks()
var gotNames []string
for _, tsk := range got {
gotNames = append(gotNames, tsk.Name())
}
assert.Equal(t, tt.expected, gotNames)
})
}
}
func Test_set_Remove(t *testing.T) {
tests := []struct {
name string
initialTasks []Task
tasksToRemove []Task
expectedOrder []string
}{
{
name: "remove existing tasks",
initialTasks: []Task{mockTask{"task1"}, mockTask{"task2"}, mockTask{"task3"}},
tasksToRemove: []Task{mockTask{"task2"}},
expectedOrder: []string{"task1", "task3"},
},
{
name: "remove non-existing tasks",
initialTasks: []Task{mockTask{"task1"}, mockTask{"task2"}},
tasksToRemove: []Task{mockTask{"task3"}},
expectedOrder: []string{"task1", "task2"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := newSet(tt.initialTasks...)
s.Remove(tt.tasksToRemove...)
assert.Equal(t, tt.expectedOrder, s.order)
})
}
}
func Test_set_Intersect(t *testing.T) {
tests := []struct {
name string
initialTasks []Task
intersectTasks []Task
expectedOrder []string
}{
{
name: "intersect with overlapping tasks",
initialTasks: []Task{mockTask{"task1"}, mockTask{"task2"}},
intersectTasks: []Task{mockTask{"task2"}, mockTask{"task3"}},
expectedOrder: []string{"task2"},
},
{
name: "intersect with non-overlapping tasks",
initialTasks: []Task{mockTask{"task1"}, mockTask{"task4"}},
intersectTasks: []Task{mockTask{"task2"}, mockTask{"task3"}},
expectedOrder: []string{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := newSet(tt.initialTasks...)
s.Intersect(tt.intersectTasks...)
assert.Equal(t, tt.expectedOrder, s.order)
})
}
}
func Test_set_Tasks(t *testing.T) {
tests := []struct {
name string
initialTasks []Task
expectedTasks tasks
}{
{
name: "empty set",
initialTasks: []Task{},
expectedTasks: nil,
},
{
name: "get tasks from set",
initialTasks: []Task{mockTask{"task1"}, mockTask{"task2"}},
expectedTasks: []Task{mockTask{"task1"}, mockTask{"task2"}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := newSet(tt.initialTasks...)
resultTasks := s.Tasks()
assert.Equal(t, tt.expectedTasks, resultTasks)
})
}
}
+89
View File
@@ -0,0 +1,89 @@
package task
import (
"context"
"fmt"
"sort"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/file"
)
var _ interface {
Task
Selector
} = (*task)(nil)
// Task is a function that can wrap a cataloger to populate the SBOM with data (coordinated through the mutex).
type Task interface {
Name() string
Execute(context.Context, file.Resolver, sbomsync.Builder) error
}
type Selector interface {
HasAllSelectors(...string) bool
Selectors() []string
}
type tasks []Task
type task struct {
name string
selectors *strset.Set
task func(context.Context, file.Resolver, sbomsync.Builder) error
}
func NewTask(name string, tsk func(context.Context, file.Resolver, sbomsync.Builder) error, tags ...string) Task {
if tsk == nil {
panic(fmt.Errorf("task cannot be nil"))
}
tags = append(tags, name)
return &task{
name: name,
selectors: strset.New(tags...),
task: tsk,
}
}
func (t task) HasAllSelectors(ids ...string) bool {
// tags or name
return t.selectors.Has(ids...)
}
func (t task) Selectors() []string {
return t.selectors.List()
}
func (t task) Name() string {
return t.name
}
func (t task) Execute(ctx context.Context, resolver file.Resolver, sbom sbomsync.Builder) error {
return t.task(ctx, resolver, sbom)
}
func (ts tasks) Names() []string {
var names []string
for _, td := range ts {
names = append(names, td.Name())
}
return names
}
func (ts tasks) Tags() []string {
tags := strset.New()
for _, td := range ts {
if s, ok := td.(Selector); ok {
tags.Add(s.Selectors()...)
}
tags.Remove(td.Name())
}
tagsList := tags.List()
sort.Strings(tagsList)
return tagsList
}