mirror of
https://github.com/anchore/syft.git
synced 2026-10-30 22:12:28 +01:00
Replace core SBOM-creation API with builder pattern (#1383)
* remove existing cataloging API Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add file cataloging config Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add package cataloging config Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add configs for cross-cutting concerns Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename CLI option configs to not require import aliases later Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update all nested structs for the Catalog struct Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update Catalog cli options - add new cataloger selection options (selection and default) - remove the excludeBinaryOverlapByOwnership - deprecate "catalogers" flag - add new javascript configuration Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * migrate relationship capabilities to separate internal package Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * refactor golang cataloger to use configuration options when creating packages Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * create internal object to facilitate reading from and writing to an SBOM Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * create a command-like object (task) to facilitate partial SBOM creation Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add cataloger selection capability - be able to parse string expressions into a set of resolved actions against sets - be able to use expressions to select/add/remove tasks to/from the final set of tasks to run Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add package, file, and environment related tasks Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update existing file catalogers to use nested UI elements Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add CreateSBOMConfig that drives the SBOM creation process Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * capture SBOM creation info as a struct Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add CreateSBOM() function Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix tests Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update docs with SBOM selection help + breaking changes Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix multiple override default inputs Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix deprecation flag printing to stdout Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * refactor cataloger selection description to separate object Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address review comments Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * keep expression errors and show specific suggestions only Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address additional review feedback Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address more review comments Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * addressed additional PR review feedback Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix file selection references Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * remove guess language data generation option Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add tests for coordinatesForSelection Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename relationship attributes Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add descriptions to relationships config fields Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * improve documentation around configuration options Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add explicit errors around legacy config entries Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/linux"
|
||||
)
|
||||
|
||||
// TODO: add tui element here?
|
||||
|
||||
func NewEnvironmentTask() Task {
|
||||
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
|
||||
release := linux.IdentifyRelease(resolver)
|
||||
if release != nil {
|
||||
builder.SetLinuxDistribution(*release)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return NewTask("environment-cataloger", fn)
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
type Executor struct {
|
||||
numWorkers int
|
||||
tasks chan Task
|
||||
}
|
||||
|
||||
func NewTaskExecutor(tasks []Task, numWorkers int) *Executor {
|
||||
p := &Executor{
|
||||
numWorkers: numWorkers,
|
||||
tasks: make(chan Task, len(tasks)),
|
||||
}
|
||||
|
||||
for i := range tasks {
|
||||
p.tasks <- tasks[i]
|
||||
}
|
||||
close(p.tasks)
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *Executor) Execute(ctx context.Context, resolver file.Resolver, s sbomsync.Builder, prog *monitor.CatalogerTaskProgress) error {
|
||||
var errs error
|
||||
wg := &sync.WaitGroup{}
|
||||
for i := 0; i < p.numWorkers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
for {
|
||||
tsk, ok := <-p.tasks
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if err := runTaskSafely(ctx, tsk, resolver, s); err != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("failed to run task: %w", err))
|
||||
prog.SetError(err)
|
||||
}
|
||||
prog.Increment()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
func runTaskSafely(ctx context.Context, t Task, resolver file.Resolver, s sbomsync.Builder) (err error) {
|
||||
// handle individual cataloger panics
|
||||
defer func() {
|
||||
if e := recover(); e != nil {
|
||||
err = fmt.Errorf("%v at:\n%s", e, string(debug.Stack()))
|
||||
}
|
||||
}()
|
||||
|
||||
return t.Execute(ctx, resolver, s)
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/wagoodman/go-progress"
|
||||
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
func Test_TaskExecutor_PanicHandling(t *testing.T) {
|
||||
tsk := NewTask("panicking-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
panic("something bad happened")
|
||||
})
|
||||
ex := NewTaskExecutor([]Task{tsk}, 1)
|
||||
|
||||
err := ex.Execute(context.Background(), nil, nil, &monitor.CatalogerTaskProgress{
|
||||
Manual: progress.NewManual(-1),
|
||||
})
|
||||
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "executor_test.go")
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
)
|
||||
|
||||
var expressionNodePattern = regexp.MustCompile(`^([a-zA-Z0-9][a-zA-Z0-9-+]*)+$`)
|
||||
|
||||
const (
|
||||
SetOperation Operation = "set"
|
||||
AddOperation Operation = "add"
|
||||
SubSelectOperation Operation = "sub-select"
|
||||
RemoveOperation Operation = "remove"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrEmptyToken = fmt.Errorf("no value given")
|
||||
ErrInvalidToken = fmt.Errorf("invalid token given: only alphanumeric characters and hyphens are allowed")
|
||||
ErrInvalidOperator = fmt.Errorf("invalid operator given")
|
||||
ErrUnknownNameOrTag = fmt.Errorf("unknown name or tag given")
|
||||
ErrTagsNotAllowed = fmt.Errorf("tags are not allowed with this operation (must use exact names)")
|
||||
ErrNamesNotAllowed = fmt.Errorf("names are not allowed with this operation (must use tags)")
|
||||
ErrAllNotAllowed = fmt.Errorf("cannot use the 'all' operand in this context")
|
||||
)
|
||||
|
||||
// ErrInvalidExpression represents an expression that cannot be parsed or can be parsed but is logically invalid.
|
||||
type ErrInvalidExpression struct {
|
||||
Expression string
|
||||
Operation Operation
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e ErrInvalidExpression) Error() string {
|
||||
return fmt.Sprintf("invalid expression: %q: %s", e.Expression, e.Err.Error())
|
||||
}
|
||||
|
||||
func newErrInvalidExpression(exp string, op Operation, err error) ErrInvalidExpression {
|
||||
return ErrInvalidExpression{
|
||||
Expression: exp,
|
||||
Operation: op,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
||||
// Expression represents a single operation-operand pair with (all validation errors).
|
||||
// E.g. "+foo", "-bar", or "something" are all expressions. Some validations are relevant to not only the
|
||||
// syntax (operation and operator) but other are sensitive to the context of the operand (e.g. if a given operand
|
||||
// is a tag or a name, validated against the operation).
|
||||
type Expression struct {
|
||||
Operation Operation
|
||||
Operand string
|
||||
Errors []error
|
||||
}
|
||||
|
||||
// Operation represents the type of operation to perform on the operand (set, add, remove, sub-select).
|
||||
type Operation string
|
||||
|
||||
// Expressions represents a list of expressions.
|
||||
type Expressions []Expression
|
||||
|
||||
// expressionContext represents all information needed to validate an expression (e.g. the set of all tasks and their tags).
|
||||
type expressionContext struct {
|
||||
Names *strset.Set
|
||||
Tags *strset.Set
|
||||
}
|
||||
|
||||
func newExpressionContext(ts []Task) *expressionContext {
|
||||
ec := &expressionContext{
|
||||
Names: strset.New(tasks(ts).Names()...),
|
||||
Tags: strset.New(tasks(ts).Tags()...),
|
||||
}
|
||||
|
||||
ec.Tags.Add("all")
|
||||
|
||||
return ec
|
||||
}
|
||||
|
||||
// newExpression creates a new validated Expression object relative to the task names and tags.
|
||||
func (ec expressionContext) newExpression(exp string, operation Operation, token string) Expression {
|
||||
if token == "" {
|
||||
return Expression{
|
||||
Operation: operation,
|
||||
Operand: token,
|
||||
Errors: []error{newErrInvalidExpression(exp, operation, ErrEmptyToken)},
|
||||
}
|
||||
}
|
||||
|
||||
if !isValidNode(token) {
|
||||
return Expression{
|
||||
Operation: operation,
|
||||
Operand: token,
|
||||
Errors: []error{newErrInvalidExpression(exp, operation, ErrInvalidToken)},
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
switch operation {
|
||||
case SetOperation, RemoveOperation:
|
||||
// names and tags allowed
|
||||
if !ec.Tags.Has(token) && !ec.Names.Has(token) {
|
||||
err = newErrInvalidExpression(exp, operation, ErrUnknownNameOrTag)
|
||||
}
|
||||
case AddOperation:
|
||||
// only names are allowed
|
||||
if !ec.Names.Has(token) {
|
||||
if ec.Tags.Has(token) {
|
||||
err = newErrInvalidExpression(exp, operation, ErrTagsNotAllowed)
|
||||
} else {
|
||||
err = newErrInvalidExpression(exp, operation, ErrUnknownNameOrTag)
|
||||
}
|
||||
}
|
||||
case SubSelectOperation:
|
||||
if token == "all" {
|
||||
// special case: we cannot sub-select all (this is most likely a misconfiguration and the user intended to use the set operation)
|
||||
err = newErrInvalidExpression(exp, operation, ErrAllNotAllowed)
|
||||
} else if !ec.Tags.Has(token) {
|
||||
// only tags are allowed...
|
||||
if ec.Names.Has(token) {
|
||||
err = newErrInvalidExpression(exp, operation, ErrNamesNotAllowed)
|
||||
} else {
|
||||
err = newErrInvalidExpression(exp, operation, ErrUnknownNameOrTag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var errs []error
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
return Expression{
|
||||
Operation: operation,
|
||||
Operand: token,
|
||||
Errors: errs,
|
||||
}
|
||||
}
|
||||
|
||||
func newExpressionsFromSelectionRequest(nc *expressionContext, selectionRequest pkgcataloging.SelectionRequest) Expressions {
|
||||
var all Expressions
|
||||
|
||||
for _, exp := range selectionRequest.DefaultNamesOrTags {
|
||||
all = append(all, nc.newExpression(exp, SetOperation, exp))
|
||||
}
|
||||
|
||||
for _, exp := range selectionRequest.SubSelectTags {
|
||||
all = append(all, nc.newExpression(exp, SubSelectOperation, exp))
|
||||
}
|
||||
|
||||
for _, exp := range selectionRequest.AddNames {
|
||||
all = append(all, nc.newExpression(exp, AddOperation, exp))
|
||||
}
|
||||
|
||||
for _, exp := range selectionRequest.RemoveNamesOrTags {
|
||||
all = append(all, nc.newExpression(exp, RemoveOperation, exp))
|
||||
}
|
||||
|
||||
sort.Sort(all)
|
||||
return all
|
||||
}
|
||||
|
||||
func isValidNode(s string) bool {
|
||||
return expressionNodePattern.Match([]byte(s))
|
||||
}
|
||||
|
||||
func (e Expressions) Clone() Expressions {
|
||||
clone := make(Expressions, len(e))
|
||||
copy(clone, e)
|
||||
return clone
|
||||
}
|
||||
|
||||
func (e Expression) String() string {
|
||||
var op string
|
||||
switch e.Operation {
|
||||
case AddOperation:
|
||||
op = "+"
|
||||
case RemoveOperation:
|
||||
op = "-"
|
||||
case SubSelectOperation:
|
||||
op = ""
|
||||
case SetOperation:
|
||||
op = ""
|
||||
default:
|
||||
op = "?"
|
||||
}
|
||||
return op + e.Operand
|
||||
}
|
||||
|
||||
func (e Expressions) Len() int {
|
||||
return len(e)
|
||||
}
|
||||
|
||||
func (e Expressions) Swap(i, j int) {
|
||||
e[i], e[j] = e[j], e[i]
|
||||
}
|
||||
|
||||
// order of operations
|
||||
var orderOfOps = map[Operation]int{
|
||||
SetOperation: 1,
|
||||
SubSelectOperation: 2,
|
||||
RemoveOperation: 3,
|
||||
AddOperation: 4,
|
||||
}
|
||||
|
||||
func (e Expressions) Less(i, j int) bool {
|
||||
ooi := orderOfOps[e[i].Operation]
|
||||
ooj := orderOfOps[e[j].Operation]
|
||||
|
||||
if ooi != ooj {
|
||||
return ooi < ooj
|
||||
}
|
||||
|
||||
return i < j
|
||||
}
|
||||
|
||||
func (e Expressions) Errors() (errs []error) {
|
||||
for _, n := range e {
|
||||
if len(n.Errors) > 0 {
|
||||
errs = append(errs, n.Errors...)
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func (e Expressions) Validate() error {
|
||||
errs := e.Errors()
|
||||
if len(errs) == 0 {
|
||||
return nil
|
||||
}
|
||||
var err error
|
||||
return multierror.Append(err, e.Errors()...)
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
)
|
||||
|
||||
func Test_newExpressionsFromSelectionRequest(t *testing.T) {
|
||||
ts := []Task{
|
||||
dummyTask("1", "t1"),
|
||||
dummyTask("2", "t2"),
|
||||
dummyTask("3", "t3"),
|
||||
dummyTask("4", "t4"),
|
||||
dummyTask("5"),
|
||||
dummyTask("6"),
|
||||
}
|
||||
|
||||
nc := newExpressionContext(ts)
|
||||
|
||||
var tests = []struct {
|
||||
name string
|
||||
basis []string
|
||||
expressions []string
|
||||
expected Expressions
|
||||
expectedErrors []error
|
||||
}{
|
||||
{
|
||||
name: "empty input",
|
||||
basis: []string{},
|
||||
expressions: []string{},
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
name: "valid single set operation",
|
||||
basis: []string{"1"},
|
||||
expressions: []string{},
|
||||
expected: []Expression{
|
||||
{Operation: SetOperation, Operand: "1"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "add operation",
|
||||
basis: []string{},
|
||||
expressions: []string{"+4"},
|
||||
expected: []Expression{
|
||||
{Operation: AddOperation, Operand: "4"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "remove operation",
|
||||
basis: []string{},
|
||||
expressions: []string{"-3"},
|
||||
expected: []Expression{
|
||||
{Operation: RemoveOperation, Operand: "3"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "select operation",
|
||||
basis: []string{},
|
||||
expressions: []string{"t2"},
|
||||
expected: []Expression{
|
||||
{Operation: SubSelectOperation, Operand: "t2"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "mixed operations order",
|
||||
basis: []string{"1"},
|
||||
expressions: []string{"+4", "-3", "t2"},
|
||||
expected: []Expression{
|
||||
// note they are sorted by operation
|
||||
{Operation: SetOperation, Operand: "1"},
|
||||
{Operation: SubSelectOperation, Operand: "t2"},
|
||||
{Operation: RemoveOperation, Operand: "3"},
|
||||
{Operation: AddOperation, Operand: "4"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid token",
|
||||
basis: []string{"!1"},
|
||||
expressions: []string{},
|
||||
expected: nil,
|
||||
expectedErrors: []error{ErrInvalidToken},
|
||||
},
|
||||
{
|
||||
name: "use + operator in basis",
|
||||
basis: []string{"+1"},
|
||||
expressions: []string{},
|
||||
expected: nil,
|
||||
expectedErrors: []error{ErrInvalidToken},
|
||||
},
|
||||
{
|
||||
name: "use - operator in basis",
|
||||
basis: []string{"-1"},
|
||||
expressions: []string{},
|
||||
expected: nil,
|
||||
expectedErrors: []error{ErrInvalidToken},
|
||||
},
|
||||
{
|
||||
name: "invalid name",
|
||||
basis: []string{},
|
||||
expressions: []string{"+t1"},
|
||||
expected: nil,
|
||||
expectedErrors: []error{ErrTagsNotAllowed},
|
||||
},
|
||||
{
|
||||
name: "invalid tag",
|
||||
basis: []string{},
|
||||
expressions: []string{"1"},
|
||||
expected: nil,
|
||||
expectedErrors: []error{ErrNamesNotAllowed},
|
||||
},
|
||||
{
|
||||
name: "invalid use of all",
|
||||
basis: []string{},
|
||||
expressions: []string{"all"},
|
||||
expected: nil,
|
||||
expectedErrors: []error{ErrAllNotAllowed},
|
||||
},
|
||||
{
|
||||
name: "allow all operand",
|
||||
basis: []string{"all"},
|
||||
expressions: []string{},
|
||||
expected: []Expression{
|
||||
// note they are sorted by operation
|
||||
{Operation: SetOperation, Operand: "all"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
req := pkgcataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...)
|
||||
|
||||
result := newExpressionsFromSelectionRequest(nc, req)
|
||||
if tt.expectedErrors != nil {
|
||||
errs := result.Errors()
|
||||
require.Len(t, errs, len(tt.expectedErrors))
|
||||
for i, err := range tt.expectedErrors {
|
||||
var target ErrInvalidExpression
|
||||
require.ErrorAs(t, errs[i], &target)
|
||||
assert.Equal(t, err, target.Err)
|
||||
}
|
||||
} else {
|
||||
assert.Empty(t, result.Errors())
|
||||
assert.Equal(t, tt.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_expressionNodes_sort(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
subject Expressions
|
||||
want Expressions
|
||||
}{
|
||||
{
|
||||
name: "sort operations but keep token order",
|
||||
subject: []Expression{
|
||||
{
|
||||
Operation: AddOperation,
|
||||
Operand: "8",
|
||||
},
|
||||
{
|
||||
Operation: AddOperation,
|
||||
Operand: "7",
|
||||
},
|
||||
{
|
||||
Operation: RemoveOperation,
|
||||
Operand: "6",
|
||||
},
|
||||
{
|
||||
Operation: RemoveOperation,
|
||||
Operand: "5",
|
||||
},
|
||||
{
|
||||
Operation: SetOperation,
|
||||
Operand: "2",
|
||||
},
|
||||
{
|
||||
Operation: SetOperation,
|
||||
Operand: "1",
|
||||
},
|
||||
{
|
||||
Operation: SubSelectOperation,
|
||||
Operand: "4",
|
||||
},
|
||||
{
|
||||
Operation: SubSelectOperation,
|
||||
Operand: "3",
|
||||
},
|
||||
},
|
||||
want: []Expression{
|
||||
{
|
||||
Operation: SetOperation,
|
||||
Operand: "2",
|
||||
},
|
||||
{
|
||||
Operation: SetOperation,
|
||||
Operand: "1",
|
||||
},
|
||||
{
|
||||
Operation: SubSelectOperation,
|
||||
Operand: "4",
|
||||
},
|
||||
{
|
||||
Operation: SubSelectOperation,
|
||||
Operand: "3",
|
||||
},
|
||||
{
|
||||
Operation: RemoveOperation,
|
||||
Operand: "6",
|
||||
},
|
||||
{
|
||||
Operation: RemoveOperation,
|
||||
Operand: "5",
|
||||
},
|
||||
{
|
||||
Operation: AddOperation,
|
||||
Operand: "8",
|
||||
},
|
||||
{
|
||||
Operation: AddOperation,
|
||||
Operand: "7",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := tt.subject.Clone()
|
||||
sort.Sort(s)
|
||||
assert.Equal(t, tt.want, s)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"fmt"
|
||||
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/file/cataloger/filecontent"
|
||||
"github.com/anchore/syft/syft/file/cataloger/filedigest"
|
||||
"github.com/anchore/syft/syft/file/cataloger/filemetadata"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash) Task {
|
||||
if selection == file.NoFilesSelection || len(hashers) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
digestsCataloger := filedigest.NewCataloger(hashers)
|
||||
|
||||
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
|
||||
accessor := builder.(sbomsync.Accessor)
|
||||
|
||||
coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor))
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := digestsCataloger.Catalog(resolver, coordinates...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to catalog file digests: %w", err)
|
||||
}
|
||||
|
||||
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
|
||||
sbom.Artifacts.FileDigests = result
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return NewTask("file-digest-cataloger", fn)
|
||||
}
|
||||
|
||||
func NewFileMetadataCatalogerTask(selection file.Selection) Task {
|
||||
if selection == file.NoFilesSelection {
|
||||
return nil
|
||||
}
|
||||
|
||||
metadataCataloger := filemetadata.NewCataloger()
|
||||
|
||||
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
|
||||
accessor := builder.(sbomsync.Accessor)
|
||||
|
||||
coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor))
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := metadataCataloger.Catalog(resolver, coordinates...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
|
||||
sbom.Artifacts.FileMetadata = result
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return NewTask("file-metadata-cataloger", fn)
|
||||
}
|
||||
|
||||
func NewFileContentCatalogerTask(cfg filecontent.Config) Task {
|
||||
if len(cfg.Globs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
cat := filecontent.NewCataloger(cfg)
|
||||
|
||||
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
|
||||
accessor := builder.(sbomsync.Accessor)
|
||||
|
||||
result, err := cat.Catalog(resolver)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
|
||||
sbom.Artifacts.FileContents = result
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return NewTask("file-content-cataloger", fn)
|
||||
}
|
||||
|
||||
// TODO: this should be replaced with a fix that allows passing a coordinate or location iterator to the cataloger
|
||||
// Today internal to both cataloger this functions differently: a slice of coordinates vs a channel of locations
|
||||
func coordinatesForSelection(selection file.Selection, accessor sbomsync.Accessor) ([]file.Coordinates, bool) {
|
||||
if selection == file.AllFilesSelection {
|
||||
return nil, true
|
||||
}
|
||||
|
||||
if selection == file.FilesOwnedByPackageSelection {
|
||||
var coordinates []file.Coordinates
|
||||
|
||||
accessor.ReadFromSBOM(func(sbom *sbom.SBOM) {
|
||||
for _, r := range sbom.Relationships {
|
||||
if r.Type != artifact.ContainsRelationship {
|
||||
continue
|
||||
}
|
||||
if _, ok := r.From.(pkg.Package); !ok {
|
||||
continue
|
||||
}
|
||||
if c, ok := r.To.(file.Coordinates); ok {
|
||||
coordinates = append(coordinates, c)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if len(coordinates) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
return coordinates, true
|
||||
}
|
||||
|
||||
return nil, false
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
func Test_coordinatesForSelection(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
selection file.Selection
|
||||
sbom *sbom.SBOM
|
||||
files []file.Coordinates
|
||||
ok bool
|
||||
}{
|
||||
{
|
||||
name: "all files",
|
||||
selection: file.AllFilesSelection,
|
||||
files: nil,
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "no files",
|
||||
selection: file.NoFilesSelection,
|
||||
files: nil,
|
||||
ok: false,
|
||||
},
|
||||
{
|
||||
name: "specific files with hits",
|
||||
selection: file.FilesOwnedByPackageSelection,
|
||||
sbom: &sbom.SBOM{
|
||||
Relationships: []artifact.Relationship{
|
||||
{
|
||||
From: pkg.Package{},
|
||||
To: file.Coordinates{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
Type: artifact.ContainsRelationship,
|
||||
},
|
||||
},
|
||||
},
|
||||
files: []file.Coordinates{
|
||||
{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
},
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "specific files no hits (by wrong type)",
|
||||
selection: file.FilesOwnedByPackageSelection,
|
||||
sbom: &sbom.SBOM{
|
||||
Relationships: []artifact.Relationship{
|
||||
{
|
||||
From: pkg.Package{},
|
||||
To: file.Coordinates{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
// wrong type
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
},
|
||||
},
|
||||
files: nil,
|
||||
ok: false,
|
||||
},
|
||||
{
|
||||
name: "specific files no hits (by wrong node types)",
|
||||
selection: file.FilesOwnedByPackageSelection,
|
||||
sbom: &sbom.SBOM{
|
||||
Relationships: []artifact.Relationship{
|
||||
{
|
||||
From: file.Coordinates{}, // wrong type
|
||||
To: file.Coordinates{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
Type: artifact.ContainsRelationship,
|
||||
},
|
||||
},
|
||||
},
|
||||
files: nil,
|
||||
ok: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
files, ok := coordinatesForSelection(tt.selection, sbomsync.NewBuilder(tt.sbom).(sbomsync.Accessor))
|
||||
assert.Equal(t, tt.files, files)
|
||||
assert.Equal(t, tt.ok, ok)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common/cpe"
|
||||
)
|
||||
|
||||
type packageTaskFactory func(cfg CatalogingFactoryConfig) Task
|
||||
|
||||
type PackageTaskFactories []packageTaskFactory
|
||||
|
||||
type CatalogingFactoryConfig struct {
|
||||
SearchConfig cataloging.SearchConfig
|
||||
RelationshipsConfig cataloging.RelationshipsConfig
|
||||
DataGenerationConfig cataloging.DataGenerationConfig
|
||||
PackagesConfig pkgcataloging.Config
|
||||
}
|
||||
|
||||
func DefaultCatalogingFactoryConfig() CatalogingFactoryConfig {
|
||||
return CatalogingFactoryConfig{
|
||||
SearchConfig: cataloging.DefaultSearchConfig(),
|
||||
RelationshipsConfig: cataloging.DefaultRelationshipsConfig(),
|
||||
DataGenerationConfig: cataloging.DefaultDataGenerationConfig(),
|
||||
PackagesConfig: pkgcataloging.DefaultConfig(),
|
||||
}
|
||||
}
|
||||
|
||||
func newPackageTaskFactory(catalogerFactory func(CatalogingFactoryConfig) pkg.Cataloger, tags ...string) packageTaskFactory {
|
||||
return func(cfg CatalogingFactoryConfig) Task {
|
||||
return NewPackageTask(cfg, catalogerFactory(cfg), tags...)
|
||||
}
|
||||
}
|
||||
|
||||
func newSimplePackageTaskFactory(catalogerFactory func() pkg.Cataloger, tags ...string) packageTaskFactory {
|
||||
return func(cfg CatalogingFactoryConfig) Task {
|
||||
return NewPackageTask(cfg, catalogerFactory(), tags...)
|
||||
}
|
||||
}
|
||||
|
||||
func (f PackageTaskFactories) Tasks(cfg CatalogingFactoryConfig) ([]Task, error) {
|
||||
var allTasks []Task
|
||||
taskNames := strset.New()
|
||||
duplicateTaskNames := strset.New()
|
||||
var err error
|
||||
for _, factory := range f {
|
||||
tsk := factory(cfg)
|
||||
if tsk == nil {
|
||||
continue
|
||||
}
|
||||
tskName := tsk.Name()
|
||||
if taskNames.Has(tskName) {
|
||||
duplicateTaskNames.Add(tskName)
|
||||
}
|
||||
|
||||
allTasks = append(allTasks, tsk)
|
||||
taskNames.Add(tskName)
|
||||
}
|
||||
if duplicateTaskNames.Size() > 0 {
|
||||
names := duplicateTaskNames.List()
|
||||
sort.Strings(names)
|
||||
err = fmt.Errorf("duplicate cataloger task names: %v", strings.Join(names, ", "))
|
||||
}
|
||||
|
||||
return allTasks, err
|
||||
}
|
||||
|
||||
// NewPackageTask creates a Task function for a generic pkg.Cataloger, honoring the common configuration options.
|
||||
//
|
||||
//nolint:funlen
|
||||
func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string) Task {
|
||||
fn := func(ctx context.Context, resolver file.Resolver, sbom sbomsync.Builder) error {
|
||||
catalogerName := c.Name()
|
||||
log.WithFields("name", catalogerName).Trace("starting package cataloger")
|
||||
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: prettyName(catalogerName),
|
||||
},
|
||||
ID: catalogerName,
|
||||
ParentID: monitor.PackageCatalogingTaskID,
|
||||
Context: "",
|
||||
HideOnSuccess: true,
|
||||
}
|
||||
|
||||
t := bus.StartCatalogerTask(info, -1, "")
|
||||
|
||||
pkgs, relationships, err := c.Catalog(resolver)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to catalog packages with %q: %w", c.Name(), err)
|
||||
}
|
||||
|
||||
log.WithFields("cataloger", c.Name()).Debugf("discovered %d packages", len(pkgs))
|
||||
|
||||
for i, p := range pkgs {
|
||||
if cfg.DataGenerationConfig.GenerateCPEs {
|
||||
// generate CPEs (note: this is excluded from package ID, so is safe to mutate)
|
||||
// we might have binary classified CPE already with the package so we want to append here
|
||||
dictionaryCPE, ok := cpe.DictionaryFind(p)
|
||||
if ok {
|
||||
log.Tracef("used CPE dictionary to find CPE for %s package %q: %s", p.Type, p.Name, dictionaryCPE.BindToFmtString())
|
||||
p.CPEs = append(p.CPEs, dictionaryCPE)
|
||||
} else {
|
||||
p.CPEs = append(p.CPEs, cpe.Generate(p)...)
|
||||
}
|
||||
}
|
||||
|
||||
// if we were not able to identify the language we have an opportunity
|
||||
// to try and get this value from the PURL. Worst case we assert that
|
||||
// we could not identify the language at either stage and set UnknownLanguage
|
||||
if p.Language == "" {
|
||||
p.Language = pkg.LanguageFromPURL(p.PURL)
|
||||
}
|
||||
|
||||
if cfg.RelationshipsConfig.PackageFileOwnership {
|
||||
// create file-to-package relationships for files owned by the package
|
||||
owningRelationships, err := packageFileOwnershipRelationships(p, resolver)
|
||||
if err != nil {
|
||||
log.Warnf("unable to create any package-file relationships for package name=%q type=%q: %w", p.Name, p.Type, err)
|
||||
} else {
|
||||
relationships = append(relationships, owningRelationships...)
|
||||
}
|
||||
}
|
||||
|
||||
pkgs[i] = p
|
||||
}
|
||||
|
||||
sbom.AddPackages(pkgs...)
|
||||
sbom.AddRelationships(relationships...)
|
||||
t.Add(int64(len(pkgs)))
|
||||
|
||||
t.SetCompleted()
|
||||
log.WithFields("name", c.Name()).Trace("package cataloger completed")
|
||||
|
||||
return nil
|
||||
}
|
||||
tags = append(tags, pkgcataloging.PackageTag)
|
||||
|
||||
return NewTask(c.Name(), fn, tags...)
|
||||
}
|
||||
|
||||
func prettyName(s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Convert first character to uppercase
|
||||
r := []rune(s)
|
||||
r[0] = unicode.ToUpper(r[0])
|
||||
|
||||
return strings.ReplaceAll(string(r), "-", " ")
|
||||
}
|
||||
|
||||
func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver) ([]artifact.Relationship, error) {
|
||||
fileOwner, ok := p.Metadata.(pkg.FileOwner)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
locations := map[artifact.ID]file.Location{}
|
||||
|
||||
for _, path := range fileOwner.OwnedFiles() {
|
||||
pathRefs, err := resolver.FilesByPath(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to find path for path=%q: %w", path, err)
|
||||
}
|
||||
|
||||
if len(pathRefs) == 0 {
|
||||
// ideally we want to warn users about missing files from a package, however, it is very common for
|
||||
// container image authors to delete files that are not needed in order to keep image sizes small. Adding
|
||||
// a warning here would be needlessly noisy (even for popular base images).
|
||||
continue
|
||||
}
|
||||
|
||||
for _, ref := range pathRefs {
|
||||
if oldRef, ok := locations[ref.Coordinates.ID()]; ok {
|
||||
log.Debugf("found path duplicate of %s", oldRef.RealPath)
|
||||
}
|
||||
locations[ref.Coordinates.ID()] = ref
|
||||
}
|
||||
}
|
||||
|
||||
var relationships []artifact.Relationship
|
||||
for _, location := range locations {
|
||||
relationships = append(relationships, artifact.Relationship{
|
||||
From: p,
|
||||
To: location.Coordinates,
|
||||
Type: artifact.ContainsRelationship,
|
||||
})
|
||||
}
|
||||
return relationships, nil
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/alpine"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/arch"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/binary"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/cpp"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/dart"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/debian"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/dotnet"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/elixir"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/erlang"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/gentoo"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/githubactions"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/golang"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/haskell"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/javascript"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/nix"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/php"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/python"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/r"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/redhat"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/ruby"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/rust"
|
||||
sbomCataloger "github.com/anchore/syft/syft/pkg/cataloger/sbom"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/swift"
|
||||
)
|
||||
|
||||
//nolint:funlen
|
||||
func DefaultPackageTaskFactories() PackageTaskFactories {
|
||||
return []packageTaskFactory{
|
||||
// OS package installed catalogers ///////////////////////////////////////////////////////////////////////////
|
||||
newSimplePackageTaskFactory(arch.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "alpm", "archlinux"),
|
||||
newSimplePackageTaskFactory(alpine.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "apk", "alpine"),
|
||||
newSimplePackageTaskFactory(debian.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "dpkg", "debian"),
|
||||
newSimplePackageTaskFactory(gentoo.NewPortageCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "portage", "gentoo"),
|
||||
newSimplePackageTaskFactory(redhat.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "rpm", "redhat"),
|
||||
|
||||
// OS package declared catalogers ///////////////////////////////////////////////////////////////////////////
|
||||
newSimplePackageTaskFactory(redhat.NewArchiveCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.OSTag, "linux", "rpm", "redhat"),
|
||||
|
||||
// language-specific package installed catalogers ///////////////////////////////////////////////////////////////////////////
|
||||
newSimplePackageTaskFactory(cpp.NewConanInfoCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "cpp", "conan"),
|
||||
newSimplePackageTaskFactory(javascript.NewPackageCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "javascript", "node"),
|
||||
newSimplePackageTaskFactory(php.NewComposerInstalledCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "php", "composer"),
|
||||
newSimplePackageTaskFactory(r.NewPackageCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "r"),
|
||||
newSimplePackageTaskFactory(ruby.NewInstalledGemSpecCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "ruby", "gem", "gemspec"),
|
||||
newSimplePackageTaskFactory(rust.NewAuditBinaryCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "rust", "binary"),
|
||||
|
||||
// language-specific package declared catalogers ///////////////////////////////////////////////////////////////////////////
|
||||
newSimplePackageTaskFactory(cpp.NewConanCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "cpp", "conan"),
|
||||
newSimplePackageTaskFactory(dart.NewPubspecLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "dart"),
|
||||
newSimplePackageTaskFactory(dotnet.NewDotnetDepsCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "dotnet", "c#"),
|
||||
newSimplePackageTaskFactory(elixir.NewMixLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "elixir"),
|
||||
newSimplePackageTaskFactory(erlang.NewRebarLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "erlang"),
|
||||
newSimplePackageTaskFactory(haskell.NewHackageCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "haskell", "hackage", "cabal"),
|
||||
newPackageTaskFactory(
|
||||
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
|
||||
return golang.NewGoModuleFileCataloger(cfg.PackagesConfig.Golang)
|
||||
},
|
||||
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "go", "golang", "gomod",
|
||||
),
|
||||
newSimplePackageTaskFactory(java.NewGradleLockfileCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "java", "gradle"),
|
||||
newPackageTaskFactory(
|
||||
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
|
||||
return java.NewPomCataloger(cfg.PackagesConfig.JavaArchive)
|
||||
},
|
||||
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "java", "maven",
|
||||
),
|
||||
newPackageTaskFactory(
|
||||
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
|
||||
return javascript.NewLockCataloger(cfg.PackagesConfig.JavaScript)
|
||||
},
|
||||
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "javascript", "node", "npm",
|
||||
),
|
||||
newSimplePackageTaskFactory(php.NewComposerLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "php", "composer"),
|
||||
newPackageTaskFactory(
|
||||
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
|
||||
return python.NewPackageCataloger(cfg.PackagesConfig.Python)
|
||||
},
|
||||
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "python",
|
||||
),
|
||||
newSimplePackageTaskFactory(ruby.NewGemFileLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "ruby", "gem"),
|
||||
newSimplePackageTaskFactory(ruby.NewGemSpecCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "ruby", "gem", "gemspec"),
|
||||
newSimplePackageTaskFactory(rust.NewCargoLockCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "rust", "cargo"),
|
||||
newSimplePackageTaskFactory(swift.NewCocoapodsCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "swift", "cocoapods"),
|
||||
newSimplePackageTaskFactory(swift.NewSwiftPackageManagerCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, "swift", "spm"),
|
||||
|
||||
// language-specific package for both image and directory scans (but not necessarily declared) ////////////////////////////////////////
|
||||
newSimplePackageTaskFactory(dotnet.NewDotnetPortableExecutableCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "dotnet", "c#", "binary"),
|
||||
newSimplePackageTaskFactory(python.NewInstalledPackageCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "python"),
|
||||
newPackageTaskFactory(
|
||||
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
|
||||
return golang.NewGoModuleBinaryCataloger(cfg.PackagesConfig.Golang)
|
||||
},
|
||||
pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "go", "golang", "gomod", "binary",
|
||||
),
|
||||
newPackageTaskFactory(
|
||||
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
|
||||
return java.NewArchiveCataloger(cfg.PackagesConfig.JavaArchive)
|
||||
},
|
||||
pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "java", "maven",
|
||||
),
|
||||
newSimplePackageTaskFactory(java.NewNativeImageCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "java"),
|
||||
newSimplePackageTaskFactory(nix.NewStoreCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "nix"),
|
||||
|
||||
// other package catalogers ///////////////////////////////////////////////////////////////////////////
|
||||
newPackageTaskFactory(
|
||||
func(cfg CatalogingFactoryConfig) pkg.Cataloger {
|
||||
return binary.NewCataloger(cfg.PackagesConfig.Binary)
|
||||
},
|
||||
pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, "binary"),
|
||||
newSimplePackageTaskFactory(githubactions.NewActionUsageCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, "github", "github-actions"),
|
||||
newSimplePackageTaskFactory(githubactions.NewWorkflowUsageCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, "github", "github-actions"),
|
||||
newSimplePackageTaskFactory(sbomCataloger.NewCataloger, pkgcataloging.ImageTag, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.ImageTag, "sbom"), // note: not evidence of installed packages
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/anchore/syft/internal/relationship"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
var _ artifact.Identifiable = (*sourceIdentifierAdapter)(nil)
|
||||
|
||||
type sourceIdentifierAdapter struct {
|
||||
desc source.Description
|
||||
}
|
||||
|
||||
func (s sourceIdentifierAdapter) ID() artifact.ID {
|
||||
return artifact.ID(s.desc.ID)
|
||||
}
|
||||
|
||||
func NewRelationshipsTask(cfg cataloging.RelationshipsConfig, src source.Description) Task {
|
||||
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
|
||||
relationship.Finalize(
|
||||
builder,
|
||||
cfg,
|
||||
&sourceIdentifierAdapter{desc: src})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return NewTask("relationships-cataloger", fn)
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
)
|
||||
|
||||
// Selection represents the users request for a subset of tasks to run and the resulting set of task names that were
|
||||
// selected. Additionally, all tokens that were matched on to reach the returned conclusion are also provided.
|
||||
type Selection struct {
|
||||
Request pkgcataloging.SelectionRequest
|
||||
Result *strset.Set
|
||||
TokensByTask map[string]TokenSelection
|
||||
}
|
||||
|
||||
// TokenSelection represents the tokens that were matched on to either include or exclude a given task (based on expression evaluation).
|
||||
type TokenSelection struct {
|
||||
SelectedOn *strset.Set
|
||||
DeselectedOn *strset.Set
|
||||
}
|
||||
|
||||
func newTokenSelection(selected, deselected []string) TokenSelection {
|
||||
return TokenSelection{
|
||||
SelectedOn: strset.New(selected...),
|
||||
DeselectedOn: strset.New(deselected...),
|
||||
}
|
||||
}
|
||||
|
||||
func (ts *TokenSelection) merge(other ...TokenSelection) {
|
||||
for _, o := range other {
|
||||
if ts.SelectedOn != nil {
|
||||
ts.SelectedOn.Add(o.SelectedOn.List()...)
|
||||
}
|
||||
if ts.DeselectedOn != nil {
|
||||
ts.DeselectedOn.Add(o.DeselectedOn.List()...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newSelection() Selection {
|
||||
return Selection{
|
||||
Result: strset.New(),
|
||||
TokensByTask: make(map[string]TokenSelection),
|
||||
}
|
||||
}
|
||||
|
||||
// Select parses the given expressions as two sets: expressions that represent a "set" operation, and expressions that
|
||||
// represent all other operations. The parsed expressions are then evaluated against the given tasks to return
|
||||
// a subset (or the same) set of tasks.
|
||||
func Select(allTasks []Task, selectionRequest pkgcataloging.SelectionRequest) ([]Task, Selection, error) {
|
||||
nodes := newExpressionsFromSelectionRequest(newExpressionContext(allTasks), selectionRequest)
|
||||
|
||||
finalTasks, selection := selectByExpressions(allTasks, nodes)
|
||||
|
||||
selection.Request = selectionRequest
|
||||
|
||||
return finalTasks, selection, nodes.Validate()
|
||||
}
|
||||
|
||||
// selectByExpressions the set of tasks to run based on the given expression(s).
|
||||
func selectByExpressions(ts tasks, nodes Expressions) (tasks, Selection) {
|
||||
if len(nodes) == 0 {
|
||||
return ts, newSelection()
|
||||
}
|
||||
|
||||
finalSet := newSet()
|
||||
selectionSet := newSet()
|
||||
addSet := newSet()
|
||||
removeSet := newSet()
|
||||
|
||||
allSelections := make(map[string]TokenSelection)
|
||||
|
||||
nodes = nodes.Clone()
|
||||
sort.Sort(nodes)
|
||||
|
||||
for i, node := range nodes {
|
||||
if len(node.Errors) > 0 {
|
||||
continue
|
||||
}
|
||||
selectedTasks, selections := evaluateExpression(ts, node)
|
||||
|
||||
for name, ss := range selections {
|
||||
if selection, exists := allSelections[name]; exists {
|
||||
ss.merge(selection)
|
||||
}
|
||||
allSelections[name] = ss
|
||||
}
|
||||
|
||||
if len(selectedTasks) == 0 {
|
||||
log.WithFields("selection", fmt.Sprintf("%q", node.String())).Warn("no cataloger tasks selected found for given selection (this might be a misconfiguration)")
|
||||
}
|
||||
|
||||
switch node.Operation {
|
||||
case SetOperation:
|
||||
finalSet.Add(selectedTasks...)
|
||||
case AddOperation, "":
|
||||
addSet.Add(selectedTasks...)
|
||||
case RemoveOperation:
|
||||
removeSet.Add(selectedTasks...)
|
||||
case SubSelectOperation:
|
||||
selectionSet.Add(selectedTasks...)
|
||||
default:
|
||||
nodes[i].Errors = append(nodes[i].Errors, ErrInvalidOperator)
|
||||
}
|
||||
}
|
||||
|
||||
if len(selectionSet.tasks) > 0 {
|
||||
finalSet.Intersect(selectionSet.Tasks()...)
|
||||
}
|
||||
finalSet.Remove(removeSet.Tasks()...)
|
||||
finalSet.Add(addSet.Tasks()...)
|
||||
|
||||
finalTasks := finalSet.Tasks()
|
||||
|
||||
return finalTasks, Selection{
|
||||
Result: strset.New(finalTasks.Names()...),
|
||||
TokensByTask: allSelections,
|
||||
}
|
||||
}
|
||||
|
||||
// evaluateExpression returns the set of tasks that match the given expression (as well as all tokens that were matched
|
||||
// on to reach the returned conclusion).
|
||||
func evaluateExpression(ts tasks, node Expression) ([]Task, map[string]TokenSelection) {
|
||||
selection := make(map[string]TokenSelection)
|
||||
var finalTasks []Task
|
||||
|
||||
for _, t := range ts {
|
||||
if !isSelected(t, node.Operand) {
|
||||
continue
|
||||
}
|
||||
|
||||
s := newTokenSelection(nil, nil)
|
||||
|
||||
switch node.Operation {
|
||||
case SetOperation, SubSelectOperation, AddOperation:
|
||||
s.SelectedOn.Add(node.Operand)
|
||||
case RemoveOperation:
|
||||
s.DeselectedOn.Add(node.Operand)
|
||||
}
|
||||
|
||||
finalTasks = append(finalTasks, t)
|
||||
|
||||
if og, exists := selection[t.Name()]; exists {
|
||||
s.merge(og)
|
||||
}
|
||||
|
||||
selection[t.Name()] = s
|
||||
}
|
||||
return finalTasks, selection
|
||||
}
|
||||
|
||||
// isSelected returns true if the given task matches the given token. If the token is "all" then the task is always selected.
|
||||
func isSelected(td Task, token string) bool {
|
||||
if token == "all" {
|
||||
return true
|
||||
}
|
||||
|
||||
if ts, ok := td.(Selector); ok {
|
||||
// use the selector to verify all tags
|
||||
if ts.HasAllSelectors(token) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// only do exact name matching
|
||||
if td.Name() == token {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
func dummyTask(name string, tags ...string) Task {
|
||||
return NewTask(name, func(ctx context.Context, resolver file.Resolver, sbom sbomsync.Builder) error {
|
||||
panic("not implemented")
|
||||
}, tags...)
|
||||
}
|
||||
|
||||
// note: this test fixture does not need to be kept up to date here, but makes a great test subject
|
||||
func createDummyTasks() tasks {
|
||||
return []Task{
|
||||
// OS package installed catalogers
|
||||
dummyTask("alpm-db-cataloger", "directory", "installed", "image", "os", "alpm", "archlinux"),
|
||||
dummyTask("apk-db-cataloger", "directory", "installed", "image", "os", "apk", "alpine"),
|
||||
dummyTask("dpkg-db-cataloger", "directory", "installed", "image", "os", "dpkg", "debian"),
|
||||
dummyTask("portage-cataloger", "directory", "installed", "image", "os", "portage", "gentoo"),
|
||||
dummyTask("rpm-db-cataloger", "directory", "installed", "image", "os", "rpm", "redhat"),
|
||||
|
||||
// OS package declared catalogers
|
||||
dummyTask("rpm-archive-cataloger", "declared", "directory", "os", "rpm", "redhat"),
|
||||
|
||||
// language-specific package installed catalogers
|
||||
dummyTask("conan-info-cataloger", "installed", "image", "language", "cpp", "conan"),
|
||||
dummyTask("javascript-package-cataloger", "installed", "image", "language", "javascript", "node"),
|
||||
dummyTask("php-composer-installed-cataloger", "installed", "image", "language", "php", "composer"),
|
||||
dummyTask("ruby-installed-gemspec-cataloger", "installed", "image", "language", "ruby", "gem", "gemspec"),
|
||||
dummyTask("rust-cargo-lock-cataloger", "installed", "image", "language", "rust", "binary"),
|
||||
|
||||
// language-specific package declared catalogers
|
||||
dummyTask("conan-cataloger", "declared", "directory", "language", "cpp", "conan"),
|
||||
dummyTask("dart-pubspec-lock-cataloger", "declared", "directory", "language", "dart"),
|
||||
dummyTask("dotnet-deps-cataloger", "declared", "directory", "language", "dotnet", "c#"),
|
||||
dummyTask("elixir-mix-lock-cataloger", "declared", "directory", "language", "elixir"),
|
||||
dummyTask("erlang-rebar-lock-cataloger", "declared", "directory", "language", "erlang"),
|
||||
dummyTask("javascript-lock-cataloger", "declared", "directory", "language", "javascript", "node", "npm"),
|
||||
|
||||
// language-specific package for both image and directory scans (but not necessarily declared)
|
||||
dummyTask("dotnet-portable-executable-cataloger", "directory", "installed", "image", "language", "dotnet", "c#"),
|
||||
dummyTask("python-installed-package-cataloger", "directory", "installed", "image", "language", "python"),
|
||||
dummyTask("go-module-binary-cataloger", "directory", "installed", "image", "language", "go", "golang", "gomod", "binary"),
|
||||
dummyTask("java-archive-cataloger", "directory", "installed", "image", "language", "java", "maven"),
|
||||
dummyTask("graalvm-native-image-cataloger", "directory", "installed", "image", "language", "java"),
|
||||
|
||||
// other package catalogers
|
||||
dummyTask("binary-cataloger", "declared", "directory", "image", "binary"),
|
||||
dummyTask("github-actions-usage-cataloger", "declared", "directory", "github", "github-actions"),
|
||||
dummyTask("github-action-workflow-usage-cataloger", "declared", "directory", "github", "github-actions"),
|
||||
dummyTask("sbom-cataloger", "declared", "directory", "image", "sbom"),
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelect(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
allTasks []Task
|
||||
basis []string
|
||||
expressions []string
|
||||
wantNames []string
|
||||
wantTokens map[string]TokenSelection
|
||||
wantRequest pkgcataloging.SelectionRequest
|
||||
wantErr assert.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "empty input",
|
||||
allTasks: []Task{},
|
||||
basis: []string{},
|
||||
expressions: []string{},
|
||||
wantNames: []string{},
|
||||
wantTokens: map[string]TokenSelection{},
|
||||
wantRequest: pkgcataloging.SelectionRequest{},
|
||||
},
|
||||
{
|
||||
name: "use default tasks",
|
||||
allTasks: createDummyTasks(),
|
||||
basis: []string{
|
||||
"image",
|
||||
},
|
||||
expressions: []string{},
|
||||
wantNames: []string{
|
||||
"alpm-db-cataloger",
|
||||
"apk-db-cataloger",
|
||||
"dpkg-db-cataloger",
|
||||
"portage-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
"conan-info-cataloger",
|
||||
"javascript-package-cataloger",
|
||||
"php-composer-installed-cataloger",
|
||||
"ruby-installed-gemspec-cataloger",
|
||||
"rust-cargo-lock-cataloger",
|
||||
"dotnet-portable-executable-cataloger",
|
||||
"python-installed-package-cataloger",
|
||||
"go-module-binary-cataloger",
|
||||
"java-archive-cataloger",
|
||||
"graalvm-native-image-cataloger",
|
||||
"binary-cataloger",
|
||||
"sbom-cataloger",
|
||||
},
|
||||
wantTokens: map[string]TokenSelection{
|
||||
"alpm-db-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"apk-db-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"dpkg-db-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"portage-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"rpm-db-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
},
|
||||
wantRequest: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"image"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "select, add, and remove tasks",
|
||||
allTasks: createDummyTasks(),
|
||||
basis: []string{
|
||||
"image",
|
||||
},
|
||||
expressions: []string{
|
||||
"+github-actions-usage-cataloger",
|
||||
"-dpkg",
|
||||
"os",
|
||||
},
|
||||
wantNames: []string{
|
||||
"alpm-db-cataloger",
|
||||
"apk-db-cataloger",
|
||||
"portage-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
"github-actions-usage-cataloger",
|
||||
},
|
||||
wantTokens: map[string]TokenSelection{
|
||||
// selected
|
||||
"alpm-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
|
||||
"apk-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
|
||||
"dpkg-db-cataloger": newTokenSelection([]string{"image", "os"}, []string{"dpkg"}),
|
||||
"portage-cataloger": newTokenSelection([]string{"image", "os"}, nil),
|
||||
"rpm-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
|
||||
"github-actions-usage-cataloger": newTokenSelection([]string{"github-actions-usage-cataloger"}, nil),
|
||||
|
||||
// ultimately not selected
|
||||
"rpm-archive-cataloger": newTokenSelection([]string{"os"}, nil),
|
||||
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
},
|
||||
wantRequest: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"image"},
|
||||
SubSelectTags: []string{"os"},
|
||||
RemoveNamesOrTags: []string{"dpkg"},
|
||||
AddNames: []string{"github-actions-usage-cataloger"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allow for partial selections",
|
||||
allTasks: createDummyTasks(),
|
||||
basis: []string{
|
||||
"image",
|
||||
},
|
||||
expressions: []string{
|
||||
// valid...
|
||||
"+github-actions-usage-cataloger",
|
||||
"-dpkg",
|
||||
"os",
|
||||
// invalid...
|
||||
"+python",
|
||||
"rust-cargo-lock-cataloger",
|
||||
},
|
||||
wantNames: []string{
|
||||
"alpm-db-cataloger",
|
||||
"apk-db-cataloger",
|
||||
"portage-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
"github-actions-usage-cataloger",
|
||||
},
|
||||
wantTokens: map[string]TokenSelection{
|
||||
// selected
|
||||
"alpm-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
|
||||
"apk-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
|
||||
"dpkg-db-cataloger": newTokenSelection([]string{"image", "os"}, []string{"dpkg"}),
|
||||
"portage-cataloger": newTokenSelection([]string{"image", "os"}, nil),
|
||||
"rpm-db-cataloger": newTokenSelection([]string{"image", "os"}, nil),
|
||||
"github-actions-usage-cataloger": newTokenSelection([]string{"github-actions-usage-cataloger"}, nil),
|
||||
|
||||
// ultimately not selected
|
||||
"rpm-archive-cataloger": newTokenSelection([]string{"os"}, nil),
|
||||
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil), // note: there is no python token used for selection
|
||||
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
|
||||
},
|
||||
wantRequest: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"image"},
|
||||
SubSelectTags: []string{"os", "rust-cargo-lock-cataloger"},
|
||||
RemoveNamesOrTags: []string{"dpkg"},
|
||||
AddNames: []string{"github-actions-usage-cataloger", "python"},
|
||||
},
|
||||
wantErr: assert.Error, // !important!
|
||||
},
|
||||
{
|
||||
name: "select all tasks",
|
||||
allTasks: createDummyTasks(),
|
||||
basis: []string{
|
||||
"all",
|
||||
},
|
||||
expressions: []string{},
|
||||
wantNames: []string{
|
||||
"alpm-db-cataloger",
|
||||
"apk-db-cataloger",
|
||||
"dpkg-db-cataloger",
|
||||
"portage-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
"rpm-archive-cataloger",
|
||||
"conan-info-cataloger",
|
||||
"javascript-package-cataloger",
|
||||
"php-composer-installed-cataloger",
|
||||
"ruby-installed-gemspec-cataloger",
|
||||
"rust-cargo-lock-cataloger",
|
||||
"conan-cataloger",
|
||||
"dart-pubspec-lock-cataloger",
|
||||
"dotnet-deps-cataloger",
|
||||
"elixir-mix-lock-cataloger",
|
||||
"erlang-rebar-lock-cataloger",
|
||||
"javascript-lock-cataloger",
|
||||
"dotnet-portable-executable-cataloger",
|
||||
"python-installed-package-cataloger",
|
||||
"go-module-binary-cataloger",
|
||||
"java-archive-cataloger",
|
||||
"graalvm-native-image-cataloger",
|
||||
"binary-cataloger",
|
||||
"github-actions-usage-cataloger",
|
||||
"github-action-workflow-usage-cataloger",
|
||||
"sbom-cataloger",
|
||||
},
|
||||
wantTokens: map[string]TokenSelection{
|
||||
"alpm-db-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"apk-db-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"dpkg-db-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"portage-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"rpm-db-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"rpm-archive-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"conan-info-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"javascript-package-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"php-composer-installed-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"rust-cargo-lock-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"conan-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"dart-pubspec-lock-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"dotnet-deps-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"elixir-mix-lock-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"erlang-rebar-lock-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"javascript-lock-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"python-installed-package-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"go-module-binary-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"java-archive-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"graalvm-native-image-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"binary-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"github-actions-usage-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"github-action-workflow-usage-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
"sbom-cataloger": newTokenSelection([]string{"all"}, nil),
|
||||
},
|
||||
wantRequest: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"all"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "set default with multiple tags",
|
||||
allTasks: createDummyTasks(),
|
||||
basis: []string{
|
||||
"gemspec",
|
||||
"python",
|
||||
},
|
||||
expressions: []string{},
|
||||
wantNames: []string{
|
||||
"ruby-installed-gemspec-cataloger",
|
||||
"python-installed-package-cataloger",
|
||||
},
|
||||
wantTokens: map[string]TokenSelection{
|
||||
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"gemspec"}, nil),
|
||||
"python-installed-package-cataloger": newTokenSelection([]string{"python"}, nil),
|
||||
},
|
||||
wantRequest: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"gemspec", "python"},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.wantErr == nil {
|
||||
tt.wantErr = assert.NoError
|
||||
}
|
||||
|
||||
req := pkgcataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...)
|
||||
|
||||
got, gotEvidence, err := Select(tt.allTasks, req)
|
||||
tt.wantErr(t, err)
|
||||
if err != nil {
|
||||
// dev note: this is useful for debugging when needed...
|
||||
//for _, e := range gotEvidence.Request.Expressions {
|
||||
// t.Logf("expression (errors %q): %#v", e.Errors, e)
|
||||
//}
|
||||
|
||||
// note: we DON'T bail early in validations... this is because we should always return the full set of
|
||||
// of selected tasks and surrounding evidence.
|
||||
}
|
||||
|
||||
gotNames := make([]string, 0)
|
||||
for _, g := range got {
|
||||
gotNames = append(gotNames, g.Name())
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.wantNames, gotNames)
|
||||
|
||||
// names in selection should match all tasks returned
|
||||
require.Len(t, tt.wantNames, gotEvidence.Result.Size(), "selected tasks should match all tasks returned (but does not)")
|
||||
assert.ElementsMatch(t, tt.wantNames, gotEvidence.Result.List(), "selected tasks should match all tasks returned (but does not)")
|
||||
|
||||
setCompare := cmp.Comparer(func(x, y *strset.Set) bool {
|
||||
return x.IsEqual(y)
|
||||
})
|
||||
|
||||
if d := cmp.Diff(tt.wantTokens, gotEvidence.TokensByTask, setCompare); d != "" {
|
||||
t.Errorf("unexpected tokens by task (-want +got):\n%s", d)
|
||||
}
|
||||
assert.Equal(t, tt.wantRequest, gotEvidence.Request)
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package task
|
||||
|
||||
type set struct {
|
||||
order []string
|
||||
tasks map[string]Task
|
||||
}
|
||||
|
||||
func newSet(tasks ...Task) *set {
|
||||
s := &set{
|
||||
order: []string{},
|
||||
tasks: make(map[string]Task),
|
||||
}
|
||||
|
||||
s.Add(tasks...)
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func (ts *set) Len() int {
|
||||
return len(ts.tasks)
|
||||
}
|
||||
|
||||
func (ts *set) Add(tasks ...Task) {
|
||||
for _, t := range tasks {
|
||||
taskName := t.Name()
|
||||
if _, exists := ts.tasks[taskName]; exists {
|
||||
continue
|
||||
}
|
||||
ts.tasks[taskName] = t
|
||||
ts.order = append(ts.order, taskName)
|
||||
}
|
||||
}
|
||||
|
||||
func (ts *set) Remove(tasks ...Task) {
|
||||
for _, t := range tasks {
|
||||
taskName := t.Name()
|
||||
if _, exists := ts.tasks[taskName]; !exists {
|
||||
continue
|
||||
}
|
||||
|
||||
delete(ts.tasks, taskName)
|
||||
for i, t := range ts.order {
|
||||
if t == taskName {
|
||||
ts.order = append(ts.order[:i], ts.order[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ts *set) Intersect(tasks ...Task) {
|
||||
other := newSet(tasks...)
|
||||
result := newSet()
|
||||
for _, taskName := range ts.order {
|
||||
// we make a new set to prevent the original set from being modified while we are iterating over "order"
|
||||
if _, exists := other.tasks[taskName]; exists {
|
||||
// note: keep the original task and ordering
|
||||
result.Add(ts.tasks[taskName])
|
||||
}
|
||||
}
|
||||
*ts = *result
|
||||
}
|
||||
|
||||
func (ts set) Tasks() tasks {
|
||||
var result []Task
|
||||
for _, name := range ts.order {
|
||||
result = append(result, ts.tasks[name])
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
var _ Task = (*mockTask)(nil)
|
||||
|
||||
type mockTask struct {
|
||||
name string
|
||||
}
|
||||
|
||||
func (m mockTask) Execute(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (m mockTask) Name() string {
|
||||
return m.name
|
||||
}
|
||||
|
||||
func Test_set_Add(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
initialTasks []Task
|
||||
newTasks []Task
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "add unique tasks",
|
||||
initialTasks: []Task{mockTask{"task2"}, mockTask{"task1"}},
|
||||
newTasks: []Task{mockTask{"task3"}},
|
||||
expected: []string{
|
||||
"task2", // note order is honored
|
||||
"task1",
|
||||
"task3",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "add duplicate tasks",
|
||||
initialTasks: []Task{mockTask{"task1"}, mockTask{"task2"}},
|
||||
newTasks: []Task{mockTask{"task1"}},
|
||||
expected: []string{
|
||||
"task1",
|
||||
"task2",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := newSet(tt.initialTasks...)
|
||||
s.Add(tt.newTasks...)
|
||||
got := s.Tasks()
|
||||
var gotNames []string
|
||||
for _, tsk := range got {
|
||||
gotNames = append(gotNames, tsk.Name())
|
||||
}
|
||||
assert.Equal(t, tt.expected, gotNames)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_set_Remove(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
initialTasks []Task
|
||||
tasksToRemove []Task
|
||||
expectedOrder []string
|
||||
}{
|
||||
{
|
||||
name: "remove existing tasks",
|
||||
initialTasks: []Task{mockTask{"task1"}, mockTask{"task2"}, mockTask{"task3"}},
|
||||
tasksToRemove: []Task{mockTask{"task2"}},
|
||||
expectedOrder: []string{"task1", "task3"},
|
||||
},
|
||||
{
|
||||
name: "remove non-existing tasks",
|
||||
initialTasks: []Task{mockTask{"task1"}, mockTask{"task2"}},
|
||||
tasksToRemove: []Task{mockTask{"task3"}},
|
||||
expectedOrder: []string{"task1", "task2"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := newSet(tt.initialTasks...)
|
||||
s.Remove(tt.tasksToRemove...)
|
||||
assert.Equal(t, tt.expectedOrder, s.order)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_set_Intersect(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
initialTasks []Task
|
||||
intersectTasks []Task
|
||||
expectedOrder []string
|
||||
}{
|
||||
{
|
||||
name: "intersect with overlapping tasks",
|
||||
initialTasks: []Task{mockTask{"task1"}, mockTask{"task2"}},
|
||||
intersectTasks: []Task{mockTask{"task2"}, mockTask{"task3"}},
|
||||
expectedOrder: []string{"task2"},
|
||||
},
|
||||
{
|
||||
name: "intersect with non-overlapping tasks",
|
||||
initialTasks: []Task{mockTask{"task1"}, mockTask{"task4"}},
|
||||
intersectTasks: []Task{mockTask{"task2"}, mockTask{"task3"}},
|
||||
expectedOrder: []string{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := newSet(tt.initialTasks...)
|
||||
s.Intersect(tt.intersectTasks...)
|
||||
assert.Equal(t, tt.expectedOrder, s.order)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_set_Tasks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
initialTasks []Task
|
||||
expectedTasks tasks
|
||||
}{
|
||||
{
|
||||
name: "empty set",
|
||||
initialTasks: []Task{},
|
||||
expectedTasks: nil,
|
||||
},
|
||||
{
|
||||
name: "get tasks from set",
|
||||
initialTasks: []Task{mockTask{"task1"}, mockTask{"task2"}},
|
||||
expectedTasks: []Task{mockTask{"task1"}, mockTask{"task2"}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := newSet(tt.initialTasks...)
|
||||
resultTasks := s.Tasks()
|
||||
assert.Equal(t, tt.expectedTasks, resultTasks)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
var _ interface {
|
||||
Task
|
||||
Selector
|
||||
} = (*task)(nil)
|
||||
|
||||
// Task is a function that can wrap a cataloger to populate the SBOM with data (coordinated through the mutex).
|
||||
type Task interface {
|
||||
Name() string
|
||||
Execute(context.Context, file.Resolver, sbomsync.Builder) error
|
||||
}
|
||||
|
||||
type Selector interface {
|
||||
HasAllSelectors(...string) bool
|
||||
Selectors() []string
|
||||
}
|
||||
|
||||
type tasks []Task
|
||||
|
||||
type task struct {
|
||||
name string
|
||||
selectors *strset.Set
|
||||
task func(context.Context, file.Resolver, sbomsync.Builder) error
|
||||
}
|
||||
|
||||
func NewTask(name string, tsk func(context.Context, file.Resolver, sbomsync.Builder) error, tags ...string) Task {
|
||||
if tsk == nil {
|
||||
panic(fmt.Errorf("task cannot be nil"))
|
||||
}
|
||||
tags = append(tags, name)
|
||||
return &task{
|
||||
name: name,
|
||||
selectors: strset.New(tags...),
|
||||
task: tsk,
|
||||
}
|
||||
}
|
||||
|
||||
func (t task) HasAllSelectors(ids ...string) bool {
|
||||
// tags or name
|
||||
return t.selectors.Has(ids...)
|
||||
}
|
||||
|
||||
func (t task) Selectors() []string {
|
||||
return t.selectors.List()
|
||||
}
|
||||
|
||||
func (t task) Name() string {
|
||||
return t.name
|
||||
}
|
||||
|
||||
func (t task) Execute(ctx context.Context, resolver file.Resolver, sbom sbomsync.Builder) error {
|
||||
return t.task(ctx, resolver, sbom)
|
||||
}
|
||||
|
||||
func (ts tasks) Names() []string {
|
||||
var names []string
|
||||
for _, td := range ts {
|
||||
names = append(names, td.Name())
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func (ts tasks) Tags() []string {
|
||||
tags := strset.New()
|
||||
for _, td := range ts {
|
||||
if s, ok := td.(Selector); ok {
|
||||
tags.Add(s.Selectors()...)
|
||||
}
|
||||
|
||||
tags.Remove(td.Name())
|
||||
}
|
||||
|
||||
tagsList := tags.List()
|
||||
sort.Strings(tagsList)
|
||||
|
||||
return tagsList
|
||||
}
|
||||
Reference in New Issue
Block a user