Add file catalogers to selection configuration (#3505)

* add file catalogers to selection configuration

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix typos

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* warn when there is conflicting file cataloging configuration

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* allow for explicit removal of all package and file tasks

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address PR feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman 2025-02-03 14:10:17 -05:00 committed by GitHub
parent 5e2ba43328
commit 684b6e3f98
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 1206 additions and 353 deletions

View File

@ -6,7 +6,7 @@ import (
"github.com/anchore/clio" "github.com/anchore/clio"
"github.com/anchore/stereoscope" "github.com/anchore/stereoscope"
ui2 "github.com/anchore/syft/cmd/syft/cli/ui" handler "github.com/anchore/syft/cmd/syft/cli/ui"
"github.com/anchore/syft/cmd/syft/internal/ui" "github.com/anchore/syft/cmd/syft/internal/ui"
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
@ -28,7 +28,7 @@ func AppClioSetupConfig(id clio.Identification, out io.Writer) *clio.SetupConfig
return clio.NewUICollection( return clio.NewUICollection(
ui.New(out, cfg.Log.Quiet, ui.New(out, cfg.Log.Quiet,
ui2.New(ui2.DefaultHandlerConfig()), handler.New(handler.DefaultHandlerConfig()),
), ),
noUI, noUI,
), nil ), nil

View File

@ -3,6 +3,7 @@ package commands
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"os"
"sort" "sort"
"strings" "strings"
@ -14,7 +15,15 @@ import (
"github.com/anchore/clio" "github.com/anchore/clio"
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/task" "github.com/anchore/syft/internal/task"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging"
)
var (
activelyAddedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("10")) // hi green
deselectedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey
activelyRemovedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("9")) // high red
defaultStyle = lipgloss.NewStyle().Underline(true)
deselectedDefaultStyle = lipgloss.NewStyle().Inherit(deselectedStyle).Underline(true)
) )
type catalogerListOptions struct { type catalogerListOptions struct {
@ -46,6 +55,7 @@ func CatalogerList(app clio.Application) *cobra.Command {
return app.SetupCommand(&cobra.Command{ return app.SetupCommand(&cobra.Command{
Use: "list [OPTIONS]", Use: "list [OPTIONS]",
Short: "List available catalogers", Short: "List available catalogers",
PreRunE: disableUI(app, os.Stdout),
RunE: func(_ *cobra.Command, _ []string) error { RunE: func(_ *cobra.Command, _ []string) error {
return runCatalogerList(opts) return runCatalogerList(opts)
}, },
@ -53,13 +63,19 @@ func CatalogerList(app clio.Application) *cobra.Command {
} }
func runCatalogerList(opts *catalogerListOptions) error { func runCatalogerList(opts *catalogerListOptions) error {
factories := task.DefaultPackageTaskFactories() pkgTaskFactories := task.DefaultPackageTaskFactories()
allTasks, err := factories.Tasks(task.DefaultCatalogingFactoryConfig()) fileTaskFactories := task.DefaultFileTaskFactories()
allPkgTasks, err := pkgTaskFactories.Tasks(task.DefaultCatalogingFactoryConfig())
if err != nil { if err != nil {
return fmt.Errorf("unable to create cataloger tasks: %w", err) return fmt.Errorf("unable to create pkg cataloger tasks: %w", err)
} }
report, err := catalogerListReport(opts, allTasks) allFileTasks, err := fileTaskFactories.Tasks(task.DefaultCatalogingFactoryConfig())
if err != nil {
return fmt.Errorf("unable to create file cataloger tasks: %w", err)
}
report, err := catalogerListReport(opts, [][]task.Task{allPkgTasks, allFileTasks})
if err != nil { if err != nil {
return fmt.Errorf("unable to generate cataloger list report: %w", err) return fmt.Errorf("unable to generate cataloger list report: %w", err)
} }
@ -69,9 +85,10 @@ func runCatalogerList(opts *catalogerListOptions) error {
return nil return nil
} }
func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (string, error) { func catalogerListReport(opts *catalogerListOptions, allTaskGroups [][]task.Task) (string, error) {
selectedTasks, selectionEvidence, err := task.Select(allTasks, selectedTaskGroups, selectionEvidence, err := task.SelectInGroups(
pkgcataloging.NewSelectionRequest(). allTaskGroups,
cataloging.NewSelectionRequest().
WithDefaults(opts.DefaultCatalogers...). WithDefaults(opts.DefaultCatalogers...).
WithExpression(opts.SelectCatalogers...), WithExpression(opts.SelectCatalogers...),
) )
@ -82,12 +99,12 @@ func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (stri
switch opts.Output { switch opts.Output {
case "json": case "json":
report, err = renderCatalogerListJSON(selectedTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers) report, err = renderCatalogerListJSON(flattenTaskGroups(selectedTaskGroups), selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers)
case "table", "": case "table", "":
if opts.ShowHidden { if opts.ShowHidden {
report = renderCatalogerListTable(allTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers) report = renderCatalogerListTables(allTaskGroups, selectionEvidence)
} else { } else {
report = renderCatalogerListTable(selectedTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers) report = renderCatalogerListTables(selectedTaskGroups, selectionEvidence)
} }
} }
@ -98,6 +115,14 @@ func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (stri
return report, nil return report, nil
} }
func flattenTaskGroups(taskGroups [][]task.Task) []task.Task {
var allTasks []task.Task
for _, tasks := range taskGroups {
allTasks = append(allTasks, tasks...)
}
return allTasks
}
func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaultSelections, selections []string) (string, error) { func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaultSelections, selections []string) (string, error) {
type node struct { type node struct {
Name string `json:"name"` Name string `json:"name"`
@ -109,7 +134,12 @@ func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaul
nodesByName := make(map[string]node) nodesByName := make(map[string]node)
for name := range tagsByName { for name := range tagsByName {
tagsSelected := selection.TokensByTask[name].SelectedOn.List() tokensByTask, ok := selection.TokensByTask[name]
var tagsSelected []string
if ok {
tagsSelected = tokensByTask.SelectedOn.List()
}
if len(tagsSelected) == 1 && tagsSelected[0] == "all" { if len(tagsSelected) == 1 && tagsSelected[0] == "all" {
tagsSelected = tagsByName[name] tagsSelected = tagsByName[name]
@ -153,10 +183,56 @@ func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaul
return string(by), err return string(by), err
} }
func renderCatalogerListTable(tasks []task.Task, selection task.Selection, defaultSelections, selections []string) string { func renderCatalogerListTables(taskGroups [][]task.Task, selection task.Selection) string {
pkgCatalogerTable := renderCatalogerListTable(taskGroups[0], selection, "Package Cataloger")
fileCatalogerTable := renderCatalogerListTable(taskGroups[1], selection, "File Cataloger")
report := fileCatalogerTable + "\n" + pkgCatalogerTable + "\n"
hasAdditions := len(selection.Request.AddNames) > 0
hasDefaults := len(selection.Request.DefaultNamesOrTags) > 0
hasRemovals := len(selection.Request.RemoveNamesOrTags) > 0
hasSubSelections := len(selection.Request.SubSelectTags) > 0
expressions := len(selection.Request.SubSelectTags) + len(selection.Request.AddNames) + len(selection.Request.RemoveNamesOrTags)
var header string
header += fmt.Sprintf("Default selections: %d\n", len(selection.Request.DefaultNamesOrTags))
if hasDefaults {
for _, expr := range selection.Request.DefaultNamesOrTags {
header += fmt.Sprintf(" • '%s'\n", expr)
}
}
header += fmt.Sprintf("Selection expressions: %d\n", expressions)
if hasSubSelections {
for _, n := range selection.Request.SubSelectTags {
header += fmt.Sprintf(" • '%s' (intersect)\n", n)
}
}
if hasRemovals {
for _, n := range selection.Request.RemoveNamesOrTags {
header += fmt.Sprintf(" • '-%s' (remove)\n", n)
}
}
if hasAdditions {
for _, n := range selection.Request.AddNames {
header += fmt.Sprintf(" • '+%s' (add)\n", n)
}
}
return header + report
}
func renderCatalogerListTable(tasks []task.Task, selection task.Selection, kindTitle string) string {
if len(tasks) == 0 {
return activelyRemovedStyle.Render(fmt.Sprintf("No %ss selected", strings.ToLower(kindTitle)))
}
t := table.NewWriter() t := table.NewWriter()
t.SetStyle(table.StyleLight) t.SetStyle(table.StyleLight)
t.AppendHeader(table.Row{"Cataloger", "Tags"}) t.AppendHeader(table.Row{kindTitle, "Tags"})
names, tagsByName := extractTaskInfo(tasks) names, tagsByName := extractTaskInfo(tasks)
@ -172,27 +248,12 @@ func renderCatalogerListTable(tasks []task.Task, selection task.Selection, defau
report := t.Render() report := t.Render()
if len(selections) > 0 {
header := "Selected by expressions:\n"
for _, expr := range selections {
header += fmt.Sprintf(" - %q\n", expr)
}
report = header + report
}
if len(defaultSelections) > 0 {
header := "Default selections:\n"
for _, expr := range defaultSelections {
header += fmt.Sprintf(" - %q\n", expr)
}
report = header + report
}
return report return report
} }
func formatRow(name string, tags []string, selection task.Selection) table.Row { func formatRow(name string, tags []string, selection task.Selection) table.Row {
isIncluded := selection.Result.Has(name) isIncluded := selection.Result.Has(name)
defaults := strset.New(selection.Request.DefaultNamesOrTags...)
var selections *task.TokenSelection var selections *task.TokenSelection
if s, exists := selection.TokensByTask[name]; exists { if s, exists := selection.TokensByTask[name]; exists {
selections = &s selections = &s
@ -200,35 +261,32 @@ func formatRow(name string, tags []string, selection task.Selection) table.Row {
var formattedTags []string var formattedTags []string
for _, tag := range tags { for _, tag := range tags {
formattedTags = append(formattedTags, formatToken(tag, selections, isIncluded)) formattedTags = append(formattedTags, formatToken(tag, selections, isIncluded, defaults))
} }
var tagStr string var tagStr string
if isIncluded { if isIncluded {
tagStr = strings.Join(formattedTags, ", ") tagStr = strings.Join(formattedTags, ", ")
} else { } else {
tagStr = strings.Join(formattedTags, grey.Render(", ")) tagStr = strings.Join(formattedTags, deselectedStyle.Render(", "))
} }
// TODO: selection should keep warnings (non-selections) in struct // TODO: selection should keep warnings (non-selections) in struct
return table.Row{ return table.Row{
formatToken(name, selections, isIncluded), formatToken(name, selections, isIncluded, defaults),
tagStr, tagStr,
} }
} }
var ( func formatToken(token string, selection *task.TokenSelection, included bool, defaults *strset.Set) string {
green = lipgloss.NewStyle().Foreground(lipgloss.Color("10")) // hi green
grey = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey
red = lipgloss.NewStyle().Foreground(lipgloss.Color("9")) // high red
)
func formatToken(token string, selection *task.TokenSelection, included bool) string {
if included && selection != nil { if included && selection != nil {
// format all tokens in selection in green // format all tokens in selection in green
if selection.SelectedOn.Has(token) { if selection.SelectedOn.Has(token) {
return green.Render(token) if defaults.Has(token) {
return defaultStyle.Render(token)
}
return activelyAddedStyle.Render(token)
} }
return token return token
@ -236,10 +294,12 @@ func formatToken(token string, selection *task.TokenSelection, included bool) st
// format all tokens in selection in red, all others in grey // format all tokens in selection in red, all others in grey
if selection != nil && selection.DeselectedOn.Has(token) { if selection != nil && selection.DeselectedOn.Has(token) {
return red.Render(token) return activelyRemovedStyle.Render(token)
} }
if defaults.Has(token) {
return grey.Render(token) return deselectedDefaultStyle.Render(token)
}
return deselectedStyle.Render(token)
} }
func extractTaskInfo(tasks []task.Task) ([]string, map[string][]string) { func extractTaskInfo(tasks []task.Task) ([]string, map[string][]string) {

View File

@ -40,8 +40,9 @@ func (d dummyTask) Execute(_ context.Context, _ file.Resolver, _ sbomsync.Builde
panic("implement me") panic("implement me")
} }
func testTasks() []task.Task { func testTasks() [][]task.Task {
return []task.Task{ return [][]task.Task{
{
dummyTask{ dummyTask{
name: "task1", name: "task1",
selectors: []string{"image", "a", "b", "1"}, selectors: []string{"image", "a", "b", "1"},
@ -58,6 +59,17 @@ func testTasks() []task.Task {
name: "task4", name: "task4",
selectors: []string{"directory", "d", "e", "4"}, selectors: []string{"directory", "d", "e", "4"},
}, },
},
{
dummyTask{
name: "file-task1",
selectors: []string{"file", "ft", "ft-1-b"},
},
dummyTask{
name: "file-task2",
selectors: []string{"file", "ft", "ft-2-b"},
},
},
} }
} }
@ -76,16 +88,23 @@ func Test_catalogerListReport(t *testing.T) {
return c return c
}(), }(),
want: ` want: `
Default selections: Default selections: 1
- "all" 'all'
Selection expressions: 0
CATALOGER TAGS
FILE CATALOGER TAGS
file-task1 file, ft, ft-1-b
file-task2 file, ft, ft-2-b
PACKAGE CATALOGER TAGS
task1 1, a, b, image task1 1, a, b, image
task2 2, b, c, image task2 2, b, c, image
task3 3, c, d, directory task3 3, c, d, directory
task4 4, d, directory, e task4 4, d, directory, e
`, `,
}, },
{ {
@ -96,7 +115,7 @@ Default selections:
return c return c
}(), }(),
want: ` want: `
{"default":["all"],"selection":[],"catalogers":[{"name":"task1","tags":["1","a","b","image"]},{"name":"task2","tags":["2","b","c","image"]},{"name":"task3","tags":["3","c","d","directory"]},{"name":"task4","tags":["4","d","directory","e"]}]} {"default":["all"],"selection":[],"catalogers":[{"name":"file-task1","tags":["file","ft","ft-1-b"]},{"name":"file-task2","tags":["file","ft","ft-2-b"]},{"name":"task1","tags":["1","a","b","image"]},{"name":"task2","tags":["2","b","c","image"]},{"name":"task3","tags":["3","c","d","directory"]},{"name":"task4","tags":["4","d","directory","e"]}]}
`, `,
}, },
{ {
@ -105,19 +124,27 @@ Default selections:
c := defaultCatalogerListOptions() c := defaultCatalogerListOptions()
c.Output = "table" c.Output = "table"
c.DefaultCatalogers = []string{ c.DefaultCatalogers = []string{
"image", "image", // note: for backwards compatibility file will automatically be added
} }
return c return c
}(), }(),
want: ` want: `
Default selections: Default selections: 2
- "image" 'image'
'file'
CATALOGER TAGS Selection expressions: 0
FILE CATALOGER TAGS
file-task1 file, ft, ft-1-b
file-task2 file, ft, ft-2-b
PACKAGE CATALOGER TAGS
task1 1, a, b, image task1 1, a, b, image
task2 2, b, c, image task2 2, b, c, image
`, `,
}, },
{ {
@ -131,7 +158,7 @@ Default selections:
return c return c
}(), }(),
want: ` want: `
{"default":["image"],"selection":[],"catalogers":[{"name":"task1","tags":["image"]},{"name":"task2","tags":["image"]}]} {"default":["image"],"selection":[],"catalogers":[{"name":"file-task1","tags":["file"]},{"name":"file-task2","tags":["file"]},{"name":"task1","tags":["image"]},{"name":"task2","tags":["image"]}]}
`, `,
}, },
{ {
@ -147,23 +174,32 @@ Default selections:
"+task3", "+task3",
"-c", "-c",
"b", "b",
"-file",
"+file-task1",
} }
return c return c
}(), }(),
want: ` want: `
Default selections: Default selections: 1
- "image" 'image'
Selected by expressions: Selection expressions: 6
- "-directory" 'b' (intersect)
- "+task3" '-directory' (remove)
- "-c" '-c' (remove)
- "b" '-file' (remove)
'+task3' (add)
CATALOGER TAGS '+file-task1' (add)
FILE CATALOGER TAGS
file-task1 file, ft, ft-1-b
PACKAGE CATALOGER TAGS
task1 1, a, b, image task1 1, a, b, image
task3 3, c, d, directory task3 3, c, d, directory
`, `,
}, },
{ {
@ -183,7 +219,7 @@ Selected by expressions:
return c return c
}(), }(),
want: ` want: `
{"default":["image"],"selection":["-directory","+task3","-c","b"],"catalogers":[{"name":"task1","tags":["b","image"]},{"name":"task3","tags":["task3"]}]} {"default":["image"],"selection":["-directory","+task3","-c","b"],"catalogers":[{"name":"file-task1","tags":["file"]},{"name":"file-task2","tags":["file"]},{"name":"task1","tags":["b","image"]},{"name":"task3","tags":["task3"]}]}
`, `,
}, },
} }

View File

@ -0,0 +1,23 @@
package commands
import (
"io"
"github.com/spf13/cobra"
"github.com/anchore/clio"
"github.com/anchore/syft/cmd/syft/internal/ui"
)
func disableUI(app clio.Application, out io.Writer) func(*cobra.Command, []string) error {
return func(_ *cobra.Command, _ []string) error {
type Stater interface {
State() *clio.State
}
state := app.(Stater).State()
state.UI = clio.NewUICollection(ui.None(out, state.Config.Log.Quiet))
return nil
}
}

View File

@ -91,7 +91,7 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
WithPackagesConfig(cfg.ToPackagesConfig()). WithPackagesConfig(cfg.ToPackagesConfig()).
WithFilesConfig(cfg.ToFilesConfig()). WithFilesConfig(cfg.ToFilesConfig()).
WithCatalogerSelection( WithCatalogerSelection(
pkgcataloging.NewSelectionRequest(). cataloging.NewSelectionRequest().
WithDefaults(cfg.DefaultCatalogers...). WithDefaults(cfg.DefaultCatalogers...).
WithExpression(cfg.SelectCatalogers...), WithExpression(cfg.SelectCatalogers...),
) )

View File

@ -10,6 +10,7 @@ import (
"github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/cmd/syft/internal/options" "github.com/anchore/syft/cmd/syft/internal/options"
"github.com/anchore/syft/syft" "github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging/pkgcataloging"
"github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
@ -20,7 +21,7 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string, scope source.Sco
Name: "syft-tester", Name: "syft-tester",
Version: "v0.99.0", Version: "v0.99.0",
}).WithCatalogerSelection( }).WithCatalogerSelection(
pkgcataloging.NewSelectionRequest(). cataloging.NewSelectionRequest().
WithExpression(catalogerSelection...), WithExpression(catalogerSelection...),
) )
cfg.Search.Scope = scope cfg.Search.Scope = scope
@ -55,7 +56,7 @@ func catalogDirectory(t *testing.T, dir string, catalogerSelection ...string) (s
Name: "syft-tester", Name: "syft-tester",
Version: "v0.99.0", Version: "v0.99.0",
}).WithCatalogerSelection( }).WithCatalogerSelection(
pkgcataloging.NewSelectionRequest(). cataloging.NewSelectionRequest().
WithExpression(catalogerSelection...), WithExpression(catalogerSelection...),
) )

View File

@ -74,7 +74,7 @@ func getSBOM(src source.Source) sbom.SBOM {
// only use OS related catalogers that would have been used with the kind of // only use OS related catalogers that would have been used with the kind of
// source type (container image or directory), but also add a specific python cataloger // source type (container image or directory), but also add a specific python cataloger
WithCatalogerSelection( WithCatalogerSelection(
pkgcataloging.NewSelectionRequest(). cataloging.NewSelectionRequest().
WithSubSelections("os"). WithSubSelections("os").
WithAdditions("python-package-cataloger"), WithAdditions("python-package-cataloger"),
). ).

View File

@ -6,6 +6,7 @@ import (
"os" "os"
"github.com/anchore/syft/syft" "github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging/pkgcataloging"
"github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
@ -55,7 +56,7 @@ func getSBOM(src source.Source, defaultTags ...string) sbom.SBOM {
WithCatalogerSelection( WithCatalogerSelection(
// here you can sub-select, add, remove catalogers from the default selection... // here you can sub-select, add, remove catalogers from the default selection...
// or replace the default selection entirely! // or replace the default selection entirely!
pkgcataloging.NewSelectionRequest(). cataloging.NewSelectionRequest().
WithDefaults(defaultTags...), WithDefaults(defaultTags...),
) )

View File

@ -0,0 +1,27 @@
package task
import (
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/filecataloging"
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
)
type CatalogingFactoryConfig struct {
ComplianceConfig cataloging.ComplianceConfig
SearchConfig cataloging.SearchConfig
RelationshipsConfig cataloging.RelationshipsConfig
DataGenerationConfig cataloging.DataGenerationConfig
PackagesConfig pkgcataloging.Config
FilesConfig filecataloging.Config
}
func DefaultCatalogingFactoryConfig() CatalogingFactoryConfig {
return CatalogingFactoryConfig{
ComplianceConfig: cataloging.DefaultComplianceConfig(),
SearchConfig: cataloging.DefaultSearchConfig(),
RelationshipsConfig: cataloging.DefaultRelationshipsConfig(),
DataGenerationConfig: cataloging.DefaultDataGenerationConfig(),
PackagesConfig: pkgcataloging.DefaultConfig(),
FilesConfig: filecataloging.DefaultConfig(),
}
}

View File

@ -8,7 +8,7 @@ import (
"github.com/hashicorp/go-multierror" "github.com/hashicorp/go-multierror"
"github.com/scylladb/go-set/strset" "github.com/scylladb/go-set/strset"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging"
) )
var expressionNodePattern = regexp.MustCompile(`^([a-zA-Z0-9][a-zA-Z0-9-+]*)+$`) var expressionNodePattern = regexp.MustCompile(`^([a-zA-Z0-9][a-zA-Z0-9-+]*)+$`)
@ -142,7 +142,7 @@ func (ec expressionContext) newExpression(exp string, operation Operation, token
} }
} }
func newExpressionsFromSelectionRequest(nc *expressionContext, selectionRequest pkgcataloging.SelectionRequest) Expressions { func newExpressionsFromSelectionRequest(nc *expressionContext, selectionRequest cataloging.SelectionRequest) Expressions {
var all Expressions var all Expressions
for _, exp := range selectionRequest.DefaultNamesOrTags { for _, exp := range selectionRequest.DefaultNamesOrTags {

View File

@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging"
) )
func Test_newExpressionsFromSelectionRequest(t *testing.T) { func Test_newExpressionsFromSelectionRequest(t *testing.T) {
@ -135,7 +135,7 @@ func Test_newExpressionsFromSelectionRequest(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
req := pkgcataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...) req := cataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...)
result := newExpressionsFromSelectionRequest(nc, req) result := newExpressionsFromSelectionRequest(nc, req)
if tt.expectedErrors != nil { if tt.expectedErrors != nil {

40
internal/task/factory.go Normal file
View File

@ -0,0 +1,40 @@
package task
import (
"fmt"
"sort"
"strings"
"github.com/scylladb/go-set/strset"
)
type factory func(cfg CatalogingFactoryConfig) Task
type Factories []factory
func (f Factories) Tasks(cfg CatalogingFactoryConfig) ([]Task, error) {
var allTasks []Task
taskNames := strset.New()
duplicateTaskNames := strset.New()
var err error
for _, fact := range f {
tsk := fact(cfg)
if tsk == nil {
continue
}
tskName := tsk.Name()
if taskNames.Has(tskName) {
duplicateTaskNames.Add(tskName)
}
allTasks = append(allTasks, tsk)
taskNames.Add(tskName)
}
if duplicateTaskNames.Size() > 0 {
names := duplicateTaskNames.List()
sort.Strings(names)
err = fmt.Errorf("duplicate cataloger task names: %v", strings.Join(names, ", "))
}
return allTasks, err
}

View File

@ -6,6 +6,7 @@ import (
"github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging/filecataloging"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/file/cataloger/executable" "github.com/anchore/syft/syft/file/cataloger/executable"
"github.com/anchore/syft/syft/file/cataloger/filecontent" "github.com/anchore/syft/syft/file/cataloger/filecontent"
@ -15,14 +16,27 @@ import (
"github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/sbom"
) )
func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash) Task { func DefaultFileTaskFactories() Factories {
return Factories{
newFileDigestCatalogerTaskFactory("digest"),
newFileMetadataCatalogerTaskFactory("file-metadata"),
newFileContentCatalogerTaskFactory("content"),
newExecutableCatalogerTaskFactory("binary-metadata"),
}
}
func newFileDigestCatalogerTaskFactory(tags ...string) factory {
return func(cfg CatalogingFactoryConfig) Task {
return newFileDigestCatalogerTask(cfg.FilesConfig.Selection, cfg.FilesConfig.Hashers, tags...)
}
}
func newFileDigestCatalogerTask(selection file.Selection, hashers []crypto.Hash, tags ...string) Task {
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
if selection == file.NoFilesSelection || len(hashers) == 0 { if selection == file.NoFilesSelection || len(hashers) == 0 {
return nil return nil
} }
digestsCataloger := filedigest.NewCataloger(hashers)
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
accessor := builder.(sbomsync.Accessor) accessor := builder.(sbomsync.Accessor)
coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor)) coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor))
@ -30,7 +44,7 @@ func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash
return nil return nil
} }
result, err := digestsCataloger.Catalog(ctx, resolver, coordinates...) result, err := filedigest.NewCataloger(hashers).Catalog(ctx, resolver, coordinates...)
accessor.WriteToSBOM(func(sbom *sbom.SBOM) { accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileDigests = result sbom.Artifacts.FileDigests = result
@ -39,17 +53,21 @@ func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash
return err return err
} }
return NewTask("file-digest-cataloger", fn) return NewTask("file-digest-cataloger", fn, commonFileTags(tags)...)
} }
func NewFileMetadataCatalogerTask(selection file.Selection) Task { func newFileMetadataCatalogerTaskFactory(tags ...string) factory {
return func(cfg CatalogingFactoryConfig) Task {
return newFileMetadataCatalogerTask(cfg.FilesConfig.Selection, tags...)
}
}
func newFileMetadataCatalogerTask(selection file.Selection, tags ...string) Task {
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
if selection == file.NoFilesSelection { if selection == file.NoFilesSelection {
return nil return nil
} }
metadataCataloger := filemetadata.NewCataloger()
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
accessor := builder.(sbomsync.Accessor) accessor := builder.(sbomsync.Accessor)
coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor)) coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor))
@ -57,7 +75,7 @@ func NewFileMetadataCatalogerTask(selection file.Selection) Task {
return nil return nil
} }
result, err := metadataCataloger.Catalog(ctx, resolver, coordinates...) result, err := filemetadata.NewCataloger().Catalog(ctx, resolver, coordinates...)
accessor.WriteToSBOM(func(sbom *sbom.SBOM) { accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileMetadata = result sbom.Artifacts.FileMetadata = result
@ -66,20 +84,24 @@ func NewFileMetadataCatalogerTask(selection file.Selection) Task {
return err return err
} }
return NewTask("file-metadata-cataloger", fn) return NewTask("file-metadata-cataloger", fn, commonFileTags(tags)...)
} }
func NewFileContentCatalogerTask(cfg filecontent.Config) Task { func newFileContentCatalogerTaskFactory(tags ...string) factory {
return func(cfg CatalogingFactoryConfig) Task {
return newFileContentCatalogerTask(cfg.FilesConfig.Content, tags...)
}
}
func newFileContentCatalogerTask(cfg filecontent.Config, tags ...string) Task {
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
if len(cfg.Globs) == 0 { if len(cfg.Globs) == 0 {
return nil return nil
} }
cat := filecontent.NewCataloger(cfg)
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
accessor := builder.(sbomsync.Accessor) accessor := builder.(sbomsync.Accessor)
result, err := cat.Catalog(ctx, resolver) result, err := filecontent.NewCataloger(cfg).Catalog(ctx, resolver)
accessor.WriteToSBOM(func(sbom *sbom.SBOM) { accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileContents = result sbom.Artifacts.FileContents = result
@ -88,20 +110,24 @@ func NewFileContentCatalogerTask(cfg filecontent.Config) Task {
return err return err
} }
return NewTask("file-content-cataloger", fn) return NewTask("file-content-cataloger", fn, commonFileTags(tags)...)
} }
func NewExecutableCatalogerTask(selection file.Selection, cfg executable.Config) Task { func newExecutableCatalogerTaskFactory(tags ...string) factory {
return func(cfg CatalogingFactoryConfig) Task {
return newExecutableCatalogerTask(cfg.FilesConfig.Selection, cfg.FilesConfig.Executable, tags...)
}
}
func newExecutableCatalogerTask(selection file.Selection, cfg executable.Config, tags ...string) Task {
fn := func(_ context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
if selection == file.NoFilesSelection { if selection == file.NoFilesSelection {
return nil return nil
} }
cat := executable.NewCataloger(cfg)
fn := func(_ context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
accessor := builder.(sbomsync.Accessor) accessor := builder.(sbomsync.Accessor)
result, err := cat.Catalog(resolver) result, err := executable.NewCataloger(cfg).Catalog(resolver)
accessor.WriteToSBOM(func(sbom *sbom.SBOM) { accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.Executables = result sbom.Artifacts.Executables = result
@ -110,7 +136,7 @@ func NewExecutableCatalogerTask(selection file.Selection, cfg executable.Config)
return err return err
} }
return NewTask("file-executable-cataloger", fn) return NewTask("file-executable-cataloger", fn, commonFileTags(tags)...)
} }
// TODO: this should be replaced with a fix that allows passing a coordinate or location iterator to the cataloger // TODO: this should be replaced with a fix that allows passing a coordinate or location iterator to the cataloger
@ -154,3 +180,8 @@ func coordinatesForSelection(selection file.Selection, accessor sbomsync.Accesso
return nil, false return nil, false
} }
func commonFileTags(tags []string) []string {
tags = append(tags, filecataloging.FileTag)
return tags
}

View File

@ -3,12 +3,9 @@ package task
import ( import (
"context" "context"
"fmt" "fmt"
"sort"
"strings" "strings"
"unicode" "unicode"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/relationship" "github.com/anchore/syft/internal/relationship"
@ -23,67 +20,18 @@ import (
cpeutils "github.com/anchore/syft/syft/pkg/cataloger/common/cpe" cpeutils "github.com/anchore/syft/syft/pkg/cataloger/common/cpe"
) )
type packageTaskFactory func(cfg CatalogingFactoryConfig) Task func newPackageTaskFactory(catalogerFactory func(CatalogingFactoryConfig) pkg.Cataloger, tags ...string) factory {
type PackageTaskFactories []packageTaskFactory
type CatalogingFactoryConfig struct {
ComplianceConfig cataloging.ComplianceConfig
SearchConfig cataloging.SearchConfig
RelationshipsConfig cataloging.RelationshipsConfig
DataGenerationConfig cataloging.DataGenerationConfig
PackagesConfig pkgcataloging.Config
}
func DefaultCatalogingFactoryConfig() CatalogingFactoryConfig {
return CatalogingFactoryConfig{
ComplianceConfig: cataloging.DefaultComplianceConfig(),
SearchConfig: cataloging.DefaultSearchConfig(),
RelationshipsConfig: cataloging.DefaultRelationshipsConfig(),
DataGenerationConfig: cataloging.DefaultDataGenerationConfig(),
PackagesConfig: pkgcataloging.DefaultConfig(),
}
}
func newPackageTaskFactory(catalogerFactory func(CatalogingFactoryConfig) pkg.Cataloger, tags ...string) packageTaskFactory {
return func(cfg CatalogingFactoryConfig) Task { return func(cfg CatalogingFactoryConfig) Task {
return NewPackageTask(cfg, catalogerFactory(cfg), tags...) return NewPackageTask(cfg, catalogerFactory(cfg), tags...)
} }
} }
func newSimplePackageTaskFactory(catalogerFactory func() pkg.Cataloger, tags ...string) packageTaskFactory { func newSimplePackageTaskFactory(catalogerFactory func() pkg.Cataloger, tags ...string) factory {
return func(cfg CatalogingFactoryConfig) Task { return func(cfg CatalogingFactoryConfig) Task {
return NewPackageTask(cfg, catalogerFactory(), tags...) return NewPackageTask(cfg, catalogerFactory(), tags...)
} }
} }
func (f PackageTaskFactories) Tasks(cfg CatalogingFactoryConfig) ([]Task, error) {
var allTasks []Task
taskNames := strset.New()
duplicateTaskNames := strset.New()
var err error
for _, factory := range f {
tsk := factory(cfg)
if tsk == nil {
continue
}
tskName := tsk.Name()
if taskNames.Has(tskName) {
duplicateTaskNames.Add(tskName)
}
allTasks = append(allTasks, tsk)
taskNames.Add(tskName)
}
if duplicateTaskNames.Size() > 0 {
names := duplicateTaskNames.List()
sort.Strings(names)
err = fmt.Errorf("duplicate cataloger task names: %v", strings.Join(names, ", "))
}
return allTasks, err
}
// NewPackageTask creates a Task function for a generic pkg.Cataloger, honoring the common configuration options. // NewPackageTask creates a Task function for a generic pkg.Cataloger, honoring the common configuration options.
func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string) Task { func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string) Task {
fn := func(ctx context.Context, resolver file.Resolver, sbom sbomsync.Builder) error { fn := func(ctx context.Context, resolver file.Resolver, sbom sbomsync.Builder) error {

View File

@ -51,8 +51,8 @@ const (
) )
//nolint:funlen //nolint:funlen
func DefaultPackageTaskFactories() PackageTaskFactories { func DefaultPackageTaskFactories() Factories {
return []packageTaskFactory{ return []factory{
// OS package installed catalogers /////////////////////////////////////////////////////////////////////////// // OS package installed catalogers ///////////////////////////////////////////////////////////////////////////
newSimplePackageTaskFactory(arch.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "alpm", "archlinux"), newSimplePackageTaskFactory(arch.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "alpm", "archlinux"),
newSimplePackageTaskFactory(alpine.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "apk", "alpine"), newSimplePackageTaskFactory(alpine.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "apk", "alpine"),

View File

@ -3,17 +3,19 @@ package task
import ( import (
"fmt" "fmt"
"sort" "sort"
"strings"
"github.com/scylladb/go-set/strset" "github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/filecataloging"
) )
// Selection represents the users request for a subset of tasks to run and the resulting set of task names that were // Selection represents the users request for a subset of tasks to run and the resulting set of task names that were
// selected. Additionally, all tokens that were matched on to reach the returned conclusion are also provided. // selected. Additionally, all tokens that were matched on to reach the returned conclusion are also provided.
type Selection struct { type Selection struct {
Request pkgcataloging.SelectionRequest Request cataloging.SelectionRequest
Result *strset.Set Result *strset.Set
TokensByTask map[string]TokenSelection TokensByTask map[string]TokenSelection
} }
@ -52,7 +54,18 @@ func newSelection() Selection {
// Select parses the given expressions as two sets: expressions that represent a "set" operation, and expressions that // Select parses the given expressions as two sets: expressions that represent a "set" operation, and expressions that
// represent all other operations. The parsed expressions are then evaluated against the given tasks to return // represent all other operations. The parsed expressions are then evaluated against the given tasks to return
// a subset (or the same) set of tasks. // a subset (or the same) set of tasks.
func Select(allTasks []Task, selectionRequest pkgcataloging.SelectionRequest) ([]Task, Selection, error) { func Select(allTasks []Task, selectionRequest cataloging.SelectionRequest) ([]Task, Selection, error) {
ensureDefaultSelectionHasFiles(&selectionRequest, allTasks)
return _select(allTasks, selectionRequest)
}
func _select(allTasks []Task, selectionRequest cataloging.SelectionRequest) ([]Task, Selection, error) {
if selectionRequest.IsEmpty() {
selection := newSelection()
selection.Request = selectionRequest
return nil, selection, nil
}
nodes := newExpressionsFromSelectionRequest(newExpressionContext(allTasks), selectionRequest) nodes := newExpressionsFromSelectionRequest(newExpressionContext(allTasks), selectionRequest)
finalTasks, selection := selectByExpressions(allTasks, nodes) finalTasks, selection := selectByExpressions(allTasks, nodes)
@ -62,6 +75,142 @@ func Select(allTasks []Task, selectionRequest pkgcataloging.SelectionRequest) ([
return finalTasks, selection, nodes.Validate() return finalTasks, selection, nodes.Validate()
} }
// ensureDefaultSelectionHasFiles ensures that the default selection request has the "file" tag, as this is a required
// for backwards compatibility (when catalogers were only for packages and not for separate groups of tasks).
func ensureDefaultSelectionHasFiles(selectionRequest *cataloging.SelectionRequest, allTasks ...[]Task) {
for _, ts := range allTasks {
_, leftOver := tagsOrNamesThatTaskGroupRespondsTo(ts, strset.New(filecataloging.FileTag))
if leftOver.Has(filecataloging.FileTag) {
// the given set of tasks do not respond to file, so don't include it in the default selection
continue
}
defaultNamesOrTags := strset.New(selectionRequest.DefaultNamesOrTags...)
removals := strset.New(selectionRequest.RemoveNamesOrTags...)
missingFileIshTag := !defaultNamesOrTags.Has(filecataloging.FileTag) && !defaultNamesOrTags.Has("all") && !defaultNamesOrTags.Has("default")
if missingFileIshTag && !removals.Has(filecataloging.FileTag) {
log.Warnf("adding '%s' tag to the default cataloger selection, to override add '-%s' to the cataloger selection request", filecataloging.FileTag, filecataloging.FileTag)
selectionRequest.DefaultNamesOrTags = append(selectionRequest.DefaultNamesOrTags, filecataloging.FileTag)
}
}
}
// SelectInGroups is a convenience function that allows for selecting tasks from multiple groups of tasks. The original
// request is split into sub-requests, where only tokens that are relevant to the given group of tasks are considered.
// If tokens are passed that are not relevant to any group of tasks, an error is returned.
func SelectInGroups(taskGroups [][]Task, selectionRequest cataloging.SelectionRequest) ([][]Task, Selection, error) {
ensureDefaultSelectionHasFiles(&selectionRequest, taskGroups...)
reqs, errs := splitCatalogerSelectionRequest(selectionRequest, taskGroups)
if errs != nil {
return nil, Selection{
Request: selectionRequest,
}, errs
}
var finalTasks [][]Task
var selections []Selection
for idx, req := range reqs {
tskGroup := taskGroups[idx]
subFinalTasks, subSelection, err := _select(tskGroup, req)
if err != nil {
return nil, Selection{
Request: selectionRequest,
}, err
}
finalTasks = append(finalTasks, subFinalTasks)
selections = append(selections, subSelection)
}
return finalTasks, mergeSelections(selections, selectionRequest), nil
}
func mergeSelections(selections []Selection, ogRequest cataloging.SelectionRequest) Selection {
finalSelection := newSelection()
for _, s := range selections {
finalSelection.Result.Add(s.Result.List()...)
for name, tokenSelection := range s.TokensByTask {
if existing, exists := finalSelection.TokensByTask[name]; exists {
existing.merge(tokenSelection)
finalSelection.TokensByTask[name] = existing
} else {
finalSelection.TokensByTask[name] = tokenSelection
}
}
}
finalSelection.Request = ogRequest
return finalSelection
}
func splitCatalogerSelectionRequest(req cataloging.SelectionRequest, selectablePkgTaskGroups [][]Task) ([]cataloging.SelectionRequest, error) {
requestTagsOrNames := allRequestReferences(req)
leftoverTags := strset.New()
usedTagsAndNames := strset.New()
var usedTagGroups []*strset.Set
for _, taskGroup := range selectablePkgTaskGroups {
selectedTagOrNames, remainingTagsOrNames := tagsOrNamesThatTaskGroupRespondsTo(taskGroup, requestTagsOrNames)
leftoverTags = strset.Union(leftoverTags, remainingTagsOrNames)
usedTagGroups = append(usedTagGroups, selectedTagOrNames)
usedTagsAndNames.Add(selectedTagOrNames.List()...)
}
leftoverTags = strset.Difference(leftoverTags, usedTagsAndNames)
leftoverTags.Remove("all")
if leftoverTags.Size() > 0 {
l := leftoverTags.List()
sort.Strings(l)
return nil, fmt.Errorf("no cataloger tasks respond to the following selections: %v", strings.Join(l, ", "))
}
var newSelections []cataloging.SelectionRequest
for _, tags := range usedTagGroups {
newSelections = append(newSelections, newSelectionWithTags(req, tags))
}
return newSelections, nil
}
func newSelectionWithTags(req cataloging.SelectionRequest, tags *strset.Set) cataloging.SelectionRequest {
return cataloging.SelectionRequest{
DefaultNamesOrTags: filterTags(req.DefaultNamesOrTags, tags),
SubSelectTags: filterTags(req.SubSelectTags, tags),
AddNames: filterTags(req.AddNames, tags),
RemoveNamesOrTags: filterTags(req.RemoveNamesOrTags, tags),
}
}
func filterTags(reqTags []string, filterTags *strset.Set) []string {
var filtered []string
for _, tag := range reqTags {
if filterTags.Has(tag) {
filtered = append(filtered, tag)
}
}
return filtered
}
func tagsOrNamesThatTaskGroupRespondsTo(tasks []Task, requestTagsOrNames *strset.Set) (*strset.Set, *strset.Set) {
positiveRefs := strset.New()
for _, t := range tasks {
if sel, ok := t.(Selector); ok {
positiveRefs.Add("all") // everything responds to "all"
positiveRefs.Add(strset.Intersection(requestTagsOrNames, strset.New(sel.Selectors()...)).List()...)
}
positiveRefs.Add(t.Name())
}
return positiveRefs, strset.Difference(requestTagsOrNames, positiveRefs)
}
func allRequestReferences(s cataloging.SelectionRequest) *strset.Set {
st := strset.New()
st.Add(s.DefaultNamesOrTags...)
st.Add(s.SubSelectTags...)
st.Add(s.AddNames...)
st.Add(s.RemoveNamesOrTags...)
return st
}
// selectByExpressions the set of tasks to run based on the given expression(s). // selectByExpressions the set of tasks to run based on the given expression(s).
func selectByExpressions(ts tasks, nodes Expressions) (tasks, Selection) { func selectByExpressions(ts tasks, nodes Expressions) (tasks, Selection) {
if len(nodes) == 0 { if len(nodes) == 0 {

View File

@ -10,7 +10,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
) )
@ -21,45 +21,54 @@ func dummyTask(name string, tags ...string) Task {
} }
// note: this test fixture does not need to be kept up to date here, but makes a great test subject // note: this test fixture does not need to be kept up to date here, but makes a great test subject
func createDummyTasks() tasks { func createDummyPackageTasks() tasks {
return []Task{ return []Task{
// OS package installed catalogers // OS package installed catalogers
dummyTask("alpm-db-cataloger", "directory", "installed", "image", "os", "alpm", "archlinux"), dummyTask("alpm-db-cataloger", "package", "directory", "installed", "image", "os", "alpm", "archlinux"),
dummyTask("apk-db-cataloger", "directory", "installed", "image", "os", "apk", "alpine"), dummyTask("apk-db-cataloger", "package", "directory", "installed", "image", "os", "apk", "alpine"),
dummyTask("dpkg-db-cataloger", "directory", "installed", "image", "os", "dpkg", "debian"), dummyTask("dpkg-db-cataloger", "package", "directory", "installed", "image", "os", "dpkg", "debian"),
dummyTask("portage-cataloger", "directory", "installed", "image", "os", "portage", "gentoo"), dummyTask("portage-cataloger", "package", "directory", "installed", "image", "os", "portage", "gentoo"),
dummyTask("rpm-db-cataloger", "directory", "installed", "image", "os", "rpm", "redhat"), dummyTask("rpm-db-cataloger", "package", "directory", "installed", "image", "os", "rpm", "redhat"),
// OS package declared catalogers // OS package declared catalogers
dummyTask("rpm-archive-cataloger", "declared", "directory", "os", "rpm", "redhat"), dummyTask("rpm-archive-cataloger", "package", "declared", "directory", "os", "rpm", "redhat"),
// language-specific package installed catalogers // language-specific package installed catalogers
dummyTask("conan-info-cataloger", "installed", "image", "language", "cpp", "conan"), dummyTask("conan-info-cataloger", "package", "installed", "image", "language", "cpp", "conan"),
dummyTask("javascript-package-cataloger", "installed", "image", "language", "javascript", "node"), dummyTask("javascript-package-cataloger", "package", "installed", "image", "language", "javascript", "node"),
dummyTask("php-composer-installed-cataloger", "installed", "image", "language", "php", "composer"), dummyTask("php-composer-installed-cataloger", "package", "installed", "image", "language", "php", "composer"),
dummyTask("ruby-installed-gemspec-cataloger", "installed", "image", "language", "ruby", "gem", "gemspec"), dummyTask("ruby-installed-gemspec-cataloger", "package", "installed", "image", "language", "ruby", "gem", "gemspec"),
dummyTask("rust-cargo-lock-cataloger", "installed", "image", "language", "rust", "binary"), dummyTask("rust-cargo-lock-cataloger", "package", "installed", "image", "language", "rust", "binary"),
// language-specific package declared catalogers // language-specific package declared catalogers
dummyTask("conan-cataloger", "declared", "directory", "language", "cpp", "conan"), dummyTask("conan-cataloger", "package", "declared", "directory", "language", "cpp", "conan"),
dummyTask("dart-pubspec-lock-cataloger", "declared", "directory", "language", "dart"), dummyTask("dart-pubspec-lock-cataloger", "package", "declared", "directory", "language", "dart"),
dummyTask("dotnet-deps-cataloger", "declared", "directory", "language", "dotnet", "c#"), dummyTask("dotnet-deps-cataloger", "package", "declared", "directory", "language", "dotnet", "c#"),
dummyTask("elixir-mix-lock-cataloger", "declared", "directory", "language", "elixir"), dummyTask("elixir-mix-lock-cataloger", "package", "declared", "directory", "language", "elixir"),
dummyTask("erlang-rebar-lock-cataloger", "declared", "directory", "language", "erlang"), dummyTask("erlang-rebar-lock-cataloger", "package", "declared", "directory", "language", "erlang"),
dummyTask("javascript-lock-cataloger", "declared", "directory", "language", "javascript", "node", "npm"), dummyTask("javascript-lock-cataloger", "package", "declared", "directory", "language", "javascript", "node", "npm"),
// language-specific package for both image and directory scans (but not necessarily declared) // language-specific package for both image and directory scans (but not necessarily declared)
dummyTask("dotnet-portable-executable-cataloger", "directory", "installed", "image", "language", "dotnet", "c#"), dummyTask("dotnet-portable-executable-cataloger", "package", "directory", "installed", "image", "language", "dotnet", "c#"),
dummyTask("python-installed-package-cataloger", "directory", "installed", "image", "language", "python"), dummyTask("python-installed-package-cataloger", "package", "directory", "installed", "image", "language", "python"),
dummyTask("go-module-binary-cataloger", "directory", "installed", "image", "language", "go", "golang", "gomod", "binary"), dummyTask("go-module-binary-cataloger", "package", "directory", "installed", "image", "language", "go", "golang", "gomod", "binary"),
dummyTask("java-archive-cataloger", "directory", "installed", "image", "language", "java", "maven"), dummyTask("java-archive-cataloger", "package", "directory", "installed", "image", "language", "java", "maven"),
dummyTask("graalvm-native-image-cataloger", "directory", "installed", "image", "language", "java"), dummyTask("graalvm-native-image-cataloger", "package", "directory", "installed", "image", "language", "java"),
// other package catalogers // other package catalogers
dummyTask("binary-cataloger", "declared", "directory", "image", "binary"), dummyTask("binary-cataloger", "package", "declared", "directory", "image", "binary"),
dummyTask("github-actions-usage-cataloger", "declared", "directory", "github", "github-actions"), dummyTask("github-actions-usage-cataloger", "package", "declared", "directory", "github", "github-actions"),
dummyTask("github-action-workflow-usage-cataloger", "declared", "directory", "github", "github-actions"), dummyTask("github-action-workflow-usage-cataloger", "package", "declared", "directory", "github", "github-actions"),
dummyTask("sbom-cataloger", "declared", "directory", "image", "sbom"), dummyTask("sbom-cataloger", "package", "declared", "directory", "image", "sbom"),
}
}
func createDummyFileTasks() tasks {
return []Task{
dummyTask("file-content-cataloger", "file", "content"),
dummyTask("file-metadata-cataloger", "file", "metadata"),
dummyTask("file-digest-cataloger", "file", "digest"),
dummyTask("file-executable-cataloger", "file", "binary-metadata"),
} }
} }
@ -72,7 +81,7 @@ func TestSelect(t *testing.T) {
expressions []string expressions []string
wantNames []string wantNames []string
wantTokens map[string]TokenSelection wantTokens map[string]TokenSelection
wantRequest pkgcataloging.SelectionRequest wantRequest cataloging.SelectionRequest
wantErr assert.ErrorAssertionFunc wantErr assert.ErrorAssertionFunc
}{ }{
{ {
@ -82,11 +91,11 @@ func TestSelect(t *testing.T) {
expressions: []string{}, expressions: []string{},
wantNames: []string{}, wantNames: []string{},
wantTokens: map[string]TokenSelection{}, wantTokens: map[string]TokenSelection{},
wantRequest: pkgcataloging.SelectionRequest{}, wantRequest: cataloging.SelectionRequest{},
}, },
{ {
name: "use default tasks", name: "use default tasks",
allTasks: createDummyTasks(), allTasks: createDummyPackageTasks(),
basis: []string{ basis: []string{
"image", "image",
}, },
@ -129,13 +138,13 @@ func TestSelect(t *testing.T) {
"binary-cataloger": newTokenSelection([]string{"image"}, nil), "binary-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil), "sbom-cataloger": newTokenSelection([]string{"image"}, nil),
}, },
wantRequest: pkgcataloging.SelectionRequest{ wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image"},
}, },
}, },
{ {
name: "select, add, and remove tasks", name: "select, add, and remove tasks",
allTasks: createDummyTasks(), allTasks: createDummyPackageTasks(),
basis: []string{ basis: []string{
"image", "image",
}, },
@ -175,7 +184,7 @@ func TestSelect(t *testing.T) {
"binary-cataloger": newTokenSelection([]string{"image"}, nil), "binary-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil), "sbom-cataloger": newTokenSelection([]string{"image"}, nil),
}, },
wantRequest: pkgcataloging.SelectionRequest{ wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image"},
SubSelectTags: []string{"os"}, SubSelectTags: []string{"os"},
RemoveNamesOrTags: []string{"dpkg"}, RemoveNamesOrTags: []string{"dpkg"},
@ -184,7 +193,7 @@ func TestSelect(t *testing.T) {
}, },
{ {
name: "allow for partial selections", name: "allow for partial selections",
allTasks: createDummyTasks(), allTasks: createDummyPackageTasks(),
basis: []string{ basis: []string{
"image", "image",
}, },
@ -228,7 +237,7 @@ func TestSelect(t *testing.T) {
"binary-cataloger": newTokenSelection([]string{"image"}, nil), "binary-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil), "sbom-cataloger": newTokenSelection([]string{"image"}, nil),
}, },
wantRequest: pkgcataloging.SelectionRequest{ wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image"},
SubSelectTags: []string{"os", "rust-cargo-lock-cataloger"}, SubSelectTags: []string{"os", "rust-cargo-lock-cataloger"},
RemoveNamesOrTags: []string{"dpkg"}, RemoveNamesOrTags: []string{"dpkg"},
@ -238,7 +247,7 @@ func TestSelect(t *testing.T) {
}, },
{ {
name: "select all tasks", name: "select all tasks",
allTasks: createDummyTasks(), allTasks: createDummyPackageTasks(),
basis: []string{ basis: []string{
"all", "all",
}, },
@ -299,13 +308,13 @@ func TestSelect(t *testing.T) {
"github-action-workflow-usage-cataloger": newTokenSelection([]string{"all"}, nil), "github-action-workflow-usage-cataloger": newTokenSelection([]string{"all"}, nil),
"sbom-cataloger": newTokenSelection([]string{"all"}, nil), "sbom-cataloger": newTokenSelection([]string{"all"}, nil),
}, },
wantRequest: pkgcataloging.SelectionRequest{ wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"all"}, DefaultNamesOrTags: []string{"all"},
}, },
}, },
{ {
name: "set default with multiple tags", name: "set default with multiple tags",
allTasks: createDummyTasks(), allTasks: createDummyPackageTasks(),
basis: []string{ basis: []string{
"gemspec", "gemspec",
"python", "python",
@ -319,10 +328,31 @@ func TestSelect(t *testing.T) {
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"gemspec"}, nil), "ruby-installed-gemspec-cataloger": newTokenSelection([]string{"gemspec"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"python"}, nil), "python-installed-package-cataloger": newTokenSelection([]string{"python"}, nil),
}, },
wantRequest: pkgcataloging.SelectionRequest{ wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"gemspec", "python"}, DefaultNamesOrTags: []string{"gemspec", "python"},
}, },
}, },
{
name: "automatically add file to default tags",
allTasks: createDummyFileTasks(),
basis: []string{},
expressions: []string{},
wantNames: []string{
"file-content-cataloger",
"file-metadata-cataloger",
"file-digest-cataloger",
"file-executable-cataloger",
},
wantTokens: map[string]TokenSelection{
"file-content-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"file"}, nil),
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"file"},
},
},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
@ -330,7 +360,7 @@ func TestSelect(t *testing.T) {
tt.wantErr = assert.NoError tt.wantErr = assert.NoError
} }
req := pkgcataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...) req := cataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...)
got, gotEvidence, err := Select(tt.allTasks, req) got, gotEvidence, err := Select(tt.allTasks, req)
tt.wantErr(t, err) tt.wantErr(t, err)
@ -367,3 +397,391 @@ func TestSelect(t *testing.T) {
}) })
} }
} }
func TestSelectInGroups(t *testing.T) {
tests := []struct {
name string
taskGroups [][]Task
selectionReq cataloging.SelectionRequest
wantGroups [][]string
wantTokens map[string]TokenSelection
wantRequest cataloging.SelectionRequest
wantErr assert.ErrorAssertionFunc
}{
{
name: "select only within the file tasks (leave package tasks alone)",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().
WithDefaults("image"). // note: file missing
WithSubSelections("content", "digest"),
wantGroups: [][]string{
{
// this is the original, untouched package task list
"alpm-db-cataloger",
"apk-db-cataloger",
"dpkg-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
"conan-info-cataloger",
"javascript-package-cataloger",
"php-composer-installed-cataloger",
"ruby-installed-gemspec-cataloger",
"rust-cargo-lock-cataloger",
"dotnet-portable-executable-cataloger",
"python-installed-package-cataloger",
"go-module-binary-cataloger",
"java-archive-cataloger",
"graalvm-native-image-cataloger",
"binary-cataloger",
"sbom-cataloger",
},
{
// this has been filtered based on the request
"file-content-cataloger",
"file-digest-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// packages
"alpm-db-cataloger": newTokenSelection([]string{"image"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"image"}, nil),
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"image"}, nil),
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
"portage-cataloger": newTokenSelection([]string{"image"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil),
"rpm-db-cataloger": newTokenSelection([]string{"image"}, nil),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
// files
"file-content-cataloger": newTokenSelection([]string{"content", "file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"digest", "file"}, nil),
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image", "file"}, // note: file automatically added
SubSelectTags: []string{"content", "digest"},
},
wantErr: assert.NoError,
},
{
name: "select package tasks (leave file tasks alone)",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().WithDefaults("image").WithSubSelections("os"),
wantGroups: [][]string{
{
// filtered based on the request
"alpm-db-cataloger",
"apk-db-cataloger",
"dpkg-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
},
{
// this is the original, untouched file task list
"file-content-cataloger",
"file-metadata-cataloger",
"file-digest-cataloger",
"file-executable-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// packages - os
"alpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"rpm-archive-cataloger": newTokenSelection([]string{"os"}, nil),
"rpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"portage-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
// packages - remaining
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
// files
"file-content-cataloger": newTokenSelection([]string{"file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"file"}, nil),
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image", "file"},
SubSelectTags: []string{"os"},
},
wantErr: assert.NoError,
},
{
name: "select only file tasks (default)",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().WithDefaults("file"),
wantGroups: [][]string{
// filtered based on the request
nil,
{
// this is the original, untouched file task list
"file-content-cataloger",
"file-metadata-cataloger",
"file-digest-cataloger",
"file-executable-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// files
"file-content-cataloger": newTokenSelection([]string{"file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"file"}, nil),
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"file"},
},
wantErr: assert.NoError,
},
{
name: "select only file tasks (via removal of package)",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().WithDefaults("file", "image").WithRemovals("package"),
wantGroups: [][]string{
// filtered based on the request
nil,
{
// this is the original, untouched file task list
"file-content-cataloger",
"file-metadata-cataloger",
"file-digest-cataloger",
"file-executable-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// packages
"alpm-db-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"apk-db-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"binary-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"conan-info-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"dpkg-db-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"java-archive-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"javascript-package-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"portage-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"rpm-db-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"sbom-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"rpm-archive-cataloger": newTokenSelection(nil, []string{"package"}),
"conan-cataloger": newTokenSelection(nil, []string{"package"}),
"dart-pubspec-lock-cataloger": newTokenSelection(nil, []string{"package"}),
"dotnet-deps-cataloger": newTokenSelection(nil, []string{"package"}),
"elixir-mix-lock-cataloger": newTokenSelection(nil, []string{"package"}),
"erlang-rebar-lock-cataloger": newTokenSelection(nil, []string{"package"}),
"javascript-lock-cataloger": newTokenSelection(nil, []string{"package"}),
"github-actions-usage-cataloger": newTokenSelection(nil, []string{"package"}),
"github-action-workflow-usage-cataloger": newTokenSelection(nil, []string{"package"}),
// files
"file-content-cataloger": newTokenSelection([]string{"file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"file"}, nil),
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"file", "image"},
RemoveNamesOrTags: []string{"package"},
},
wantErr: assert.NoError,
},
{
name: "select file and package tasks",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().
WithDefaults("image").
WithSubSelections("os", "content", "digest"),
wantGroups: [][]string{
{
// filtered based on the request
"alpm-db-cataloger",
"apk-db-cataloger",
"dpkg-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
},
{
// filtered based on the request
"file-content-cataloger",
"file-digest-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// packages - os
"alpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"rpm-archive-cataloger": newTokenSelection([]string{"os"}, nil),
"rpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"portage-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
// packages - remaining
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
// files
"file-content-cataloger": newTokenSelection([]string{"file", "content"}, nil), // note extra tags
"file-digest-cataloger": newTokenSelection([]string{"file", "digest"}, nil), // note extra tags
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image", "file"},
SubSelectTags: []string{"os", "content", "digest"},
},
wantErr: assert.NoError,
},
{
name: "complex selection with multiple operators across groups",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().
WithDefaults("os"). // note: no file tag present
WithExpression("+github-actions-usage-cataloger", "-dpkg", "-digest", "content", "+file-metadata-cataloger", "-declared"),
wantGroups: [][]string{
{
"alpm-db-cataloger",
"apk-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
"github-actions-usage-cataloger",
},
{
"file-content-cataloger",
"file-metadata-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// selected package tasks
"alpm-db-cataloger": newTokenSelection([]string{"os"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"os"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"os"}, []string{"dpkg"}),
"portage-cataloger": newTokenSelection([]string{"os"}, nil),
"rpm-archive-cataloger": newTokenSelection([]string{"os"}, []string{"declared"}),
"rpm-db-cataloger": newTokenSelection([]string{"os"}, nil),
"github-actions-usage-cataloger": newTokenSelection([]string{"github-actions-usage-cataloger"}, []string{"declared"}),
// selected file tasks
"file-content-cataloger": newTokenSelection([]string{"content", "file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file-metadata-cataloger", "file"}, nil),
// removed package tasks
"binary-cataloger": newTokenSelection(nil, []string{"declared"}),
"conan-cataloger": newTokenSelection(nil, []string{"declared"}),
"dart-pubspec-lock-cataloger": newTokenSelection(nil, []string{"declared"}),
"dotnet-deps-cataloger": newTokenSelection(nil, []string{"declared"}),
"elixir-mix-lock-cataloger": newTokenSelection(nil, []string{"declared"}),
"erlang-rebar-lock-cataloger": newTokenSelection(nil, []string{"declared"}),
"github-action-workflow-usage-cataloger": newTokenSelection(nil, []string{"declared"}),
"javascript-lock-cataloger": newTokenSelection(nil, []string{"declared"}),
"sbom-cataloger": newTokenSelection(nil, []string{"declared"}),
// removed file tasks
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"file"}, []string{"digest"}),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"os", "file"}, // note: file added automatically
SubSelectTags: []string{"content"},
RemoveNamesOrTags: []string{"dpkg", "digest", "declared"},
AddNames: []string{"github-actions-usage-cataloger", "file-metadata-cataloger"},
},
wantErr: assert.NoError,
},
{
name: "invalid tag",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().WithDefaults("invalid"),
wantGroups: nil,
wantTokens: nil,
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"invalid", "file"},
},
wantErr: assert.Error,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = assert.NoError
}
gotGroups, gotSelection, err := SelectInGroups(tt.taskGroups, tt.selectionReq)
tt.wantErr(t, err)
if err != nil {
// dev note: this is useful for debugging when needed...
//for _, e := range gotEvidence.Request.Expressions {
// t.Logf("expression (errors %q): %#v", e.Errors, e)
//}
// note: we DON'T bail early in validations... this is because we should always return the full set of
// of selected tasks and surrounding evidence.
}
var gotGroupNames [][]string
for _, group := range gotGroups {
var names []string
for _, task := range group {
names = append(names, task.Name())
}
gotGroupNames = append(gotGroupNames, names)
}
assert.Equal(t, tt.wantGroups, gotGroupNames)
assert.Equal(t, tt.wantTokens, gotSelection.TokensByTask)
assert.Equal(t, tt.wantRequest, gotSelection.Request)
})
}
}

View File

@ -0,0 +1,6 @@
package filecataloging
const (
// FileTag should be used to identify catalogers that primarily discover information about files (as opposed to packages).
FileTag = "file"
)

View File

@ -0,0 +1,11 @@
package pkgcataloging
import (
"github.com/anchore/syft/syft/cataloging"
)
// SelectionRequest is deprecated: use cataloging.SelectionRequest instead
type SelectionRequest = cataloging.SelectionRequest
// NewSelectionRequest is deprecated: use cataloging.NewSelectionRequest instead
var NewSelectionRequest = cataloging.NewSelectionRequest

View File

@ -1,8 +1,6 @@
package pkgcataloging package cataloging
import ( import "strings"
"strings"
)
type SelectionRequest struct { type SelectionRequest struct {
DefaultNamesOrTags []string `json:"default,omitempty"` DefaultNamesOrTags []string `json:"default,omitempty"`
@ -50,6 +48,10 @@ func (s SelectionRequest) WithRemovals(nameOrTags ...string) SelectionRequest {
return s return s
} }
func (s SelectionRequest) IsEmpty() bool {
return len(s.AddNames) == 0 && len(s.RemoveNamesOrTags) == 0 && len(s.SubSelectTags) == 0 && len(s.DefaultNamesOrTags) == 0
}
func cleanSelection(tags []string) []string { func cleanSelection(tags []string) []string {
var cleaned []string var cleaned []string
for _, tag := range tags { for _, tag := range tags {

View File

@ -21,7 +21,7 @@ type configurationAuditTrail struct {
} }
type catalogerManifest struct { type catalogerManifest struct {
Requested pkgcataloging.SelectionRequest `json:"requested" yaml:"requested" mapstructure:"requested"` Requested cataloging.SelectionRequest `json:"requested" yaml:"requested" mapstructure:"requested"`
Used []string `json:"used" yaml:"used" mapstructure:"used"` Used []string `json:"used" yaml:"used" mapstructure:"used"`
} }

View File

@ -125,6 +125,9 @@ func monitorCatalogingTask(srcID artifact.ID, tasks [][]task.Task) *monitor.Cata
func formatTaskNames(tasks []task.Task) []string { func formatTaskNames(tasks []task.Task) []string {
set := strset.New() set := strset.New()
for _, td := range tasks { for _, td := range tasks {
if td == nil {
continue
}
set.Add(td.Name()) set.Add(td.Name())
} }
list := set.List() list := set.List()

View File

@ -28,14 +28,14 @@ type CreateSBOMConfig struct {
Packages pkgcataloging.Config Packages pkgcataloging.Config
Files filecataloging.Config Files filecataloging.Config
Parallelism int Parallelism int
CatalogerSelection pkgcataloging.SelectionRequest CatalogerSelection cataloging.SelectionRequest
// audit what tool is being used to generate the SBOM // audit what tool is being used to generate the SBOM
ToolName string ToolName string
ToolVersion string ToolVersion string
ToolConfiguration interface{} ToolConfiguration interface{}
packageTaskFactories task.PackageTaskFactories packageTaskFactories task.Factories
packageCatalogerReferences []pkgcataloging.CatalogerReference packageCatalogerReferences []pkgcataloging.CatalogerReference
} }
@ -150,7 +150,7 @@ func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig {
} }
// WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag. // WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag.
func (c *CreateSBOMConfig) WithCatalogerSelection(selection pkgcataloging.SelectionRequest) *CreateSBOMConfig { func (c *CreateSBOMConfig) WithCatalogerSelection(selection cataloging.SelectionRequest) *CreateSBOMConfig {
c.CatalogerSelection = selection c.CatalogerSelection = selection
return c return c
} }
@ -166,6 +166,10 @@ func (c *CreateSBOMConfig) WithoutCatalogers() *CreateSBOMConfig {
// WithCatalogers allows for adding user-provided catalogers to the final set of catalogers that will always be run // WithCatalogers allows for adding user-provided catalogers to the final set of catalogers that will always be run
// regardless of the source type or any cataloger selections provided. // regardless of the source type or any cataloger selections provided.
func (c *CreateSBOMConfig) WithCatalogers(catalogerRefs ...pkgcataloging.CatalogerReference) *CreateSBOMConfig { func (c *CreateSBOMConfig) WithCatalogers(catalogerRefs ...pkgcataloging.CatalogerReference) *CreateSBOMConfig {
for i := range catalogerRefs {
// ensure that all package catalogers have the package tag
catalogerRefs[i].Tags = append(catalogerRefs[i].Tags, pkgcataloging.PackageTag)
}
c.packageCatalogerReferences = append(c.packageCatalogerReferences, catalogerRefs...) c.packageCatalogerReferences = append(c.packageCatalogerReferences, catalogerRefs...)
return c return c
@ -182,8 +186,8 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task
environmentTasks := c.environmentTasks() environmentTasks := c.environmentTasks()
relationshipsTasks := c.relationshipTasks(src) relationshipsTasks := c.relationshipTasks(src)
unknownTasks := c.unknownsTasks() unknownTasks := c.unknownsTasks()
fileTasks := c.fileTasks()
pkgTasks, selectionEvidence, err := c.packageTasks(src) pkgTasks, fileTasks, selectionEvidence, err := c.selectTasks(src)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -214,80 +218,117 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task
taskGroups..., taskGroups...,
) )
var allTasks []task.Task
allTasks = append(allTasks, pkgTasks...)
allTasks = append(allTasks, fileTasks...)
return taskGroups, &catalogerManifest{ return taskGroups, &catalogerManifest{
Requested: selectionEvidence.Request, Requested: selectionEvidence.Request,
Used: formatTaskNames(pkgTasks), Used: formatTaskNames(allTasks),
}, nil }, nil
} }
// fileTasks returns the set of tasks that should be run to catalog files. // fileTasks returns the set of tasks that should be run to catalog files.
func (c *CreateSBOMConfig) fileTasks() []task.Task { func (c *CreateSBOMConfig) fileTasks(cfg task.CatalogingFactoryConfig) ([]task.Task, error) {
var tsks []task.Task tsks, err := task.DefaultFileTaskFactories().Tasks(cfg)
if err != nil {
if t := task.NewFileDigestCatalogerTask(c.Files.Selection, c.Files.Hashers...); t != nil { return nil, fmt.Errorf("unable to create file cataloger tasks: %w", err)
tsks = append(tsks, t)
}
if t := task.NewFileMetadataCatalogerTask(c.Files.Selection); t != nil {
tsks = append(tsks, t)
}
if t := task.NewFileContentCatalogerTask(c.Files.Content); t != nil {
tsks = append(tsks, t)
}
if t := task.NewExecutableCatalogerTask(c.Files.Selection, c.Files.Executable); t != nil {
tsks = append(tsks, t)
} }
return tsks return tsks, nil
} }
// packageTasks returns the set of tasks that should be run to catalog packages. // selectTasks returns the set of tasks that should be run to catalog packages and files.
func (c *CreateSBOMConfig) packageTasks(src source.Description) ([]task.Task, *task.Selection, error) { func (c *CreateSBOMConfig) selectTasks(src source.Description) ([]task.Task, []task.Task, *task.Selection, error) {
cfg := task.CatalogingFactoryConfig{ cfg := task.CatalogingFactoryConfig{
SearchConfig: c.Search, SearchConfig: c.Search,
RelationshipsConfig: c.Relationships, RelationshipsConfig: c.Relationships,
DataGenerationConfig: c.DataGeneration, DataGenerationConfig: c.DataGeneration,
PackagesConfig: c.Packages, PackagesConfig: c.Packages,
ComplianceConfig: c.Compliance, ComplianceConfig: c.Compliance,
FilesConfig: c.Files,
} }
persistentTasks, selectableTasks, err := c.allPackageTasks(cfg) persistentPkgTasks, selectablePkgTasks, err := c.allPackageTasks(cfg)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("unable to create package cataloger tasks: %w", err) return nil, nil, nil, fmt.Errorf("unable to create package cataloger tasks: %w", err)
} }
req, err := finalSelectionRequest(c.CatalogerSelection, src) req, err := finalTaskSelectionRequest(c.CatalogerSelection, src)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, nil, err
} }
finalTasks, selection, err := task.Select(selectableTasks, *req) selectableFileTasks, err := c.fileTasks(cfg)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, nil, err
} }
finalTasks = append(finalTasks, persistentTasks...) taskGroups := [][]task.Task{
selectablePkgTasks,
if len(finalTasks) == 0 { selectableFileTasks,
log.Warn("no catalogers selected")
return finalTasks, &selection, nil
} }
return finalTasks, &selection, nil finalTaskGroups, selection, err := task.SelectInGroups(taskGroups, *req)
if err != nil {
return nil, nil, nil, err
}
finalPkgTasks := finalTaskGroups[0]
finalFileTasks := finalTaskGroups[1]
finalPkgTasks = append(finalPkgTasks, persistentPkgTasks...)
if len(finalPkgTasks) == 0 && len(finalFileTasks) == 0 {
return nil, nil, nil, fmt.Errorf("no catalogers selected")
}
logTaskNames(finalPkgTasks, "package cataloger")
logTaskNames(finalFileTasks, "file cataloger")
if len(finalPkgTasks) == 0 && len(finalFileTasks) == 0 {
return nil, nil, nil, fmt.Errorf("no catalogers selected")
}
if len(finalPkgTasks) == 0 {
log.Debug("no package catalogers selected")
}
if len(finalFileTasks) == 0 {
if c.Files.Selection != file.NoFilesSelection {
log.Warnf("no file catalogers selected but file selection is configured as %q (this may be unintentional)", c.Files.Selection)
} else {
log.Debug("no file catalogers selected")
}
}
return finalPkgTasks, finalFileTasks, &selection, nil
} }
func finalSelectionRequest(req pkgcataloging.SelectionRequest, src source.Description) (*pkgcataloging.SelectionRequest, error) { func logTaskNames(tasks []task.Task, kind string) {
// log as tree output (like tree command)
log.Debugf("selected %d %s tasks", len(tasks), kind)
names := formatTaskNames(tasks)
for idx, t := range names {
if idx == len(tasks)-1 {
log.Tracef("└── %s", t)
} else {
log.Tracef("├── %s", t)
}
}
}
func finalTaskSelectionRequest(req cataloging.SelectionRequest, src source.Description) (*cataloging.SelectionRequest, error) {
if len(req.DefaultNamesOrTags) == 0 { if len(req.DefaultNamesOrTags) == 0 {
defaultTag, err := findDefaultTag(src) defaultTags, err := findDefaultTags(src)
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to determine default cataloger tag: %w", err) return nil, fmt.Errorf("unable to determine default cataloger tag: %w", err)
} }
if defaultTag != "" { req.DefaultNamesOrTags = append(req.DefaultNamesOrTags, defaultTags...)
req.DefaultNamesOrTags = append(req.DefaultNamesOrTags, defaultTag)
}
req.RemoveNamesOrTags = replaceDefaultTagReferences(defaultTag, req.RemoveNamesOrTags) req.RemoveNamesOrTags = replaceDefaultTagReferences(defaultTags, req.RemoveNamesOrTags)
req.SubSelectTags = replaceDefaultTagReferences(defaultTag, req.SubSelectTags) req.SubSelectTags = replaceDefaultTagReferences(defaultTags, req.SubSelectTags)
} }
return &req, nil return &req, nil
@ -379,21 +420,29 @@ func (c *CreateSBOMConfig) Create(ctx context.Context, src source.Source) (*sbom
return CreateSBOM(ctx, src, c) return CreateSBOM(ctx, src, c)
} }
func findDefaultTag(src source.Description) (string, error) { func findDefaultTags(src source.Description) ([]string, error) {
switch m := src.Metadata.(type) { switch m := src.Metadata.(type) {
case source.ImageMetadata: case source.ImageMetadata:
return pkgcataloging.ImageTag, nil return []string{pkgcataloging.ImageTag, filecataloging.FileTag}, nil
case source.FileMetadata, source.DirectoryMetadata: case source.FileMetadata, source.DirectoryMetadata:
return pkgcataloging.DirectoryTag, nil return []string{pkgcataloging.DirectoryTag, filecataloging.FileTag}, nil
default: default:
return "", fmt.Errorf("unable to determine default cataloger tag for source type=%T", m) return nil, fmt.Errorf("unable to determine default cataloger tag for source type=%T", m)
} }
} }
func replaceDefaultTagReferences(defaultTag string, lst []string) []string { func replaceDefaultTagReferences(defaultTags []string, lst []string) []string {
for i, tag := range lst { for i, tag := range lst {
if strings.ToLower(tag) == "default" { if strings.ToLower(tag) == "default" {
lst[i] = defaultTag switch len(defaultTags) {
case 0:
lst[i] = ""
case 1:
lst[i] = defaultTags[0]
default:
// remove the default tag and add the individual tags
lst = append(lst[:i], append(defaultTags, lst[i+1:]...)...)
}
} }
} }
return lst return lst

View File

@ -88,15 +88,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"), pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "image"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "image"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -107,15 +107,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "directory"), pkgCatalogerNamesWithTagOrName(t, "directory"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"directory"}, DefaultNamesOrTags: []string{"directory", "file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "directory"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "directory"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -127,51 +127,53 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "directory"), pkgCatalogerNamesWithTagOrName(t, "directory"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"directory"}, DefaultNamesOrTags: []string{"directory", "file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "directory"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "directory"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
{ {
name: "no file digest cataloger", name: "no file digest cataloger",
src: imgSrc, src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithHashers()), cfg: DefaultCreateSBOMConfig().WithCatalogerSelection(cataloging.NewSelectionRequest().WithRemovals("digest")),
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"), pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(false, true, true), // note: the digest cataloger is not included fileCatalogerNames("file-metadata", "content", "binary-metadata"),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
RemoveNamesOrTags: []string{"digest"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "image"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "image"), fileCatalogerNames("file-metadata", "content", "binary-metadata")),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
{ {
name: "select no file catalogers", name: "select no file catalogers",
src: imgSrc, src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.NoFilesSelection)), cfg: DefaultCreateSBOMConfig().WithCatalogerSelection(cataloging.NewSelectionRequest().WithRemovals("file")),
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"), pkgCatalogerNamesWithTagOrName(t, "image"),
// note: there are no file catalogers in their own group nil, // note: there is a file cataloging group, with no items in it
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
RemoveNamesOrTags: []string{"file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "image"), Used: pkgCatalogerNamesWithTagOrName(t, "image"),
}, },
@ -186,16 +188,16 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
// note: there is a single group of catalogers for pkgs and files // note: there is a single group of catalogers for pkgs and files
append( append(
pkgCatalogerNamesWithTagOrName(t, "image"), pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(true, true, true)..., fileCatalogerNames()...,
), ),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "image"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "image"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -208,15 +210,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"), addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
}, },
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"), Used: flatten(addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -229,15 +231,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"), addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"directory"}, DefaultNamesOrTags: []string{"directory", "file"},
}, },
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"), Used: flatten(addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -246,20 +248,20 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
src: imgSrc, src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithCatalogers( cfg: DefaultCreateSBOMConfig().WithCatalogers(
pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")), pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")),
).WithCatalogerSelection(pkgcataloging.NewSelectionRequest().WithSubSelections("javascript")), ).WithCatalogerSelection(cataloging.NewSelectionRequest().WithSubSelections("javascript")),
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
addTo(pkgIntersect("image", "javascript"), "persistent"), addTo(pkgIntersect("image", "javascript"), "persistent"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
SubSelectTags: []string{"javascript"}, SubSelectTags: []string{"javascript"},
}, },
Used: addTo(pkgIntersect("image", "javascript"), "persistent"), Used: flatten(addTo(pkgIntersect("image", "javascript"), "persistent"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -272,15 +274,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"), addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
}, },
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"), Used: flatten(addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -293,15 +295,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"), pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "image"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "image"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -314,9 +316,6 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
// sanity check // sanity check
require.NotEmpty(t, tt.wantTaskNames) require.NotEmpty(t, tt.wantTaskNames)
for _, group := range tt.wantTaskNames {
require.NotEmpty(t, group)
}
// test the subject // test the subject
gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src) gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src)
@ -378,20 +377,51 @@ func pkgCatalogerNamesWithTagOrName(t *testing.T, token string) []string {
return names return names
} }
func fileCatalogerNames(digest, metadata, executable bool) []string { func fileCatalogerNames(tokens ...string) []string {
var names []string var names []string
if digest { cfg := task.DefaultCatalogingFactoryConfig()
names = append(names, "file-digest-cataloger") topLoop:
for _, factory := range task.DefaultFileTaskFactories() {
cat := factory(cfg)
if cat == nil {
continue
} }
if executable {
names = append(names, "file-executable-cataloger") name := cat.Name()
if len(tokens) == 0 {
names = append(names, name)
continue
} }
if metadata {
names = append(names, "file-metadata-cataloger") for _, token := range tokens {
if selector, ok := cat.(task.Selector); ok {
if selector.HasAllSelectors(token) {
names = append(names, name)
continue topLoop
} }
}
if name == token {
names = append(names, name)
}
}
}
sort.Strings(names)
return names return names
} }
func flatten(lists ...[]string) []string {
var final []string
for _, lst := range lists {
final = append(final, lst...)
}
sort.Strings(final)
return final
}
func relationshipCatalogerNames() []string { func relationshipCatalogerNames() []string {
return []string{"relationships-cataloger"} return []string{"relationships-cataloger"}
} }
@ -436,7 +466,7 @@ func Test_replaceDefaultTagReferences(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, replaceDefaultTagReferences("replacement", tt.lst)) assert.Equal(t, tt.want, replaceDefaultTagReferences([]string{"replacement"}, tt.lst))
}) })
} }
} }
@ -446,7 +476,7 @@ func Test_findDefaultTag(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
src source.Description src source.Description
want string want []string
wantErr require.ErrorAssertionFunc wantErr require.ErrorAssertionFunc
}{ }{
{ {
@ -454,21 +484,21 @@ func Test_findDefaultTag(t *testing.T) {
src: source.Description{ src: source.Description{
Metadata: source.ImageMetadata{}, Metadata: source.ImageMetadata{},
}, },
want: pkgcataloging.ImageTag, want: []string{pkgcataloging.ImageTag, filecataloging.FileTag},
}, },
{ {
name: "directory", name: "directory",
src: source.Description{ src: source.Description{
Metadata: source.DirectoryMetadata{}, Metadata: source.DirectoryMetadata{},
}, },
want: pkgcataloging.DirectoryTag, want: []string{pkgcataloging.DirectoryTag, filecataloging.FileTag},
}, },
{ {
name: "file", name: "file",
src: source.Description{ src: source.Description{
Metadata: source.FileMetadata{}, Metadata: source.FileMetadata{},
}, },
want: pkgcataloging.DirectoryTag, // not a mistake... want: []string{pkgcataloging.DirectoryTag, filecataloging.FileTag}, // not a mistake...
}, },
{ {
name: "unknown", name: "unknown",
@ -483,7 +513,7 @@ func Test_findDefaultTag(t *testing.T) {
if tt.wantErr == nil { if tt.wantErr == nil {
tt.wantErr = require.NoError tt.wantErr = require.NoError
} }
got, err := findDefaultTag(tt.src) got, err := findDefaultTags(tt.src)
tt.wantErr(t, err) tt.wantErr(t, err)
if err != nil { if err != nil {
return return

View File

@ -298,6 +298,15 @@ func TestPackagesCmdFlags(t *testing.T) {
assertSuccessfulReturnCode, assertSuccessfulReturnCode,
}, },
}, },
{
name: "select-no-package-catalogers",
args: []string{"scan", "-o", "json", "--select-catalogers", "-package", coverageImage},
assertions: []traitAssertion{
assertPackageCount(0),
assertInOutput(`"used":["file-content-cataloger","file-digest-cataloger","file-executable-cataloger","file-metadata-cataloger"]`),
assertSuccessfulReturnCode,
},
},
{ {
name: "override-default-catalogers-option", name: "override-default-catalogers-option",
// This will detect enable: // This will detect enable:
@ -311,6 +320,15 @@ func TestPackagesCmdFlags(t *testing.T) {
assertSuccessfulReturnCode, assertSuccessfulReturnCode,
}, },
}, },
{
name: "override-default-catalogers-with-files",
args: []string{"packages", "-o", "json", "--override-default-catalogers", "file", coverageImage},
assertions: []traitAssertion{
assertPackageCount(0),
assertInOutput(`"used":["file-content-cataloger","file-digest-cataloger","file-executable-cataloger","file-metadata-cataloger"]`),
assertSuccessfulReturnCode,
},
},
{ {
name: "new and old cataloger options are mutually exclusive", name: "new and old cataloger options are mutually exclusive",
args: []string{"packages", "-o", "json", "--override-default-catalogers", "python", "--catalogers", "gemspec", coverageImage}, args: []string{"packages", "-o", "json", "--override-default-catalogers", "python", "--catalogers", "gemspec", coverageImage},