Add file catalogers to selection configuration (#3505)

* add file catalogers to selection configuration

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix typos

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* warn when there is conflicting file cataloging configuration

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* allow for explicit removal of all package and file tasks

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address PR feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman 2025-02-03 14:10:17 -05:00 committed by GitHub
parent 5e2ba43328
commit 684b6e3f98
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 1206 additions and 353 deletions

View File

@ -6,7 +6,7 @@ import (
"github.com/anchore/clio" "github.com/anchore/clio"
"github.com/anchore/stereoscope" "github.com/anchore/stereoscope"
ui2 "github.com/anchore/syft/cmd/syft/cli/ui" handler "github.com/anchore/syft/cmd/syft/cli/ui"
"github.com/anchore/syft/cmd/syft/internal/ui" "github.com/anchore/syft/cmd/syft/internal/ui"
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
@ -28,7 +28,7 @@ func AppClioSetupConfig(id clio.Identification, out io.Writer) *clio.SetupConfig
return clio.NewUICollection( return clio.NewUICollection(
ui.New(out, cfg.Log.Quiet, ui.New(out, cfg.Log.Quiet,
ui2.New(ui2.DefaultHandlerConfig()), handler.New(handler.DefaultHandlerConfig()),
), ),
noUI, noUI,
), nil ), nil

View File

@ -3,6 +3,7 @@ package commands
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"os"
"sort" "sort"
"strings" "strings"
@ -14,7 +15,15 @@ import (
"github.com/anchore/clio" "github.com/anchore/clio"
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/task" "github.com/anchore/syft/internal/task"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging"
)
var (
activelyAddedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("10")) // hi green
deselectedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey
activelyRemovedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("9")) // high red
defaultStyle = lipgloss.NewStyle().Underline(true)
deselectedDefaultStyle = lipgloss.NewStyle().Inherit(deselectedStyle).Underline(true)
) )
type catalogerListOptions struct { type catalogerListOptions struct {
@ -44,8 +53,9 @@ func CatalogerList(app clio.Application) *cobra.Command {
opts := defaultCatalogerListOptions() opts := defaultCatalogerListOptions()
return app.SetupCommand(&cobra.Command{ return app.SetupCommand(&cobra.Command{
Use: "list [OPTIONS]", Use: "list [OPTIONS]",
Short: "List available catalogers", Short: "List available catalogers",
PreRunE: disableUI(app, os.Stdout),
RunE: func(_ *cobra.Command, _ []string) error { RunE: func(_ *cobra.Command, _ []string) error {
return runCatalogerList(opts) return runCatalogerList(opts)
}, },
@ -53,13 +63,19 @@ func CatalogerList(app clio.Application) *cobra.Command {
} }
func runCatalogerList(opts *catalogerListOptions) error { func runCatalogerList(opts *catalogerListOptions) error {
factories := task.DefaultPackageTaskFactories() pkgTaskFactories := task.DefaultPackageTaskFactories()
allTasks, err := factories.Tasks(task.DefaultCatalogingFactoryConfig()) fileTaskFactories := task.DefaultFileTaskFactories()
allPkgTasks, err := pkgTaskFactories.Tasks(task.DefaultCatalogingFactoryConfig())
if err != nil { if err != nil {
return fmt.Errorf("unable to create cataloger tasks: %w", err) return fmt.Errorf("unable to create pkg cataloger tasks: %w", err)
} }
report, err := catalogerListReport(opts, allTasks) allFileTasks, err := fileTaskFactories.Tasks(task.DefaultCatalogingFactoryConfig())
if err != nil {
return fmt.Errorf("unable to create file cataloger tasks: %w", err)
}
report, err := catalogerListReport(opts, [][]task.Task{allPkgTasks, allFileTasks})
if err != nil { if err != nil {
return fmt.Errorf("unable to generate cataloger list report: %w", err) return fmt.Errorf("unable to generate cataloger list report: %w", err)
} }
@ -69,9 +85,10 @@ func runCatalogerList(opts *catalogerListOptions) error {
return nil return nil
} }
func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (string, error) { func catalogerListReport(opts *catalogerListOptions, allTaskGroups [][]task.Task) (string, error) {
selectedTasks, selectionEvidence, err := task.Select(allTasks, selectedTaskGroups, selectionEvidence, err := task.SelectInGroups(
pkgcataloging.NewSelectionRequest(). allTaskGroups,
cataloging.NewSelectionRequest().
WithDefaults(opts.DefaultCatalogers...). WithDefaults(opts.DefaultCatalogers...).
WithExpression(opts.SelectCatalogers...), WithExpression(opts.SelectCatalogers...),
) )
@ -82,12 +99,12 @@ func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (stri
switch opts.Output { switch opts.Output {
case "json": case "json":
report, err = renderCatalogerListJSON(selectedTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers) report, err = renderCatalogerListJSON(flattenTaskGroups(selectedTaskGroups), selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers)
case "table", "": case "table", "":
if opts.ShowHidden { if opts.ShowHidden {
report = renderCatalogerListTable(allTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers) report = renderCatalogerListTables(allTaskGroups, selectionEvidence)
} else { } else {
report = renderCatalogerListTable(selectedTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers) report = renderCatalogerListTables(selectedTaskGroups, selectionEvidence)
} }
} }
@ -98,6 +115,14 @@ func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (stri
return report, nil return report, nil
} }
func flattenTaskGroups(taskGroups [][]task.Task) []task.Task {
var allTasks []task.Task
for _, tasks := range taskGroups {
allTasks = append(allTasks, tasks...)
}
return allTasks
}
func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaultSelections, selections []string) (string, error) { func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaultSelections, selections []string) (string, error) {
type node struct { type node struct {
Name string `json:"name"` Name string `json:"name"`
@ -109,7 +134,12 @@ func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaul
nodesByName := make(map[string]node) nodesByName := make(map[string]node)
for name := range tagsByName { for name := range tagsByName {
tagsSelected := selection.TokensByTask[name].SelectedOn.List() tokensByTask, ok := selection.TokensByTask[name]
var tagsSelected []string
if ok {
tagsSelected = tokensByTask.SelectedOn.List()
}
if len(tagsSelected) == 1 && tagsSelected[0] == "all" { if len(tagsSelected) == 1 && tagsSelected[0] == "all" {
tagsSelected = tagsByName[name] tagsSelected = tagsByName[name]
@ -153,10 +183,56 @@ func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaul
return string(by), err return string(by), err
} }
func renderCatalogerListTable(tasks []task.Task, selection task.Selection, defaultSelections, selections []string) string { func renderCatalogerListTables(taskGroups [][]task.Task, selection task.Selection) string {
pkgCatalogerTable := renderCatalogerListTable(taskGroups[0], selection, "Package Cataloger")
fileCatalogerTable := renderCatalogerListTable(taskGroups[1], selection, "File Cataloger")
report := fileCatalogerTable + "\n" + pkgCatalogerTable + "\n"
hasAdditions := len(selection.Request.AddNames) > 0
hasDefaults := len(selection.Request.DefaultNamesOrTags) > 0
hasRemovals := len(selection.Request.RemoveNamesOrTags) > 0
hasSubSelections := len(selection.Request.SubSelectTags) > 0
expressions := len(selection.Request.SubSelectTags) + len(selection.Request.AddNames) + len(selection.Request.RemoveNamesOrTags)
var header string
header += fmt.Sprintf("Default selections: %d\n", len(selection.Request.DefaultNamesOrTags))
if hasDefaults {
for _, expr := range selection.Request.DefaultNamesOrTags {
header += fmt.Sprintf(" • '%s'\n", expr)
}
}
header += fmt.Sprintf("Selection expressions: %d\n", expressions)
if hasSubSelections {
for _, n := range selection.Request.SubSelectTags {
header += fmt.Sprintf(" • '%s' (intersect)\n", n)
}
}
if hasRemovals {
for _, n := range selection.Request.RemoveNamesOrTags {
header += fmt.Sprintf(" • '-%s' (remove)\n", n)
}
}
if hasAdditions {
for _, n := range selection.Request.AddNames {
header += fmt.Sprintf(" • '+%s' (add)\n", n)
}
}
return header + report
}
func renderCatalogerListTable(tasks []task.Task, selection task.Selection, kindTitle string) string {
if len(tasks) == 0 {
return activelyRemovedStyle.Render(fmt.Sprintf("No %ss selected", strings.ToLower(kindTitle)))
}
t := table.NewWriter() t := table.NewWriter()
t.SetStyle(table.StyleLight) t.SetStyle(table.StyleLight)
t.AppendHeader(table.Row{"Cataloger", "Tags"}) t.AppendHeader(table.Row{kindTitle, "Tags"})
names, tagsByName := extractTaskInfo(tasks) names, tagsByName := extractTaskInfo(tasks)
@ -172,27 +248,12 @@ func renderCatalogerListTable(tasks []task.Task, selection task.Selection, defau
report := t.Render() report := t.Render()
if len(selections) > 0 {
header := "Selected by expressions:\n"
for _, expr := range selections {
header += fmt.Sprintf(" - %q\n", expr)
}
report = header + report
}
if len(defaultSelections) > 0 {
header := "Default selections:\n"
for _, expr := range defaultSelections {
header += fmt.Sprintf(" - %q\n", expr)
}
report = header + report
}
return report return report
} }
func formatRow(name string, tags []string, selection task.Selection) table.Row { func formatRow(name string, tags []string, selection task.Selection) table.Row {
isIncluded := selection.Result.Has(name) isIncluded := selection.Result.Has(name)
defaults := strset.New(selection.Request.DefaultNamesOrTags...)
var selections *task.TokenSelection var selections *task.TokenSelection
if s, exists := selection.TokensByTask[name]; exists { if s, exists := selection.TokensByTask[name]; exists {
selections = &s selections = &s
@ -200,35 +261,32 @@ func formatRow(name string, tags []string, selection task.Selection) table.Row {
var formattedTags []string var formattedTags []string
for _, tag := range tags { for _, tag := range tags {
formattedTags = append(formattedTags, formatToken(tag, selections, isIncluded)) formattedTags = append(formattedTags, formatToken(tag, selections, isIncluded, defaults))
} }
var tagStr string var tagStr string
if isIncluded { if isIncluded {
tagStr = strings.Join(formattedTags, ", ") tagStr = strings.Join(formattedTags, ", ")
} else { } else {
tagStr = strings.Join(formattedTags, grey.Render(", ")) tagStr = strings.Join(formattedTags, deselectedStyle.Render(", "))
} }
// TODO: selection should keep warnings (non-selections) in struct // TODO: selection should keep warnings (non-selections) in struct
return table.Row{ return table.Row{
formatToken(name, selections, isIncluded), formatToken(name, selections, isIncluded, defaults),
tagStr, tagStr,
} }
} }
var ( func formatToken(token string, selection *task.TokenSelection, included bool, defaults *strset.Set) string {
green = lipgloss.NewStyle().Foreground(lipgloss.Color("10")) // hi green
grey = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey
red = lipgloss.NewStyle().Foreground(lipgloss.Color("9")) // high red
)
func formatToken(token string, selection *task.TokenSelection, included bool) string {
if included && selection != nil { if included && selection != nil {
// format all tokens in selection in green // format all tokens in selection in green
if selection.SelectedOn.Has(token) { if selection.SelectedOn.Has(token) {
return green.Render(token) if defaults.Has(token) {
return defaultStyle.Render(token)
}
return activelyAddedStyle.Render(token)
} }
return token return token
@ -236,10 +294,12 @@ func formatToken(token string, selection *task.TokenSelection, included bool) st
// format all tokens in selection in red, all others in grey // format all tokens in selection in red, all others in grey
if selection != nil && selection.DeselectedOn.Has(token) { if selection != nil && selection.DeselectedOn.Has(token) {
return red.Render(token) return activelyRemovedStyle.Render(token)
} }
if defaults.Has(token) {
return grey.Render(token) return deselectedDefaultStyle.Render(token)
}
return deselectedStyle.Render(token)
} }
func extractTaskInfo(tasks []task.Task) ([]string, map[string][]string) { func extractTaskInfo(tasks []task.Task) ([]string, map[string][]string) {

View File

@ -40,23 +40,35 @@ func (d dummyTask) Execute(_ context.Context, _ file.Resolver, _ sbomsync.Builde
panic("implement me") panic("implement me")
} }
func testTasks() []task.Task { func testTasks() [][]task.Task {
return []task.Task{ return [][]task.Task{
dummyTask{ {
name: "task1", dummyTask{
selectors: []string{"image", "a", "b", "1"}, name: "task1",
selectors: []string{"image", "a", "b", "1"},
},
dummyTask{
name: "task2",
selectors: []string{"image", "b", "c", "2"},
},
dummyTask{
name: "task3",
selectors: []string{"directory", "c", "d", "3"},
},
dummyTask{
name: "task4",
selectors: []string{"directory", "d", "e", "4"},
},
}, },
dummyTask{ {
name: "task2", dummyTask{
selectors: []string{"image", "b", "c", "2"}, name: "file-task1",
}, selectors: []string{"file", "ft", "ft-1-b"},
dummyTask{ },
name: "task3", dummyTask{
selectors: []string{"directory", "c", "d", "3"}, name: "file-task2",
}, selectors: []string{"file", "ft", "ft-2-b"},
dummyTask{ },
name: "task4",
selectors: []string{"directory", "d", "e", "4"},
}, },
} }
} }
@ -76,16 +88,23 @@ func Test_catalogerListReport(t *testing.T) {
return c return c
}(), }(),
want: ` want: `
Default selections: Default selections: 1
- "all" 'all'
Selection expressions: 0
CATALOGER TAGS
FILE CATALOGER TAGS
task1 1, a, b, image
task2 2, b, c, image file-task1 file, ft, ft-1-b
task3 3, c, d, directory file-task2 file, ft, ft-2-b
task4 4, d, directory, e
PACKAGE CATALOGER TAGS
task1 1, a, b, image
task2 2, b, c, image
task3 3, c, d, directory
task4 4, d, directory, e
`, `,
}, },
{ {
@ -96,7 +115,7 @@ Default selections:
return c return c
}(), }(),
want: ` want: `
{"default":["all"],"selection":[],"catalogers":[{"name":"task1","tags":["1","a","b","image"]},{"name":"task2","tags":["2","b","c","image"]},{"name":"task3","tags":["3","c","d","directory"]},{"name":"task4","tags":["4","d","directory","e"]}]} {"default":["all"],"selection":[],"catalogers":[{"name":"file-task1","tags":["file","ft","ft-1-b"]},{"name":"file-task2","tags":["file","ft","ft-2-b"]},{"name":"task1","tags":["1","a","b","image"]},{"name":"task2","tags":["2","b","c","image"]},{"name":"task3","tags":["3","c","d","directory"]},{"name":"task4","tags":["4","d","directory","e"]}]}
`, `,
}, },
{ {
@ -105,19 +124,27 @@ Default selections:
c := defaultCatalogerListOptions() c := defaultCatalogerListOptions()
c.Output = "table" c.Output = "table"
c.DefaultCatalogers = []string{ c.DefaultCatalogers = []string{
"image", "image", // note: for backwards compatibility file will automatically be added
} }
return c return c
}(), }(),
want: ` want: `
Default selections: Default selections: 2
- "image" 'image'
'file'
CATALOGER TAGS Selection expressions: 0
task1 1, a, b, image FILE CATALOGER TAGS
task2 2, b, c, image
file-task1 file, ft, ft-1-b
file-task2 file, ft, ft-2-b
PACKAGE CATALOGER TAGS
task1 1, a, b, image
task2 2, b, c, image
`, `,
}, },
{ {
@ -131,7 +158,7 @@ Default selections:
return c return c
}(), }(),
want: ` want: `
{"default":["image"],"selection":[],"catalogers":[{"name":"task1","tags":["image"]},{"name":"task2","tags":["image"]}]} {"default":["image"],"selection":[],"catalogers":[{"name":"file-task1","tags":["file"]},{"name":"file-task2","tags":["file"]},{"name":"task1","tags":["image"]},{"name":"task2","tags":["image"]}]}
`, `,
}, },
{ {
@ -147,23 +174,32 @@ Default selections:
"+task3", "+task3",
"-c", "-c",
"b", "b",
"-file",
"+file-task1",
} }
return c return c
}(), }(),
want: ` want: `
Default selections: Default selections: 1
- "image" 'image'
Selected by expressions: Selection expressions: 6
- "-directory" 'b' (intersect)
- "+task3" '-directory' (remove)
- "-c" '-c' (remove)
- "b" '-file' (remove)
'+task3' (add)
CATALOGER TAGS '+file-task1' (add)
task1 1, a, b, image FILE CATALOGER TAGS
task3 3, c, d, directory
file-task1 file, ft, ft-1-b
PACKAGE CATALOGER TAGS
task1 1, a, b, image
task3 3, c, d, directory
`, `,
}, },
{ {
@ -183,7 +219,7 @@ Selected by expressions:
return c return c
}(), }(),
want: ` want: `
{"default":["image"],"selection":["-directory","+task3","-c","b"],"catalogers":[{"name":"task1","tags":["b","image"]},{"name":"task3","tags":["task3"]}]} {"default":["image"],"selection":["-directory","+task3","-c","b"],"catalogers":[{"name":"file-task1","tags":["file"]},{"name":"file-task2","tags":["file"]},{"name":"task1","tags":["b","image"]},{"name":"task3","tags":["task3"]}]}
`, `,
}, },
} }

View File

@ -0,0 +1,23 @@
package commands
import (
"io"
"github.com/spf13/cobra"
"github.com/anchore/clio"
"github.com/anchore/syft/cmd/syft/internal/ui"
)
func disableUI(app clio.Application, out io.Writer) func(*cobra.Command, []string) error {
return func(_ *cobra.Command, _ []string) error {
type Stater interface {
State() *clio.State
}
state := app.(Stater).State()
state.UI = clio.NewUICollection(ui.None(out, state.Config.Log.Quiet))
return nil
}
}

View File

@ -91,7 +91,7 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
WithPackagesConfig(cfg.ToPackagesConfig()). WithPackagesConfig(cfg.ToPackagesConfig()).
WithFilesConfig(cfg.ToFilesConfig()). WithFilesConfig(cfg.ToFilesConfig()).
WithCatalogerSelection( WithCatalogerSelection(
pkgcataloging.NewSelectionRequest(). cataloging.NewSelectionRequest().
WithDefaults(cfg.DefaultCatalogers...). WithDefaults(cfg.DefaultCatalogers...).
WithExpression(cfg.SelectCatalogers...), WithExpression(cfg.SelectCatalogers...),
) )

View File

@ -10,6 +10,7 @@ import (
"github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/cmd/syft/internal/options" "github.com/anchore/syft/cmd/syft/internal/options"
"github.com/anchore/syft/syft" "github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging/pkgcataloging"
"github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
@ -20,7 +21,7 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string, scope source.Sco
Name: "syft-tester", Name: "syft-tester",
Version: "v0.99.0", Version: "v0.99.0",
}).WithCatalogerSelection( }).WithCatalogerSelection(
pkgcataloging.NewSelectionRequest(). cataloging.NewSelectionRequest().
WithExpression(catalogerSelection...), WithExpression(catalogerSelection...),
) )
cfg.Search.Scope = scope cfg.Search.Scope = scope
@ -55,7 +56,7 @@ func catalogDirectory(t *testing.T, dir string, catalogerSelection ...string) (s
Name: "syft-tester", Name: "syft-tester",
Version: "v0.99.0", Version: "v0.99.0",
}).WithCatalogerSelection( }).WithCatalogerSelection(
pkgcataloging.NewSelectionRequest(). cataloging.NewSelectionRequest().
WithExpression(catalogerSelection...), WithExpression(catalogerSelection...),
) )

View File

@ -74,7 +74,7 @@ func getSBOM(src source.Source) sbom.SBOM {
// only use OS related catalogers that would have been used with the kind of // only use OS related catalogers that would have been used with the kind of
// source type (container image or directory), but also add a specific python cataloger // source type (container image or directory), but also add a specific python cataloger
WithCatalogerSelection( WithCatalogerSelection(
pkgcataloging.NewSelectionRequest(). cataloging.NewSelectionRequest().
WithSubSelections("os"). WithSubSelections("os").
WithAdditions("python-package-cataloger"), WithAdditions("python-package-cataloger"),
). ).

View File

@ -6,6 +6,7 @@ import (
"os" "os"
"github.com/anchore/syft/syft" "github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging/pkgcataloging"
"github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
@ -55,7 +56,7 @@ func getSBOM(src source.Source, defaultTags ...string) sbom.SBOM {
WithCatalogerSelection( WithCatalogerSelection(
// here you can sub-select, add, remove catalogers from the default selection... // here you can sub-select, add, remove catalogers from the default selection...
// or replace the default selection entirely! // or replace the default selection entirely!
pkgcataloging.NewSelectionRequest(). cataloging.NewSelectionRequest().
WithDefaults(defaultTags...), WithDefaults(defaultTags...),
) )

View File

@ -0,0 +1,27 @@
package task
import (
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/filecataloging"
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
)
type CatalogingFactoryConfig struct {
ComplianceConfig cataloging.ComplianceConfig
SearchConfig cataloging.SearchConfig
RelationshipsConfig cataloging.RelationshipsConfig
DataGenerationConfig cataloging.DataGenerationConfig
PackagesConfig pkgcataloging.Config
FilesConfig filecataloging.Config
}
func DefaultCatalogingFactoryConfig() CatalogingFactoryConfig {
return CatalogingFactoryConfig{
ComplianceConfig: cataloging.DefaultComplianceConfig(),
SearchConfig: cataloging.DefaultSearchConfig(),
RelationshipsConfig: cataloging.DefaultRelationshipsConfig(),
DataGenerationConfig: cataloging.DefaultDataGenerationConfig(),
PackagesConfig: pkgcataloging.DefaultConfig(),
FilesConfig: filecataloging.DefaultConfig(),
}
}

View File

@ -8,7 +8,7 @@ import (
"github.com/hashicorp/go-multierror" "github.com/hashicorp/go-multierror"
"github.com/scylladb/go-set/strset" "github.com/scylladb/go-set/strset"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging"
) )
var expressionNodePattern = regexp.MustCompile(`^([a-zA-Z0-9][a-zA-Z0-9-+]*)+$`) var expressionNodePattern = regexp.MustCompile(`^([a-zA-Z0-9][a-zA-Z0-9-+]*)+$`)
@ -142,7 +142,7 @@ func (ec expressionContext) newExpression(exp string, operation Operation, token
} }
} }
func newExpressionsFromSelectionRequest(nc *expressionContext, selectionRequest pkgcataloging.SelectionRequest) Expressions { func newExpressionsFromSelectionRequest(nc *expressionContext, selectionRequest cataloging.SelectionRequest) Expressions {
var all Expressions var all Expressions
for _, exp := range selectionRequest.DefaultNamesOrTags { for _, exp := range selectionRequest.DefaultNamesOrTags {

View File

@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging"
) )
func Test_newExpressionsFromSelectionRequest(t *testing.T) { func Test_newExpressionsFromSelectionRequest(t *testing.T) {
@ -135,7 +135,7 @@ func Test_newExpressionsFromSelectionRequest(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
req := pkgcataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...) req := cataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...)
result := newExpressionsFromSelectionRequest(nc, req) result := newExpressionsFromSelectionRequest(nc, req)
if tt.expectedErrors != nil { if tt.expectedErrors != nil {

40
internal/task/factory.go Normal file
View File

@ -0,0 +1,40 @@
package task
import (
"fmt"
"sort"
"strings"
"github.com/scylladb/go-set/strset"
)
type factory func(cfg CatalogingFactoryConfig) Task
type Factories []factory
func (f Factories) Tasks(cfg CatalogingFactoryConfig) ([]Task, error) {
var allTasks []Task
taskNames := strset.New()
duplicateTaskNames := strset.New()
var err error
for _, fact := range f {
tsk := fact(cfg)
if tsk == nil {
continue
}
tskName := tsk.Name()
if taskNames.Has(tskName) {
duplicateTaskNames.Add(tskName)
}
allTasks = append(allTasks, tsk)
taskNames.Add(tskName)
}
if duplicateTaskNames.Size() > 0 {
names := duplicateTaskNames.List()
sort.Strings(names)
err = fmt.Errorf("duplicate cataloger task names: %v", strings.Join(names, ", "))
}
return allTasks, err
}

View File

@ -6,6 +6,7 @@ import (
"github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging/filecataloging"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/file/cataloger/executable" "github.com/anchore/syft/syft/file/cataloger/executable"
"github.com/anchore/syft/syft/file/cataloger/filecontent" "github.com/anchore/syft/syft/file/cataloger/filecontent"
@ -15,14 +16,27 @@ import (
"github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/sbom"
) )
func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash) Task { func DefaultFileTaskFactories() Factories {
if selection == file.NoFilesSelection || len(hashers) == 0 { return Factories{
return nil newFileDigestCatalogerTaskFactory("digest"),
newFileMetadataCatalogerTaskFactory("file-metadata"),
newFileContentCatalogerTaskFactory("content"),
newExecutableCatalogerTaskFactory("binary-metadata"),
} }
}
digestsCataloger := filedigest.NewCataloger(hashers) func newFileDigestCatalogerTaskFactory(tags ...string) factory {
return func(cfg CatalogingFactoryConfig) Task {
return newFileDigestCatalogerTask(cfg.FilesConfig.Selection, cfg.FilesConfig.Hashers, tags...)
}
}
func newFileDigestCatalogerTask(selection file.Selection, hashers []crypto.Hash, tags ...string) Task {
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error { fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
if selection == file.NoFilesSelection || len(hashers) == 0 {
return nil
}
accessor := builder.(sbomsync.Accessor) accessor := builder.(sbomsync.Accessor)
coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor)) coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor))
@ -30,7 +44,7 @@ func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash
return nil return nil
} }
result, err := digestsCataloger.Catalog(ctx, resolver, coordinates...) result, err := filedigest.NewCataloger(hashers).Catalog(ctx, resolver, coordinates...)
accessor.WriteToSBOM(func(sbom *sbom.SBOM) { accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileDigests = result sbom.Artifacts.FileDigests = result
@ -39,17 +53,21 @@ func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash
return err return err
} }
return NewTask("file-digest-cataloger", fn) return NewTask("file-digest-cataloger", fn, commonFileTags(tags)...)
} }
func NewFileMetadataCatalogerTask(selection file.Selection) Task { func newFileMetadataCatalogerTaskFactory(tags ...string) factory {
if selection == file.NoFilesSelection { return func(cfg CatalogingFactoryConfig) Task {
return nil return newFileMetadataCatalogerTask(cfg.FilesConfig.Selection, tags...)
} }
}
metadataCataloger := filemetadata.NewCataloger() func newFileMetadataCatalogerTask(selection file.Selection, tags ...string) Task {
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error { fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
if selection == file.NoFilesSelection {
return nil
}
accessor := builder.(sbomsync.Accessor) accessor := builder.(sbomsync.Accessor)
coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor)) coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor))
@ -57,7 +75,7 @@ func NewFileMetadataCatalogerTask(selection file.Selection) Task {
return nil return nil
} }
result, err := metadataCataloger.Catalog(ctx, resolver, coordinates...) result, err := filemetadata.NewCataloger().Catalog(ctx, resolver, coordinates...)
accessor.WriteToSBOM(func(sbom *sbom.SBOM) { accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileMetadata = result sbom.Artifacts.FileMetadata = result
@ -66,20 +84,24 @@ func NewFileMetadataCatalogerTask(selection file.Selection) Task {
return err return err
} }
return NewTask("file-metadata-cataloger", fn) return NewTask("file-metadata-cataloger", fn, commonFileTags(tags)...)
} }
func NewFileContentCatalogerTask(cfg filecontent.Config) Task { func newFileContentCatalogerTaskFactory(tags ...string) factory {
if len(cfg.Globs) == 0 { return func(cfg CatalogingFactoryConfig) Task {
return nil return newFileContentCatalogerTask(cfg.FilesConfig.Content, tags...)
} }
}
cat := filecontent.NewCataloger(cfg) func newFileContentCatalogerTask(cfg filecontent.Config, tags ...string) Task {
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error { fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
if len(cfg.Globs) == 0 {
return nil
}
accessor := builder.(sbomsync.Accessor) accessor := builder.(sbomsync.Accessor)
result, err := cat.Catalog(ctx, resolver) result, err := filecontent.NewCataloger(cfg).Catalog(ctx, resolver)
accessor.WriteToSBOM(func(sbom *sbom.SBOM) { accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.FileContents = result sbom.Artifacts.FileContents = result
@ -88,20 +110,24 @@ func NewFileContentCatalogerTask(cfg filecontent.Config) Task {
return err return err
} }
return NewTask("file-content-cataloger", fn) return NewTask("file-content-cataloger", fn, commonFileTags(tags)...)
} }
func NewExecutableCatalogerTask(selection file.Selection, cfg executable.Config) Task { func newExecutableCatalogerTaskFactory(tags ...string) factory {
if selection == file.NoFilesSelection { return func(cfg CatalogingFactoryConfig) Task {
return nil return newExecutableCatalogerTask(cfg.FilesConfig.Selection, cfg.FilesConfig.Executable, tags...)
} }
}
cat := executable.NewCataloger(cfg) func newExecutableCatalogerTask(selection file.Selection, cfg executable.Config, tags ...string) Task {
fn := func(_ context.Context, resolver file.Resolver, builder sbomsync.Builder) error { fn := func(_ context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
if selection == file.NoFilesSelection {
return nil
}
accessor := builder.(sbomsync.Accessor) accessor := builder.(sbomsync.Accessor)
result, err := cat.Catalog(resolver) result, err := executable.NewCataloger(cfg).Catalog(resolver)
accessor.WriteToSBOM(func(sbom *sbom.SBOM) { accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
sbom.Artifacts.Executables = result sbom.Artifacts.Executables = result
@ -110,7 +136,7 @@ func NewExecutableCatalogerTask(selection file.Selection, cfg executable.Config)
return err return err
} }
return NewTask("file-executable-cataloger", fn) return NewTask("file-executable-cataloger", fn, commonFileTags(tags)...)
} }
// TODO: this should be replaced with a fix that allows passing a coordinate or location iterator to the cataloger // TODO: this should be replaced with a fix that allows passing a coordinate or location iterator to the cataloger
@ -154,3 +180,8 @@ func coordinatesForSelection(selection file.Selection, accessor sbomsync.Accesso
return nil, false return nil, false
} }
func commonFileTags(tags []string) []string {
tags = append(tags, filecataloging.FileTag)
return tags
}

View File

@ -3,12 +3,9 @@ package task
import ( import (
"context" "context"
"fmt" "fmt"
"sort"
"strings" "strings"
"unicode" "unicode"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/relationship" "github.com/anchore/syft/internal/relationship"
@ -23,67 +20,18 @@ import (
cpeutils "github.com/anchore/syft/syft/pkg/cataloger/common/cpe" cpeutils "github.com/anchore/syft/syft/pkg/cataloger/common/cpe"
) )
type packageTaskFactory func(cfg CatalogingFactoryConfig) Task func newPackageTaskFactory(catalogerFactory func(CatalogingFactoryConfig) pkg.Cataloger, tags ...string) factory {
type PackageTaskFactories []packageTaskFactory
type CatalogingFactoryConfig struct {
ComplianceConfig cataloging.ComplianceConfig
SearchConfig cataloging.SearchConfig
RelationshipsConfig cataloging.RelationshipsConfig
DataGenerationConfig cataloging.DataGenerationConfig
PackagesConfig pkgcataloging.Config
}
func DefaultCatalogingFactoryConfig() CatalogingFactoryConfig {
return CatalogingFactoryConfig{
ComplianceConfig: cataloging.DefaultComplianceConfig(),
SearchConfig: cataloging.DefaultSearchConfig(),
RelationshipsConfig: cataloging.DefaultRelationshipsConfig(),
DataGenerationConfig: cataloging.DefaultDataGenerationConfig(),
PackagesConfig: pkgcataloging.DefaultConfig(),
}
}
func newPackageTaskFactory(catalogerFactory func(CatalogingFactoryConfig) pkg.Cataloger, tags ...string) packageTaskFactory {
return func(cfg CatalogingFactoryConfig) Task { return func(cfg CatalogingFactoryConfig) Task {
return NewPackageTask(cfg, catalogerFactory(cfg), tags...) return NewPackageTask(cfg, catalogerFactory(cfg), tags...)
} }
} }
func newSimplePackageTaskFactory(catalogerFactory func() pkg.Cataloger, tags ...string) packageTaskFactory { func newSimplePackageTaskFactory(catalogerFactory func() pkg.Cataloger, tags ...string) factory {
return func(cfg CatalogingFactoryConfig) Task { return func(cfg CatalogingFactoryConfig) Task {
return NewPackageTask(cfg, catalogerFactory(), tags...) return NewPackageTask(cfg, catalogerFactory(), tags...)
} }
} }
func (f PackageTaskFactories) Tasks(cfg CatalogingFactoryConfig) ([]Task, error) {
var allTasks []Task
taskNames := strset.New()
duplicateTaskNames := strset.New()
var err error
for _, factory := range f {
tsk := factory(cfg)
if tsk == nil {
continue
}
tskName := tsk.Name()
if taskNames.Has(tskName) {
duplicateTaskNames.Add(tskName)
}
allTasks = append(allTasks, tsk)
taskNames.Add(tskName)
}
if duplicateTaskNames.Size() > 0 {
names := duplicateTaskNames.List()
sort.Strings(names)
err = fmt.Errorf("duplicate cataloger task names: %v", strings.Join(names, ", "))
}
return allTasks, err
}
// NewPackageTask creates a Task function for a generic pkg.Cataloger, honoring the common configuration options. // NewPackageTask creates a Task function for a generic pkg.Cataloger, honoring the common configuration options.
func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string) Task { func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string) Task {
fn := func(ctx context.Context, resolver file.Resolver, sbom sbomsync.Builder) error { fn := func(ctx context.Context, resolver file.Resolver, sbom sbomsync.Builder) error {

View File

@ -51,8 +51,8 @@ const (
) )
//nolint:funlen //nolint:funlen
func DefaultPackageTaskFactories() PackageTaskFactories { func DefaultPackageTaskFactories() Factories {
return []packageTaskFactory{ return []factory{
// OS package installed catalogers /////////////////////////////////////////////////////////////////////////// // OS package installed catalogers ///////////////////////////////////////////////////////////////////////////
newSimplePackageTaskFactory(arch.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "alpm", "archlinux"), newSimplePackageTaskFactory(arch.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "alpm", "archlinux"),
newSimplePackageTaskFactory(alpine.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "apk", "alpine"), newSimplePackageTaskFactory(alpine.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "apk", "alpine"),

View File

@ -3,17 +3,19 @@ package task
import ( import (
"fmt" "fmt"
"sort" "sort"
"strings"
"github.com/scylladb/go-set/strset" "github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/filecataloging"
) )
// Selection represents the users request for a subset of tasks to run and the resulting set of task names that were // Selection represents the users request for a subset of tasks to run and the resulting set of task names that were
// selected. Additionally, all tokens that were matched on to reach the returned conclusion are also provided. // selected. Additionally, all tokens that were matched on to reach the returned conclusion are also provided.
type Selection struct { type Selection struct {
Request pkgcataloging.SelectionRequest Request cataloging.SelectionRequest
Result *strset.Set Result *strset.Set
TokensByTask map[string]TokenSelection TokensByTask map[string]TokenSelection
} }
@ -52,7 +54,18 @@ func newSelection() Selection {
// Select parses the given expressions as two sets: expressions that represent a "set" operation, and expressions that // Select parses the given expressions as two sets: expressions that represent a "set" operation, and expressions that
// represent all other operations. The parsed expressions are then evaluated against the given tasks to return // represent all other operations. The parsed expressions are then evaluated against the given tasks to return
// a subset (or the same) set of tasks. // a subset (or the same) set of tasks.
func Select(allTasks []Task, selectionRequest pkgcataloging.SelectionRequest) ([]Task, Selection, error) { func Select(allTasks []Task, selectionRequest cataloging.SelectionRequest) ([]Task, Selection, error) {
ensureDefaultSelectionHasFiles(&selectionRequest, allTasks)
return _select(allTasks, selectionRequest)
}
func _select(allTasks []Task, selectionRequest cataloging.SelectionRequest) ([]Task, Selection, error) {
if selectionRequest.IsEmpty() {
selection := newSelection()
selection.Request = selectionRequest
return nil, selection, nil
}
nodes := newExpressionsFromSelectionRequest(newExpressionContext(allTasks), selectionRequest) nodes := newExpressionsFromSelectionRequest(newExpressionContext(allTasks), selectionRequest)
finalTasks, selection := selectByExpressions(allTasks, nodes) finalTasks, selection := selectByExpressions(allTasks, nodes)
@ -62,6 +75,142 @@ func Select(allTasks []Task, selectionRequest pkgcataloging.SelectionRequest) ([
return finalTasks, selection, nodes.Validate() return finalTasks, selection, nodes.Validate()
} }
// ensureDefaultSelectionHasFiles ensures that the default selection request has the "file" tag, as this is a required
// for backwards compatibility (when catalogers were only for packages and not for separate groups of tasks).
func ensureDefaultSelectionHasFiles(selectionRequest *cataloging.SelectionRequest, allTasks ...[]Task) {
for _, ts := range allTasks {
_, leftOver := tagsOrNamesThatTaskGroupRespondsTo(ts, strset.New(filecataloging.FileTag))
if leftOver.Has(filecataloging.FileTag) {
// the given set of tasks do not respond to file, so don't include it in the default selection
continue
}
defaultNamesOrTags := strset.New(selectionRequest.DefaultNamesOrTags...)
removals := strset.New(selectionRequest.RemoveNamesOrTags...)
missingFileIshTag := !defaultNamesOrTags.Has(filecataloging.FileTag) && !defaultNamesOrTags.Has("all") && !defaultNamesOrTags.Has("default")
if missingFileIshTag && !removals.Has(filecataloging.FileTag) {
log.Warnf("adding '%s' tag to the default cataloger selection, to override add '-%s' to the cataloger selection request", filecataloging.FileTag, filecataloging.FileTag)
selectionRequest.DefaultNamesOrTags = append(selectionRequest.DefaultNamesOrTags, filecataloging.FileTag)
}
}
}
// SelectInGroups is a convenience function that allows for selecting tasks from multiple groups of tasks. The original
// request is split into sub-requests, where only tokens that are relevant to the given group of tasks are considered.
// If tokens are passed that are not relevant to any group of tasks, an error is returned.
func SelectInGroups(taskGroups [][]Task, selectionRequest cataloging.SelectionRequest) ([][]Task, Selection, error) {
ensureDefaultSelectionHasFiles(&selectionRequest, taskGroups...)
reqs, errs := splitCatalogerSelectionRequest(selectionRequest, taskGroups)
if errs != nil {
return nil, Selection{
Request: selectionRequest,
}, errs
}
var finalTasks [][]Task
var selections []Selection
for idx, req := range reqs {
tskGroup := taskGroups[idx]
subFinalTasks, subSelection, err := _select(tskGroup, req)
if err != nil {
return nil, Selection{
Request: selectionRequest,
}, err
}
finalTasks = append(finalTasks, subFinalTasks)
selections = append(selections, subSelection)
}
return finalTasks, mergeSelections(selections, selectionRequest), nil
}
func mergeSelections(selections []Selection, ogRequest cataloging.SelectionRequest) Selection {
finalSelection := newSelection()
for _, s := range selections {
finalSelection.Result.Add(s.Result.List()...)
for name, tokenSelection := range s.TokensByTask {
if existing, exists := finalSelection.TokensByTask[name]; exists {
existing.merge(tokenSelection)
finalSelection.TokensByTask[name] = existing
} else {
finalSelection.TokensByTask[name] = tokenSelection
}
}
}
finalSelection.Request = ogRequest
return finalSelection
}
func splitCatalogerSelectionRequest(req cataloging.SelectionRequest, selectablePkgTaskGroups [][]Task) ([]cataloging.SelectionRequest, error) {
requestTagsOrNames := allRequestReferences(req)
leftoverTags := strset.New()
usedTagsAndNames := strset.New()
var usedTagGroups []*strset.Set
for _, taskGroup := range selectablePkgTaskGroups {
selectedTagOrNames, remainingTagsOrNames := tagsOrNamesThatTaskGroupRespondsTo(taskGroup, requestTagsOrNames)
leftoverTags = strset.Union(leftoverTags, remainingTagsOrNames)
usedTagGroups = append(usedTagGroups, selectedTagOrNames)
usedTagsAndNames.Add(selectedTagOrNames.List()...)
}
leftoverTags = strset.Difference(leftoverTags, usedTagsAndNames)
leftoverTags.Remove("all")
if leftoverTags.Size() > 0 {
l := leftoverTags.List()
sort.Strings(l)
return nil, fmt.Errorf("no cataloger tasks respond to the following selections: %v", strings.Join(l, ", "))
}
var newSelections []cataloging.SelectionRequest
for _, tags := range usedTagGroups {
newSelections = append(newSelections, newSelectionWithTags(req, tags))
}
return newSelections, nil
}
func newSelectionWithTags(req cataloging.SelectionRequest, tags *strset.Set) cataloging.SelectionRequest {
return cataloging.SelectionRequest{
DefaultNamesOrTags: filterTags(req.DefaultNamesOrTags, tags),
SubSelectTags: filterTags(req.SubSelectTags, tags),
AddNames: filterTags(req.AddNames, tags),
RemoveNamesOrTags: filterTags(req.RemoveNamesOrTags, tags),
}
}
func filterTags(reqTags []string, filterTags *strset.Set) []string {
var filtered []string
for _, tag := range reqTags {
if filterTags.Has(tag) {
filtered = append(filtered, tag)
}
}
return filtered
}
func tagsOrNamesThatTaskGroupRespondsTo(tasks []Task, requestTagsOrNames *strset.Set) (*strset.Set, *strset.Set) {
positiveRefs := strset.New()
for _, t := range tasks {
if sel, ok := t.(Selector); ok {
positiveRefs.Add("all") // everything responds to "all"
positiveRefs.Add(strset.Intersection(requestTagsOrNames, strset.New(sel.Selectors()...)).List()...)
}
positiveRefs.Add(t.Name())
}
return positiveRefs, strset.Difference(requestTagsOrNames, positiveRefs)
}
func allRequestReferences(s cataloging.SelectionRequest) *strset.Set {
st := strset.New()
st.Add(s.DefaultNamesOrTags...)
st.Add(s.SubSelectTags...)
st.Add(s.AddNames...)
st.Add(s.RemoveNamesOrTags...)
return st
}
// selectByExpressions the set of tasks to run based on the given expression(s). // selectByExpressions the set of tasks to run based on the given expression(s).
func selectByExpressions(ts tasks, nodes Expressions) (tasks, Selection) { func selectByExpressions(ts tasks, nodes Expressions) (tasks, Selection) {
if len(nodes) == 0 { if len(nodes) == 0 {

View File

@ -10,7 +10,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
) )
@ -21,45 +21,54 @@ func dummyTask(name string, tags ...string) Task {
} }
// note: this test fixture does not need to be kept up to date here, but makes a great test subject // note: this test fixture does not need to be kept up to date here, but makes a great test subject
func createDummyTasks() tasks { func createDummyPackageTasks() tasks {
return []Task{ return []Task{
// OS package installed catalogers // OS package installed catalogers
dummyTask("alpm-db-cataloger", "directory", "installed", "image", "os", "alpm", "archlinux"), dummyTask("alpm-db-cataloger", "package", "directory", "installed", "image", "os", "alpm", "archlinux"),
dummyTask("apk-db-cataloger", "directory", "installed", "image", "os", "apk", "alpine"), dummyTask("apk-db-cataloger", "package", "directory", "installed", "image", "os", "apk", "alpine"),
dummyTask("dpkg-db-cataloger", "directory", "installed", "image", "os", "dpkg", "debian"), dummyTask("dpkg-db-cataloger", "package", "directory", "installed", "image", "os", "dpkg", "debian"),
dummyTask("portage-cataloger", "directory", "installed", "image", "os", "portage", "gentoo"), dummyTask("portage-cataloger", "package", "directory", "installed", "image", "os", "portage", "gentoo"),
dummyTask("rpm-db-cataloger", "directory", "installed", "image", "os", "rpm", "redhat"), dummyTask("rpm-db-cataloger", "package", "directory", "installed", "image", "os", "rpm", "redhat"),
// OS package declared catalogers // OS package declared catalogers
dummyTask("rpm-archive-cataloger", "declared", "directory", "os", "rpm", "redhat"), dummyTask("rpm-archive-cataloger", "package", "declared", "directory", "os", "rpm", "redhat"),
// language-specific package installed catalogers // language-specific package installed catalogers
dummyTask("conan-info-cataloger", "installed", "image", "language", "cpp", "conan"), dummyTask("conan-info-cataloger", "package", "installed", "image", "language", "cpp", "conan"),
dummyTask("javascript-package-cataloger", "installed", "image", "language", "javascript", "node"), dummyTask("javascript-package-cataloger", "package", "installed", "image", "language", "javascript", "node"),
dummyTask("php-composer-installed-cataloger", "installed", "image", "language", "php", "composer"), dummyTask("php-composer-installed-cataloger", "package", "installed", "image", "language", "php", "composer"),
dummyTask("ruby-installed-gemspec-cataloger", "installed", "image", "language", "ruby", "gem", "gemspec"), dummyTask("ruby-installed-gemspec-cataloger", "package", "installed", "image", "language", "ruby", "gem", "gemspec"),
dummyTask("rust-cargo-lock-cataloger", "installed", "image", "language", "rust", "binary"), dummyTask("rust-cargo-lock-cataloger", "package", "installed", "image", "language", "rust", "binary"),
// language-specific package declared catalogers // language-specific package declared catalogers
dummyTask("conan-cataloger", "declared", "directory", "language", "cpp", "conan"), dummyTask("conan-cataloger", "package", "declared", "directory", "language", "cpp", "conan"),
dummyTask("dart-pubspec-lock-cataloger", "declared", "directory", "language", "dart"), dummyTask("dart-pubspec-lock-cataloger", "package", "declared", "directory", "language", "dart"),
dummyTask("dotnet-deps-cataloger", "declared", "directory", "language", "dotnet", "c#"), dummyTask("dotnet-deps-cataloger", "package", "declared", "directory", "language", "dotnet", "c#"),
dummyTask("elixir-mix-lock-cataloger", "declared", "directory", "language", "elixir"), dummyTask("elixir-mix-lock-cataloger", "package", "declared", "directory", "language", "elixir"),
dummyTask("erlang-rebar-lock-cataloger", "declared", "directory", "language", "erlang"), dummyTask("erlang-rebar-lock-cataloger", "package", "declared", "directory", "language", "erlang"),
dummyTask("javascript-lock-cataloger", "declared", "directory", "language", "javascript", "node", "npm"), dummyTask("javascript-lock-cataloger", "package", "declared", "directory", "language", "javascript", "node", "npm"),
// language-specific package for both image and directory scans (but not necessarily declared) // language-specific package for both image and directory scans (but not necessarily declared)
dummyTask("dotnet-portable-executable-cataloger", "directory", "installed", "image", "language", "dotnet", "c#"), dummyTask("dotnet-portable-executable-cataloger", "package", "directory", "installed", "image", "language", "dotnet", "c#"),
dummyTask("python-installed-package-cataloger", "directory", "installed", "image", "language", "python"), dummyTask("python-installed-package-cataloger", "package", "directory", "installed", "image", "language", "python"),
dummyTask("go-module-binary-cataloger", "directory", "installed", "image", "language", "go", "golang", "gomod", "binary"), dummyTask("go-module-binary-cataloger", "package", "directory", "installed", "image", "language", "go", "golang", "gomod", "binary"),
dummyTask("java-archive-cataloger", "directory", "installed", "image", "language", "java", "maven"), dummyTask("java-archive-cataloger", "package", "directory", "installed", "image", "language", "java", "maven"),
dummyTask("graalvm-native-image-cataloger", "directory", "installed", "image", "language", "java"), dummyTask("graalvm-native-image-cataloger", "package", "directory", "installed", "image", "language", "java"),
// other package catalogers // other package catalogers
dummyTask("binary-cataloger", "declared", "directory", "image", "binary"), dummyTask("binary-cataloger", "package", "declared", "directory", "image", "binary"),
dummyTask("github-actions-usage-cataloger", "declared", "directory", "github", "github-actions"), dummyTask("github-actions-usage-cataloger", "package", "declared", "directory", "github", "github-actions"),
dummyTask("github-action-workflow-usage-cataloger", "declared", "directory", "github", "github-actions"), dummyTask("github-action-workflow-usage-cataloger", "package", "declared", "directory", "github", "github-actions"),
dummyTask("sbom-cataloger", "declared", "directory", "image", "sbom"), dummyTask("sbom-cataloger", "package", "declared", "directory", "image", "sbom"),
}
}
func createDummyFileTasks() tasks {
return []Task{
dummyTask("file-content-cataloger", "file", "content"),
dummyTask("file-metadata-cataloger", "file", "metadata"),
dummyTask("file-digest-cataloger", "file", "digest"),
dummyTask("file-executable-cataloger", "file", "binary-metadata"),
} }
} }
@ -72,7 +81,7 @@ func TestSelect(t *testing.T) {
expressions []string expressions []string
wantNames []string wantNames []string
wantTokens map[string]TokenSelection wantTokens map[string]TokenSelection
wantRequest pkgcataloging.SelectionRequest wantRequest cataloging.SelectionRequest
wantErr assert.ErrorAssertionFunc wantErr assert.ErrorAssertionFunc
}{ }{
{ {
@ -82,11 +91,11 @@ func TestSelect(t *testing.T) {
expressions: []string{}, expressions: []string{},
wantNames: []string{}, wantNames: []string{},
wantTokens: map[string]TokenSelection{}, wantTokens: map[string]TokenSelection{},
wantRequest: pkgcataloging.SelectionRequest{}, wantRequest: cataloging.SelectionRequest{},
}, },
{ {
name: "use default tasks", name: "use default tasks",
allTasks: createDummyTasks(), allTasks: createDummyPackageTasks(),
basis: []string{ basis: []string{
"image", "image",
}, },
@ -129,13 +138,13 @@ func TestSelect(t *testing.T) {
"binary-cataloger": newTokenSelection([]string{"image"}, nil), "binary-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil), "sbom-cataloger": newTokenSelection([]string{"image"}, nil),
}, },
wantRequest: pkgcataloging.SelectionRequest{ wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image"},
}, },
}, },
{ {
name: "select, add, and remove tasks", name: "select, add, and remove tasks",
allTasks: createDummyTasks(), allTasks: createDummyPackageTasks(),
basis: []string{ basis: []string{
"image", "image",
}, },
@ -175,7 +184,7 @@ func TestSelect(t *testing.T) {
"binary-cataloger": newTokenSelection([]string{"image"}, nil), "binary-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil), "sbom-cataloger": newTokenSelection([]string{"image"}, nil),
}, },
wantRequest: pkgcataloging.SelectionRequest{ wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image"},
SubSelectTags: []string{"os"}, SubSelectTags: []string{"os"},
RemoveNamesOrTags: []string{"dpkg"}, RemoveNamesOrTags: []string{"dpkg"},
@ -184,7 +193,7 @@ func TestSelect(t *testing.T) {
}, },
{ {
name: "allow for partial selections", name: "allow for partial selections",
allTasks: createDummyTasks(), allTasks: createDummyPackageTasks(),
basis: []string{ basis: []string{
"image", "image",
}, },
@ -228,7 +237,7 @@ func TestSelect(t *testing.T) {
"binary-cataloger": newTokenSelection([]string{"image"}, nil), "binary-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil), "sbom-cataloger": newTokenSelection([]string{"image"}, nil),
}, },
wantRequest: pkgcataloging.SelectionRequest{ wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image"},
SubSelectTags: []string{"os", "rust-cargo-lock-cataloger"}, SubSelectTags: []string{"os", "rust-cargo-lock-cataloger"},
RemoveNamesOrTags: []string{"dpkg"}, RemoveNamesOrTags: []string{"dpkg"},
@ -238,7 +247,7 @@ func TestSelect(t *testing.T) {
}, },
{ {
name: "select all tasks", name: "select all tasks",
allTasks: createDummyTasks(), allTasks: createDummyPackageTasks(),
basis: []string{ basis: []string{
"all", "all",
}, },
@ -299,13 +308,13 @@ func TestSelect(t *testing.T) {
"github-action-workflow-usage-cataloger": newTokenSelection([]string{"all"}, nil), "github-action-workflow-usage-cataloger": newTokenSelection([]string{"all"}, nil),
"sbom-cataloger": newTokenSelection([]string{"all"}, nil), "sbom-cataloger": newTokenSelection([]string{"all"}, nil),
}, },
wantRequest: pkgcataloging.SelectionRequest{ wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"all"}, DefaultNamesOrTags: []string{"all"},
}, },
}, },
{ {
name: "set default with multiple tags", name: "set default with multiple tags",
allTasks: createDummyTasks(), allTasks: createDummyPackageTasks(),
basis: []string{ basis: []string{
"gemspec", "gemspec",
"python", "python",
@ -319,10 +328,31 @@ func TestSelect(t *testing.T) {
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"gemspec"}, nil), "ruby-installed-gemspec-cataloger": newTokenSelection([]string{"gemspec"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"python"}, nil), "python-installed-package-cataloger": newTokenSelection([]string{"python"}, nil),
}, },
wantRequest: pkgcataloging.SelectionRequest{ wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"gemspec", "python"}, DefaultNamesOrTags: []string{"gemspec", "python"},
}, },
}, },
{
name: "automatically add file to default tags",
allTasks: createDummyFileTasks(),
basis: []string{},
expressions: []string{},
wantNames: []string{
"file-content-cataloger",
"file-metadata-cataloger",
"file-digest-cataloger",
"file-executable-cataloger",
},
wantTokens: map[string]TokenSelection{
"file-content-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"file"}, nil),
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"file"},
},
},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
@ -330,7 +360,7 @@ func TestSelect(t *testing.T) {
tt.wantErr = assert.NoError tt.wantErr = assert.NoError
} }
req := pkgcataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...) req := cataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...)
got, gotEvidence, err := Select(tt.allTasks, req) got, gotEvidence, err := Select(tt.allTasks, req)
tt.wantErr(t, err) tt.wantErr(t, err)
@ -367,3 +397,391 @@ func TestSelect(t *testing.T) {
}) })
} }
} }
func TestSelectInGroups(t *testing.T) {
tests := []struct {
name string
taskGroups [][]Task
selectionReq cataloging.SelectionRequest
wantGroups [][]string
wantTokens map[string]TokenSelection
wantRequest cataloging.SelectionRequest
wantErr assert.ErrorAssertionFunc
}{
{
name: "select only within the file tasks (leave package tasks alone)",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().
WithDefaults("image"). // note: file missing
WithSubSelections("content", "digest"),
wantGroups: [][]string{
{
// this is the original, untouched package task list
"alpm-db-cataloger",
"apk-db-cataloger",
"dpkg-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
"conan-info-cataloger",
"javascript-package-cataloger",
"php-composer-installed-cataloger",
"ruby-installed-gemspec-cataloger",
"rust-cargo-lock-cataloger",
"dotnet-portable-executable-cataloger",
"python-installed-package-cataloger",
"go-module-binary-cataloger",
"java-archive-cataloger",
"graalvm-native-image-cataloger",
"binary-cataloger",
"sbom-cataloger",
},
{
// this has been filtered based on the request
"file-content-cataloger",
"file-digest-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// packages
"alpm-db-cataloger": newTokenSelection([]string{"image"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"image"}, nil),
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"image"}, nil),
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
"portage-cataloger": newTokenSelection([]string{"image"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil),
"rpm-db-cataloger": newTokenSelection([]string{"image"}, nil),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
// files
"file-content-cataloger": newTokenSelection([]string{"content", "file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"digest", "file"}, nil),
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image", "file"}, // note: file automatically added
SubSelectTags: []string{"content", "digest"},
},
wantErr: assert.NoError,
},
{
name: "select package tasks (leave file tasks alone)",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().WithDefaults("image").WithSubSelections("os"),
wantGroups: [][]string{
{
// filtered based on the request
"alpm-db-cataloger",
"apk-db-cataloger",
"dpkg-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
},
{
// this is the original, untouched file task list
"file-content-cataloger",
"file-metadata-cataloger",
"file-digest-cataloger",
"file-executable-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// packages - os
"alpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"rpm-archive-cataloger": newTokenSelection([]string{"os"}, nil),
"rpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"portage-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
// packages - remaining
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
// files
"file-content-cataloger": newTokenSelection([]string{"file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"file"}, nil),
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image", "file"},
SubSelectTags: []string{"os"},
},
wantErr: assert.NoError,
},
{
name: "select only file tasks (default)",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().WithDefaults("file"),
wantGroups: [][]string{
// filtered based on the request
nil,
{
// this is the original, untouched file task list
"file-content-cataloger",
"file-metadata-cataloger",
"file-digest-cataloger",
"file-executable-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// files
"file-content-cataloger": newTokenSelection([]string{"file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"file"}, nil),
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"file"},
},
wantErr: assert.NoError,
},
{
name: "select only file tasks (via removal of package)",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().WithDefaults("file", "image").WithRemovals("package"),
wantGroups: [][]string{
// filtered based on the request
nil,
{
// this is the original, untouched file task list
"file-content-cataloger",
"file-metadata-cataloger",
"file-digest-cataloger",
"file-executable-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// packages
"alpm-db-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"apk-db-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"binary-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"conan-info-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"dpkg-db-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"java-archive-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"javascript-package-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"portage-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"rpm-db-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"sbom-cataloger": newTokenSelection([]string{"image"}, []string{"package"}),
"rpm-archive-cataloger": newTokenSelection(nil, []string{"package"}),
"conan-cataloger": newTokenSelection(nil, []string{"package"}),
"dart-pubspec-lock-cataloger": newTokenSelection(nil, []string{"package"}),
"dotnet-deps-cataloger": newTokenSelection(nil, []string{"package"}),
"elixir-mix-lock-cataloger": newTokenSelection(nil, []string{"package"}),
"erlang-rebar-lock-cataloger": newTokenSelection(nil, []string{"package"}),
"javascript-lock-cataloger": newTokenSelection(nil, []string{"package"}),
"github-actions-usage-cataloger": newTokenSelection(nil, []string{"package"}),
"github-action-workflow-usage-cataloger": newTokenSelection(nil, []string{"package"}),
// files
"file-content-cataloger": newTokenSelection([]string{"file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"file"}, nil),
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"file", "image"},
RemoveNamesOrTags: []string{"package"},
},
wantErr: assert.NoError,
},
{
name: "select file and package tasks",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().
WithDefaults("image").
WithSubSelections("os", "content", "digest"),
wantGroups: [][]string{
{
// filtered based on the request
"alpm-db-cataloger",
"apk-db-cataloger",
"dpkg-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
},
{
// filtered based on the request
"file-content-cataloger",
"file-digest-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// packages - os
"alpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"rpm-archive-cataloger": newTokenSelection([]string{"os"}, nil),
"rpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"portage-cataloger": newTokenSelection([]string{"os", "image"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"os", "image"}, nil),
// packages - remaining
"binary-cataloger": newTokenSelection([]string{"image"}, nil),
"conan-info-cataloger": newTokenSelection([]string{"image"}, nil),
"dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil),
"go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil),
"graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil),
"java-archive-cataloger": newTokenSelection([]string{"image"}, nil),
"javascript-package-cataloger": newTokenSelection([]string{"image"}, nil),
"php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil),
"python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil),
"ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil),
"rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil),
"sbom-cataloger": newTokenSelection([]string{"image"}, nil),
// files
"file-content-cataloger": newTokenSelection([]string{"file", "content"}, nil), // note extra tags
"file-digest-cataloger": newTokenSelection([]string{"file", "digest"}, nil), // note extra tags
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file"}, nil),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image", "file"},
SubSelectTags: []string{"os", "content", "digest"},
},
wantErr: assert.NoError,
},
{
name: "complex selection with multiple operators across groups",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().
WithDefaults("os"). // note: no file tag present
WithExpression("+github-actions-usage-cataloger", "-dpkg", "-digest", "content", "+file-metadata-cataloger", "-declared"),
wantGroups: [][]string{
{
"alpm-db-cataloger",
"apk-db-cataloger",
"portage-cataloger",
"rpm-db-cataloger",
"github-actions-usage-cataloger",
},
{
"file-content-cataloger",
"file-metadata-cataloger",
},
},
wantTokens: map[string]TokenSelection{
// selected package tasks
"alpm-db-cataloger": newTokenSelection([]string{"os"}, nil),
"apk-db-cataloger": newTokenSelection([]string{"os"}, nil),
"dpkg-db-cataloger": newTokenSelection([]string{"os"}, []string{"dpkg"}),
"portage-cataloger": newTokenSelection([]string{"os"}, nil),
"rpm-archive-cataloger": newTokenSelection([]string{"os"}, []string{"declared"}),
"rpm-db-cataloger": newTokenSelection([]string{"os"}, nil),
"github-actions-usage-cataloger": newTokenSelection([]string{"github-actions-usage-cataloger"}, []string{"declared"}),
// selected file tasks
"file-content-cataloger": newTokenSelection([]string{"content", "file"}, nil),
"file-metadata-cataloger": newTokenSelection([]string{"file-metadata-cataloger", "file"}, nil),
// removed package tasks
"binary-cataloger": newTokenSelection(nil, []string{"declared"}),
"conan-cataloger": newTokenSelection(nil, []string{"declared"}),
"dart-pubspec-lock-cataloger": newTokenSelection(nil, []string{"declared"}),
"dotnet-deps-cataloger": newTokenSelection(nil, []string{"declared"}),
"elixir-mix-lock-cataloger": newTokenSelection(nil, []string{"declared"}),
"erlang-rebar-lock-cataloger": newTokenSelection(nil, []string{"declared"}),
"github-action-workflow-usage-cataloger": newTokenSelection(nil, []string{"declared"}),
"javascript-lock-cataloger": newTokenSelection(nil, []string{"declared"}),
"sbom-cataloger": newTokenSelection(nil, []string{"declared"}),
// removed file tasks
"file-executable-cataloger": newTokenSelection([]string{"file"}, nil),
"file-digest-cataloger": newTokenSelection([]string{"file"}, []string{"digest"}),
},
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"os", "file"}, // note: file added automatically
SubSelectTags: []string{"content"},
RemoveNamesOrTags: []string{"dpkg", "digest", "declared"},
AddNames: []string{"github-actions-usage-cataloger", "file-metadata-cataloger"},
},
wantErr: assert.NoError,
},
{
name: "invalid tag",
taskGroups: [][]Task{
createDummyPackageTasks(),
createDummyFileTasks(),
},
selectionReq: cataloging.NewSelectionRequest().WithDefaults("invalid"),
wantGroups: nil,
wantTokens: nil,
wantRequest: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"invalid", "file"},
},
wantErr: assert.Error,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = assert.NoError
}
gotGroups, gotSelection, err := SelectInGroups(tt.taskGroups, tt.selectionReq)
tt.wantErr(t, err)
if err != nil {
// dev note: this is useful for debugging when needed...
//for _, e := range gotEvidence.Request.Expressions {
// t.Logf("expression (errors %q): %#v", e.Errors, e)
//}
// note: we DON'T bail early in validations... this is because we should always return the full set of
// of selected tasks and surrounding evidence.
}
var gotGroupNames [][]string
for _, group := range gotGroups {
var names []string
for _, task := range group {
names = append(names, task.Name())
}
gotGroupNames = append(gotGroupNames, names)
}
assert.Equal(t, tt.wantGroups, gotGroupNames)
assert.Equal(t, tt.wantTokens, gotSelection.TokensByTask)
assert.Equal(t, tt.wantRequest, gotSelection.Request)
})
}
}

View File

@ -0,0 +1,6 @@
package filecataloging
const (
// FileTag should be used to identify catalogers that primarily discover information about files (as opposed to packages).
FileTag = "file"
)

View File

@ -0,0 +1,11 @@
package pkgcataloging
import (
"github.com/anchore/syft/syft/cataloging"
)
// SelectionRequest is deprecated: use cataloging.SelectionRequest instead
type SelectionRequest = cataloging.SelectionRequest
// NewSelectionRequest is deprecated: use cataloging.NewSelectionRequest instead
var NewSelectionRequest = cataloging.NewSelectionRequest

View File

@ -1,8 +1,6 @@
package pkgcataloging package cataloging
import ( import "strings"
"strings"
)
type SelectionRequest struct { type SelectionRequest struct {
DefaultNamesOrTags []string `json:"default,omitempty"` DefaultNamesOrTags []string `json:"default,omitempty"`
@ -50,6 +48,10 @@ func (s SelectionRequest) WithRemovals(nameOrTags ...string) SelectionRequest {
return s return s
} }
func (s SelectionRequest) IsEmpty() bool {
return len(s.AddNames) == 0 && len(s.RemoveNamesOrTags) == 0 && len(s.SubSelectTags) == 0 && len(s.DefaultNamesOrTags) == 0
}
func cleanSelection(tags []string) []string { func cleanSelection(tags []string) []string {
var cleaned []string var cleaned []string
for _, tag := range tags { for _, tag := range tags {

View File

@ -21,8 +21,8 @@ type configurationAuditTrail struct {
} }
type catalogerManifest struct { type catalogerManifest struct {
Requested pkgcataloging.SelectionRequest `json:"requested" yaml:"requested" mapstructure:"requested"` Requested cataloging.SelectionRequest `json:"requested" yaml:"requested" mapstructure:"requested"`
Used []string `json:"used" yaml:"used" mapstructure:"used"` Used []string `json:"used" yaml:"used" mapstructure:"used"`
} }
type marshalAPIConfiguration configurationAuditTrail type marshalAPIConfiguration configurationAuditTrail

View File

@ -125,6 +125,9 @@ func monitorCatalogingTask(srcID artifact.ID, tasks [][]task.Task) *monitor.Cata
func formatTaskNames(tasks []task.Task) []string { func formatTaskNames(tasks []task.Task) []string {
set := strset.New() set := strset.New()
for _, td := range tasks { for _, td := range tasks {
if td == nil {
continue
}
set.Add(td.Name()) set.Add(td.Name())
} }
list := set.List() list := set.List()

View File

@ -28,14 +28,14 @@ type CreateSBOMConfig struct {
Packages pkgcataloging.Config Packages pkgcataloging.Config
Files filecataloging.Config Files filecataloging.Config
Parallelism int Parallelism int
CatalogerSelection pkgcataloging.SelectionRequest CatalogerSelection cataloging.SelectionRequest
// audit what tool is being used to generate the SBOM // audit what tool is being used to generate the SBOM
ToolName string ToolName string
ToolVersion string ToolVersion string
ToolConfiguration interface{} ToolConfiguration interface{}
packageTaskFactories task.PackageTaskFactories packageTaskFactories task.Factories
packageCatalogerReferences []pkgcataloging.CatalogerReference packageCatalogerReferences []pkgcataloging.CatalogerReference
} }
@ -150,7 +150,7 @@ func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig {
} }
// WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag. // WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag.
func (c *CreateSBOMConfig) WithCatalogerSelection(selection pkgcataloging.SelectionRequest) *CreateSBOMConfig { func (c *CreateSBOMConfig) WithCatalogerSelection(selection cataloging.SelectionRequest) *CreateSBOMConfig {
c.CatalogerSelection = selection c.CatalogerSelection = selection
return c return c
} }
@ -166,6 +166,10 @@ func (c *CreateSBOMConfig) WithoutCatalogers() *CreateSBOMConfig {
// WithCatalogers allows for adding user-provided catalogers to the final set of catalogers that will always be run // WithCatalogers allows for adding user-provided catalogers to the final set of catalogers that will always be run
// regardless of the source type or any cataloger selections provided. // regardless of the source type or any cataloger selections provided.
func (c *CreateSBOMConfig) WithCatalogers(catalogerRefs ...pkgcataloging.CatalogerReference) *CreateSBOMConfig { func (c *CreateSBOMConfig) WithCatalogers(catalogerRefs ...pkgcataloging.CatalogerReference) *CreateSBOMConfig {
for i := range catalogerRefs {
// ensure that all package catalogers have the package tag
catalogerRefs[i].Tags = append(catalogerRefs[i].Tags, pkgcataloging.PackageTag)
}
c.packageCatalogerReferences = append(c.packageCatalogerReferences, catalogerRefs...) c.packageCatalogerReferences = append(c.packageCatalogerReferences, catalogerRefs...)
return c return c
@ -182,8 +186,8 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task
environmentTasks := c.environmentTasks() environmentTasks := c.environmentTasks()
relationshipsTasks := c.relationshipTasks(src) relationshipsTasks := c.relationshipTasks(src)
unknownTasks := c.unknownsTasks() unknownTasks := c.unknownsTasks()
fileTasks := c.fileTasks()
pkgTasks, selectionEvidence, err := c.packageTasks(src) pkgTasks, fileTasks, selectionEvidence, err := c.selectTasks(src)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -214,80 +218,117 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task
taskGroups..., taskGroups...,
) )
var allTasks []task.Task
allTasks = append(allTasks, pkgTasks...)
allTasks = append(allTasks, fileTasks...)
return taskGroups, &catalogerManifest{ return taskGroups, &catalogerManifest{
Requested: selectionEvidence.Request, Requested: selectionEvidence.Request,
Used: formatTaskNames(pkgTasks), Used: formatTaskNames(allTasks),
}, nil }, nil
} }
// fileTasks returns the set of tasks that should be run to catalog files. // fileTasks returns the set of tasks that should be run to catalog files.
func (c *CreateSBOMConfig) fileTasks() []task.Task { func (c *CreateSBOMConfig) fileTasks(cfg task.CatalogingFactoryConfig) ([]task.Task, error) {
var tsks []task.Task tsks, err := task.DefaultFileTaskFactories().Tasks(cfg)
if err != nil {
if t := task.NewFileDigestCatalogerTask(c.Files.Selection, c.Files.Hashers...); t != nil { return nil, fmt.Errorf("unable to create file cataloger tasks: %w", err)
tsks = append(tsks, t)
}
if t := task.NewFileMetadataCatalogerTask(c.Files.Selection); t != nil {
tsks = append(tsks, t)
}
if t := task.NewFileContentCatalogerTask(c.Files.Content); t != nil {
tsks = append(tsks, t)
}
if t := task.NewExecutableCatalogerTask(c.Files.Selection, c.Files.Executable); t != nil {
tsks = append(tsks, t)
} }
return tsks return tsks, nil
} }
// packageTasks returns the set of tasks that should be run to catalog packages. // selectTasks returns the set of tasks that should be run to catalog packages and files.
func (c *CreateSBOMConfig) packageTasks(src source.Description) ([]task.Task, *task.Selection, error) { func (c *CreateSBOMConfig) selectTasks(src source.Description) ([]task.Task, []task.Task, *task.Selection, error) {
cfg := task.CatalogingFactoryConfig{ cfg := task.CatalogingFactoryConfig{
SearchConfig: c.Search, SearchConfig: c.Search,
RelationshipsConfig: c.Relationships, RelationshipsConfig: c.Relationships,
DataGenerationConfig: c.DataGeneration, DataGenerationConfig: c.DataGeneration,
PackagesConfig: c.Packages, PackagesConfig: c.Packages,
ComplianceConfig: c.Compliance, ComplianceConfig: c.Compliance,
FilesConfig: c.Files,
} }
persistentTasks, selectableTasks, err := c.allPackageTasks(cfg) persistentPkgTasks, selectablePkgTasks, err := c.allPackageTasks(cfg)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("unable to create package cataloger tasks: %w", err) return nil, nil, nil, fmt.Errorf("unable to create package cataloger tasks: %w", err)
} }
req, err := finalSelectionRequest(c.CatalogerSelection, src) req, err := finalTaskSelectionRequest(c.CatalogerSelection, src)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, nil, err
} }
finalTasks, selection, err := task.Select(selectableTasks, *req) selectableFileTasks, err := c.fileTasks(cfg)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, nil, err
} }
finalTasks = append(finalTasks, persistentTasks...) taskGroups := [][]task.Task{
selectablePkgTasks,
if len(finalTasks) == 0 { selectableFileTasks,
log.Warn("no catalogers selected")
return finalTasks, &selection, nil
} }
return finalTasks, &selection, nil finalTaskGroups, selection, err := task.SelectInGroups(taskGroups, *req)
if err != nil {
return nil, nil, nil, err
}
finalPkgTasks := finalTaskGroups[0]
finalFileTasks := finalTaskGroups[1]
finalPkgTasks = append(finalPkgTasks, persistentPkgTasks...)
if len(finalPkgTasks) == 0 && len(finalFileTasks) == 0 {
return nil, nil, nil, fmt.Errorf("no catalogers selected")
}
logTaskNames(finalPkgTasks, "package cataloger")
logTaskNames(finalFileTasks, "file cataloger")
if len(finalPkgTasks) == 0 && len(finalFileTasks) == 0 {
return nil, nil, nil, fmt.Errorf("no catalogers selected")
}
if len(finalPkgTasks) == 0 {
log.Debug("no package catalogers selected")
}
if len(finalFileTasks) == 0 {
if c.Files.Selection != file.NoFilesSelection {
log.Warnf("no file catalogers selected but file selection is configured as %q (this may be unintentional)", c.Files.Selection)
} else {
log.Debug("no file catalogers selected")
}
}
return finalPkgTasks, finalFileTasks, &selection, nil
} }
func finalSelectionRequest(req pkgcataloging.SelectionRequest, src source.Description) (*pkgcataloging.SelectionRequest, error) { func logTaskNames(tasks []task.Task, kind string) {
// log as tree output (like tree command)
log.Debugf("selected %d %s tasks", len(tasks), kind)
names := formatTaskNames(tasks)
for idx, t := range names {
if idx == len(tasks)-1 {
log.Tracef("└── %s", t)
} else {
log.Tracef("├── %s", t)
}
}
}
func finalTaskSelectionRequest(req cataloging.SelectionRequest, src source.Description) (*cataloging.SelectionRequest, error) {
if len(req.DefaultNamesOrTags) == 0 { if len(req.DefaultNamesOrTags) == 0 {
defaultTag, err := findDefaultTag(src) defaultTags, err := findDefaultTags(src)
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to determine default cataloger tag: %w", err) return nil, fmt.Errorf("unable to determine default cataloger tag: %w", err)
} }
if defaultTag != "" { req.DefaultNamesOrTags = append(req.DefaultNamesOrTags, defaultTags...)
req.DefaultNamesOrTags = append(req.DefaultNamesOrTags, defaultTag)
}
req.RemoveNamesOrTags = replaceDefaultTagReferences(defaultTag, req.RemoveNamesOrTags) req.RemoveNamesOrTags = replaceDefaultTagReferences(defaultTags, req.RemoveNamesOrTags)
req.SubSelectTags = replaceDefaultTagReferences(defaultTag, req.SubSelectTags) req.SubSelectTags = replaceDefaultTagReferences(defaultTags, req.SubSelectTags)
} }
return &req, nil return &req, nil
@ -379,21 +420,29 @@ func (c *CreateSBOMConfig) Create(ctx context.Context, src source.Source) (*sbom
return CreateSBOM(ctx, src, c) return CreateSBOM(ctx, src, c)
} }
func findDefaultTag(src source.Description) (string, error) { func findDefaultTags(src source.Description) ([]string, error) {
switch m := src.Metadata.(type) { switch m := src.Metadata.(type) {
case source.ImageMetadata: case source.ImageMetadata:
return pkgcataloging.ImageTag, nil return []string{pkgcataloging.ImageTag, filecataloging.FileTag}, nil
case source.FileMetadata, source.DirectoryMetadata: case source.FileMetadata, source.DirectoryMetadata:
return pkgcataloging.DirectoryTag, nil return []string{pkgcataloging.DirectoryTag, filecataloging.FileTag}, nil
default: default:
return "", fmt.Errorf("unable to determine default cataloger tag for source type=%T", m) return nil, fmt.Errorf("unable to determine default cataloger tag for source type=%T", m)
} }
} }
func replaceDefaultTagReferences(defaultTag string, lst []string) []string { func replaceDefaultTagReferences(defaultTags []string, lst []string) []string {
for i, tag := range lst { for i, tag := range lst {
if strings.ToLower(tag) == "default" { if strings.ToLower(tag) == "default" {
lst[i] = defaultTag switch len(defaultTags) {
case 0:
lst[i] = ""
case 1:
lst[i] = defaultTags[0]
default:
// remove the default tag and add the individual tags
lst = append(lst[:i], append(defaultTags, lst[i+1:]...)...)
}
} }
} }
return lst return lst

View File

@ -88,15 +88,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"), pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "image"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "image"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -107,15 +107,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "directory"), pkgCatalogerNamesWithTagOrName(t, "directory"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"directory"}, DefaultNamesOrTags: []string{"directory", "file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "directory"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "directory"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -127,51 +127,53 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "directory"), pkgCatalogerNamesWithTagOrName(t, "directory"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"directory"}, DefaultNamesOrTags: []string{"directory", "file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "directory"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "directory"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
{ {
name: "no file digest cataloger", name: "no file digest cataloger",
src: imgSrc, src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithHashers()), cfg: DefaultCreateSBOMConfig().WithCatalogerSelection(cataloging.NewSelectionRequest().WithRemovals("digest")),
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"), pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(false, true, true), // note: the digest cataloger is not included fileCatalogerNames("file-metadata", "content", "binary-metadata"),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
RemoveNamesOrTags: []string{"digest"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "image"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "image"), fileCatalogerNames("file-metadata", "content", "binary-metadata")),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
{ {
name: "select no file catalogers", name: "select no file catalogers",
src: imgSrc, src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.NoFilesSelection)), cfg: DefaultCreateSBOMConfig().WithCatalogerSelection(cataloging.NewSelectionRequest().WithRemovals("file")),
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"), pkgCatalogerNamesWithTagOrName(t, "image"),
// note: there are no file catalogers in their own group nil, // note: there is a file cataloging group, with no items in it
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
RemoveNamesOrTags: []string{"file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "image"), Used: pkgCatalogerNamesWithTagOrName(t, "image"),
}, },
@ -186,16 +188,16 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
// note: there is a single group of catalogers for pkgs and files // note: there is a single group of catalogers for pkgs and files
append( append(
pkgCatalogerNamesWithTagOrName(t, "image"), pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(true, true, true)..., fileCatalogerNames()...,
), ),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "image"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "image"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -208,15 +210,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"), addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
}, },
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"), Used: flatten(addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -229,15 +231,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"), addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"directory"}, DefaultNamesOrTags: []string{"directory", "file"},
}, },
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"), Used: flatten(addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -246,20 +248,20 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
src: imgSrc, src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithCatalogers( cfg: DefaultCreateSBOMConfig().WithCatalogers(
pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")), pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")),
).WithCatalogerSelection(pkgcataloging.NewSelectionRequest().WithSubSelections("javascript")), ).WithCatalogerSelection(cataloging.NewSelectionRequest().WithSubSelections("javascript")),
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
addTo(pkgIntersect("image", "javascript"), "persistent"), addTo(pkgIntersect("image", "javascript"), "persistent"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
SubSelectTags: []string{"javascript"}, SubSelectTags: []string{"javascript"},
}, },
Used: addTo(pkgIntersect("image", "javascript"), "persistent"), Used: flatten(addTo(pkgIntersect("image", "javascript"), "persistent"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -272,15 +274,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"), addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
}, },
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"), Used: flatten(addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -293,15 +295,15 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
wantTaskNames: [][]string{ wantTaskNames: [][]string{
environmentCatalogerNames(), environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"), pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(true, true, true), fileCatalogerNames(),
relationshipCatalogerNames(), relationshipCatalogerNames(),
unknownsTaskNames(), unknownsTaskNames(),
}, },
wantManifest: &catalogerManifest{ wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{ Requested: cataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"}, DefaultNamesOrTags: []string{"image", "file"},
}, },
Used: pkgCatalogerNamesWithTagOrName(t, "image"), Used: flatten(pkgCatalogerNamesWithTagOrName(t, "image"), fileCatalogerNames()),
}, },
wantErr: require.NoError, wantErr: require.NoError,
}, },
@ -314,9 +316,6 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
// sanity check // sanity check
require.NotEmpty(t, tt.wantTaskNames) require.NotEmpty(t, tt.wantTaskNames)
for _, group := range tt.wantTaskNames {
require.NotEmpty(t, group)
}
// test the subject // test the subject
gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src) gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src)
@ -378,20 +377,51 @@ func pkgCatalogerNamesWithTagOrName(t *testing.T, token string) []string {
return names return names
} }
func fileCatalogerNames(digest, metadata, executable bool) []string { func fileCatalogerNames(tokens ...string) []string {
var names []string var names []string
if digest { cfg := task.DefaultCatalogingFactoryConfig()
names = append(names, "file-digest-cataloger") topLoop:
} for _, factory := range task.DefaultFileTaskFactories() {
if executable { cat := factory(cfg)
names = append(names, "file-executable-cataloger")
} if cat == nil {
if metadata { continue
names = append(names, "file-metadata-cataloger") }
name := cat.Name()
if len(tokens) == 0 {
names = append(names, name)
continue
}
for _, token := range tokens {
if selector, ok := cat.(task.Selector); ok {
if selector.HasAllSelectors(token) {
names = append(names, name)
continue topLoop
}
}
if name == token {
names = append(names, name)
}
}
} }
sort.Strings(names)
return names return names
} }
func flatten(lists ...[]string) []string {
var final []string
for _, lst := range lists {
final = append(final, lst...)
}
sort.Strings(final)
return final
}
func relationshipCatalogerNames() []string { func relationshipCatalogerNames() []string {
return []string{"relationships-cataloger"} return []string{"relationships-cataloger"}
} }
@ -436,7 +466,7 @@ func Test_replaceDefaultTagReferences(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, replaceDefaultTagReferences("replacement", tt.lst)) assert.Equal(t, tt.want, replaceDefaultTagReferences([]string{"replacement"}, tt.lst))
}) })
} }
} }
@ -446,7 +476,7 @@ func Test_findDefaultTag(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
src source.Description src source.Description
want string want []string
wantErr require.ErrorAssertionFunc wantErr require.ErrorAssertionFunc
}{ }{
{ {
@ -454,21 +484,21 @@ func Test_findDefaultTag(t *testing.T) {
src: source.Description{ src: source.Description{
Metadata: source.ImageMetadata{}, Metadata: source.ImageMetadata{},
}, },
want: pkgcataloging.ImageTag, want: []string{pkgcataloging.ImageTag, filecataloging.FileTag},
}, },
{ {
name: "directory", name: "directory",
src: source.Description{ src: source.Description{
Metadata: source.DirectoryMetadata{}, Metadata: source.DirectoryMetadata{},
}, },
want: pkgcataloging.DirectoryTag, want: []string{pkgcataloging.DirectoryTag, filecataloging.FileTag},
}, },
{ {
name: "file", name: "file",
src: source.Description{ src: source.Description{
Metadata: source.FileMetadata{}, Metadata: source.FileMetadata{},
}, },
want: pkgcataloging.DirectoryTag, // not a mistake... want: []string{pkgcataloging.DirectoryTag, filecataloging.FileTag}, // not a mistake...
}, },
{ {
name: "unknown", name: "unknown",
@ -483,7 +513,7 @@ func Test_findDefaultTag(t *testing.T) {
if tt.wantErr == nil { if tt.wantErr == nil {
tt.wantErr = require.NoError tt.wantErr = require.NoError
} }
got, err := findDefaultTag(tt.src) got, err := findDefaultTags(tt.src)
tt.wantErr(t, err) tt.wantErr(t, err)
if err != nil { if err != nil {
return return

View File

@ -298,6 +298,15 @@ func TestPackagesCmdFlags(t *testing.T) {
assertSuccessfulReturnCode, assertSuccessfulReturnCode,
}, },
}, },
{
name: "select-no-package-catalogers",
args: []string{"scan", "-o", "json", "--select-catalogers", "-package", coverageImage},
assertions: []traitAssertion{
assertPackageCount(0),
assertInOutput(`"used":["file-content-cataloger","file-digest-cataloger","file-executable-cataloger","file-metadata-cataloger"]`),
assertSuccessfulReturnCode,
},
},
{ {
name: "override-default-catalogers-option", name: "override-default-catalogers-option",
// This will detect enable: // This will detect enable:
@ -311,6 +320,15 @@ func TestPackagesCmdFlags(t *testing.T) {
assertSuccessfulReturnCode, assertSuccessfulReturnCode,
}, },
}, },
{
name: "override-default-catalogers-with-files",
args: []string{"packages", "-o", "json", "--override-default-catalogers", "file", coverageImage},
assertions: []traitAssertion{
assertPackageCount(0),
assertInOutput(`"used":["file-content-cataloger","file-digest-cataloger","file-executable-cataloger","file-metadata-cataloger"]`),
assertSuccessfulReturnCode,
},
},
{ {
name: "new and old cataloger options are mutually exclusive", name: "new and old cataloger options are mutually exclusive",
args: []string{"packages", "-o", "json", "--override-default-catalogers", "python", "--catalogers", "gemspec", coverageImage}, args: []string{"packages", "-o", "json", "--override-default-catalogers", "python", "--catalogers", "gemspec", coverageImage},