Add file catalogers to selection configuration (#3505)

* add file catalogers to selection configuration Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix typos Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * warn when there is conflicting file cataloging configuration Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * allow for explicit removal of all package and file tasks Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address PR feedback Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
2026-10-31 14:36:33 +01:00 · 2025-02-03 19:10:17 +00:00
parent 5e2ba43328
commit 684b6e3f98
25 changed files with 1206 additions and 353 deletions
@@ -6,7 +6,7 @@ import (

 	"github.com/anchore/clio"
 	"github.com/anchore/stereoscope"
-	ui2 "github.com/anchore/syft/cmd/syft/cli/ui"
+	handler "github.com/anchore/syft/cmd/syft/cli/ui"
 	"github.com/anchore/syft/cmd/syft/internal/ui"
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/log"
@@ -28,7 +28,7 @@ func AppClioSetupConfig(id clio.Identification, out io.Writer) *clio.SetupConfig

 				return clio.NewUICollection(
 					ui.New(out, cfg.Log.Quiet,
-						ui2.New(ui2.DefaultHandlerConfig()),
+						handler.New(handler.DefaultHandlerConfig()),
 					),
 					noUI,
 				), nil
@@ -3,6 +3,7 @@ package commands
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"sort"
 	"strings"

@@ -14,7 +15,15 @@ import (
 	"github.com/anchore/clio"
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/task"
-	"github.com/anchore/syft/syft/cataloging/pkgcataloging"
+	"github.com/anchore/syft/syft/cataloging"
+)
+
+var (
+	activelyAddedStyle     = lipgloss.NewStyle().Foreground(lipgloss.Color("10")) // hi green
+	deselectedStyle        = lipgloss.NewStyle().Foreground(lipgloss.Color("8"))  // dark grey
+	activelyRemovedStyle   = lipgloss.NewStyle().Foreground(lipgloss.Color("9"))  // high red
+	defaultStyle           = lipgloss.NewStyle().Underline(true)
+	deselectedDefaultStyle = lipgloss.NewStyle().Inherit(deselectedStyle).Underline(true)
 )

 type catalogerListOptions struct {
@@ -44,8 +53,9 @@ func CatalogerList(app clio.Application) *cobra.Command {
 	opts := defaultCatalogerListOptions()

 	return app.SetupCommand(&cobra.Command{
-		Use:   "list [OPTIONS]",
-		Short: "List available catalogers",
+		Use:     "list [OPTIONS]",
+		Short:   "List available catalogers",
+		PreRunE: disableUI(app, os.Stdout),
 		RunE: func(_ *cobra.Command, _ []string) error {
 			return runCatalogerList(opts)
 		},
@@ -53,13 +63,19 @@ func CatalogerList(app clio.Application) *cobra.Command {
 }

 func runCatalogerList(opts *catalogerListOptions) error {
-	factories := task.DefaultPackageTaskFactories()
-	allTasks, err := factories.Tasks(task.DefaultCatalogingFactoryConfig())
+	pkgTaskFactories := task.DefaultPackageTaskFactories()
+	fileTaskFactories := task.DefaultFileTaskFactories()
+	allPkgTasks, err := pkgTaskFactories.Tasks(task.DefaultCatalogingFactoryConfig())
 	if err != nil {
-		return fmt.Errorf("unable to create cataloger tasks: %w", err)
+		return fmt.Errorf("unable to create pkg cataloger tasks: %w", err)
 	}

-	report, err := catalogerListReport(opts, allTasks)
+	allFileTasks, err := fileTaskFactories.Tasks(task.DefaultCatalogingFactoryConfig())
+	if err != nil {
+		return fmt.Errorf("unable to create file cataloger tasks: %w", err)
+	}
+
+	report, err := catalogerListReport(opts, [][]task.Task{allPkgTasks, allFileTasks})
 	if err != nil {
 		return fmt.Errorf("unable to generate cataloger list report: %w", err)
 	}
@@ -69,9 +85,10 @@ func runCatalogerList(opts *catalogerListOptions) error {
 	return nil
 }

-func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (string, error) {
-	selectedTasks, selectionEvidence, err := task.Select(allTasks,
-		pkgcataloging.NewSelectionRequest().
+func catalogerListReport(opts *catalogerListOptions, allTaskGroups [][]task.Task) (string, error) {
+	selectedTaskGroups, selectionEvidence, err := task.SelectInGroups(
+		allTaskGroups,
+		cataloging.NewSelectionRequest().
 			WithDefaults(opts.DefaultCatalogers...).
 			WithExpression(opts.SelectCatalogers...),
 	)
@@ -82,12 +99,12 @@ func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (stri

 	switch opts.Output {
 	case "json":
-		report, err = renderCatalogerListJSON(selectedTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers)
+		report, err = renderCatalogerListJSON(flattenTaskGroups(selectedTaskGroups), selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers)
 	case "table", "":
 		if opts.ShowHidden {
-			report = renderCatalogerListTable(allTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers)
+			report = renderCatalogerListTables(allTaskGroups, selectionEvidence)
 		} else {
-			report = renderCatalogerListTable(selectedTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers)
+			report = renderCatalogerListTables(selectedTaskGroups, selectionEvidence)
 		}
 	}

@@ -98,6 +115,14 @@ func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (stri
 	return report, nil
 }

+func flattenTaskGroups(taskGroups [][]task.Task) []task.Task {
+	var allTasks []task.Task
+	for _, tasks := range taskGroups {
+		allTasks = append(allTasks, tasks...)
+	}
+	return allTasks
+}
+
 func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaultSelections, selections []string) (string, error) {
 	type node struct {
 		Name string   `json:"name"`
@@ -109,7 +134,12 @@ func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaul
 	nodesByName := make(map[string]node)

 	for name := range tagsByName {
-		tagsSelected := selection.TokensByTask[name].SelectedOn.List()
+		tokensByTask, ok := selection.TokensByTask[name]
+
+		var tagsSelected []string
+		if ok {
+			tagsSelected = tokensByTask.SelectedOn.List()
+		}

 		if len(tagsSelected) == 1 && tagsSelected[0] == "all" {
 			tagsSelected = tagsByName[name]
@@ -153,10 +183,56 @@ func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaul
 	return string(by), err
 }

-func renderCatalogerListTable(tasks []task.Task, selection task.Selection, defaultSelections, selections []string) string {
+func renderCatalogerListTables(taskGroups [][]task.Task, selection task.Selection) string {
+	pkgCatalogerTable := renderCatalogerListTable(taskGroups[0], selection, "Package Cataloger")
+	fileCatalogerTable := renderCatalogerListTable(taskGroups[1], selection, "File Cataloger")
+
+	report := fileCatalogerTable + "\n" + pkgCatalogerTable + "\n"
+
+	hasAdditions := len(selection.Request.AddNames) > 0
+	hasDefaults := len(selection.Request.DefaultNamesOrTags) > 0
+	hasRemovals := len(selection.Request.RemoveNamesOrTags) > 0
+	hasSubSelections := len(selection.Request.SubSelectTags) > 0
+	expressions := len(selection.Request.SubSelectTags) + len(selection.Request.AddNames) + len(selection.Request.RemoveNamesOrTags)
+
+	var header string
+
+	header += fmt.Sprintf("Default selections: %d\n", len(selection.Request.DefaultNamesOrTags))
+	if hasDefaults {
+		for _, expr := range selection.Request.DefaultNamesOrTags {
+			header += fmt.Sprintf("  • '%s'\n", expr)
+		}
+	}
+
+	header += fmt.Sprintf("Selection expressions: %d\n", expressions)
+
+	if hasSubSelections {
+		for _, n := range selection.Request.SubSelectTags {
+			header += fmt.Sprintf("  • '%s' (intersect)\n", n)
+		}
+	}
+	if hasRemovals {
+		for _, n := range selection.Request.RemoveNamesOrTags {
+			header += fmt.Sprintf("  • '-%s' (remove)\n", n)
+		}
+	}
+	if hasAdditions {
+		for _, n := range selection.Request.AddNames {
+			header += fmt.Sprintf("  • '+%s' (add)\n", n)
+		}
+	}
+
+	return header + report
+}
+
+func renderCatalogerListTable(tasks []task.Task, selection task.Selection, kindTitle string) string {
+	if len(tasks) == 0 {
+		return activelyRemovedStyle.Render(fmt.Sprintf("No %ss selected", strings.ToLower(kindTitle)))
+	}
+
 	t := table.NewWriter()
 	t.SetStyle(table.StyleLight)
-	t.AppendHeader(table.Row{"Cataloger", "Tags"})
+	t.AppendHeader(table.Row{kindTitle, "Tags"})

 	names, tagsByName := extractTaskInfo(tasks)

@@ -172,27 +248,12 @@ func renderCatalogerListTable(tasks []task.Task, selection task.Selection, defau

 	report := t.Render()

-	if len(selections) > 0 {
-		header := "Selected by expressions:\n"
-		for _, expr := range selections {
-			header += fmt.Sprintf("  - %q\n", expr)
-		}
-		report = header + report
-	}
-
-	if len(defaultSelections) > 0 {
-		header := "Default selections:\n"
-		for _, expr := range defaultSelections {
-			header += fmt.Sprintf("  - %q\n", expr)
-		}
-		report = header + report
-	}
-
 	return report
 }

 func formatRow(name string, tags []string, selection task.Selection) table.Row {
 	isIncluded := selection.Result.Has(name)
+	defaults := strset.New(selection.Request.DefaultNamesOrTags...)
 	var selections *task.TokenSelection
 	if s, exists := selection.TokensByTask[name]; exists {
 		selections = &s
@@ -200,35 +261,32 @@ func formatRow(name string, tags []string, selection task.Selection) table.Row {

 	var formattedTags []string
 	for _, tag := range tags {
-		formattedTags = append(formattedTags, formatToken(tag, selections, isIncluded))
+		formattedTags = append(formattedTags, formatToken(tag, selections, isIncluded, defaults))
 	}

 	var tagStr string
 	if isIncluded {
 		tagStr = strings.Join(formattedTags, ", ")
 	} else {
-		tagStr = strings.Join(formattedTags, grey.Render(", "))
+		tagStr = strings.Join(formattedTags, deselectedStyle.Render(", "))
 	}

 	// TODO: selection should keep warnings (non-selections) in struct

 	return table.Row{
-		formatToken(name, selections, isIncluded),
+		formatToken(name, selections, isIncluded, defaults),
 		tagStr,
 	}
 }

-var (
-	green = lipgloss.NewStyle().Foreground(lipgloss.Color("10")) // hi green
-	grey  = lipgloss.NewStyle().Foreground(lipgloss.Color("8"))  // dark grey
-	red   = lipgloss.NewStyle().Foreground(lipgloss.Color("9"))  // high red
-)
-
-func formatToken(token string, selection *task.TokenSelection, included bool) string {
+func formatToken(token string, selection *task.TokenSelection, included bool, defaults *strset.Set) string {
 	if included && selection != nil {
 		// format all tokens in selection in green
 		if selection.SelectedOn.Has(token) {
-			return green.Render(token)
+			if defaults.Has(token) {
+				return defaultStyle.Render(token)
+			}
+			return activelyAddedStyle.Render(token)
 		}

 		return token
@@ -236,10 +294,12 @@ func formatToken(token string, selection *task.TokenSelection, included bool) st

 	// format all tokens in selection in red, all others in grey
 	if selection != nil && selection.DeselectedOn.Has(token) {
-		return red.Render(token)
+		return activelyRemovedStyle.Render(token)
 	}
-
-	return grey.Render(token)
+	if defaults.Has(token) {
+		return deselectedDefaultStyle.Render(token)
+	}
+	return deselectedStyle.Render(token)
 }

 func extractTaskInfo(tasks []task.Task) ([]string, map[string][]string) {
@@ -40,23 +40,35 @@ func (d dummyTask) Execute(_ context.Context, _ file.Resolver, _ sbomsync.Builde
 	panic("implement me")
 }

-func testTasks() []task.Task {
-	return []task.Task{
-		dummyTask{
-			name:      "task1",
-			selectors: []string{"image", "a", "b", "1"},
+func testTasks() [][]task.Task {
+	return [][]task.Task{
+		{
+			dummyTask{
+				name:      "task1",
+				selectors: []string{"image", "a", "b", "1"},
+			},
+			dummyTask{
+				name:      "task2",
+				selectors: []string{"image", "b", "c", "2"},
+			},
+			dummyTask{
+				name:      "task3",
+				selectors: []string{"directory", "c", "d", "3"},
+			},
+			dummyTask{
+				name:      "task4",
+				selectors: []string{"directory", "d", "e", "4"},
+			},
 		},
-		dummyTask{
-			name:      "task2",
-			selectors: []string{"image", "b", "c", "2"},
-		},
-		dummyTask{
-			name:      "task3",
-			selectors: []string{"directory", "c", "d", "3"},
-		},
-		dummyTask{
-			name:      "task4",
-			selectors: []string{"directory", "d", "e", "4"},
+		{
+			dummyTask{
+				name:      "file-task1",
+				selectors: []string{"file", "ft", "ft-1-b"},
+			},
+			dummyTask{
+				name:      "file-task2",
+				selectors: []string{"file", "ft", "ft-2-b"},
+			},
 		},
 	}
 }
@@ -76,16 +88,23 @@ func Test_catalogerListReport(t *testing.T) {
 				return c
 			}(),
 			want: `
-Default selections:
-  - "all"
-┌───────────┬────────────────────┐
-│ CATALOGER │ TAGS               │
-├───────────┼────────────────────┤
-│ task1     │ 1, a, b, image     │
-│ task2     │ 2, b, c, image     │
-│ task3     │ 3, c, d, directory │
-│ task4     │ 4, d, directory, e │
-└───────────┴────────────────────┘
+Default selections: 1
+  • 'all'
+Selection expressions: 0
+┌────────────────┬──────────────────┐
+│ FILE CATALOGER │ TAGS             │
+├────────────────┼──────────────────┤
+│ file-task1     │ file, ft, ft-1-b │
+│ file-task2     │ file, ft, ft-2-b │
+└────────────────┴──────────────────┘
+┌───────────────────┬────────────────────┐
+│ PACKAGE CATALOGER │ TAGS               │
+├───────────────────┼────────────────────┤
+│ task1             │ 1, a, b, image     │
+│ task2             │ 2, b, c, image     │
+│ task3             │ 3, c, d, directory │
+│ task4             │ 4, d, directory, e │
+└───────────────────┴────────────────────┘
 `,
 		},
 		{
@@ -96,7 +115,7 @@ Default selections:
 				return c
 			}(),
 			want: `
-{"default":["all"],"selection":[],"catalogers":[{"name":"task1","tags":["1","a","b","image"]},{"name":"task2","tags":["2","b","c","image"]},{"name":"task3","tags":["3","c","d","directory"]},{"name":"task4","tags":["4","d","directory","e"]}]}
+{"default":["all"],"selection":[],"catalogers":[{"name":"file-task1","tags":["file","ft","ft-1-b"]},{"name":"file-task2","tags":["file","ft","ft-2-b"]},{"name":"task1","tags":["1","a","b","image"]},{"name":"task2","tags":["2","b","c","image"]},{"name":"task3","tags":["3","c","d","directory"]},{"name":"task4","tags":["4","d","directory","e"]}]}
 `,
 		},
 		{
@@ -105,19 +124,27 @@ Default selections:
 				c := defaultCatalogerListOptions()
 				c.Output = "table"
 				c.DefaultCatalogers = []string{
-					"image",
+					"image", // note: for backwards compatibility file will automatically be added
 				}
 				return c
 			}(),
 			want: `
-Default selections:
-  - "image"
-┌───────────┬────────────────┐
-│ CATALOGER │ TAGS           │
-├───────────┼────────────────┤
-│ task1     │ 1, a, b, image │
-│ task2     │ 2, b, c, image │
-└───────────┴────────────────┘
+Default selections: 2
+  • 'image'
+  • 'file'
+Selection expressions: 0
+┌────────────────┬──────────────────┐
+│ FILE CATALOGER │ TAGS             │
+├────────────────┼──────────────────┤
+│ file-task1     │ file, ft, ft-1-b │
+│ file-task2     │ file, ft, ft-2-b │
+└────────────────┴──────────────────┘
+┌───────────────────┬────────────────┐
+│ PACKAGE CATALOGER │ TAGS           │
+├───────────────────┼────────────────┤
+│ task1             │ 1, a, b, image │
+│ task2             │ 2, b, c, image │
+└───────────────────┴────────────────┘
 `,
 		},
 		{
@@ -131,7 +158,7 @@ Default selections:
 				return c
 			}(),
 			want: `
-{"default":["image"],"selection":[],"catalogers":[{"name":"task1","tags":["image"]},{"name":"task2","tags":["image"]}]}
+{"default":["image"],"selection":[],"catalogers":[{"name":"file-task1","tags":["file"]},{"name":"file-task2","tags":["file"]},{"name":"task1","tags":["image"]},{"name":"task2","tags":["image"]}]}
 `,
 		},
 		{
@@ -147,23 +174,32 @@ Default selections:
 					"+task3",
 					"-c",
 					"b",
+					"-file",
+					"+file-task1",
 				}
 				return c
 			}(),
 			want: `
-Default selections:
-  - "image"
-Selected by expressions:
-  - "-directory"
-  - "+task3"
-  - "-c"
-  - "b"
-┌───────────┬────────────────────┐
-│ CATALOGER │ TAGS               │
-├───────────┼────────────────────┤
-│ task1     │ 1, a, b, image     │
-│ task3     │ 3, c, d, directory │
-└───────────┴────────────────────┘
+Default selections: 1
+  • 'image'
+Selection expressions: 6
+  • 'b' (intersect)
+  • '-directory' (remove)
+  • '-c' (remove)
+  • '-file' (remove)
+  • '+task3' (add)
+  • '+file-task1' (add)
+┌────────────────┬──────────────────┐
+│ FILE CATALOGER │ TAGS             │
+├────────────────┼──────────────────┤
+│ file-task1     │ file, ft, ft-1-b │
+└────────────────┴──────────────────┘
+┌───────────────────┬────────────────────┐
+│ PACKAGE CATALOGER │ TAGS               │
+├───────────────────┼────────────────────┤
+│ task1             │ 1, a, b, image     │
+│ task3             │ 3, c, d, directory │
+└───────────────────┴────────────────────┘
 `,
 		},
 		{
@@ -183,7 +219,7 @@ Selected by expressions:
 				return c
 			}(),
 			want: `
-{"default":["image"],"selection":["-directory","+task3","-c","b"],"catalogers":[{"name":"task1","tags":["b","image"]},{"name":"task3","tags":["task3"]}]}
+{"default":["image"],"selection":["-directory","+task3","-c","b"],"catalogers":[{"name":"file-task1","tags":["file"]},{"name":"file-task2","tags":["file"]},{"name":"task1","tags":["b","image"]},{"name":"task3","tags":["task3"]}]}
 `,
 		},
 	}
@@ -0,0 +1,23 @@
+package commands
+
+import (
+	"io"
+
+	"github.com/spf13/cobra"
+
+	"github.com/anchore/clio"
+	"github.com/anchore/syft/cmd/syft/internal/ui"
+)
+
+func disableUI(app clio.Application, out io.Writer) func(*cobra.Command, []string) error {
+	return func(_ *cobra.Command, _ []string) error {
+		type Stater interface {
+			State() *clio.State
+		}
+
+		state := app.(Stater).State()
+		state.UI = clio.NewUICollection(ui.None(out, state.Config.Log.Quiet))
+
+		return nil
+	}
+}
@@ -91,7 +91,7 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
 		WithPackagesConfig(cfg.ToPackagesConfig()).
 		WithFilesConfig(cfg.ToFilesConfig()).
 		WithCatalogerSelection(
-			pkgcataloging.NewSelectionRequest().
+			cataloging.NewSelectionRequest().
 				WithDefaults(cfg.DefaultCatalogers...).
 				WithExpression(cfg.SelectCatalogers...),
 		)
@@ -10,6 +10,7 @@ import (
 	"github.com/anchore/stereoscope/pkg/imagetest"
 	"github.com/anchore/syft/cmd/syft/internal/options"
 	"github.com/anchore/syft/syft"
+	"github.com/anchore/syft/syft/cataloging"
 	"github.com/anchore/syft/syft/cataloging/pkgcataloging"
 	"github.com/anchore/syft/syft/sbom"
 	"github.com/anchore/syft/syft/source"
@@ -20,7 +21,7 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string, scope source.Sco
 		Name:    "syft-tester",
 		Version: "v0.99.0",
 	}).WithCatalogerSelection(
-		pkgcataloging.NewSelectionRequest().
+		cataloging.NewSelectionRequest().
 			WithExpression(catalogerSelection...),
 	)
 	cfg.Search.Scope = scope
@@ -55,7 +56,7 @@ func catalogDirectory(t *testing.T, dir string, catalogerSelection ...string) (s
 		Name:    "syft-tester",
 		Version: "v0.99.0",
 	}).WithCatalogerSelection(
-		pkgcataloging.NewSelectionRequest().
+		cataloging.NewSelectionRequest().
 			WithExpression(catalogerSelection...),
 	)