diff --git a/.github/scripts/capability-drift-check.sh b/.github/scripts/capability-drift-check.sh new file mode 100755 index 000000000..446508492 --- /dev/null +++ b/.github/scripts/capability-drift-check.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -u + +if [ "$(git status --porcelain | wc -l)" -ne "0" ]; then + echo " 🔴 there are uncommitted changes, please commit them before running this check" + exit 1 +fi + +if ! make generate-capabilities; then + echo "Generating capability descriptions failed" + exit 1 +fi + +if [ "$(git status --porcelain | wc -l)" -ne "0" ]; then + echo " 🔴 there is drift in capability descriptions! Please run 'make generate-capabilities' and commit the changes." + exit 1 +fi \ No newline at end of file diff --git a/.github/scripts/json-schema-drift-check.sh b/.github/scripts/json-schema-drift-check.sh index 3002236d6..4c370b21c 100755 --- a/.github/scripts/json-schema-drift-check.sh +++ b/.github/scripts/json-schema-drift-check.sh @@ -12,6 +12,6 @@ if ! make generate-json-schema; then fi if [ "$(git status --porcelain | wc -l)" -ne "0" ]; then - echo " 🔴 there are uncommitted changes, please commit them before running this check" + echo " 🔴 there is drift in json schema! Please run 'make generate-json-schema' and commit the changes." exit 1 fi diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index 2518a4780..6a68b6e8d 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -27,6 +27,11 @@ jobs: - name: Bootstrap environment uses: ./.github/actions/bootstrap + with: + # the self-consistency tests for the output of the capabilities code generation depends on unit test + # output from ./syft/pkg/... packages. Therefore we need to download the test fixture cache here + # so that running the few unit tests as part of static analysis works correctly. + download-test-fixture-cache: true - name: Run static analysis run: make static-analysis diff --git a/.gitignore b/.gitignore index 12578be80..24cb7b770 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ VERSION /test/results coverage.txt *.log +**/test-fixtures/test-observations.json # probable archives .images diff --git a/Taskfile.yaml b/Taskfile.yaml index 3fb15daaa..e4f636e28 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -73,6 +73,7 @@ tasks: - task: check-licenses - task: lint - task: check-json-schema-drift + - task: check-capability-drift - task: check-binary-fixture-size test: @@ -190,6 +191,11 @@ tasks: cmds: - .github/scripts/json-schema-drift-check.sh + check-capability-drift: + desc: Ensure there is no drift between the capabilities data and the code + cmds: + - .github/scripts/capability-drift-check.sh + check-binary-fixture-size: desc: Ensure that the binary test fixtures are not too large cmds: @@ -218,6 +224,7 @@ tasks: # unit test coverage threshold (in % coverage) COVERAGE_THRESHOLD: 62 cmds: + - task: clean-test-observations - "go test -coverprofile {{ .TMP_DIR }}/unit-coverage-details.txt {{ .TEST_PKGS }}" - cmd: ".github/scripts/coverage.py {{ .COVERAGE_THRESHOLD }} {{ .TMP_DIR }}/unit-coverage-details.txt" silent: true @@ -500,6 +507,7 @@ tasks: desc: Add data generation tasks cmds: - task: generate-json-schema + - task: generate-capabilities - task: generate-license-list - task: generate-cpe-dictionary-index @@ -521,6 +529,21 @@ tasks: - task: generate:cpe-index:cache:update - task: generate:cpe-index:build + generate-capabilities: + desc: Generate the capabilities data file + deps: + - tmpdir + - fixtures + cmds: + # remove all test observations prior to regenerating + - task: clean-test-observations + # this is required to update test observations; such evidence is used to update the packages/*.yaml + - "go test ./syft/pkg/... -count=1" + - "go generate ./internal/capabilities/..." + - "gofmt -s -w ./internal/capabilities" + # now that we have the latest capabilities, run completeness tests to ensure this is self-consistent + - "SYFT_ENABLE_COMPLETENESS_TESTS=true go test ./internal/capabilities/... -count=1" + ## Build-related targets ################################# @@ -646,3 +669,8 @@ tasks: done echo -e "${BOLD}Deleted all ephemeral test fixtures${RESET}" - rm -f {{ .LAST_CACHE_PULL_FILE }} {{ .CACHE_PATHS_FILE }} + + clean-test-observations: + desc: Remove all test observations (i.e. test-fixtures/test-observations.json) + cmds: + - find . -type f -wholename "**/test-fixtures/test-observations.json" | xargs rm -f diff --git a/cmd/syft/internal/commands/attest.go b/cmd/syft/internal/commands/attest.go index 526303edb..f538fa292 100644 --- a/cmd/syft/internal/commands/attest.go +++ b/cmd/syft/internal/commands/attest.go @@ -35,6 +35,7 @@ const ( attestSchemeHelp = "\n " + schemeHelpHeader + "\n" + imageSchemeHelp attestHelp = attestExample + attestSchemeHelp cosignBinName = "cosign" + jsonFormat = "json" ) type attestOptions struct { @@ -93,9 +94,7 @@ func defaultAttestOutputOptions() options.Output { string(spdxtagvalue.ID), }, Outputs: []string{syftjson.ID.String()}, - OutputFile: options.OutputFile{ //nolint:staticcheck - Enabled: false, // explicitly not allowed - }, + // OutputFile is omitted (defaults to Enabled: false, explicitly not allowed for attestation) Format: options.DefaultFormat(), } } @@ -240,7 +239,7 @@ func predicateType(outputName string) string { return "cyclonedx" case "spdx-tag-value", "spdx-tv": return "spdx" - case "spdx-json", "json": + case "spdx-json", jsonFormat: return "spdxjson" default: return "custom" diff --git a/cmd/syft/internal/commands/cataloger.go b/cmd/syft/internal/commands/cataloger.go index 0baf5b523..05e1f4058 100644 --- a/cmd/syft/internal/commands/cataloger.go +++ b/cmd/syft/internal/commands/cataloger.go @@ -1,6 +1,8 @@ package commands import ( + "os" + "github.com/spf13/cobra" "github.com/anchore/clio" @@ -16,5 +18,14 @@ func Cataloger(app clio.Application) *cobra.Command { CatalogerList(app), ) + // only add cataloger info command if experimental capabilities feature is enabled + if isCapabilitiesExperimentEnabled() { + cmd.AddCommand(CatalogerCaps(app)) + } + return cmd } + +func isCapabilitiesExperimentEnabled() bool { + return os.Getenv("SYFT_EXP_CAPABILITIES") == "true" +} diff --git a/cmd/syft/internal/commands/cataloger_info.go b/cmd/syft/internal/commands/cataloger_info.go new file mode 100644 index 000000000..87e01ba69 --- /dev/null +++ b/cmd/syft/internal/commands/cataloger_info.go @@ -0,0 +1,589 @@ +package commands + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "sort" + "strings" + + "github.com/charmbracelet/lipgloss" + "github.com/olekukonko/tablewriter" + "github.com/olekukonko/tablewriter/renderer" + "github.com/olekukonko/tablewriter/tw" + "github.com/scylladb/go-set/strset" + "github.com/spf13/cobra" + + "github.com/anchore/clio" + "github.com/anchore/syft/cmd/syft/internal/options" + "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/capabilities" + "github.com/anchore/syft/internal/task" + "github.com/anchore/syft/syft/cataloging" + _ "github.com/anchore/syft/syft/pkg/cataloger" // register all cataloger capabilities, otherwise the info command will not function +) + +var ( + yesStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("10")) // hi green + noStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey + criteriaWidth = 60 + criteriaMargin = 10 +) + +// types for JSON cataloger info output +type ( + configFieldInfo struct { + Key string `json:"key"` + Description string `json:"description"` + AppKey string `json:"app_key,omitempty"` + } + + configInfo struct { + Type string `json:"type"` + Fields []configFieldInfo `json:"fields,omitempty"` + } + + detectorPackageInfo struct { + Class string `json:"class"` + Name string `json:"name"` + PURL string `json:"purl"` + CPEs []string `json:"cpes"` + Type string `json:"type"` + } + + patternInfo struct { + Method string `json:"method"` + Criteria []string `json:"criteria"` + Conditions []capabilities.DetectorCondition `json:"conditions,omitempty"` + Packages []detectorPackageInfo `json:"packages,omitempty"` + Comment string `json:"comment,omitempty"` + PackageTypes []string `json:"package_types,omitempty"` + JSONSchemaTypes []string `json:"json_schema_types,omitempty"` + Capabilities capabilities.CapabilitySet `json:"capabilities,omitempty"` + } + + catalogerInfo struct { + Ecosystem string `json:"ecosystem,omitempty"` + Name string `json:"name"` + Type string `json:"type"` + Selectors []string `json:"selectors,omitempty"` + Deprecated bool `json:"deprecated,omitempty"` + Patterns []patternInfo `json:"patterns,omitempty"` + Capabilities capabilities.CapabilitySet `json:"capabilities,omitempty"` + Config *configInfo `json:"config,omitempty"` + } +) + +type catalogerInfoOptions struct { + Output string `yaml:"output" json:"output" mapstructure:"output"` + options.CatalogerSelection `yaml:",inline" json:",inline" mapstructure:",squash"` + Names []string // cataloger names from args +} + +func (o *catalogerInfoOptions) setNames(args []string) error { + o.Names = args + + usingLegacyCatalogers := len(o.Catalogers) > 0 + usingNewCatalogers := len(o.DefaultCatalogers) > 0 || len(o.SelectCatalogers) > 0 + usingSelection := usingNewCatalogers || usingLegacyCatalogers + + if usingSelection && len(o.Names) > 0 { + return fmt.Errorf("cannot use both cataloger name arguments and '--catalogers'/'--select-catalogers'/'--default-catalogers' flags") + } + + if usingSelection { + // get all available package cataloger tasks + pkgTaskFactories := task.DefaultPackageTaskFactories() + allPkgTasks, err := pkgTaskFactories.Tasks(task.DefaultCatalogingFactoryConfig()) + if err != nil { + return fmt.Errorf("unable to create pkg cataloger tasks: %w", err) + } + + // make the selection based on user input + defaultCatalogers := options.FlattenAndSort(o.DefaultCatalogers) + selectCatalogers := options.FlattenAndSort(o.SelectCatalogers) + selectedTaskGroups, _, err := task.SelectInGroups( + [][]task.Task{allPkgTasks}, + cataloging.NewSelectionRequest(). + WithDefaults(defaultCatalogers...). + WithExpression(selectCatalogers...), + ) + + if err != nil { + return fmt.Errorf("unable to select catalogers: %w", err) + } + + // build the list of cataloger names based on the selection + for _, g := range selectedTaskGroups { + for _, t := range g { + o.Names = append(o.Names, t.Name()) + } + } + } + return nil +} + +func (o *catalogerInfoOptions) AddFlags(flags clio.FlagSet) { + flags.StringVarP(&o.Output, "output", "o", "format to output the cataloger info (available: table, json)") +} + +func defaultCatalogerCapsOptions() *catalogerInfoOptions { + return &catalogerInfoOptions{ + CatalogerSelection: options.CatalogerSelection{ + // this is different than the default behavior where a scan will automatically detect the default set + DefaultCatalogers: []string{"all"}, + }, + } +} + +func CatalogerCaps(app clio.Application) *cobra.Command { + opts := defaultCatalogerCapsOptions() + + return app.SetupCommand(&cobra.Command{ + Use: "info [OPTIONS] [CATALOGER_NAMES...]", + Short: "Show detailed capabilities of catalogers", + Args: cobra.ArbitraryArgs, + PreRunE: disableUI(app, os.Stdout), + RunE: func(_ *cobra.Command, args []string) error { + if err := opts.setNames(args); err != nil { + return err + } + + return runCatalogerInfo(opts) + }, + }, opts) +} + +func runCatalogerInfo(opts *catalogerInfoOptions) error { + doc, err := capabilities.LoadDocument() + if err != nil { + return fmt.Errorf("unable to load cataloger capabilities: %w", err) + } + + // filter by cataloger names if provided + catalogers := doc.Catalogers + if len(opts.Names) > 0 { + catalogers = filterCatalogersByName(catalogers, opts.Names) + } + + report, err := catalogerInfoReport(opts, doc, catalogers) + if err != nil { + return fmt.Errorf("unable to generate cataloger info report: %w", err) + } + + bus.Report(report) + bus.Notify("Note: the `cataloger info` command is experimental and may change or be removed without notice. Do not depend on its output in production systems.") + + return nil +} + +func filterCatalogersByName(catalogers []capabilities.CatalogerEntry, names []string) []capabilities.CatalogerEntry { + nameSet := make(map[string]bool) + for _, name := range names { + nameSet[name] = true + } + + var filtered []capabilities.CatalogerEntry + for _, cat := range catalogers { + if nameSet[cat.Name] { + filtered = append(filtered, cat) + } + } + return filtered +} + +func catalogerInfoReport(opts *catalogerInfoOptions, doc *capabilities.Document, catalogers []capabilities.CatalogerEntry) (string, error) { + switch opts.Output { + case jsonFormat: + return renderCatalogerInfoJSON(doc, catalogers) + case "table", "": + return renderCatalogerInfoTable(doc, catalogers), nil + default: + return "", fmt.Errorf("invalid output format %q, must be one of: table, json", opts.Output) + } +} + +func renderCatalogerInfoJSON(doc *capabilities.Document, catalogers []capabilities.CatalogerEntry) (string, error) { + type document struct { + Catalogers []catalogerInfo `json:"catalogers"` + } + + docOut := document{} + + // sort catalogers by ecosystem then name + sort.Slice(catalogers, func(i, j int) bool { + if catalogers[i].Ecosystem != catalogers[j].Ecosystem { + return catalogers[i].Ecosystem < catalogers[j].Ecosystem + } + return catalogers[i].Name < catalogers[j].Name + }) + + for _, cat := range catalogers { + info := catalogerInfo{ + Ecosystem: cat.Ecosystem, + Name: cat.Name, + Type: cat.Type, + Selectors: cat.Selectors, + Deprecated: isDeprecatedCataloger(cat.Selectors), + } + + // convert parsers to patterns if available + info.Patterns = convertParsersToPatterns(cat.Parsers) + + // if no parsers, use detectors instead + if len(info.Patterns) == 0 { + info.Capabilities = cat.Capabilities + info.Patterns = convertDetectorsToPatterns(cat.Detectors, cat.PackageTypes, cat.JSONSchemaTypes) + } + + info.Config = getConfigInfoFromDocument(doc, cat.Config) + + docOut.Catalogers = append(docOut.Catalogers, info) + } + + by, err := json.Marshal(docOut) + return string(by), err +} + +// isDeprecatedCataloger checks if a cataloger is deprecated based on its selectors +func isDeprecatedCataloger(selectors []string) bool { + for _, selector := range selectors { + if selector == "deprecated" { + return true + } + } + return false +} + +// convertDetectorPackages converts detector package info to the JSON output format +func convertDetectorPackages(pkgs []capabilities.DetectorPackageInfo) []detectorPackageInfo { + var result []detectorPackageInfo + for _, pkg := range pkgs { + result = append(result, detectorPackageInfo{ + Class: pkg.Class, + Name: pkg.Name, + PURL: pkg.PURL, + CPEs: pkg.CPEs, + Type: pkg.Type, + }) + } + return result +} + +// convertParsersToPatterns converts parser entries to pattern info for JSON output +func convertParsersToPatterns(parsers []capabilities.Parser) []patternInfo { + var patterns []patternInfo + for _, parser := range parsers { + patterns = append(patterns, patternInfo{ + Method: string(parser.Detector.Method), + Criteria: parser.Detector.Criteria, + Conditions: parser.Detector.Conditions, + Packages: convertDetectorPackages(parser.Detector.Packages), + Comment: parser.Detector.Comment, + PackageTypes: parser.PackageTypes, + JSONSchemaTypes: parser.JSONSchemaTypes, + Capabilities: parser.Capabilities, + }) + } + return patterns +} + +// convertDetectorsToPatterns converts detector entries to pattern info for JSON output (for non-parser catalogers) +func convertDetectorsToPatterns(detectors []capabilities.Detector, packageTypes, jsonSchemaTypes []string) []patternInfo { + var patterns []patternInfo + for _, det := range detectors { + patterns = append(patterns, patternInfo{ + Method: string(det.Method), + Criteria: det.Criteria, + Conditions: det.Conditions, + Packages: convertDetectorPackages(det.Packages), + Comment: det.Comment, + PackageTypes: packageTypes, + JSONSchemaTypes: jsonSchemaTypes, + }) + } + return patterns +} + +// getConfigInfoFromDocument retrieves config info from the capabilities document +func getConfigInfoFromDocument(doc *capabilities.Document, configType string) *configInfo { + if configType == "" { + return nil + } + configEntry, ok := doc.Configs[configType] + if !ok { + return nil + } + cfg := &configInfo{ + Type: configType, + } + for _, field := range configEntry.Fields { + cfg.Fields = append(cfg.Fields, configFieldInfo{ + Key: field.Key, + Description: field.Description, + AppKey: field.AppKey, + }) + } + return cfg +} + +func renderCatalogerInfoTable(_ *capabilities.Document, catalogers []capabilities.CatalogerEntry) string { + if len(catalogers) == 0 { + return noStyle.Render("No catalogers found") + } + + // sort catalogers by ecosystem then name + sort.Slice(catalogers, func(i, j int) bool { + if catalogers[i].Ecosystem != catalogers[j].Ecosystem { + return catalogers[i].Ecosystem < catalogers[j].Ecosystem + } + return catalogers[i].Name < catalogers[j].Name + }) + + var buf bytes.Buffer + + // create table with hierarchical merging + table := tablewriter.NewTable(&buf, + tablewriter.WithRenderer(renderer.NewBlueprint(tw.Rendition{ + Settings: tw.Settings{Separators: tw.Separators{BetweenRows: tw.On}}, + })), + tablewriter.WithConfig(tablewriter.Config{ + Row: tw.CellConfig{ + Formatting: tw.CellFormatting{MergeMode: tw.MergeHierarchical}, + Alignment: tw.CellAlignment{Global: tw.AlignLeft}, + }, + }), + ) + + // set headers + table.Header("ECOSYSTEM", "CATALOGER", "CRITERIA", "LICENSE", "NODES", "EDGES", "KINDS", "LISTING", "DIGESTS", "HASH") + + // build rows for each cataloger + var data [][]string + for _, cat := range catalogers { + ecosystem := cat.Ecosystem + if ecosystem == "" { + ecosystem = "other" + } + + if cat.Type == "generic" { + // generic catalogers: one row per parser + for _, parser := range cat.Parsers { + criteria := formatCriteria([]capabilities.Detector{parser.Detector}) + row := buildTableRowFromCapabilities(ecosystem, cat.Name, criteria, parser.Capabilities) + data = append(data, row) + } + } else { + // custom catalogers: one row with all detectors + criteria := formatCriteria(cat.Detectors) + row := buildTableRowFromCapabilities(ecosystem, cat.Name, criteria, cat.Capabilities) + data = append(data, row) + } + } + + // add all rows at once for better merging + _ = table.Bulk(data) + _ = table.Render() + return buf.String() +} + +// buildTableRowFromCapabilities builds a table row from capability values +func buildTableRowFromCapabilities(ecosystem, name, criteria string, caps capabilities.CapabilitySet) []string { + // extract capability default values + license := extractBoolCapability(caps, "license") + nodes := extractNodesCapability(caps) + edges := extractStringCapability(caps, "dependency.edges") + kinds := extractArrayCapability(caps, "dependency.kinds") + listing := extractBoolCapability(caps, "package_manager.files.listing") + digests := extractBoolCapability(caps, "package_manager.files.digests") + hash := extractBoolCapability(caps, "package_manager.package_integrity_hash") + + return []string{ + ecosystem, + name, + criteria, + license, + nodes, + edges, + kinds, + listing, + digests, + hash, + } +} + +// extractBoolCapability extracts a boolean capability value and formats it +func extractBoolCapability(caps capabilities.CapabilitySet, name string) string { + for _, cap := range caps { + if cap.Name == name { + if b, ok := cap.Default.(bool); ok { + if b { + return yesStyle.Render("✔") + } + return noStyle.Render("·") + } + return noStyle.Render("·") + } + } + return noStyle.Render("·") +} + +// extractStringCapability extracts a string capability value +func extractStringCapability(caps capabilities.CapabilitySet, name string) string { + for _, cap := range caps { + if cap.Name == name { + if s, ok := cap.Default.(string); ok && s != "" { + return s + } + return noStyle.Render("·") + } + } + return noStyle.Render("·") +} + +// extractArrayCapability extracts an array capability value and formats it as comma-separated +func extractArrayCapability(caps capabilities.CapabilitySet, name string) string { + for _, cap := range caps { + if cap.Name == name { + // handle various array types + switch v := cap.Default.(type) { + case []string: + if len(v) > 0 { + return strings.Join(v, ", ") + } + case []interface{}: + if len(v) > 0 { + strs := make([]string, 0, len(v)) + for _, item := range v { + strs = append(strs, fmt.Sprintf("%v", item)) + } + return strings.Join(strs, ", ") + } + } + return noStyle.Render("·") + } + } + return noStyle.Render("·") +} + +// extractNodesCapability extracts dependency.depth and formats it +// maps ["direct", "indirect"] to "transitive" +func extractNodesCapability(caps capabilities.CapabilitySet) string { + for _, cap := range caps { + if cap.Name == "dependency.depth" { + switch v := cap.Default.(type) { + case []string: + return formatDepthStringArray(v) + case []interface{}: + return formatDepthInterfaceArray(v) + } + return noStyle.Render("·") + } + } + return noStyle.Render("·") +} + +// formatDepthStringArray formats a []string dependency depth value +func formatDepthStringArray(v []string) string { + if len(v) == 0 { + return noStyle.Render("·") + } + if hasBothDirectAndIndirect(v) { + return "transitive" + } + return strings.Join(v, ", ") +} + +// formatDepthInterfaceArray formats a []interface{} dependency depth value +func formatDepthInterfaceArray(v []interface{}) string { + if len(v) == 0 { + return noStyle.Render("·") + } + strs := make([]string, 0, len(v)) + for _, item := range v { + strs = append(strs, fmt.Sprintf("%v", item)) + } + if hasBothDirectAndIndirect(strs) { + return "transitive" + } + return strings.Join(strs, ", ") +} + +// hasBothDirectAndIndirect checks if a slice contains both "direct" and "indirect" strings +func hasBothDirectAndIndirect(items []string) bool { + hasDirect := false + hasIndirect := false + for _, item := range items { + if item == "direct" { + hasDirect = true + } + if item == "indirect" { + hasIndirect = true + } + } + return hasDirect && hasIndirect +} + +func formatCriteria(detectors []capabilities.Detector) string { + var allCriteria []string + methods := strset.New() + + for _, det := range detectors { + allCriteria = append(allCriteria, det.Criteria...) + methods.Add(string(det.Method)) + } + + // smart word wrapping: wrap by word (criterion) up to criteriaWidth, + // allow up to criteriaWidth+criteriaMargin before forcing a new line + var lines []string + var currentLine []string + currentLength := 0 + + for _, criterion := range allCriteria { + // calculate length including the criterion plus ", " separator (except for first item) + itemLength := len(criterion) + if len(currentLine) > 0 { + itemLength += 2 // for ", " + } + + newLength := currentLength + itemLength + + if len(currentLine) > 0 && newLength > criteriaWidth { + // check if it's within margin + if newLength <= criteriaWidth+criteriaMargin { + // close enough, add it anyway + currentLine = append(currentLine, criterion) + currentLength = newLength + } else { + // too long, finalize current line and start new one + lines = append(lines, strings.Join(currentLine, ", ")) + currentLine = []string{criterion} + currentLength = len(criterion) + } + } else { + // fits within width or first item + currentLine = append(currentLine, criterion) + currentLength = newLength + } + } + + // add final line + if len(currentLine) > 0 { + lines = append(lines, strings.Join(currentLine, ", ")) + } + + methodsList := methods.List() + sort.Strings(methodsList) + method := strings.Join(methodsList, ", ") + + if len(lines) == 0 { + return "" + } + + joined := strings.Join(lines, "\n") + if method != string(capabilities.GlobDetection) { + return fmt.Sprintf("%s (%s)", joined, method) + } + return joined +} diff --git a/cmd/syft/internal/commands/cataloger_info_test.go b/cmd/syft/internal/commands/cataloger_info_test.go new file mode 100644 index 000000000..6794c3b81 --- /dev/null +++ b/cmd/syft/internal/commands/cataloger_info_test.go @@ -0,0 +1,373 @@ +package commands + +import ( + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/capabilities" +) + +func Test_isDeprecatedCataloger(t *testing.T) { + tests := []struct { + name string + selectors []string + want bool + }{ + { + name: "empty selectors", + selectors: nil, + want: false, + }, + { + name: "no deprecated selector", + selectors: []string{"python", "pip", "package"}, + want: false, + }, + { + name: "has deprecated selector", + selectors: []string{"python", "deprecated", "package"}, + want: true, + }, + { + name: "only deprecated selector", + selectors: []string{"deprecated"}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isDeprecatedCataloger(tt.selectors) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_hasBothDirectAndIndirect(t *testing.T) { + tests := []struct { + name string + items []string + want bool + }{ + { + name: "empty slice", + items: nil, + want: false, + }, + { + name: "only direct", + items: []string{"direct"}, + want: false, + }, + { + name: "only indirect", + items: []string{"indirect"}, + want: false, + }, + { + name: "both direct and indirect", + items: []string{"direct", "indirect"}, + want: true, + }, + { + name: "both with other items", + items: []string{"other", "direct", "more", "indirect", "stuff"}, + want: true, + }, + { + name: "unrelated items", + items: []string{"foo", "bar", "baz"}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := hasBothDirectAndIndirect(tt.items) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_filterCatalogersByName(t *testing.T) { + catalogers := []capabilities.CatalogerEntry{ + {Name: "python-cataloger", Ecosystem: "python"}, + {Name: "java-cataloger", Ecosystem: "java"}, + {Name: "go-cataloger", Ecosystem: "golang"}, + {Name: "npm-cataloger", Ecosystem: "javascript"}, + } + + tests := []struct { + name string + catalogers []capabilities.CatalogerEntry + names []string + want []capabilities.CatalogerEntry + }{ + { + name: "empty names returns empty", + catalogers: catalogers, + names: nil, + want: nil, + }, + { + name: "single match", + catalogers: catalogers, + names: []string{"python-cataloger"}, + want: []capabilities.CatalogerEntry{ + {Name: "python-cataloger", Ecosystem: "python"}, + }, + }, + { + name: "multiple matches", + catalogers: catalogers, + names: []string{"python-cataloger", "java-cataloger"}, + want: []capabilities.CatalogerEntry{ + {Name: "python-cataloger", Ecosystem: "python"}, + {Name: "java-cataloger", Ecosystem: "java"}, + }, + }, + { + name: "no matches", + catalogers: catalogers, + names: []string{"nonexistent"}, + want: nil, + }, + { + name: "partial matches", + catalogers: catalogers, + names: []string{"python-cataloger", "nonexistent"}, + want: []capabilities.CatalogerEntry{ + {Name: "python-cataloger", Ecosystem: "python"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := filterCatalogersByName(tt.catalogers, tt.names) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("filterCatalogersByName() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func Test_formatCriteria(t *testing.T) { + tests := []struct { + name string + detectors []capabilities.Detector + want string + }{ + { + name: "empty detectors", + detectors: nil, + want: "", + }, + { + name: "single glob detector", + detectors: []capabilities.Detector{ + { + Method: capabilities.GlobDetection, + Criteria: []string{"**/*.py"}, + }, + }, + want: "**/*.py", + }, + { + name: "multiple criteria single detector", + detectors: []capabilities.Detector{ + { + Method: capabilities.GlobDetection, + Criteria: []string{"**/*.py", "**/*.pyc", "**/requirements.txt"}, + }, + }, + want: "**/*.py, **/*.pyc, **/requirements.txt", + }, + { + name: "path detection shows method", + detectors: []capabilities.Detector{ + { + Method: capabilities.PathDetection, + Criteria: []string{"/usr/bin/python"}, + }, + }, + want: "/usr/bin/python (path)", + }, + { + name: "mimetype detection shows method", + detectors: []capabilities.Detector{ + { + Method: capabilities.MIMETypeDetection, + Criteria: []string{"application/x-python"}, + }, + }, + want: "application/x-python (mimetype)", + }, + { + name: "multiple detectors combine criteria", + detectors: []capabilities.Detector{ + { + Method: capabilities.GlobDetection, + Criteria: []string{"**/*.py"}, + }, + { + Method: capabilities.GlobDetection, + Criteria: []string{"**/*.pyc"}, + }, + }, + want: "**/*.py, **/*.pyc", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := formatCriteria(tt.detectors) + assert.Equal(t, tt.want, got) + }) + } +} + +func testDocument() *capabilities.Document { + return &capabilities.Document{ + Configs: map[string]capabilities.CatalogerConfigEntry{ + "test-config": { + Fields: []capabilities.CatalogerConfigFieldEntry{ + {Key: "field1", Description: "test field"}, + }, + }, + }, + } +} + +func Test_catalogerInfoReport(t *testing.T) { + tests := []struct { + name string + opts *catalogerInfoOptions + doc *capabilities.Document + catalogers []capabilities.CatalogerEntry + wantErr require.ErrorAssertionFunc + assertions func(t *testing.T, got string) + }{ + { + name: "empty catalogers table", + opts: &catalogerInfoOptions{Output: "table"}, + doc: testDocument(), + assertions: func(t *testing.T, got string) { + assert.Contains(t, got, "No catalogers found") + }, + }, + { + name: "empty catalogers json", + opts: &catalogerInfoOptions{Output: "json"}, + doc: testDocument(), + assertions: func(t *testing.T, got string) { + assert.Contains(t, got, `"catalogers":null`) + }, + }, + { + name: "single cataloger table output", + opts: &catalogerInfoOptions{Output: "table"}, + doc: testDocument(), + catalogers: []capabilities.CatalogerEntry{ + { + Name: "test-cataloger", + Ecosystem: "test", + Type: "custom", + Detectors: []capabilities.Detector{ + {Method: capabilities.GlobDetection, Criteria: []string{"**/*.test"}}, + }, + }, + }, + assertions: func(t *testing.T, got string) { + assert.Contains(t, got, "test-cataloger") + assert.Contains(t, got, "ECOSYSTEM") + assert.Contains(t, got, "CATALOGER") + }, + }, + { + name: "single cataloger json output", + opts: &catalogerInfoOptions{Output: "json"}, + doc: testDocument(), + catalogers: []capabilities.CatalogerEntry{ + { + Name: "test-cataloger", + Ecosystem: "test", + Type: "custom", + Selectors: []string{"test", "custom"}, + }, + }, + assertions: func(t *testing.T, got string) { + assert.Contains(t, got, `"name":"test-cataloger"`) + assert.Contains(t, got, `"ecosystem":"test"`) + assert.Contains(t, got, `"type":"custom"`) + }, + }, + { + name: "deprecated cataloger json output", + opts: &catalogerInfoOptions{Output: "json"}, + doc: testDocument(), + catalogers: []capabilities.CatalogerEntry{ + { + Name: "old-cataloger", + Ecosystem: "legacy", + Type: "custom", + Selectors: []string{"legacy", "deprecated"}, + }, + }, + assertions: func(t *testing.T, got string) { + assert.Contains(t, got, `"deprecated":true`) + }, + }, + { + name: "invalid output format", + opts: &catalogerInfoOptions{Output: "invalid"}, + doc: testDocument(), + wantErr: require.Error, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + + got, err := catalogerInfoReport(tt.opts, tt.doc, tt.catalogers) + tt.wantErr(t, err) + + if err != nil { + return + } + + if tt.assertions != nil { + tt.assertions(t, got) + } + }) + } +} + +func Test_formatCriteria_wordWrapping(t *testing.T) { + // test word wrapping behavior with long criteria lists + longCriteria := make([]string, 20) + for i := range longCriteria { + longCriteria[i] = "**/file" + strings.Repeat("x", 10) + } + + detector := capabilities.Detector{ + Method: capabilities.GlobDetection, + Criteria: longCriteria, + } + + got := formatCriteria([]capabilities.Detector{detector}) + + // should contain newlines for wrapping + assert.Contains(t, got, "\n") + // should contain all criteria (joined with ", " and possibly split across lines) + for _, c := range longCriteria { + assert.Contains(t, got, c) + } +} diff --git a/cmd/syft/internal/commands/cataloger_list.go b/cmd/syft/internal/commands/cataloger_list.go index 18d95c307..c776a737f 100644 --- a/cmd/syft/internal/commands/cataloger_list.go +++ b/cmd/syft/internal/commands/cataloger_list.go @@ -15,6 +15,7 @@ import ( "github.com/anchore/clio" "github.com/anchore/syft/cmd/syft/internal/options" "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/capabilities" "github.com/anchore/syft/internal/task" "github.com/anchore/syft/syft/cataloging" ) @@ -28,25 +29,23 @@ var ( ) type catalogerListOptions struct { - Output string `yaml:"output" json:"output" mapstructure:"output"` - DefaultCatalogers []string `yaml:"default-catalogers" json:"default-catalogers" mapstructure:"default-catalogers"` - SelectCatalogers []string `yaml:"select-catalogers" json:"select-catalogers" mapstructure:"select-catalogers"` - ShowHidden bool `yaml:"show-hidden" json:"show-hidden" mapstructure:"show-hidden"` + Output string `yaml:"output" json:"output" mapstructure:"output"` + options.CatalogerSelection `yaml:",inline" json:",inline" mapstructure:",squash"` + ShowHidden bool `yaml:"show-hidden" json:"show-hidden" mapstructure:"show-hidden"` } func (o *catalogerListOptions) AddFlags(flags clio.FlagSet) { flags.StringVarP(&o.Output, "output", "o", "format to output the cataloger list (available: table, json)") - flags.StringArrayVarP(&o.DefaultCatalogers, "override-default-catalogers", "", "override the default catalogers with an expression") - - flags.StringArrayVarP(&o.SelectCatalogers, "select-catalogers", "", "select catalogers with an expression") - flags.BoolVarP(&o.ShowHidden, "show-hidden", "s", "show catalogers that have been de-selected") } func defaultCatalogerListOptions() *catalogerListOptions { return &catalogerListOptions{ - DefaultCatalogers: []string{"all"}, + CatalogerSelection: options.CatalogerSelection{ + // this is different than the default behavior where a scan will automatically detect the default set + DefaultCatalogers: []string{"all"}, + }, } } @@ -101,7 +100,7 @@ func catalogerListReport(opts *catalogerListOptions, allTaskGroups [][]task.Task var report string switch opts.Output { - case "json": + case jsonFormat: report, err = renderCatalogerListJSON(flattenTaskGroups(selectedTaskGroups), selectionEvidence, defaultCatalogers, selectCatalogers) case "table", "": if opts.ShowHidden { @@ -306,22 +305,14 @@ func formatToken(token string, selection *task.TokenSelection, included bool, de } func extractTaskInfo(tasks []task.Task) ([]string, map[string][]string) { + infos := capabilities.ExtractCatalogerInfo(tasks) + tagsByName := make(map[string][]string) var names []string - for _, tsk := range tasks { - var tags []string - name := tsk.Name() - - if s, ok := tsk.(task.Selector); ok { - set := strset.New(s.Selectors()...) - set.Remove(name) - tags = set.List() - sort.Strings(tags) - } - - tagsByName[name] = tags - names = append(names, name) + for _, info := range infos { + tagsByName[info.Name] = info.Selectors + names = append(names, info.Name) } sort.Strings(names) diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index 8d96a2f36..0414e921f 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -31,17 +31,16 @@ import ( type Catalog struct { // high-level cataloger configuration - Catalogers []string `yaml:"-" json:"catalogers" mapstructure:"catalogers"` // deprecated and not shown in yaml output - DefaultCatalogers []string `yaml:"default-catalogers" json:"default-catalogers" mapstructure:"default-catalogers"` - SelectCatalogers []string `yaml:"select-catalogers" json:"select-catalogers" mapstructure:"select-catalogers"` - Package packageConfig `yaml:"package" json:"package" mapstructure:"package"` - License licenseConfig `yaml:"license" json:"license" mapstructure:"license"` - File fileConfig `yaml:"file" json:"file" mapstructure:"file"` - Scope string `yaml:"scope" json:"scope" mapstructure:"scope"` - Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel - Relationships relationshipsConfig `yaml:"relationships" json:"relationships" mapstructure:"relationships"` - Compliance complianceConfig `yaml:"compliance" json:"compliance" mapstructure:"compliance"` - Enrich []string `yaml:"enrich" json:"enrich" mapstructure:"enrich"` + CatalogerSelection `yaml:",inline" json:",inline" mapstructure:",squash"` + + Package packageConfig `yaml:"package" json:"package" mapstructure:"package"` + License licenseConfig `yaml:"license" json:"license" mapstructure:"license"` + File fileConfig `yaml:"file" json:"file" mapstructure:"file"` + Scope string `yaml:"scope" json:"scope" mapstructure:"scope"` + Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel + Relationships relationshipsConfig `yaml:"relationships" json:"relationships" mapstructure:"relationships"` + Compliance complianceConfig `yaml:"compliance" json:"compliance" mapstructure:"compliance"` + Enrich []string `yaml:"enrich" json:"enrich" mapstructure:"enrich"` // ecosystem-specific cataloger configuration Dotnet dotnetConfig `yaml:"dotnet" json:"dotnet" mapstructure:"dotnet"` @@ -77,6 +76,8 @@ func DefaultCatalog() Catalog { Package: defaultPackageConfig(), License: defaultLicenseConfig(), LinuxKernel: defaultLinuxKernelConfig(), + JavaScript: defaultJavaScriptConfig(), + Python: defaultPythonConfig(), Nix: defaultNixConfig(), Dotnet: defaultDotnetConfig(), Golang: defaultGolangConfig(), @@ -230,26 +231,9 @@ func (cfg *Catalog) AddFlags(flags clio.FlagSet) { flags.StringArrayVarP(&cfg.Exclusions, "exclude", "", "exclude paths from being scanned using a glob expression") - flags.StringArrayVarP(&cfg.Catalogers, "catalogers", "", - "enable one or more package catalogers") - flags.IntVarP(&cfg.Parallelism, "parallelism", "", "number of cataloger workers to run in parallel") - if pfp, ok := flags.(fangs.PFlagSetProvider); ok { - if err := pfp.PFlagSet().MarkDeprecated("catalogers", "use: override-default-catalogers and select-catalogers"); err != nil { - panic(err) - } - } else { - panic("unable to mark flags as deprecated") - } - - flags.StringArrayVarP(&cfg.DefaultCatalogers, "override-default-catalogers", "", - "set the base set of catalogers to use (defaults to 'image' or 'directory' depending on the scan source)") - - flags.StringArrayVarP(&cfg.SelectCatalogers, "select-catalogers", "", - "add, remove, and filter the catalogers to be used") - flags.StringArrayVarP(&cfg.Enrich, "enrich", "", fmt.Sprintf("enable package data enrichment from local and online sources (options: %s)", strings.Join(publicisedEnrichmentOptions, ", "))) @@ -276,23 +260,9 @@ Available options are: %s`, strings.Join(publicisedEnrichmentOptions, ", "))) } func (cfg *Catalog) PostLoad() error { - usingLegacyCatalogers := len(cfg.Catalogers) > 0 - usingNewCatalogers := len(cfg.DefaultCatalogers) > 0 || len(cfg.SelectCatalogers) > 0 - - if usingLegacyCatalogers && usingNewCatalogers { - return fmt.Errorf("cannot use both 'catalogers' and 'select-catalogers'/'default-catalogers' flags") - } - cfg.From = Flatten(cfg.From) - - cfg.Catalogers = FlattenAndSort(cfg.Catalogers) - cfg.DefaultCatalogers = FlattenAndSort(cfg.DefaultCatalogers) - cfg.SelectCatalogers = FlattenAndSort(cfg.SelectCatalogers) cfg.Enrich = FlattenAndSort(cfg.Enrich) - // for backwards compatibility - cfg.DefaultCatalogers = append(cfg.DefaultCatalogers, cfg.Catalogers...) - s := source.ParseScope(cfg.Scope) if s == source.UnknownScope { return fmt.Errorf("bad scope value %q", cfg.Scope) diff --git a/cmd/syft/internal/options/catalog_test.go b/cmd/syft/internal/options/catalog_test.go index 7753da5c6..af82d7333 100644 --- a/cmd/syft/internal/options/catalog_test.go +++ b/cmd/syft/internal/options/catalog_test.go @@ -8,55 +8,12 @@ import ( ) func TestCatalog_PostLoad(t *testing.T) { - tests := []struct { name string options Catalog assert func(t *testing.T, options Catalog) wantErr assert.ErrorAssertionFunc }{ - { - name: "mutually exclusive cataloger flags (cat / def-cat)", - options: Catalog{ - Catalogers: []string{"foo,bar", "42"}, - DefaultCatalogers: []string{"some,thing"}, - Scope: "squashed", - }, - wantErr: assert.Error, - }, - { - name: "mutually exclusive cataloger flags (cat / sel-cat)", - options: Catalog{ - Catalogers: []string{"foo,bar", "42"}, - SelectCatalogers: []string{"some,thing"}, - Scope: "squashed", - }, - wantErr: assert.Error, - }, - { - name: "allow old cataloger flags", - options: Catalog{ - Catalogers: []string{"foo,bar"}, - Scope: "squashed", - }, - assert: func(t *testing.T, options Catalog) { - assert.Equal(t, []string{"bar", "foo"}, options.DefaultCatalogers) // note: sorted order - assert.Equal(t, []string{"bar", "foo"}, options.Catalogers) // note: sorted order - }, - }, - { - name: "allow new cataloger flags", - options: Catalog{ - SelectCatalogers: []string{"foo,bar", "42"}, - DefaultCatalogers: []string{"some,thing"}, - Scope: "squashed", - }, - assert: func(t *testing.T, options Catalog) { - assert.Equal(t, []string{"42", "bar", "foo"}, options.SelectCatalogers) // note: sorted order - assert.Equal(t, []string{"some", "thing"}, options.DefaultCatalogers) // note: sorted order - assert.Empty(t, options.Catalogers) - }, - }, { name: "must have package overlap flag when pruning binaries by overlap", options: Catalog{ diff --git a/cmd/syft/internal/options/cataloger_selection.go b/cmd/syft/internal/options/cataloger_selection.go new file mode 100644 index 000000000..0834bcfa4 --- /dev/null +++ b/cmd/syft/internal/options/cataloger_selection.go @@ -0,0 +1,57 @@ +package options + +import ( + "fmt" + + "github.com/anchore/clio" + "github.com/anchore/fangs" +) + +type CatalogerSelection struct { + // high-level cataloger configuration + Catalogers []string `yaml:"-" json:"catalogers" mapstructure:"catalogers"` // deprecated and not shown in yaml output + DefaultCatalogers []string `yaml:"default-catalogers" json:"default-catalogers" mapstructure:"default-catalogers"` + SelectCatalogers []string `yaml:"select-catalogers" json:"select-catalogers" mapstructure:"select-catalogers"` +} + +var _ interface { + clio.FlagAdder + clio.PostLoader +} = (*CatalogerSelection)(nil) + +func (cfg *CatalogerSelection) AddFlags(flags clio.FlagSet) { + flags.StringArrayVarP(&cfg.Catalogers, "catalogers", "", + "enable one or more package catalogers") + + if pfp, ok := flags.(fangs.PFlagSetProvider); ok { + if err := pfp.PFlagSet().MarkDeprecated("catalogers", "use: override-default-catalogers and select-catalogers"); err != nil { + panic(err) + } + } else { + panic("unable to mark flags as deprecated") + } + + flags.StringArrayVarP(&cfg.DefaultCatalogers, "override-default-catalogers", "", + "set the base set of catalogers to use (defaults to 'image' or 'directory' depending on the scan source)") + + flags.StringArrayVarP(&cfg.SelectCatalogers, "select-catalogers", "", + "add, remove, and filter the catalogers to be used") +} + +func (cfg *CatalogerSelection) PostLoad() error { + usingLegacyCatalogers := len(cfg.Catalogers) > 0 + usingNewCatalogers := len(cfg.DefaultCatalogers) > 0 || len(cfg.SelectCatalogers) > 0 + + if usingLegacyCatalogers && usingNewCatalogers { + return fmt.Errorf("cannot use both 'catalogers' and 'select-catalogers'/'default-catalogers' flags %q | %q %q", cfg.Catalogers, cfg.SelectCatalogers, cfg.DefaultCatalogers) + } + + cfg.Catalogers = FlattenAndSort(cfg.Catalogers) + cfg.DefaultCatalogers = FlattenAndSort(cfg.DefaultCatalogers) + cfg.SelectCatalogers = FlattenAndSort(cfg.SelectCatalogers) + + // for backwards compatibility + cfg.DefaultCatalogers = append(cfg.DefaultCatalogers, cfg.Catalogers...) + + return nil +} diff --git a/cmd/syft/internal/options/cataloger_selection_test.go b/cmd/syft/internal/options/cataloger_selection_test.go new file mode 100644 index 000000000..fe6d5f30c --- /dev/null +++ b/cmd/syft/internal/options/cataloger_selection_test.go @@ -0,0 +1,68 @@ +package options + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCatalogerSelection_PostLoad(t *testing.T) { + + tests := []struct { + name string + options CatalogerSelection + assert func(t *testing.T, options CatalogerSelection) + wantErr assert.ErrorAssertionFunc + }{ + { + name: "mutually exclusive cataloger flags (cat / def-cat)", + options: CatalogerSelection{ + Catalogers: []string{"foo,bar", "42"}, + DefaultCatalogers: []string{"some,thing"}, + }, + wantErr: assert.Error, + }, + { + name: "mutually exclusive cataloger flags (cat / sel-cat)", + options: CatalogerSelection{ + Catalogers: []string{"foo,bar", "42"}, + SelectCatalogers: []string{"some,thing"}, + }, + wantErr: assert.Error, + }, + { + name: "allow old cataloger flags", + options: CatalogerSelection{ + Catalogers: []string{"foo,bar"}, + }, + assert: func(t *testing.T, options CatalogerSelection) { + assert.Equal(t, []string{"bar", "foo"}, options.DefaultCatalogers) // note: sorted order + assert.Equal(t, []string{"bar", "foo"}, options.Catalogers) // note: sorted order + }, + }, + { + name: "allow new cataloger flags", + options: CatalogerSelection{ + SelectCatalogers: []string{"foo,bar", "42"}, + DefaultCatalogers: []string{"some,thing"}, + }, + assert: func(t *testing.T, options CatalogerSelection) { + assert.Equal(t, []string{"42", "bar", "foo"}, options.SelectCatalogers) // note: sorted order + assert.Equal(t, []string{"some", "thing"}, options.DefaultCatalogers) // note: sorted order + assert.Empty(t, options.Catalogers) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = assert.NoError + } + tt.wantErr(t, tt.options.PostLoad(), fmt.Sprintf("PostLoad()")) + if tt.assert != nil { + tt.assert(t, tt.options) + } + }) + } +} diff --git a/cmd/syft/internal/options/javascript.go b/cmd/syft/internal/options/javascript.go index 982bffa29..73019dc91 100644 --- a/cmd/syft/internal/options/javascript.go +++ b/cmd/syft/internal/options/javascript.go @@ -1,6 +1,9 @@ package options -import "github.com/anchore/clio" +import ( + "github.com/anchore/clio" + "github.com/anchore/syft/syft/pkg/cataloger/javascript" +) type javaScriptConfig struct { SearchRemoteLicenses *bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"` @@ -12,6 +15,24 @@ var _ interface { clio.FieldDescriber } = (*javaScriptConfig)(nil) +func defaultJavaScriptConfig() javaScriptConfig { + def := javascript.DefaultCatalogerConfig() + var includeDevDependencies *bool + if def.IncludeDevDependencies { + includeDevDependencies = &def.IncludeDevDependencies + } + + var searchRemoteLicenses *bool + if def.SearchRemoteLicenses { + searchRemoteLicenses = &def.SearchRemoteLicenses + } + return javaScriptConfig{ + NpmBaseURL: def.NPMBaseURL, + SearchRemoteLicenses: searchRemoteLicenses, + IncludeDevDependencies: includeDevDependencies, + } +} + func (o *javaScriptConfig) DescribeFields(descriptions clio.FieldDescriptionSet) { descriptions.Add(&o.SearchRemoteLicenses, `enables Syft to use the network to fill in more detailed license information`) descriptions.Add(&o.NpmBaseURL, `base NPM url to use`) diff --git a/cmd/syft/internal/options/linux_kernel.go b/cmd/syft/internal/options/linux_kernel.go index 03f24dbf0..fab464aa6 100644 --- a/cmd/syft/internal/options/linux_kernel.go +++ b/cmd/syft/internal/options/linux_kernel.go @@ -1,14 +1,18 @@ package options -import "github.com/anchore/clio" +import ( + "github.com/anchore/clio" + "github.com/anchore/syft/syft/pkg/cataloger/kernel" +) type linuxKernelConfig struct { CatalogModules bool `json:"catalog-modules" yaml:"catalog-modules" mapstructure:"catalog-modules"` } func defaultLinuxKernelConfig() linuxKernelConfig { + def := kernel.DefaultLinuxKernelCatalogerConfig() return linuxKernelConfig{ - CatalogModules: true, + CatalogModules: def.CatalogModules, } } diff --git a/cmd/syft/internal/options/python.go b/cmd/syft/internal/options/python.go index 97c5330c1..7ebdf9527 100644 --- a/cmd/syft/internal/options/python.go +++ b/cmd/syft/internal/options/python.go @@ -1,6 +1,9 @@ package options -import "github.com/anchore/clio" +import ( + "github.com/anchore/clio" + "github.com/anchore/syft/syft/pkg/cataloger/python" +) type pythonConfig struct { SearchRemoteLicenses *bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"` @@ -12,6 +15,13 @@ var _ interface { clio.FieldDescriber } = (*pythonConfig)(nil) +func defaultPythonConfig() pythonConfig { + def := python.DefaultCatalogerConfig() + return pythonConfig{ + GuessUnpinnedRequirements: &def.GuessUnpinnedRequirements, + } +} + func (o *pythonConfig) DescribeFields(descriptions clio.FieldDescriptionSet) { descriptions.Add(&o.SearchRemoteLicenses, `enables Syft to use the network to fill in more detailed license information`) descriptions.Add(&o.PypiBaseURL, `base Pypi url to use`) diff --git a/internal/capabilities/README.md b/internal/capabilities/README.md new file mode 100644 index 000000000..d6cd4e270 --- /dev/null +++ b/internal/capabilities/README.md @@ -0,0 +1,227 @@ +# Cataloger Capabilities Documentation + +This documentation describes the format and structure of cataloger capabilities YAML files. + +## File Organization + +Capabilities are organized as follows: +- **Cataloger capabilities**: Located in `syft/pkg/cataloger/*/capabilities.yaml` (one file per ecosystem, alongside the cataloger source code: `golang/capabilities.yaml`, `python/capabilities.yaml`, etc.) +- **Application configuration**: Located in `internal/capabilities/appconfig.yaml` + +Each `capabilities.yaml` file is partially auto-generated. Run `go generate ./internal/capabilities` to regenerate. +- Fields marked **AUTO-GENERATED** will be updated during regeneration +- All **capabilities** sections are **MANUAL** - edit these to describe cataloger behavior + +## Capability Sections + +There are two types of capability sections depending on cataloger type: + +### 1. Generic Catalogers (`type: generic`) +- Have capabilities at the **PARSER level** +- Each parser function has its own capabilities section +- Allows different parsers within the same cataloger to have different capabilities + +### 2. Custom Catalogers (`type: custom`) +- Have capabilities at the **CATALOGER level** +- Single capabilities section for the entire cataloger + +## Capabilities Format + +Capabilities use a field-based format with defaults and optional conditional overrides: + +```yaml +capabilities: + - field: # dot-notation path (e.g., "license", "dependency.depth") + default: # value when no conditions match + conditions: # optional - conditional overrides evaluated in order + - when: {ConfigField: val} # when these config fields match (AND logic) + value: # use this value instead + comment: "explanation" # optional - why this condition exists + evidence: # optional - source code references + - "StructName.FieldName" + comment: "explanation" # optional - general field explanation +``` + +## Detector Conditions + +Detectors (used by custom catalogers) can have optional conditions that control when they are active. This allows a single cataloger to have different detection behavior based on configuration. + +### Structure + +```yaml +detectors: + - method: glob # AUTO-GENERATED - detection method + criteria: ["**/*.jar"] # AUTO-GENERATED - patterns to match + comment: "always active" # MANUAL - optional explanation + - method: glob + criteria: ["**/*.zip"] + conditions: # MANUAL - when this detector is active + - when: {IncludeZipFiles: true} # config fields that must match + comment: "optional explanation" + comment: "ZIP detection requires config" +``` + +### Notes +- Conditions reference fields from the cataloger's config struct +- Multiple conditions in the array use **OR logic** (any condition can activate) +- Multiple fields in a `when` clause use **AND logic** (all must match) +- Detectors without conditions are always active +- Only custom catalogers support detectors with conditions + +## Condition Evaluation + +- Conditions are evaluated in array order (first match wins) +- Multiple fields in a `when` clause use **AND logic** (all must match) +- Multiple conditions in the array use **OR logic** (first matching condition) +- If no conditions match, the default value is used + +## Capability Fields + +Standard capability field names and their value types: + +### `license` (boolean) + +Whether license information is available. + +**Examples:** +```yaml +default: true # always available +default: false # never available +default: false # requires configuration + conditions: + - when: {SearchRemoteLicenses: true} + value: true +``` + +### `dependency.depth` (array of strings) + +Which dependency depths can be discovered. + +**Values:** `direct` (immediate deps), `indirect` (transitive deps) + +**Examples:** +```yaml +default: [direct] # only immediate dependencies +default: [direct, indirect] # full transitive closure +default: [] # no dependency information +``` + +### `dependency.edges` (string) + +Relationships between nodes and completeness of the dependency graph. + +**Values:** +- `""` - dependencies found but no edges between them +- `"flat"` - single level of dependencies with edges to root package only +- `"reduced"` - transitive reduction (redundant edges removed) +- `"complete"` - all relationships with accurate direct and indirect edges + +**Examples:** +```yaml +default: complete +default: "" +``` + +### `dependency.kinds` (array of strings) + +Types of dependencies that can be discovered. + +**Values:** `runtime`, `dev`, `build`, `test`, `optional` + +**Examples:** +```yaml +default: [runtime] # production dependencies only +default: [runtime, dev] # production and development +default: [runtime, dev, build] # all dependency types +default: [runtime] # with conditional dev deps + conditions: + - when: {IncludeDevDeps: true} + value: [runtime, dev] +``` + +### `package_manager.files.listing` (boolean) + +Whether file listings are available (which files belong to the package). + +**Examples:** +```yaml +default: true +default: false + conditions: + - when: {CaptureOwnedFiles: true} + value: true +``` + +### `package_manager.files.digests` (boolean) + +Whether file digests/checksums are included in listings. + +**Examples:** +```yaml +default: true +default: false +``` + +### `package_manager.package_integrity_hash` (boolean) + +Whether a hash for verifying package integrity is available. + +**Examples:** +```yaml +default: true +default: false +``` + +## Examples + +### Simple cataloger with no configuration + +```yaml +capabilities: + - name: license + default: true + comment: "license field always present in package.json" + - name: dependency.depth + default: [direct] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [runtime] + comment: "devDependencies not parsed by this cataloger" + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false +``` + +### Cataloger with configuration-dependent capabilities + +```yaml +capabilities: + - name: license + default: false + conditions: + - when: {SearchLocalModCacheLicenses: true} + value: true + comment: "searches for licenses in GOPATH mod cache" + - when: {SearchRemoteLicenses: true} + value: true + comment: "fetches licenses from proxy.golang.org" + comment: "license scanning requires configuration" + - name: dependency.depth + default: [direct, indirect] + - name: dependency.edges + default: flat + - name: dependency.kinds + default: [runtime, dev] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - "GolangBinaryBuildinfoEntry.H1Digest" +``` diff --git a/internal/capabilities/appconfig.yaml b/internal/capabilities/appconfig.yaml new file mode 100644 index 000000000..b42a8ee29 --- /dev/null +++ b/internal/capabilities/appconfig.yaml @@ -0,0 +1,64 @@ +# Application-level configuration. See README.md for documentation. +# This file is partially auto-generated. Run 'go generate ./internal/capabilities' to regenerate. + +application: # AUTO-GENERATED - application-level config keys + - key: dotnet.dep-packages-must-claim-dll + description: only keep dep.json packages which have a runtime/resource DLL claimed in the deps.json targets section (but not necessarily found on disk). The package is also included if any child package claims a DLL, even if the package itself does not claim a DLL. + - key: dotnet.dep-packages-must-have-dll + description: only keep dep.json packages which an executable on disk is found. The package is also included if a DLL is found for any child package, even if the package itself does not have a DLL. + - key: dotnet.exclude-project-references + description: exclude packages with type "project" from deps.json output (these are internal project references, not NuGet packages) + - key: dotnet.propagate-dll-claims-to-parents + description: treat DLL claims or on-disk evidence for child packages as DLL claims or on-disk evidence for any parent package + - key: dotnet.relax-dll-claims-when-bundling-detected + description: show all packages from the deps.json if bundling tooling is present as a dependency (e.g. ILRepack) + - key: golang.local-mod-cache-dir + description: specify an explicit go mod cache directory, if unset this defaults to $GOPATH/pkg/mod or $HOME/go/pkg/mod + - key: golang.local-vendor-dir + description: specify an explicit go vendor directory, if unset this defaults to ./vendor + - key: golang.main-module-version.from-build-settings + description: use the build settings (e.g. vcs.version & vcs.time) to craft a v0 pseudo version (e.g. v0.0.0-20220308212642-53e6d0aaf6fb) when a more accurate version cannot be found otherwise + - key: golang.main-module-version.from-contents + description: search for semver-like strings in the binary contents + - key: golang.main-module-version.from-ld-flags + description: look for LD flags that appear to be setting a version (e.g. -X main.version=1.0.0) + - key: golang.no-proxy + description: specifies packages which should not be fetched by proxy if unset this defaults to $GONOPROXY + - key: golang.proxy + description: remote proxy to use when retrieving go packages from the network, if unset this defaults to $GOPROXY followed by https://proxy.golang.org + - key: golang.search-local-mod-cache-licenses + description: search for go package licences in the GOPATH of the system running Syft, note that this is outside the container filesystem and potentially outside the root of a local directory scan + - key: golang.search-local-vendor-licenses + description: search for go package licences in the vendor folder on the system running Syft, note that this is outside the container filesystem and potentially outside the root of a local directory scan + - key: golang.search-remote-licenses + description: search for go package licences by retrieving the package from a network proxy + - key: golang.use-packages-lib + description: use the golang.org/x/tools/go/packages library, which executes golang tooling found on the path in addition to potential network access to get the most accurate results + - key: java.maven-local-repository-dir + description: override the default location of the local Maven repository. the default is the subdirectory '.m2/repository' in your home directory + - key: java.maven-url + description: maven repository to use, defaults to Maven central + - key: java.max-parent-recursive-depth + description: depth to recursively resolve parent POMs, no limit if <= 0 + - key: java.resolve-transitive-dependencies + description: resolve transient dependencies such as those defined in a dependency's POM on Maven central + - key: java.use-maven-local-repository + description: 'use the local Maven repository to retrieve pom files. When Maven is installed and was previously used for building the software that is being scanned, then most pom files will be available in this repository on the local file system. this greatly speeds up scans. when all pom files are available in the local repository, then ''use-network'' is not needed. TIP: If you want to download all required pom files to the local repository without running a full build, run ''mvn help:effective-pom'' before performing the scan with syft.' + - key: java.use-network + description: enables Syft to use the network to fetch version and license information for packages when a parent or imported pom file is not found in the local maven repository. the pom files are downloaded from the remote Maven repository at 'maven-url' + - key: javascript.include-dev-dependencies + description: include development-scoped dependencies + - key: javascript.npm-base-url + description: base NPM url to use + - key: javascript.search-remote-licenses + description: enables Syft to use the network to fill in more detailed license information + - key: linux-kernel.catalog-modules + description: whether to catalog linux kernel modules found within lib/modules/** directories + - key: nix.capture-owned-files + description: enumerate all files owned by packages found within Nix store paths + - key: python.guess-unpinned-requirements + description: when running across entries in requirements.txt that do not specify a specific version (e.g. "sqlalchemy >= 1.0.0, <= 2.0.0, != 3.0.0, <= 3.0.0"), attempt to guess what the version could be based on the version requirements specified (e.g. "1.0.0"). When enabled the lowest expressible version when given an arbitrary constraint will be used (even if that version may not be available/published). + - key: python.pypi-base-url + description: base Pypi url to use + - key: python.search-remote-licenses + description: enables Syft to use the network to fill in more detailed license information diff --git a/internal/capabilities/capabilities.go b/internal/capabilities/capabilities.go new file mode 100644 index 000000000..492cdd2c4 --- /dev/null +++ b/internal/capabilities/capabilities.go @@ -0,0 +1,132 @@ +// Package capabilities provides discovery and tracking of cataloger capabilities. +package capabilities + +import ( + "embed" + "fmt" + "io/fs" + "path/filepath" + "sort" + + "github.com/scylladb/go-set/strset" + "gopkg.in/yaml.v3" + + "github.com/anchore/syft/internal/task" +) + +//go:generate go run ./generate + +//go:embed appconfig.yaml +var appconfigYAML []byte + +var catalogerFiles *embed.FS + +func RegisterCatalogerFiles(f embed.FS) { + catalogerFiles = &f +} + +// LoadDocument loads and returns the complete document including configs and app-configs +func LoadDocument() (*Document, error) { + if catalogerFiles == nil { + return nil, fmt.Errorf("cataloger files not registered") + } + + // parse application config + var appDoc struct { + Application []ApplicationConfigField `yaml:"application"` + } + if err := yaml.Unmarshal(appconfigYAML, &appDoc); err != nil { + return nil, fmt.Errorf("failed to parse appconfig.yaml: %w", err) + } + + // walk the embedded filesystem to find all cataloger capabilities.yaml files + var catalogersDoc Document + catalogersDoc.ApplicationConfig = appDoc.Application + catalogersDoc.Configs = make(map[string]CatalogerConfigEntry) + + err := fs.WalkDir(catalogerFiles, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + // skip non-yaml files and directories + if d.IsDir() || filepath.Ext(path) != ".yaml" || path == "." { + return nil + } + + // read the file + data, err := catalogerFiles.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read %s: %w", path, err) + } + + // parse the file + var capDoc struct { + Configs map[string]CatalogerConfigEntry `yaml:"configs"` + Catalogers []CatalogerEntry `yaml:"catalogers"` + } + if err := yaml.Unmarshal(data, &capDoc); err != nil { + return fmt.Errorf("failed to parse %s: %w", path, err) + } + + // merge configs + for k, v := range capDoc.Configs { + catalogersDoc.Configs[k] = v + } + + // merge catalogers + catalogersDoc.Catalogers = append(catalogersDoc.Catalogers, capDoc.Catalogers...) + + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to walk cataloger capabilities: %w", err) + } + + // sort catalogers by name for consistency + sort.Slice(catalogersDoc.Catalogers, func(i, j int) bool { + return catalogersDoc.Catalogers[i].Name < catalogersDoc.Catalogers[j].Name + }) + + return &catalogersDoc, nil +} + +// Packages loads and returns all cataloger capabilities from the embedded YAML file +func Packages() ([]CatalogerEntry, error) { + doc, err := LoadDocument() + if err != nil { + return nil, err + } + return doc.Catalogers, nil +} + +// CatalogerInfo represents a cataloger's name and selection tags +type CatalogerInfo struct { + Name string + Selectors []string // tags for cataloger name selection +} + +// ExtractCatalogerInfo extracts cataloger names and their selection tags from tasks +func ExtractCatalogerInfo(tasks []task.Task) []CatalogerInfo { + var infos []CatalogerInfo + + for _, tsk := range tasks { + var selectors []string + name := tsk.Name() + + if s, ok := tsk.(task.Selector); ok { + set := strset.New(s.Selectors()...) + set.Remove(name) + selectors = set.List() + sort.Strings(selectors) + } + + infos = append(infos, CatalogerInfo{ + Name: name, + Selectors: selectors, + }) + } + + return infos +} diff --git a/internal/capabilities/evaluation.go b/internal/capabilities/evaluation.go new file mode 100644 index 000000000..0dfdc48d7 --- /dev/null +++ b/internal/capabilities/evaluation.go @@ -0,0 +1,60 @@ +package capabilities + +import "reflect" + +// EvaluateCapabilities evaluates a capability set against a given configuration +// and returns the effective capability values as a flat map. +// Example: {"license": false, "dependency.depth": ["direct", "indirect"]} +func EvaluateCapabilities(caps CapabilitySet, config map[string]interface{}) map[string]interface{} { + result := make(map[string]interface{}) + for _, capField := range caps { + result[capField.Name] = EvaluateField(capField, config) + } + return result +} + +// EvaluateField evaluates a single capability field against a configuration. +// Conditions are evaluated in order, and the first matching condition's value is returned. +// If no conditions match, the default value is returned. +func EvaluateField(capField CapabilityField, config map[string]interface{}) interface{} { + // check conditions in order (first match wins) + for _, cond := range capField.Conditions { + if ConditionMatches(cond.When, config) { + return cond.Value + } + } + // no condition matched, return default + return capField.Default +} + +// ConditionMatches checks if a condition's when clause matches the given configuration. +// All fields in the when clause must match the config (AND logic). +// Returns true if all key-value pairs in when match the config. +func ConditionMatches(when map[string]interface{}, config map[string]interface{}) bool { + // all fields in when must match config (AND logic) + for key, expectedValue := range when { + actualValue, exists := config[key] + if !exists { + return false + } + if !valuesEqual(actualValue, expectedValue) { + return false + } + } + return true +} + +// valuesEqual compares two values for equality, handling different types appropriately. +// Uses reflect.DeepEqual for complex types like slices and maps. +func valuesEqual(a, b interface{}) bool { + // handle nil cases + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + + // use reflect.DeepEqual for reliable comparison across types + return reflect.DeepEqual(a, b) +} diff --git a/internal/capabilities/evaluation_test.go b/internal/capabilities/evaluation_test.go new file mode 100644 index 000000000..afcf72a67 --- /dev/null +++ b/internal/capabilities/evaluation_test.go @@ -0,0 +1,544 @@ +package capabilities + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" +) + +func Test_valuesEqual(t *testing.T) { + tests := []struct { + name string + a interface{} + b interface{} + want bool + }{ + { + name: "both nil", + a: nil, + b: nil, + want: true, + }, + { + name: "first nil second non-nil", + a: nil, + b: "value", + want: false, + }, + { + name: "first non-nil second nil", + a: "value", + b: nil, + want: false, + }, + { + name: "equal strings", + a: "hello", + b: "hello", + want: true, + }, + { + name: "different strings", + a: "hello", + b: "world", + want: false, + }, + { + name: "equal booleans true", + a: true, + b: true, + want: true, + }, + { + name: "equal booleans false", + a: false, + b: false, + want: true, + }, + { + name: "different booleans", + a: true, + b: false, + want: false, + }, + { + name: "equal integers", + a: 42, + b: 42, + want: true, + }, + { + name: "different integers", + a: 42, + b: 43, + want: false, + }, + { + name: "equal slices", + a: []string{"a", "b", "c"}, + b: []string{"a", "b", "c"}, + want: true, + }, + { + name: "different slices", + a: []string{"a", "b", "c"}, + b: []string{"a", "b", "d"}, + want: false, + }, + { + name: "slices different length", + a: []string{"a", "b"}, + b: []string{"a", "b", "c"}, + want: false, + }, + { + name: "equal maps", + a: map[string]int{"x": 1, "y": 2}, + b: map[string]int{"x": 1, "y": 2}, + want: true, + }, + { + name: "different maps", + a: map[string]int{"x": 1, "y": 2}, + b: map[string]int{"x": 1, "y": 3}, + want: false, + }, + { + name: "different types string vs int", + a: "42", + b: 42, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := valuesEqual(tt.a, tt.b) + require.Equal(t, tt.want, got) + }) + } +} + +func TestConditionMatches(t *testing.T) { + tests := []struct { + name string + when map[string]interface{} + config map[string]interface{} + want bool + }{ + { + name: "empty when clause matches anything", + when: map[string]interface{}{}, + config: map[string]interface{}{"key": "value"}, + want: true, + }, + { + name: "empty when clause with empty config", + when: map[string]interface{}{}, + config: map[string]interface{}{}, + want: true, + }, + { + name: "single key match", + when: map[string]interface{}{"SearchLocalModCacheLicenses": true}, + config: map[string]interface{}{"SearchLocalModCacheLicenses": true}, + want: true, + }, + { + name: "single key mismatch", + when: map[string]interface{}{"SearchLocalModCacheLicenses": true}, + config: map[string]interface{}{"SearchLocalModCacheLicenses": false}, + want: false, + }, + { + name: "key missing from config", + when: map[string]interface{}{"SearchLocalModCacheLicenses": true}, + config: map[string]interface{}{}, + want: false, + }, + { + name: "multiple keys all match", + when: map[string]interface{}{ + "SearchLocalModCacheLicenses": true, + "UseNetwork": true, + }, + config: map[string]interface{}{ + "SearchLocalModCacheLicenses": true, + "UseNetwork": true, + "ExtraKey": "ignored", + }, + want: true, + }, + { + name: "multiple keys one mismatch", + when: map[string]interface{}{ + "SearchLocalModCacheLicenses": true, + "UseNetwork": true, + }, + config: map[string]interface{}{ + "SearchLocalModCacheLicenses": true, + "UseNetwork": false, + }, + want: false, + }, + { + name: "multiple keys one missing", + when: map[string]interface{}{ + "SearchLocalModCacheLicenses": true, + "UseNetwork": true, + }, + config: map[string]interface{}{ + "SearchLocalModCacheLicenses": true, + }, + want: false, + }, + { + name: "string value match", + when: map[string]interface{}{"mode": "fast"}, + config: map[string]interface{}{"mode": "fast"}, + want: true, + }, + { + name: "slice value match", + when: map[string]interface{}{"formats": []string{"json", "yaml"}}, + config: map[string]interface{}{"formats": []string{"json", "yaml"}}, + want: true, + }, + { + name: "slice value mismatch", + when: map[string]interface{}{"formats": []string{"json", "yaml"}}, + config: map[string]interface{}{"formats": []string{"json", "xml"}}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ConditionMatches(tt.when, tt.config) + require.Equal(t, tt.want, got) + }) + } +} + +func TestEvaluateField(t *testing.T) { + tests := []struct { + name string + capField CapabilityField + config map[string]interface{} + want interface{} + }{ + { + name: "no conditions returns default", + capField: CapabilityField{ + Name: "license", + Default: false, + Conditions: nil, + }, + config: map[string]interface{}{}, + want: false, + }, + { + name: "empty conditions returns default", + capField: CapabilityField{ + Name: "license", + Default: false, + Conditions: []CapabilityCondition{}, + }, + config: map[string]interface{}{}, + want: false, + }, + { + name: "single condition matches", + capField: CapabilityField{ + Name: "license", + Default: false, + Conditions: []CapabilityCondition{ + { + When: map[string]any{"SearchLocalModCacheLicenses": true}, + Value: true, + }, + }, + }, + config: map[string]interface{}{"SearchLocalModCacheLicenses": true}, + want: true, + }, + { + name: "single condition does not match", + capField: CapabilityField{ + Name: "license", + Default: false, + Conditions: []CapabilityCondition{ + { + When: map[string]any{"SearchLocalModCacheLicenses": true}, + Value: true, + }, + }, + }, + config: map[string]interface{}{"SearchLocalModCacheLicenses": false}, + want: false, + }, + { + name: "multiple conditions first match wins", + capField: CapabilityField{ + Name: "license", + Default: false, + Conditions: []CapabilityCondition{ + { + When: map[string]any{"SearchLocalModCacheLicenses": true}, + Value: "local", + }, + { + When: map[string]any{"SearchRemoteLicenses": true}, + Value: "remote", + }, + }, + }, + config: map[string]interface{}{ + "SearchLocalModCacheLicenses": true, + "SearchRemoteLicenses": true, + }, + want: "local", + }, + { + name: "multiple conditions second matches", + capField: CapabilityField{ + Name: "license", + Default: false, + Conditions: []CapabilityCondition{ + { + When: map[string]any{"SearchLocalModCacheLicenses": true}, + Value: "local", + }, + { + When: map[string]any{"SearchRemoteLicenses": true}, + Value: "remote", + }, + }, + }, + config: map[string]interface{}{ + "SearchLocalModCacheLicenses": false, + "SearchRemoteLicenses": true, + }, + want: "remote", + }, + { + name: "no conditions match returns default", + capField: CapabilityField{ + Name: "license", + Default: "none", + Conditions: []CapabilityCondition{ + { + When: map[string]any{"SearchLocalModCacheLicenses": true}, + Value: "local", + }, + { + When: map[string]any{"SearchRemoteLicenses": true}, + Value: "remote", + }, + }, + }, + config: map[string]interface{}{ + "SearchLocalModCacheLicenses": false, + "SearchRemoteLicenses": false, + }, + want: "none", + }, + { + name: "slice default value", + capField: CapabilityField{ + Name: "dependency.depth", + Default: []string{"direct", "indirect"}, + Conditions: nil, + }, + config: map[string]interface{}{}, + want: []string{"direct", "indirect"}, + }, + { + name: "condition with multiple when keys", + capField: CapabilityField{ + Name: "feature", + Default: false, + Conditions: []CapabilityCondition{ + { + When: map[string]any{ + "EnableFeatureA": true, + "EnableFeatureB": true, + }, + Value: true, + }, + }, + }, + config: map[string]interface{}{ + "EnableFeatureA": true, + "EnableFeatureB": true, + }, + want: true, + }, + { + name: "condition with multiple when keys partial match fails", + capField: CapabilityField{ + Name: "feature", + Default: false, + Conditions: []CapabilityCondition{ + { + When: map[string]any{ + "EnableFeatureA": true, + "EnableFeatureB": true, + }, + Value: true, + }, + }, + }, + config: map[string]interface{}{ + "EnableFeatureA": true, + "EnableFeatureB": false, + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := EvaluateField(tt.capField, tt.config) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("EvaluateField() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestEvaluateCapabilities(t *testing.T) { + tests := []struct { + name string + caps CapabilitySet + config map[string]interface{} + want map[string]interface{} + }{ + { + name: "empty capability set", + caps: CapabilitySet{}, + config: map[string]interface{}{}, + want: map[string]interface{}{}, + }, + { + name: "single capability no conditions", + caps: CapabilitySet{ + { + Name: "license", + Default: false, + }, + }, + config: map[string]interface{}{}, + want: map[string]interface{}{ + "license": false, + }, + }, + { + name: "single capability with matching condition", + caps: CapabilitySet{ + { + Name: "license", + Default: false, + Conditions: []CapabilityCondition{ + { + When: map[string]any{"SearchLocalModCacheLicenses": true}, + Value: true, + }, + }, + }, + }, + config: map[string]interface{}{"SearchLocalModCacheLicenses": true}, + want: map[string]interface{}{ + "license": true, + }, + }, + { + name: "multiple capabilities mixed conditions", + caps: CapabilitySet{ + { + Name: "license", + Default: false, + Conditions: []CapabilityCondition{ + { + When: map[string]any{"SearchLocalModCacheLicenses": true}, + Value: true, + }, + }, + }, + { + Name: "dependency.depth", + Default: []string{"direct", "indirect"}, + }, + { + Name: "dependency.edges", + Default: "flat", + }, + }, + config: map[string]interface{}{"SearchLocalModCacheLicenses": true}, + want: map[string]interface{}{ + "license": true, + "dependency.depth": []string{"direct", "indirect"}, + "dependency.edges": "flat", + }, + }, + { + name: "real-world go module binary cataloger example", + caps: CapabilitySet{ + { + Name: "license", + Default: false, + Conditions: []CapabilityCondition{ + { + When: map[string]any{"SearchLocalModCacheLicenses": true}, + Value: true, + }, + { + When: map[string]any{"SearchRemoteLicenses": true}, + Value: true, + }, + }, + }, + { + Name: "dependency.depth", + Default: []string{"direct", "indirect"}, + }, + { + Name: "dependency.edges", + Default: "flat", + }, + { + Name: "package_manager.files.listing", + Default: false, + }, + }, + config: map[string]interface{}{ + "SearchLocalModCacheLicenses": false, + "SearchRemoteLicenses": true, + }, + want: map[string]interface{}{ + "license": true, + "dependency.depth": []string{"direct", "indirect"}, + "dependency.edges": "flat", + "package_manager.files.listing": false, + }, + }, + { + name: "nil capability set", + caps: nil, + config: map[string]interface{}{ + "anything": true, + }, + want: map[string]interface{}{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := EvaluateCapabilities(tt.caps, tt.config) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("EvaluateCapabilities() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/internal/capabilities/generate/.gitignore b/internal/capabilities/generate/.gitignore new file mode 100644 index 000000000..e6242fbc5 --- /dev/null +++ b/internal/capabilities/generate/.gitignore @@ -0,0 +1 @@ +/generate \ No newline at end of file diff --git a/internal/capabilities/generate/README.md b/internal/capabilities/generate/README.md new file mode 100644 index 000000000..a9a7833ce --- /dev/null +++ b/internal/capabilities/generate/README.md @@ -0,0 +1,1402 @@ +# Capabilities Generation System + +This internal tool is responsible for: +- partially generating the `syft/pkg/cataloger/*/capabilities.yaml` ecosystem files, which document what capabilities each cataloger in syft has +- running completeness / consistency tests of the claims from capability YAML files against actual test observations + +Syft has dozens of catalogers across many ecosystems. Each cataloger has different capabilities, such as: +- Some provide license information, others don't +- Some detect transitive dependencies, others only direct +- Some capabilities depend on configuration + +The capability YAML files (located alongside cataloger source code) contain all of these capability claims, organized by ecosystem. + +The `capabilities` generation system itself: +1. **Discovers** cataloger information from source code using AST parsing +2. **Extracts** metadata about parsers, detectors, and configuration from code and tests +3. **Merges** discovered information with manually-maintained capability documentation +4. **Validates** that the generated document is complete and in sync with the codebase + +**Why do this?** +The short answer is to provide a foundation for the OSS documentation, where the source of truth for facts about the capabilities of Syft can be derived from verifiable claims from the tool itself. + + +## Quick Start + +**To regenerate capability YAML files after code changes:** +```bash +make generate-capabilities +``` + +This will: +- Run unit tests under ./syft/pkg... to gather up-to-date cataloger behavior +- Regenerate code that describes cataloger capabilities +- Run completeness tests against the newly generated code to ensure all capability YAML claims are consistent with test observations + +If you have made cataloger code changes you may see completeness tests fail, that's ok! +That only means that you need to manually update the capability YAML files that were generated. + +## Data Flow + +```mermaid +graph LR + subgraph "Source Code Inputs" + A1[syft/pkg/cataloger/*/
cataloger.go] + A2[syft/pkg/cataloger/*/
config.go] + A3[cmd/syft/internal/options/
catalog.go, ecosystem.go] + A4[syft task factories
AllCatalogers] + end + + subgraph "Test Inputs" + B1[test-fixtures/
test-observations.json] + end + + subgraph "Discovery Processes" + C1[discover_catalogers.go
AST Parse Catalogers] + C2[discover_cataloger_configs.go
AST Parse Configs] + C3[discover_app_config.go
AST Parse App Configs] + C4[discover_metadata.go
Read Observations] + C5[cataloger_config_linking.go
Link Catalogers to Configs] + C6[cataloger_names.go
Query Task Factories] + end + + subgraph "Discovered Data" + D1[Generic Catalogers
name, parsers, detectors] + D2[Config Structs
fields, app-config keys] + D3[App Config Fields
keys, descriptions, defaults] + D4[Metadata Types
per parser/cataloger] + D5[Package Types
per parser/cataloger] + D6[Cataloger-Config Links
mapping] + D7[Selectors
tags per cataloger] + end + + subgraph "Configuration/Overrides" + E2[metadataTypeCoverageExceptions
packageTypeCoverageExceptions
observationExceptions] + E1[catalogerTypeOverrides
catalogerConfigOverrides
catalogerConfigExceptions] + end + + subgraph "Capabilities YAML processing" + F1[io.go
Load Existing YAML] + F2[merge.go
Merge Logic] + F3[Preserve MANUAL fields
Update AUTO-GENERATED] + end + + subgraph "Output" + G1[syft/pkg/cataloger/*/
capabilities.yaml] + end + + subgraph "Validation" + H1[Distributed Tests
Comprehensive Tests] + H2[metadata_check.go
Type Coverage] + end + + A1 --> C1 + A2 --> C2 + A3 --> C3 + A4 --> C6 + B1 --> C4 + + C1 --> D1 + C2 --> D2 + C3 --> D3 + C4 --> D4 + C4 --> D5 + C5 --> D6 + C6 --> D7 + + D1 --> F2 + D2 --> F2 + D3 --> F2 + D4 --> F2 + D5 --> F2 + D6 --> F2 + D7 --> F2 + + E1 -.configure.-> F2 + E2 -.configure.-> H1 + + F1 --> F3 + F2 --> F3 + F3 --> G1 + + G1 --> H1 + G1 --> H2 + + style D1 fill:#e1f5ff + style D2 fill:#e1f5ff + style D3 fill:#e1f5ff + style D4 fill:#e1f5ff + style D5 fill:#e1f5ff + style D6 fill:#e1f5ff + style D7 fill:#e1f5ff + style G1 fill:#c8e6c9 + style E1 fill:#fff9c4 + style E2 fill:#fff9c4 +``` + +1. **Cataloger Discovery**: AST parser walks `syft/pkg/cataloger/` to find `generic.NewCataloger()` calls and extract parser information +2. **Cataloger Config Discovery**: AST parser finds config structs and extracts fields with `// app-config:` annotations +3. **App Config Discovery**: AST parser extracts ecosystem configurations from options package, including descriptions and defaults +4. **Metadata Discovery**: JSON reader loads test observations that record what metadata/package types each parser produces +5. **Linking**: AST analyzer connects catalogers to their config structs by examining constructor parameters +6. **Merge**: Discovered data combines with existing YAML, preserving all manually-maintained capability sections +7. **Validation**: Comprehensive tests ensure the output is complete and synchronized with codebase + +## The Capability YAML Files + +### Purpose + +The `syft/pkg/cataloger/*/capabilities.yaml` files are the canonical documentation of: +- Every cataloger in syft +- What files/patterns each cataloger detects +- What metadata and package types each cataloger produces +- What capabilities each cataloger has (licenses, dependencies, etc.) +- How configuration affects these capabilities + +### Structure + +Each ecosystem has its own file (e.g., `syft/pkg/cataloger/golang/capabilities.yaml`, `syft/pkg/cataloger/python/capabilities.yaml`): + +```yaml +# syft/pkg/cataloger/golang/capabilities.yaml - Cataloger capabilities for the golang ecosystem + +configs: # AUTO-GENERATED - config structs for this ecosystem + golang.CatalogerConfig: + fields: + - key: SearchLocalModCacheLicenses + description: SearchLocalModCacheLicenses enables... + app_key: golang.search-local-mod-cache-licenses + +catalogers: # Mixed AUTO-GENERATED structure, MANUAL capabilities + - ecosystem: golang # MANUAL + name: go-module-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/golang/cataloger.go + function: NewGoModuleBinaryCataloger + config: golang.CatalogerConfig # AUTO-GENERATED + selectors: [go, golang, ...] # AUTO-GENERATED + parsers: # AUTO-GENERATED structure + - function: parseGoMod # AUTO-GENERATED + detector: # AUTO-GENERATED + method: glob + criteria: ["**/go.mod"] + metadata_types: # AUTO-GENERATED + - pkg.GolangModuleEntry + package_types: # AUTO-GENERATED + - go-module + json_schema_types: # AUTO-GENERATED + - GolangModEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + conditions: + - when: {SearchRemoteLicenses: true} + value: true + comment: fetches licenses from proxy.golang.org + - name: dependency.depth + default: [direct, indirect] + - name: dependency.edges + default: complete +``` + +### AUTO-GENERATED vs MANUAL Fields + +#### AUTO-GENERATED Fields +These are updated on every regeneration: + +**Cataloger Level:** +- `name` - cataloger identifier +- `type` - "generic" or "custom" +- `source.file` - source file path +- `source.function` - constructor function name +- `config` - linked config struct name +- `selectors` - tags from task factories + +**Parser Level (generic catalogers):** +- `function` - parser function name (as used in the generic cataloger) +- `detector.method` - glob/path/mimetype +- `detector.criteria` - patterns matched +- `metadata_types` - from test-observations.json +- `package_types` - from test-observations.json +- `json_schema_types` - converted from metadata_types + +**Custom Cataloger Level:** +- `metadata_types` - from test-observations.json +- `package_types` - from test-observations.json +- `json_schema_types` - converted from metadata_types + +**Sections:** +- `appconfig.yaml`: contains the `application:` section with app-level config keys relevant to catalogers +- `configs:` section in each ecosystem file: config structs used by catalogers in that ecosystem + +#### MANUAL Fields +These are preserved across regeneration and must be edited by hand: + +- `ecosystem` - ecosystem/language identifier (cataloger level) +- `capabilities` - capability definitions with conditions +- `detectors` - for custom catalogers (except binary-classifier-cataloger) +- `conditions` on detectors - when detector is active based on config + +### How Regeneration Works + +When you run `go generate ./internal/capabilities`: + +1. **Loads existing YAML files** from `syft/pkg/cataloger/*/capabilities.yaml` and `internal/capabilities/appconfig.yaml` into both structs (for logic) and node trees (for comment preservation) +2. **Discovers all cataloger data** from source code and tests +3. **Merges** discovered data with existing: + - Updates AUTO-GENERATED fields + - **Preserves** all MANUAL fields (capabilities, ecosystem, etc.) + - Adds annotations (`# AUTO-GENERATED`, `# MANUAL`) to field comments +4. **Writes back** to `syft/pkg/cataloger/*/capabilities.yaml` using the node tree to preserve all comments +5. **Validates** the result with completeness tests + +> [!NOTE] +> Don't forget to update test observation evidence with `go test ./syft/pkg/...` before regeneration. + +## Generation Process + +### High-Level Workflow + +``` +1. Discovery Phase + ├─ Parse cataloger source code (AST) + ├─ Find all parsers and detectors + ├─ Read test observations for metadata types + ├─ Discover config structs and fields + ├─ Discover app-level configurations + └─ Link catalogers to their configs + +2. Merge Phase + ├─ Load existing syft/pkg/cataloger/*/capabilities.yaml files + ├─ Process each cataloger: + │ ├─ Update AUTO-GENERATED fields + │ └─ Preserve MANUAL fields + ├─ Add new catalogers + └─ Detect orphaned entries + +3. Write Phase + ├─ Group catalogers by ecosystem + ├─ Update YAML node trees per ecosystem + ├─ Add field annotations + └─ Write to syft/pkg/cataloger/*/capabilities.yaml and appconfig.yaml + +4. Validation Phase + ├─ Check all catalogers present + ├─ Check metadata/package type coverage + └─ Run completeness tests +``` + +### Detailed Discovery Processes + +#### 1. Generic Cataloger Discovery (`discover_catalogers.go`) + +**What it finds:** catalogers using the `generic.NewCataloger()` pattern + +**Process:** +1. Walk `syft/pkg/cataloger/` recursively for `.go` files +2. Parse each file with Go AST parser (`go/ast`, `go/parser`) +3. Find functions matching pattern: `New*Cataloger() pkg.Cataloger` +4. Within function body, find `generic.NewCataloger(name, ...)` call +5. Extract cataloger name from first argument +6. Find all chained `WithParserBy*()` calls: + ```go + generic.NewCataloger("my-cataloger"). + WithParserByGlobs(parseMyFormat, "**/*.myformat"). + WithParserByMimeTypes(parseMyBinary, "application/x-mytype") + ``` +7. For each parser call: + - Extract parser function name (e.g., `parseMyFormat`) + - Extract detection method (Globs/Path/MimeTypes) + - Extract criteria (patterns or mime types) + - Resolve constant references across files if needed + +**Output:** `map[string]DiscoveredCataloger` with full parser information + +#### 2. Config Discovery (`discover_cataloger_configs.go`) + +**What it finds:** cataloger configuration structs + +**Process:** +1. Find all `.go` files in `syft/pkg/cataloger/*/` +2. Look for structs with "Config" in their name +3. For each config struct: + - Extract struct fields + - Look for `// app-config: key.name` annotations in field comments + - Extract field descriptions from doc comments +4. Filter results by whitelist (only configs referenced in `pkgcataloging.Config`) + +**Example source:** +```go +type CatalogerConfig struct { + // SearchLocalModCacheLicenses enables searching for go package licenses + // in the local GOPATH mod cache. + // app-config: golang.search-local-mod-cache-licenses + SearchLocalModCacheLicenses bool +} +``` + +**Output:** `map[string]ConfigInfo` with field details and app-config keys + +#### 3. App Config Discovery (`discover_app_config.go`) + +**What it finds:** application-level configuration from the options package + +**Process:** +1. Parse `cmd/syft/internal/options/catalog.go` to find `Catalog` struct +2. Extract ecosystem config fields (e.g., `Golang golangConfig`) +3. For each ecosystem: + - Find the config file (e.g., `golang.go`) + - Parse the config struct + - Find `DescribeFields() []FieldDescription` method + - Extract field descriptions from the returned descriptions + - Find `default*Config()` function and extract default values +4. Build full key paths (e.g., `golang.search-local-mod-cache-licenses`) + +**Example source:** +```go +// golang.go +type golangConfig struct { + SearchLocalModCacheLicenses bool `yaml:"search-local-mod-cache-licenses" ...` +} + +func (c golangConfig) DescribeFields(opts ...options.DescribeFieldsOption) []options.FieldDescription { + return []options.FieldDescription{ + { + Name: "search-local-mod-cache-licenses", + Description: "search for go package licences in the GOPATH...", + }, + } +} +``` + +**Output:** `[]AppConfigField` with keys, descriptions, and defaults + +#### 4. Cataloger-Config Linking (`cataloger_config_linking.go`) + +**What it finds:** which config struct each cataloger uses + +**Process:** +1. For each discovered cataloger, find its constructor function +2. Extract the first parameter type from the function signature +3. Filter for types that look like configs (contain "Config") +4. Build mapping: cataloger name → config struct name +5. Apply manual overrides from `catalogerConfigOverrides` map +6. Apply exceptions from `catalogerConfigExceptions` set + +**Example:** +```go +// Constructor signature: +func NewGoModuleBinaryCataloger(cfg golang.CatalogerConfig) pkg.Cataloger + +// Results in link: +"go-module-binary-cataloger" → "golang.CatalogerConfig" +``` + +**Output:** `map[string]string` (cataloger → config mapping) + +#### 5. Metadata Discovery (`discover_metadata.go`) + +**What it finds:** metadata types and package types each parser produces + +**Process:** +1. Find all `test-fixtures/test-observations.json` files +2. Parse JSON which contains: + ```json + { + "package": "golang", + "parsers": { + "parseGoMod": { + "metadata_types": ["pkg.GolangModuleEntry"], + "package_types": ["go-module"] + } + }, + "catalogers": { + "linux-kernel-cataloger": { + "metadata_types": ["pkg.LinuxKernel"], + "package_types": ["linux-kernel"] + } + } + } + ``` +3. Build index by package name and parser function +4. Apply to discovered catalogers: + - Parser-level observations → attached to specific parsers + - Cataloger-level observations → for custom catalogers +5. Convert metadata types to JSON schema types using `packagemetadata` registry + +**Why this exists:** the AST parser can't determine what types a parser produces just by reading code. This information comes from test execution. + +**Output:** populated `MetadataTypes` and `PackageTypes` on catalogers/parsers + +## Input Sources + +### 1. Source Code Inputs + +#### Cataloger Constructors (`syft/pkg/cataloger/*/cataloger.go`) + +**What's extracted:** +- Cataloger names +- Parser function names +- Detection methods (glob, path, mimetype) +- Detection criteria (patterns) + +**Example:** +```go +func NewGoModuleBinaryCataloger() pkg.Cataloger { + return generic.NewCataloger("go-module-binary-cataloger"). + WithParserByGlobs(parseGoBin, "**/go.mod"). + WithParserByMimeTypes(parseGoArchive, "application/x-archive") +} +``` + +#### Config Structs (`syft/pkg/cataloger/*/config.go`) + +**What's extracted:** +- Config struct fields +- Field types +- Field descriptions from comments +- App-config key mappings from annotations + +**Example:** +```go +type CatalogerConfig struct { + // SearchRemoteLicenses enables downloading go package licenses from the upstream + // go proxy (typically proxy.golang.org). + // app-config: golang.search-remote-licenses + SearchRemoteLicenses bool + + // LocalModCacheDir specifies the location of the local go module cache directory. + // When not set, syft will attempt to discover the GOPATH env or default to $HOME/go. + // app-config: golang.local-mod-cache-dir + LocalModCacheDir string +} +``` + +#### Options Package (`cmd/syft/internal/options/`) + +**What's extracted:** +- Ecosystem config structs +- App-level configuration keys +- Field descriptions from `DescribeFields()` methods +- Default values from `default*Config()` functions + +**Example:** +```go +// catalog.go +type Catalog struct { + Golang golangConfig `yaml:"golang" json:"golang" mapstructure:"golang"` +} + +// golang.go +func (c golangConfig) DescribeFields(opts ...options.DescribeFieldsOption) []options.FieldDescription { + return []options.FieldDescription{ + { + Name: "search-remote-licenses", + Description: "search for go package licences by retrieving the package from a network proxy", + }, + } +} +``` + +### 2. Test-Driven Inputs + +#### test-observations.json Files + +**Location:** `syft/pkg/cataloger/*/test-fixtures/test-observations.json` + +**Purpose:** records what metadata and package types each parser produces during test execution + +**How they're generated:** automatically by the `pkgtest.CatalogTester` helpers when tests run + +**Example test code:** +```go +func TestGoModuleCataloger(t *testing.T) { + tester := NewGoModuleBinaryCataloger() + + pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/go-module-fixture"). + TestCataloger(t, tester) // Auto-writes observations on first run +} +``` + +**Example observations file:** +```json +{ + "package": "golang", + "parsers": { + "parseGoMod": { + "metadata_types": ["pkg.GolangModuleEntry"], + "package_types": ["go-module"] + }, + "parseGoSum": { + "metadata_types": ["pkg.GolangModuleEntry"], + "package_types": ["go-module"] + } + } +} +``` + +**Why this exists:** +- Metadata types can't be determined from AST parsing alone +- Ensures tests use the pkgtest helpers (enforced by `TestAllCatalogers HaveObservations`) +- Provides test coverage visibility + +### 3. Syft Runtime Inputs + +#### Task Factories (`allPackageCatalogerInfo()`) + +**What's extracted:** +- Canonical list of all catalogers (ensures sync with binary) +- Selectors (tags) for each cataloger + +**Example:** +```go +info := cataloger.CatalogerInfo{ + Name: "go-module-binary-cataloger", + Selectors: []string{"go", "golang", "binary", "language", "package"}, +} +``` + +### 4. Global Configuration Variables + +#### Merge Logic Overrides (`merge.go`) + +```go +// catalogerTypeOverrides forces a specific cataloger type when discovery gets it wrong +var catalogerTypeOverrides = map[string]string{ + "java-archive-cataloger": "custom", // technically generic but treated as custom +} + +// catalogerConfigExceptions lists catalogers that should NOT have config linked +var catalogerConfigExceptions = strset.New( + "binary-classifier-cataloger", +) + +// catalogerConfigOverrides manually specifies config when linking fails +var catalogerConfigOverrides = map[string]string{ + "dotnet-portable-executable-cataloger": "dotnet.CatalogerConfig", + "nix-store-cataloger": "nix.Config", +} +``` + +**When to update:** +- Add to `catalogerTypeOverrides` when a cataloger's type is misdetected +- Add to `catalogerConfigExceptions` when a cataloger shouldn't have config +- Add to `catalogerConfigOverrides` when automatic config linking fails + +#### Completeness Test Configuration (distributed across `*_test.go` files) + +```go +// requireParserObservations controls observation validation strictness +// - true: fail if ANY parser is missing observations (strict) +// - false: only check custom catalogers (lenient, current mode) +const requireParserObservations = false + +// metadataTypeCoverageExceptions lists metadata types allowed to not be documented +var metadataTypeCoverageExceptions = strset.New( + reflect.TypeOf(pkg.MicrosoftKbPatch{}).Name(), +) + +// packageTypeCoverageExceptions lists package types allowed to not be documented +var packageTypeCoverageExceptions = strset.New( + string(pkg.JenkinsPluginPkg), + string(pkg.KbPkg), +) + +// observationExceptions maps cataloger/parser names to observation types to skip +// - nil value: skip ALL observation checks for this cataloger/parser +// - set value: skip only specified observation types +var observationExceptions = map[string]*strset.Set{ + "graalvm-native-image-cataloger": nil, // skip all checks + "linux-kernel-cataloger": strset.New("relationships"), // skip only relationships +} +``` + +**When to update:** +- Add to exceptions when a type is intentionally not documented +- Add to `observationExceptions` when a cataloger lacks reliable test fixtures +- Set `requireParserObservations = true` when ready to enforce full parser coverage + +## Completeness Tests + +### Purpose + +Completeness tests ensure capability YAML files stay in perfect sync with the codebase. These tests catch: +- New catalogers that haven't been documented +- Orphaned cataloger entries (cataloger was removed but YAML wasn't updated) +- Missing metadata/package type documentation +- Invalid capability field references +- Catalogers not using test helpers + +### Test Organization + +Completeness tests are distributed across multiple packages: +- `internal/capabilities/` +- `internal/capabilities/internal/` +- `internal/capabilities/generate/` + +**Guard Clause**: All completeness tests are protected by `checkCompletenessTestsEnabled()` which checks the `SYFT_ENABLE_COMPLETENESS_TESTS=true` environment variable. + +**Why the guard?** There's a chicken-and-egg problem: +1. Package cataloger tests (`go test ./syft/pkg/...`) must run first to generate observation JSON files +2. Those observation JSONs are then used by the capabilities generation and completeness tests +3. Therefore, completeness tests cannot run as normal unit tests - they must run after the pkg tests and generation steps + +### Test Categories + +#### 1. Synchronization Tests + +**`TestCatalogersInSync`** +- Ensures all catalogers from `syft cataloger list` appear in YAML +- Ensures all catalogers in YAML exist in the binary +- Ensures all capabilities sections are filled (no TODOs/nulls) + +**Failure means:** you added/removed a cataloger but didn't regenerate the capability YAML files + +**Fix:** run `go generate ./internal/capabilities` + +--- + +**`TestCapabilitiesAreUpToDate`** +- Runs only in CI +- Ensures regeneration succeeds +- Ensures generated files have no uncommitted changes + +**Failure means:** capability YAML files weren't regenerated after code changes + +**Fix:** run `go generate ./internal/capabilities` and commit changes + +#### 2. Coverage Tests + +**`TestPackageTypeCoverage`** +- Ensures all types in `pkg.AllPkgs` are documented in some cataloger +- Allows exceptions via `packageTypeCoverageExceptions` + +**Failure means:** you added a new package type but no cataloger documents it + +**Fix:** either add a cataloger entry or add to exceptions if intentionally not supported + +--- + +**`TestMetadataTypeCoverage`** +- Ensures all types in `packagemetadata.AllTypes()` are documented +- Allows exceptions via `metadataTypeCoverageExceptions` + +**Failure means:** you added a new metadata type but no cataloger produces it + +**Fix:** either add metadata_types to a cataloger or add to exceptions + +--- + +**`TestMetadataTypesHaveJSONSchemaTypes`** +- Ensures metadata_types and json_schema_types are synchronized +- Validates every metadata type has a corresponding json_schema_type with correct conversion +- Checks both cataloger-level and parser-level types + +**Failure means:** metadata_types and json_schema_types are out of sync + +**Fix:** run `go generate ./internal/capabilities` to regenerate synchronized types + +#### 3. Structure Tests + +**`TestCatalogerStructure`** +- Validates generic vs custom cataloger structure rules: + - Generic catalogers must have parsers, no cataloger-level capabilities + - Custom catalogers must have detectors and cataloger-level capabilities +- Ensures ecosystem is always set + +**Failure means:** cataloger structure doesn't follow conventions + +**Fix:** correct the cataloger structure in the appropriate `syft/pkg/cataloger/*/capabilities.yaml` file + +--- + +**`TestCatalogerDataQuality`** +- Checks for duplicate cataloger names +- Validates detector formats for custom catalogers +- Checks for duplicate parser functions within catalogers + +**Failure means:** data integrity issue in capability YAML files + +**Fix:** remove duplicates or fix detector definitions + +#### 4. Config Tests + +**`TestConfigCompleteness`** +- Ensures all configs in the `configs:` section are referenced by a cataloger +- Ensures all cataloger config references exist +- Ensures all app-key references exist in `application:` section + +**Failure means:** orphaned config or broken reference + +**Fix:** remove unused configs or add missing entries + +--- + +**`TestAppConfigFieldsHaveDescriptions`** +- Ensures all application config fields have descriptions + +**Failure means:** missing `DescribeFields()` entry + +**Fix:** add description in the ecosystem's `DescribeFields()` method + +--- + +**`TestAppConfigKeyFormat`** +- Validates config keys follow format: `ecosystem.field-name` +- Ensures kebab-case (no underscores or spaces) + +**Failure means:** malformed config key + +**Fix:** rename the config key to follow conventions + +#### 5. Capability Tests + +**`TestCapabilityConfigFieldReferences`** +- Validates that config fields referenced in capability conditions actually exist +- Checks both cataloger-level and parser-level capabilities + +**Example failure:** +```yaml +capabilities: + - name: license + conditions: + - when: {NonExistentField: true} # ← this field doesn't exist in config struct + value: true +``` + +**Fix:** correct the field name to match the actual config struct + +--- + +**`TestCapabilityFieldNaming`** +- Ensures capability field names follow known patterns: + - `license` + - `dependency.depth` + - `dependency.edges` + - `dependency.kinds` + - `package_manager.files.listing` + - `package_manager.files.digests` + - `package_manager.package_integrity_hash` + +**Failure means:** typo in capability field name + +**Fix:** correct the typo or add new field to known list + +--- + +**`TestCapabilityValueTypes`** +- Validates capability values match expected types: + - Boolean fields: `license`, `package_manager.*` + - Array fields: `dependency.depth`, `dependency.kinds` + - String fields: `dependency.edges` + +**Example failure:** +```yaml +capabilities: + - name: license + default: "yes" # ← should be boolean true/false +``` + +**Fix:** use correct type for the field + +--- + +**`TestCapabilityEvidenceFieldReferences`** +- Validates that evidence references point to real struct fields +- Uses AST parsing to verify field paths exist + +**Example:** +```yaml +capabilities: + - name: package_manager.files.digests + default: true + evidence: + - AlpmDBEntry.Files[].Digests # ← validates this path exists +``` + +**Failure means:** typo in evidence reference or struct was changed + +**Fix:** correct the evidence reference or update after struct changes + +#### 6. Observations Test + +**`TestCatalogersHaveTestObservations`** +- Ensures all custom catalogers have test observations +- Optionally checks parsers (controlled by `requireParserObservations`) +- Allows exceptions via `observationExceptions` + +**Failure means:** cataloger tests aren't using pkgtest helpers + +**Fix:** update tests to use `pkgtest.CatalogTester`: +```go +pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/my-fixture"). + TestCataloger(t, myCataloger) +``` + +### How to Fix Test Failures + +#### General Approach +1. **Read the test error message** - it usually tells you exactly what's wrong +2. **Check if regeneration needed** - most failures fixed by: `go generate ./internal/capabilities` +3. **Check for code/test changes** - did you add/modify a cataloger? +4. **Consider exceptions** - is this intentionally unsupported? + +#### Common Failures and Fixes + +| Failure | Most Likely Cause | Fix | +|---------|------------------|-----| +| Cataloger not in YAML | Added new cataloger | Regenerate | +| Orphaned YAML entry | Removed cataloger | Regenerate | +| Missing metadata type | Added type but no test observations | Add pkgtest usage or exception | +| Missing observations | Test not using pkgtest | Update test to use `CatalogTester` | +| Config field reference | Typo in capability condition | Fix field name in capability YAML | +| Incomplete capabilities | Missing capability definition | Add capabilities section to capability YAML | + +## Manual Maintenance + +### What Requires Manual Editing + +These fields in capability YAML files are **MANUAL** and must be maintained by hand: + +#### 1. Ecosystem Field (Cataloger Level) +```yaml +catalogers: + - ecosystem: golang # MANUAL - identify the ecosystem +``` + +**Guidelines:** use the ecosystem/language name (golang, python, java, rust, etc.) + +#### 2. Capabilities Sections + +**For Generic Catalogers** (parser level): +```yaml +parsers: + - function: parseGoMod + capabilities: # MANUAL + - name: license + default: false + conditions: + - when: {SearchRemoteLicenses: true} + value: true + comment: fetches licenses from proxy.golang.org + - name: dependency.depth + default: [direct, indirect] + - name: dependency.edges + default: complete +``` + +**For Custom Catalogers** (cataloger level): +```yaml +catalogers: + - name: linux-kernel-cataloger + type: custom + capabilities: # MANUAL + - name: license + default: true +``` + +#### 3. Detectors for Custom Catalogers + +**For most custom catalogers:** +```yaml +detectors: # MANUAL + - method: glob + criteria: + - '**/lib/modules/**/modules.builtin' + comment: kernel modules directory +``` + +**Exception:** `binary-classifier-cataloger` has AUTO-GENERATED detectors extracted from source + +#### 4. Detector Conditions + +when a detector should only be active with certain configuration: +```yaml +detectors: + - method: glob + criteria: ['**/*.zip'] + conditions: # MANUAL + - when: {IncludeZipFiles: true} + comment: ZIP detection requires explicit config +``` + +### Capabilities Format and Guidelines + +#### Standard Capability Fields + +**Boolean Fields:** +```yaml +- name: license + default: true # always available + # OR + default: false # never available + # OR + default: false + conditions: + - when: {SearchRemoteLicenses: true} + value: true + comment: requires network access to fetch licenses +``` + +**Array Fields (dependency.depth):** +```yaml +- name: dependency.depth + default: [direct] # only immediate dependencies + # OR + default: [direct, indirect] # full transitive closure + # OR + default: [] # no dependency information +``` + +**String Fields (dependency.edges):** +```yaml +- name: dependency.edges + default: "" # dependencies found but no edges between them + # OR + default: flat # single level of dependencies with edges to root only + # OR + default: reduced # transitive reduction (redundant edges removed) + # OR + default: complete # all relationships with accurate direct/indirect edges +``` + +**Array Fields (dependency.kinds):** +```yaml +- name: dependency.kinds + default: [runtime] # production dependencies only + # OR + default: [runtime, dev] # production and development + # OR + default: [runtime, dev, build, test] # all dependency types +``` + +#### Using Conditions + +Conditions allow capabilities to vary based on configuration values: + +```yaml +capabilities: + - name: license + default: false + conditions: + - when: {SearchLocalModCacheLicenses: true} + value: true + comment: searches for licenses in GOPATH mod cache + - when: {SearchRemoteLicenses: true} + value: true + comment: fetches licenses from proxy.golang.org + comment: license scanning requires configuration +``` + +**Rules:** +- Conditions are evaluated in array order (first match wins) +- Multiple fields WITHIN a `when` clause use AND logic (all must match) +- Multiple conditions in the array use OR logic (first matching condition) +- If no conditions match, the `default` value is used + +#### Adding Evidence + +evidence documents which struct fields provide the capability: + +```yaml +- name: package_manager.files.listing + default: true + evidence: + - AlpmDBEntry.Files + comment: file listings stored in Files array +``` + +**For nested fields:** +```yaml +evidence: + - CondaMetaPackage.PathsData.Paths +``` + +**For array element fields:** +```yaml +evidence: + - AlpmDBEntry.Files[].Digests +``` + +### Best Practices + +1. **Be specific in comments:** explain WHY, not just WHAT +2. **Document conditions clearly:** explain what configuration enables the capability +3. **Use evidence references:** helps verify capabilities are accurate +4. **Test after edits:** run `SYFT_ENABLE_COMPLETENESS_TESTS=true go test ./internal/capabilities/...` to validate + +## Development Workflows + +### Adding a New Cataloger + +#### If Using `generic.NewCataloger()`: + +**What happens automatically:** +1. Generator discovers the cataloger via AST parsing +2. Extracts parsers, detectors, and patterns +3. Adds entry to the appropriate `syft/pkg/cataloger/*/capabilities.yaml` file +4. Links to config (if constructor has config parameter) +5. Extracts metadata types from test-observations.json (if test uses pkgtest) + +**What you must do manually:** +1. Set the `ecosystem` field in the capability YAML file +2. Add `capabilities` sections to each parser +3. Run `go generate ./internal/capabilities` +4. Commit the updated capability YAML file + +**Example workflow:** +```bash +# 1. Write cataloger code +vim syft/pkg/cataloger/mynew/cataloger.go + +# 2. Write tests using pkgtest (generates observations) +vim syft/pkg/cataloger/mynew/cataloger_test.go + +# 3. Run tests to generate observations +go test ./syft/pkg/cataloger/mynew + +# 4. Regenerate capability YAML files +go generate ./internal/capabilities + +# 5. Edit the capability file manually +vim syft/pkg/cataloger/mynew/capabilities.yaml +# - Set ecosystem field +# - Add capabilities sections + +# 6. Validate +SYFT_ENABLE_COMPLETENESS_TESTS=true go test ./internal/capabilities/... + +# 7. Commit +git add syft/pkg/cataloger/mynew/capabilities.yaml +git add syft/pkg/cataloger/mynew/test-fixtures/test-observations.json +git commit +``` + +#### If Writing a Custom Cataloger: + +**What happens automatically:** +1. Generator creates entry with name and type +2. Extracts metadata types from test-observations.json + +**What you must do manually:** +1. Set `ecosystem` +2. Add `detectors` array with detection methods +3. Add `capabilities` section (cataloger level, not parser level) +4. Run `go generate ./internal/capabilities` + +### Modifying an Existing Cataloger + +#### If Changing Parser Detection Patterns: + +**Impact:** AUTO-GENERATED field, automatically updated + +**Workflow:** +```bash +# 1. Change the code +vim syft/pkg/cataloger/something/cataloger.go + +# 2. Regenerate +go generate ./internal/capabilities + +# 3. Review changes +git diff syft/pkg/cataloger/*/capabilities.yaml + +# 4. Commit +git add syft/pkg/cataloger/*/capabilities.yaml +git commit +``` + +#### If Changing Metadata Type: + +**Impact:** AUTO-GENERATED field, updated via test observations + +**Workflow:** +```bash +# 1. Change the code +vim syft/pkg/cataloger/something/parser.go + +# 2. Update tests (if needed) +vim syft/pkg/cataloger/something/parser_test.go + +# 3. Run tests to update observations +go test ./syft/pkg/cataloger/something + +# 4. Regenerate +go generate ./internal/capabilities + +# 5. Commit +git add syft/pkg/cataloger/*/capabilities.yaml +git add syft/pkg/cataloger/something/test-fixtures/test-observations.json +git commit +``` + +#### If Changing Capabilities: + +**Impact:** MANUAL field, preserved across regeneration + +**Workflow:** +```bash +# 1. Edit the capability file directly +vim syft/pkg/cataloger/something/capabilities.yaml + +# 2. Validate +SYFT_ENABLE_COMPLETENESS_TESTS=true go test ./internal/capabilities/... + +# 3. Commit +git commit syft/pkg/cataloger/something/capabilities.yaml +``` + +### Adding New Capability Fields + +if you need to add a completely new capability field (e.g., `package_manager.build_tool_info`): + +**Steps:** +1. Add field name to known fields in the appropriate test file +2. Add value type validation to the validation logic +3. Update file header documentation in capability YAML files +4. Add the field to relevant catalogers in the appropriate `syft/pkg/cataloger/*/capabilities.yaml` files +5. Update any runtime code that consumes capabilities + +### When to Update Exceptions + +#### Add to `catalogerTypeOverrides`: +- Discovery incorrectly classifies a cataloger's type +- Example: cataloger uses generic framework but behaves like custom + +#### Add to `catalogerConfigExceptions`: +- Cataloger should not have config linked +- Example: simple catalogers with no configuration + +#### Add to `catalogerConfigOverrides`: +- Automatic config linking fails +- Cataloger in a subpackage or unusual structure +- Example: dotnet catalogers split across multiple packages + +#### Add to `metadataTypeCoverageExceptions`: +- Metadata type is deprecated or intentionally unused +- Example: `MicrosoftKbPatch` (special case type) + +#### Add to `packageTypeCoverageExceptions`: +- Package type is deprecated or special case +- Example: `JenkinsPluginPkg`, `KbPkg` + +#### Add to `observationExceptions`: +- Cataloger lacks reliable test fixtures (e.g., requires specific binaries) +- Cataloger produces relationships but they're not standard dependencies +- Example: `graalvm-native-image-cataloger` (requires native images) + +## File Inventory + +### Core Generation (`internal/capabilities/generate/`) + +- **`main.go`**: entry point, orchestrates regeneration, prints status messages +- **`merge.go`**: core merging logic, preserves manual sections while updating auto-generated +- **`io.go`**: YAML reading/writing with comment preservation using gopkg.in/yaml.v3 + +### Discovery (`internal/capabilities/generate/`) + +- **`discover_catalogers.go`**: AST parsing to discover generic catalogers and parsers from source code +- **`discover_cataloger_configs.go`**: AST parsing to discover cataloger config structs +- **`discover_app_config.go`**: AST parsing to discover application-level config from options package +- **`cataloger_config_linking.go`**: links catalogers to config structs by analyzing constructors +- **`discover_metadata.go`**: reads test-observations.json files to get metadata/package types + +### Shared Utilities (`internal/capabilities/internal/`) + +- **`paths.go`**: single source of truth for all capability file paths +- **`load_capabilities.go`**: loading and parsing capability YAML files +- **`cataloger_names.go`**: helper to get all cataloger names from syft task factories +- **`repo_root.go`**: helpers to find the repository root + +### Validation + +- **`metadata_check.go`**: validates metadata and package type coverage + +### Completeness Tests + +Tests are distributed across packages with `checkCompletenessTestsEnabled()` guard: +- `internal/capabilities/*_test.go` +- `internal/capabilities/internal/*_test.go` +- `internal/capabilities/generate/*_test.go` + +### Unit Tests + +- **`discover_catalogers_test.go`**: tests for cataloger discovery +- **`discover_cataloger_configs_test.go`**: tests for config discovery +- **`cataloger_config_linking_test.go`**: tests for config linking +- **`merge_test.go`**: tests for merge logic +- **`io_test.go`**: tests for YAML I/O + +## Troubleshooting + +### "Cataloger X not found in capability YAML" + +**Cause:** you added a new cataloger but didn't regenerate the capability YAML files + +**Fix:** +```bash +go generate ./internal/capabilities +``` + +### "Cataloger X in YAML but not in binary" + +**Cause:** you removed a cataloger but didn't regenerate + +**Fix:** +```bash +go generate ./internal/capabilities +# Review the diff - the cataloger entry should be removed +``` + +### "Metadata type X not represented in any cataloger" + +**Cause:** you added a new metadata type but: +- No cataloger produces it yet, OR +- Tests don't use pkgtest helpers (so observations aren't generated) + +**Fix Option 1 - Add test observations:** +```go +// Update test to use pkgtest +pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/my-fixture"). + TestCataloger(t, myCataloger) + +// Run tests +go test ./syft/pkg/cataloger/mypackage + +// Regenerate +go generate ./internal/capabilities +``` + +**Fix Option 2 - Add exception (if intentionally unused):** +```go +// in metadata_check.go +var metadataTypeExceptions = map[string]bool{ + "pkg.MyNewType": true, +} +``` + +### "Parser X has no test observations" + +**Cause:** test doesn't use pkgtest helpers + +**Fix:** +```go +// Before: +func TestMyParser(t *testing.T) { + // manual test code +} + +// After: +func TestMyParser(t *testing.T) { + cataloger := NewMyCataloger() + pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/my-fixture"). + TestCataloger(t, cataloger) +} +``` + +### "Config field X not found in struct Y" + +**Cause:** capability condition references a non-existent config field + +**Fix:** edit the capability YAML file and correct the field name: +```yaml +# Before: +conditions: + - when: {SerachRemoteLicenses: true} # typo! + +# After: +conditions: + - when: {SearchRemoteLicenses: true} +``` + +### "Evidence field X.Y not found in struct X" + +**Cause:** +- Typo in evidence reference, OR +- Struct was refactored and field moved/renamed + +**Fix:** edit the capability YAML file and correct the evidence reference: +```yaml +# Before: +evidence: + - AlpmDBEntry.FileListing # wrong field name + +# After: +evidence: + - AlpmDBEntry.Files +``` + +### "Capability YAML files have uncommitted changes after regeneration" + +**Cause:** capability YAML files are out of date (usually caught in CI) + +**Fix:** +```bash +go generate ./internal/capabilities +git add syft/pkg/cataloger/*/capabilities.yaml +git commit -m "chore: regenerate capabilities" +``` + +### Generator Fails with "struct X not found" + +**Cause:** config linking trying to link to a non-existent struct + +**Fix Option 1 - Add override:** +```go +// merge.go +var catalogerConfigOverrides = map[string]string{ + "my-cataloger": "mypackage.MyConfig", +} +``` + +**Fix Option 2 - Add exception:** +```go +// merge.go +var catalogerConfigExceptions = strset.New( + "my-cataloger", // doesn't use config +) +``` + +### "Parser capabilities must be defined" + +**Cause:** parser in capability YAML file has no capabilities section + +**Fix:** add capabilities to the parser: +```yaml +parsers: + - function: parseMyFormat + capabilities: + - name: license + default: false + - name: dependency.depth + default: [] + # ... (add all required capability fields) +``` + +### Understanding Error Messages + +most test failures include detailed guidance. Look for: +- **List of missing items:** tells you exactly what to add/remove +- **Suggestions:** usually includes the command to fix (e.g., "Run 'go generate ./internal/capabilities'") +- **File locations:** tells you which file to edit + +**General debugging approach:** +1. Read the full error message +2. Check if it's fixed by regeneration +3. Check for recent code/test changes +4. Consider if it should be an exception +5. Ask for help if still stuck (include full error message) + +--- + +## Questions or Issues? + +if you encounter problems not covered here: +1. Check test error messages (they're usually quite helpful) +2. Look at recent commits for examples of similar changes +3. Ask in the team chat with the full error message diff --git a/internal/capabilities/generate/cataloger_config_linking.go b/internal/capabilities/generate/cataloger_config_linking.go new file mode 100644 index 000000000..41b5b5db8 --- /dev/null +++ b/internal/capabilities/generate/cataloger_config_linking.go @@ -0,0 +1,339 @@ +// this file links catalogers to their configuration structs by analyzing constructor function signatures to determine which config struct each cataloger uses. +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" +) + +// LinkCatalogersToConfigs analyzes cataloger constructor functions to determine which config struct +// each cataloger uses. Returns a map where key is the cataloger name (e.g., "go-module-binary-cataloger") +// and value is the config struct reference (e.g., "golang.CatalogerConfig"). +// Returns empty string for catalogers that don't take a config parameter. +func LinkCatalogersToConfigs(repoRoot string) (map[string]string, error) { + catalogerRoot := filepath.Join(repoRoot, "syft", "pkg", "cataloger") + return LinkCatalogersToConfigsFromPath(catalogerRoot, repoRoot) +} + +// LinkCatalogersToConfigsFromPath analyzes cataloger constructor functions in the specified directory +// to determine which config struct each cataloger uses. This is the parameterized version that allows +// testing with custom fixture directories. +// Returns a map where key is the cataloger name (e.g., "go-module-binary-cataloger") +// and value is the config struct reference (e.g., "golang.CatalogerConfig"). +// Returns empty string for catalogers that don't take a config parameter. +// The baseRoot parameter is used for relative path calculation to determine package names. +func LinkCatalogersToConfigsFromPath(catalogerRoot, baseRoot string) (map[string]string, error) { + // find all .go files under the cataloger root recursively + var files []string + err := filepath.Walk(catalogerRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, "_test.go") { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk cataloger directory: %w", err) + } + + linkages := make(map[string]string) + + for _, file := range files { + links, err := linkCatalogersInFile(file, baseRoot) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", file, err) + } + + for catalogerName, configType := range links { + if existing, ok := linkages[catalogerName]; ok && existing != configType { + return nil, fmt.Errorf("conflicting config types for cataloger %q: %s vs %s", catalogerName, existing, configType) + } + linkages[catalogerName] = configType + } + } + + return linkages, nil +} + +func linkCatalogersInFile(path, repoRoot string) (map[string]string, error) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + return nil, err + } + + // extract package name from file path + relPath, err := filepath.Rel(repoRoot, path) + if err != nil { + relPath = path + } + packageName := extractPackageNameFromPath(relPath) + if packageName == "" { + return nil, nil + } + + linkages := make(map[string]string) + + // find all function declarations + for _, decl := range f.Decls { + funcDecl, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + + // check if function name matches New*Cataloger pattern + if !strings.HasPrefix(funcDecl.Name.Name, "New") || !strings.HasSuffix(funcDecl.Name.Name, "Cataloger") { + continue + } + + // check if return type is pkg.Cataloger + if !returnsPackageCataloger(funcDecl) { + continue + } + + // extract the cataloger name from the function body + catalogerName := extractCatalogerName(funcDecl, f, path, repoRoot) + if catalogerName == "" { + // couldn't determine cataloger name, skip + continue + } + + // extract config type from function parameters + configType := extractConfigParameter(funcDecl, packageName) + + // store the linkage (empty string means no config) + linkages[catalogerName] = configType + } + + return linkages, nil +} + +// extractCatalogerName extracts the cataloger name from the function body. +// It looks for: +// 1. generic.NewCataloger("name") calls +// 2. Cataloger implementations that define Name() method +// 3. Hardcoded name constants +func extractCatalogerName(funcDecl *ast.FuncDecl, file *ast.File, filePath, repoRoot string) string { + if funcDecl.Body == nil { + return "" + } + + ctx := &parseContext{ + file: file, + filePath: filePath, + repoRoot: repoRoot, + } + + var catalogerName string + + // walk the function body to find generic.NewCataloger call or name assignment + ast.Inspect(funcDecl.Body, func(n ast.Node) bool { + // look for call expressions + callExpr, ok := n.(*ast.CallExpr) + if !ok { + return true + } + + // check if this is a call to generic.NewCataloger + if isGenericNewCatalogerCall(callExpr) { + // extract the cataloger name from the first argument + if len(callExpr.Args) > 0 { + // handle string literals + if lit, ok := callExpr.Args[0].(*ast.BasicLit); ok && lit.Kind == token.STRING { + catalogerName = strings.Trim(lit.Value, `"`) + return false // found it, stop searching + } + // handle constant identifiers + if ident, ok := callExpr.Args[0].(*ast.Ident); ok { + catalogerName = resolveLocalConstant(ident.Name, ctx) + return false // found it, stop searching + } + } + } + + return true + }) + + // if we didn't find a generic.NewCataloger call, try to infer from custom cataloger + if catalogerName == "" { + catalogerName = inferCatalogerNameFromCustomImpl(funcDecl, file, ctx) + } + + return catalogerName +} + +// inferCatalogerNameFromCustomImpl tries to infer the cataloger name from custom cataloger implementations +// by looking for Name() method implementations or hardcoded name variables +func inferCatalogerNameFromCustomImpl(funcDecl *ast.FuncDecl, file *ast.File, ctx *parseContext) string { + typeName := extractReturnTypeName(funcDecl) + if typeName == "" { + return "" + } + return findNameMethodReturn(file, typeName, ctx) +} + +// extractReturnTypeName extracts the type name from the return statement of a constructor function +func extractReturnTypeName(funcDecl *ast.FuncDecl) string { + var typeName string + + ast.Inspect(funcDecl.Body, func(n ast.Node) bool { + returnStmt, ok := n.(*ast.ReturnStmt) + if !ok || len(returnStmt.Results) == 0 { + return true + } + + result := returnStmt.Results[0] + + // handle &Type{...} + if unaryExpr, ok := result.(*ast.UnaryExpr); ok && unaryExpr.Op == token.AND { + if compLit, ok := unaryExpr.X.(*ast.CompositeLit); ok { + if ident, ok := compLit.Type.(*ast.Ident); ok { + typeName = ident.Name + return false + } + } + } + + // handle Type{...} + if compLit, ok := result.(*ast.CompositeLit); ok { + if ident, ok := compLit.Type.(*ast.Ident); ok { + typeName = ident.Name + return false + } + } + + return true + }) + + return typeName +} + +// findNameMethodReturn finds the Name() method for the given type and extracts its return value +func findNameMethodReturn(file *ast.File, typeName string, ctx *parseContext) string { + for _, decl := range file.Decls { + funcDecl, ok := decl.(*ast.FuncDecl) + if !ok || funcDecl.Name.Name != "Name" { + continue + } + + if funcDecl.Recv == nil || len(funcDecl.Recv.List) == 0 { + continue + } + + recvTypeName := extractReceiverTypeName(funcDecl.Recv.List[0].Type) + if recvTypeName != typeName { + continue + } + + // found the Name() method, extract the return value + if funcDecl.Body != nil { + if name := extractNameFromMethodBody(funcDecl.Body, ctx); name != "" { + return name + } + } + } + + return "" +} + +// extractReceiverTypeName extracts the type name from a receiver type expression +func extractReceiverTypeName(recvType ast.Expr) string { + // handle both T and *T receivers + if ident, ok := recvType.(*ast.Ident); ok { + return ident.Name + } + if starExpr, ok := recvType.(*ast.StarExpr); ok { + if ident, ok := starExpr.X.(*ast.Ident); ok { + return ident.Name + } + } + return "" +} + +// extractNameFromMethodBody extracts the cataloger name from a Name() method body +func extractNameFromMethodBody(body *ast.BlockStmt, ctx *parseContext) string { + var name string + ast.Inspect(body, func(n ast.Node) bool { + returnStmt, ok := n.(*ast.ReturnStmt) + if !ok || len(returnStmt.Results) == 0 { + return true + } + + // handle string literal + if lit, ok := returnStmt.Results[0].(*ast.BasicLit); ok && lit.Kind == token.STRING { + name = strings.Trim(lit.Value, `"`) + return false + } + + // handle constant identifier (e.g., pomCatalogerName) + if ident, ok := returnStmt.Results[0].(*ast.Ident); ok { + name = resolveLocalConstant(ident.Name, ctx) + return false + } + + return true + }) + return name +} + +// extractConfigParameter extracts the config type from the first parameter of a cataloger constructor. +// Returns empty string if no config parameter is found. +// Returns format: "packageName.StructName" (e.g., "golang.CatalogerConfig") +func extractConfigParameter(funcDecl *ast.FuncDecl, localPackageName string) string { + if funcDecl.Type.Params == nil || len(funcDecl.Type.Params.List) == 0 { + // no parameters, no config + return "" + } + + // check the first parameter + firstParam := funcDecl.Type.Params.List[0] + if firstParam.Type == nil { + return "" + } + + // extract the type name + configType := extractConfigTypeName(firstParam.Type, localPackageName) + + // filter out non-config types + if configType != "" && !looksLikeConfigType(configType) { + return "" + } + + return configType +} + +// extractConfigTypeName extracts the full type name from a parameter type expression +func extractConfigTypeName(typeExpr ast.Expr, localPackageName string) string { + switch t := typeExpr.(type) { + case *ast.Ident: + // local type (e.g., CatalogerConfig) + return localPackageName + "." + t.Name + case *ast.SelectorExpr: + // imported type (e.g., java.ArchiveCatalogerConfig) + if pkgIdent, ok := t.X.(*ast.Ident); ok { + return pkgIdent.Name + "." + t.Sel.Name + } + } + return "" +} + +// looksLikeConfigType checks if a type name looks like a configuration struct +func looksLikeConfigType(typeName string) bool { + // remove package prefix + parts := strings.Split(typeName, ".") + if len(parts) < 2 { + return false + } + + structName := parts[len(parts)-1] + + // check for common config patterns + return strings.Contains(structName, "Config") +} diff --git a/internal/capabilities/generate/cataloger_config_linking_test.go b/internal/capabilities/generate/cataloger_config_linking_test.go new file mode 100644 index 000000000..5b2bce7c6 --- /dev/null +++ b/internal/capabilities/generate/cataloger_config_linking_test.go @@ -0,0 +1,401 @@ +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLinkCatalogersToConfigsFromPath(t *testing.T) { + tests := []struct { + name string + fixturePath string + expectedLinkages map[string]string + wantErr require.ErrorAssertionFunc + }{ + { + name: "simple generic cataloger with local config", + fixturePath: "simple-generic-cataloger", + expectedLinkages: map[string]string{ + "go-module-cataloger": "golang.CatalogerConfig", + }, + }, + { + name: "cataloger name from constant", + fixturePath: "cataloger-with-constant", + expectedLinkages: map[string]string{ + "python-package-cataloger": "python.CatalogerConfig", + }, + }, + { + name: "custom cataloger with Name() in same file", + fixturePath: "custom-cataloger-same-file", + expectedLinkages: map[string]string{ + "java-pom-cataloger": "java.ArchiveCatalogerConfig", + }, + }, + { + name: "custom cataloger with Name() in different file - not detected", + fixturePath: "custom-cataloger-different-file", + expectedLinkages: map[string]string{ + // empty - current limitation, cannot detect cross-file Names + }, + }, + { + name: "cataloger without config parameter", + fixturePath: "no-config-cataloger", + expectedLinkages: map[string]string{ + "javascript-cataloger": "", // empty string means no config + }, + }, + { + name: "imported config type", + fixturePath: "imported-config-type", + expectedLinkages: map[string]string{ + "linux-kernel-cataloger": "kernel.LinuxKernelCatalogerConfig", + }, + }, + { + name: "non-config first parameter", + fixturePath: "non-config-first-param", + expectedLinkages: map[string]string{ + "binary-cataloger": "", // Parser not a config type + }, + }, + { + name: "conflicting cataloger names", + fixturePath: "conflicting-names", + wantErr: require.Error, + }, + { + name: "mixed naming patterns", + fixturePath: "mixed-naming-patterns", + expectedLinkages: map[string]string{ + "ruby-cataloger": "ruby.Config", + }, + }, + { + name: "selector expression config", + fixturePath: "selector-expression-config", + expectedLinkages: map[string]string{ + "rust-cataloger": "rust.CatalogerConfig", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + + fixtureDir := filepath.Join("testdata", "cataloger", tt.fixturePath) + catalogerRoot := filepath.Join(fixtureDir, "cataloger") + linkages, err := LinkCatalogersToConfigsFromPath(catalogerRoot, fixtureDir) + tt.wantErr(t, err) + + if err != nil { + return + } + + require.Equal(t, tt.expectedLinkages, linkages) + }) + } +} + +func TestExtractConfigTypeName(t *testing.T) { + tests := []struct { + name string + fixturePath string + catalogerName string + expectedConfig string + expectedNoConfig bool + }{ + { + name: "golang config", + fixturePath: "simple-generic-cataloger", + catalogerName: "go-module-cataloger", + expectedConfig: "golang.CatalogerConfig", + }, + { + name: "python config with constant", + fixturePath: "cataloger-with-constant", + catalogerName: "python-package-cataloger", + expectedConfig: "python.CatalogerConfig", + }, + { + name: "java archive config same file", + fixturePath: "custom-cataloger-same-file", + catalogerName: "java-pom-cataloger", + expectedConfig: "java.ArchiveCatalogerConfig", + }, + { + name: "kernel config imported type", + fixturePath: "imported-config-type", + catalogerName: "linux-kernel-cataloger", + expectedConfig: "kernel.LinuxKernelCatalogerConfig", + }, + { + name: "javascript - no config", + fixturePath: "no-config-cataloger", + catalogerName: "javascript-cataloger", + expectedNoConfig: true, + }, + { + name: "ruby with mixed naming", + fixturePath: "mixed-naming-patterns", + catalogerName: "ruby-cataloger", + expectedConfig: "ruby.Config", + }, + { + name: "rust with selector expression", + fixturePath: "selector-expression-config", + catalogerName: "rust-cataloger", + expectedConfig: "rust.CatalogerConfig", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixtureDir := filepath.Join("testdata", "cataloger", tt.fixturePath) + catalogerRoot := filepath.Join(fixtureDir, "cataloger") + linkages, err := LinkCatalogersToConfigsFromPath(catalogerRoot, fixtureDir) + require.NoError(t, err) + + config, ok := linkages[tt.catalogerName] + + if tt.expectedNoConfig { + if ok { + require.Empty(t, config, "expected no config for %s", tt.catalogerName) + } + } else { + require.True(t, ok, "should find cataloger %s", tt.catalogerName) + require.Equal(t, tt.expectedConfig, config) + } + }) + } +} + +func TestLooksLikeConfigType(t *testing.T) { + tests := []struct { + name string + typeName string + want bool + }{ + { + name: "golang config", + typeName: "golang.CatalogerConfig", + want: true, + }, + { + name: "python config", + typeName: "python.CatalogerConfig", + want: true, + }, + { + name: "java archive config", + typeName: "java.ArchiveCatalogerConfig", + want: true, + }, + { + name: "kernel config", + typeName: "kernel.LinuxKernelCatalogerConfig", + want: true, + }, + { + name: "nix config", + typeName: "nix.Config", + want: true, + }, + { + name: "config prefix", + typeName: "package.ConfigOptions", + want: true, + }, + { + name: "not a config type", + typeName: "package.Parser", + want: false, + }, + { + name: "not a config type - resolver", + typeName: "file.Resolver", + want: false, + }, + { + name: "no package prefix", + typeName: "CatalogerConfig", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := looksLikeConfigType(tt.typeName) + require.Equal(t, tt.want, got) + }) + } +} + +func TestExtractReceiverTypeName(t *testing.T) { + tests := []struct { + name string + receiver string // receiver code snippet + want string + }{ + { + name: "value receiver", + receiver: "func (c Cataloger) Name() string { return \"\" }", + want: "Cataloger", + }, + { + name: "pointer receiver", + receiver: "func (c *Cataloger) Name() string { return \"\" }", + want: "Cataloger", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // parse the function to get the receiver type + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "", "package test\n"+tt.receiver, 0) + require.NoError(t, err) + + // extract the function declaration + require.Len(t, file.Decls, 1) + funcDecl, ok := file.Decls[0].(*ast.FuncDecl) + require.True(t, ok) + + // get receiver type + var recvType ast.Expr + if funcDecl.Recv != nil && len(funcDecl.Recv.List) > 0 { + recvType = funcDecl.Recv.List[0].Type + } + + got := extractReceiverTypeName(recvType) + require.Equal(t, tt.want, got) + }) + } +} + +func TestExtractConfigTypeNameHelper(t *testing.T) { + tests := []struct { + name string + funcSig string // function signature with parameter + localPackageName string + want string + }{ + { + name: "local type", + funcSig: "func New(cfg CatalogerConfig) pkg.Cataloger { return nil }", + localPackageName: "python", + want: "python.CatalogerConfig", + }, + { + name: "imported type", + funcSig: "func New(cfg java.ArchiveCatalogerConfig) pkg.Cataloger { return nil }", + localPackageName: "python", + want: "java.ArchiveCatalogerConfig", + }, + { + name: "imported type - kernel package", + funcSig: "func New(cfg kernel.LinuxKernelCatalogerConfig) pkg.Cataloger { return nil }", + localPackageName: "other", + want: "kernel.LinuxKernelCatalogerConfig", + }, + { + name: "no parameters", + funcSig: "func New() pkg.Cataloger { return nil }", + localPackageName: "python", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // parse the function to get parameter type + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "", "package test\n"+tt.funcSig, 0) + require.NoError(t, err) + + // extract the function declaration + require.Len(t, file.Decls, 1) + funcDecl, ok := file.Decls[0].(*ast.FuncDecl) + require.True(t, ok) + + // get first parameter type + var paramType ast.Expr + if funcDecl.Type.Params != nil && len(funcDecl.Type.Params.List) > 0 { + paramType = funcDecl.Type.Params.List[0].Type + } + + got := extractConfigTypeName(paramType, tt.localPackageName) + require.Equal(t, tt.want, got) + }) + } +} + +func TestExtractReturnTypeName(t *testing.T) { + tests := []struct { + name string + funcDef string // complete function definition + want string + }{ + { + name: "pointer to composite literal", + funcDef: `func New() pkg.Cataloger { + return &javaCataloger{name: "test"} + }`, + want: "javaCataloger", + }, + { + name: "composite literal", + funcDef: `func New() pkg.Cataloger { + return pythonCataloger{name: "test"} + }`, + want: "pythonCataloger", + }, + { + name: "variable return", + funcDef: `func New() pkg.Cataloger { + c := &Cataloger{} + return c + }`, + want: "", + }, + { + name: "nil return", + funcDef: `func New() pkg.Cataloger { + return nil + }`, + want: "", + }, + { + name: "empty function body", + funcDef: `func New() pkg.Cataloger {}`, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // parse the function + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "", "package test\n"+tt.funcDef, 0) + require.NoError(t, err) + + // extract the function declaration + require.Len(t, file.Decls, 1) + funcDecl, ok := file.Decls[0].(*ast.FuncDecl) + require.True(t, ok) + + got := extractReturnTypeName(funcDecl) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/internal/capabilities/generate/discover_app_config.go b/internal/capabilities/generate/discover_app_config.go new file mode 100644 index 000000000..0604d4c6d --- /dev/null +++ b/internal/capabilities/generate/discover_app_config.go @@ -0,0 +1,655 @@ +// this file discovers application-level configuration from cmd/syft/internal/options/ by parsing ecosystem config structs, their DescribeFields() methods, and default value functions. +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "reflect" + "sort" + "strings" +) + +// AppConfigField represents an application-level configuration field for catalogers +type AppConfigField struct { + Key string // e.g., "golang.search-local-mod-cache-licenses" + Description string // extracted from DescribeFields() method + DefaultValue interface{} // extracted from Default*() functions +} + +// extractEcosystemConfigFieldsFromCatalog parses catalog.go and extracts the ecosystem-specific +// config fields from the Catalog struct, returning a map of struct type name to YAML tag +func extractEcosystemConfigFieldsFromCatalog(catalogFilePath string) (map[string]string, error) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, catalogFilePath, nil, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("failed to parse catalog.go: %w", err) + } + + // find the Catalog struct + catalogStruct := findConfigStruct(f, "Catalog") + if catalogStruct == nil { + return nil, fmt.Errorf("catalog struct not found in %s", catalogFilePath) + } + + // extract ecosystem config fields from the Catalog struct + // these are between the "ecosystem-specific cataloger configuration" comment and the next section + ecosystemConfigs := make(map[string]string) + inEcosystemSection := false + + for _, field := range catalogStruct.Fields.List { + // check for ecosystem section marker comment + if field.Doc != nil { + for _, comment := range field.Doc.List { + if strings.Contains(comment.Text, "ecosystem-specific cataloger configuration") { + inEcosystemSection = true + break + } + // check if we've hit the next section (any comment marking a new section) + if inEcosystemSection && strings.HasPrefix(comment.Text, "// configuration for") { + inEcosystemSection = false + break + } + } + } + + if !inEcosystemSection { + continue + } + + // extract field type and yaml tag + if len(field.Names) == 0 { + continue + } + + // get the type name (e.g., "golangConfig") + var typeName string + if ident, ok := field.Type.(*ast.Ident); ok { + typeName = ident.Name + } else { + continue + } + + // get the yaml tag + yamlTag := extractYAMLTag(field) + if yamlTag == "" || yamlTag == "-" { + continue + } + + ecosystemConfigs[typeName] = yamlTag + } + + return ecosystemConfigs, nil +} + +// findFilesWithCatalogerImports scans the options directory for .go files that import +// from "github.com/anchore/syft/syft/pkg/cataloger/*" packages +func findFilesWithCatalogerImports(optionsDir string) ([]string, error) { + entries, err := os.ReadDir(optionsDir) + if err != nil { + return nil, fmt.Errorf("failed to read options directory: %w", err) + } + + var candidateFiles []string + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") { + continue + } + + filePath := filepath.Join(optionsDir, entry.Name()) + + // parse the file to check imports + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, filePath, nil, parser.ImportsOnly) + if err != nil { + continue // skip files that can't be parsed + } + + // check if file imports from cataloger packages + for _, imp := range f.Imports { + importPath := strings.Trim(imp.Path.Value, `"`) + if strings.HasPrefix(importPath, "github.com/anchore/syft/syft/pkg/cataloger/") { + candidateFiles = append(candidateFiles, filePath) + break + } + } + } + + return candidateFiles, nil +} + +// extractConfigStructTypes parses a Go file and returns all struct type names defined in it +func extractConfigStructTypes(filePath string) ([]string, error) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, filePath, nil, 0) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", filePath, err) + } + + var structTypes []string + for _, decl := range f.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.TYPE { + continue + } + + for _, spec := range genDecl.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + + // check if it's a struct type + if _, ok := typeSpec.Type.(*ast.StructType); ok { + structTypes = append(structTypes, typeSpec.Name.Name) + } + } + } + + return structTypes, nil +} + +// discoverCatalogerConfigs discovers cataloger config files by: +// 1. Finding files with cataloger imports in options directory +// 2. Extracting ecosystem config fields from Catalog struct +// 3. Matching file structs against Catalog fields +// Returns a map of file path to top-level YAML key and a reverse lookup map of YAML key to struct name +func discoverCatalogerConfigs(repoRoot string) (map[string]string, map[string]string, error) { + optionsDir := filepath.Join(repoRoot, "cmd", "syft", "internal", "options") + catalogFilePath := filepath.Join(optionsDir, "catalog.go") + + // get ecosystem config fields from Catalog struct + ecosystemConfigs, err := extractEcosystemConfigFieldsFromCatalog(catalogFilePath) + if err != nil { + return nil, nil, err + } + + if len(ecosystemConfigs) == 0 { + return nil, nil, fmt.Errorf("no ecosystem config fields found in Catalog struct") + } + + // find files with cataloger imports + candidateFiles, err := findFilesWithCatalogerImports(optionsDir) + if err != nil { + return nil, nil, err + } + + // match candidate files against Catalog ecosystem fields + fileToKey := make(map[string]string) + foundStructs := make(map[string]bool) + + for _, filePath := range candidateFiles { + structTypes, err := extractConfigStructTypes(filePath) + if err != nil { + return nil, nil, err + } + + // check if any struct type matches an ecosystem config + for _, structType := range structTypes { + if yamlKey, exists := ecosystemConfigs[structType]; exists { + fileToKey[filePath] = yamlKey + foundStructs[structType] = true + break + } + } + } + + // validate that all ecosystem configs were found + var missingConfigs []string + for structType := range ecosystemConfigs { + if !foundStructs[structType] { + missingConfigs = append(missingConfigs, structType) + } + } + + if len(missingConfigs) > 0 { + sort.Strings(missingConfigs) + return nil, nil, fmt.Errorf("could not find files for ecosystem configs: %s", strings.Join(missingConfigs, ", ")) + } + + // build reverse lookup map (yamlKey -> structName) + keyToStruct := make(map[string]string) + for structName, yamlKey := range ecosystemConfigs { + keyToStruct[yamlKey] = structName + } + + return fileToKey, keyToStruct, nil +} + +// DiscoverAppConfigs discovers all application-level cataloger configuration fields +// from the options package +func DiscoverAppConfigs(repoRoot string) ([]AppConfigField, error) { + // discover cataloger config files dynamically + configFiles, keyToStruct, err := discoverCatalogerConfigs(repoRoot) + if err != nil { + return nil, fmt.Errorf("failed to discover cataloger configs: %w", err) + } + + // extract configuration fields from each discovered file + var configs []AppConfigField + for filePath, topLevelKey := range configFiles { + fields, err := extractAppConfigFields(filePath, topLevelKey, keyToStruct) + if err != nil { + return nil, fmt.Errorf("failed to extract config from %s: %w", filePath, err) + } + configs = append(configs, fields...) + } + + // sort by key for consistent output + sort.Slice(configs, func(i, j int) bool { + return configs[i].Key < configs[j].Key + }) + + return configs, nil +} + +// extractAppConfigFields extracts config fields from an options file +func extractAppConfigFields(filePath, topLevelKey string, keyToStruct map[string]string) ([]AppConfigField, error) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + return nil, err + } + + var configs []AppConfigField + + // find the main config struct (not nested ones) + configStruct, descriptions := findAppConfigStructAndDescriptions(f, topLevelKey, keyToStruct) + if configStruct == nil { + return nil, fmt.Errorf("no config struct found in %s", filePath) + } + + // extract default values from the default function + defaults := extractAppDefaultValues(f) + + // build config fields from struct fields + for _, field := range configStruct.Fields.List { + // extract yaml tag to get the field key + yamlKey := extractYAMLTag(field) + if yamlKey == "" || yamlKey == "-" { + continue + } + + var fieldName string + if len(field.Names) > 0 { + fieldName = field.Names[0].Name + } else { + continue + } + + // build full key path + fullKey := topLevelKey + "." + yamlKey + + // handle nested structs (e.g., golang.MainModuleVersion) + if isNestedStruct(field.Type) { + nestedConfigs := extractNestedAppConfigs(f, fullKey, fieldName, field.Type, descriptions, defaults) + configs = append(configs, nestedConfigs...) + continue + } + + // get description from DescribeFields + description := descriptions[fieldName] + + // get default value + defaultValue := defaults[fieldName] + + configs = append(configs, AppConfigField{ + Key: fullKey, + Description: description, + DefaultValue: defaultValue, + }) + } + + return configs, nil +} + +// findAppConfigStructAndDescriptions finds the main config struct and extracts field descriptions +// from the DescribeFields method +func findAppConfigStructAndDescriptions(f *ast.File, topLevelKey string, keyToStruct map[string]string) (*ast.StructType, map[string]string) { + structName := keyToStruct[topLevelKey] + configStruct := findConfigStruct(f, structName) + descriptions := extractDescriptionsFromDescribeFields(f) + return configStruct, descriptions +} + +// findConfigStruct searches for the config struct with the expected name in the AST +func findConfigStruct(f *ast.File, expectedName string) *ast.StructType { + for _, decl := range f.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.TYPE { + continue + } + + for _, spec := range genDecl.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + + structType, ok := typeSpec.Type.(*ast.StructType) + if !ok { + continue + } + + if typeSpec.Name.Name == expectedName { + return structType + } + } + } + return nil +} + +// extractDescriptionsFromDescribeFields extracts field descriptions from the DescribeFields method +func extractDescriptionsFromDescribeFields(f *ast.File) map[string]string { + descriptions := make(map[string]string) + + for _, decl := range f.Decls { + funcDecl, ok := decl.(*ast.FuncDecl) + if !ok || funcDecl.Name.Name != "DescribeFields" { + continue + } + + // extract descriptions from descriptions.Add calls + ast.Inspect(funcDecl.Body, func(n ast.Node) bool { + callExpr, ok := n.(*ast.CallExpr) + if !ok { + return true + } + + // check if this is a descriptions.Add call + selector, ok := callExpr.Fun.(*ast.SelectorExpr) + if !ok || selector.Sel.Name != "Add" { + return true + } + + // first argument should be a field reference (&o.FieldName or &o.Parent.FieldName) + if len(callExpr.Args) < 2 { + return true + } + + fieldPath := extractFieldPathFromRef(callExpr.Args[0]) + if fieldPath == "" { + return true + } + + // second argument is the description string + description := extractStringLiteral(callExpr.Args[1]) + if description != "" { + description = cleanDescription(description) + descriptions[fieldPath] = description + } + + return true + }) + } + + return descriptions +} + +// extractNestedAppConfigs handles nested config structs like golang.MainModuleVersion +func extractNestedAppConfigs(f *ast.File, parentKey, parentFieldName string, fieldType ast.Expr, descriptions map[string]string, defaults map[string]interface{}) []AppConfigField { + var configs []AppConfigField + + // find the nested struct type + var nestedStructName string + switch t := fieldType.(type) { + case *ast.Ident: + nestedStructName = t.Name + default: + return nil + } + + // find the struct definition + var nestedStruct *ast.StructType + for _, decl := range f.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.TYPE { + continue + } + + for _, spec := range genDecl.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok || typeSpec.Name.Name != nestedStructName { + continue + } + + var structOk bool + nestedStruct, structOk = typeSpec.Type.(*ast.StructType) + if structOk { + break + } + } + } + + if nestedStruct == nil { + return nil + } + + // extract fields from nested struct + for _, field := range nestedStruct.Fields.List { + yamlKey := extractYAMLTag(field) + if yamlKey == "" || yamlKey == "-" { + continue + } + + var fieldName string + if len(field.Names) > 0 { + fieldName = field.Names[0].Name + } else { + continue + } + + fullKey := parentKey + "." + yamlKey + + // get description using the nested path (e.g., "MainModuleVersion.FromLDFlags") + nestedPath := parentFieldName + "." + fieldName + description := descriptions[nestedPath] + + // try to get default value from nested defaults + var defaultValue interface{} + if nestedDefaults, ok := defaults[parentFieldName].(map[string]interface{}); ok { + defaultValue = nestedDefaults[fieldName] + } + + configs = append(configs, AppConfigField{ + Key: fullKey, + Description: description, + DefaultValue: defaultValue, + }) + } + + return configs +} + +// extractAppDefaultValues extracts default values from the default*Config function +func extractAppDefaultValues(f *ast.File) map[string]interface{} { + defaults := make(map[string]interface{}) + + for _, decl := range f.Decls { + funcDecl, ok := decl.(*ast.FuncDecl) + if !ok || !strings.HasPrefix(funcDecl.Name.Name, "default") { + continue + } + + // look for return statements that construct the config struct + ast.Inspect(funcDecl.Body, func(n ast.Node) bool { + // look for return statements + returnStmt, ok := n.(*ast.ReturnStmt) + if !ok || len(returnStmt.Results) == 0 { + return true + } + + // check if returning a struct literal + compositeLit, ok := returnStmt.Results[0].(*ast.CompositeLit) + if !ok { + return true + } + + // extract field values from the composite literal + for _, elt := range compositeLit.Elts { + kvExpr, ok := elt.(*ast.KeyValueExpr) + if !ok { + continue + } + + // get field name + ident, ok := kvExpr.Key.(*ast.Ident) + if !ok { + continue + } + + fieldName := ident.Name + + // extract the value + value := extractAppValue(kvExpr.Value) + if value != nil { + defaults[fieldName] = value + } + } + + return true + }) + } + + return defaults +} + +// extractAppValue extracts a Go value from an AST expression +func extractAppValue(expr ast.Expr) interface{} { + switch v := expr.(type) { + case *ast.BasicLit: + // string, int, bool literals + switch v.Kind { + case token.STRING: + return strings.Trim(v.Value, `"`) + case token.INT: + return v.Value + case token.FLOAT: + return v.Value + } + case *ast.Ident: + // boolean values + if v.Name == "true" { + return true + } + if v.Name == "false" { + return false + } + if v.Name == "nil" { + return nil + } + case *ast.CompositeLit: + // nested struct literal + nested := make(map[string]interface{}) + for _, elt := range v.Elts { + kvExpr, ok := elt.(*ast.KeyValueExpr) + if !ok { + continue + } + ident, ok := kvExpr.Key.(*ast.Ident) + if !ok { + continue + } + nested[ident.Name] = extractAppValue(kvExpr.Value) + } + if len(nested) > 0 { + return nested + } + } + return nil +} + +// extractYAMLTag extracts the yaml tag value from a struct field +func extractYAMLTag(field *ast.Field) string { + if field.Tag == nil { + return "" + } + + tag := strings.Trim(field.Tag.Value, "`") + tags := reflect.StructTag(tag) + yamlTag := tags.Get("yaml") + + // handle tags like "field-name,omitempty" + if idx := strings.Index(yamlTag, ","); idx != -1 { + yamlTag = yamlTag[:idx] + } + + return yamlTag +} + +// extractFieldPathFromRef extracts field path from & o.FieldName or &o.Parent.FieldName expression +func extractFieldPathFromRef(expr ast.Expr) string { + unaryExpr, ok := expr.(*ast.UnaryExpr) + if !ok { + return "" + } + + // handle nested field references like &o.MainModuleVersion.FromLDFlags + var parts []string + current := unaryExpr.X + + for { + selectorExpr, ok := current.(*ast.SelectorExpr) + if !ok { + break + } + + // add this selector to the path + parts = append([]string{selectorExpr.Sel.Name}, parts...) + + // move to the next level + current = selectorExpr.X + } + + // join the parts with dots to create the full path + // e.g., ["MainModuleVersion", "FromLDFlags"] -> "MainModuleVersion.FromLDFlags" + if len(parts) > 0 { + return strings.Join(parts, ".") + } + + return "" +} + +// extractStringLiteral extracts a string value from a BasicLit node +func extractStringLiteral(expr ast.Expr) string { + lit, ok := expr.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return "" + } + + // remove backticks or quotes + value := strings.Trim(lit.Value, "`\"") + return value +} + +// cleanDescription cleans up multi-line descriptions +func cleanDescription(desc string) string { + // replace multiple whitespace with single space + desc = strings.Join(strings.Fields(desc), " ") + return strings.TrimSpace(desc) +} + +// isNestedStruct checks if a field type is a nested struct (not a pointer or basic type) +func isNestedStruct(expr ast.Expr) bool { + switch t := expr.(type) { + case *ast.Ident: + // check if it's a struct type (not a basic type) + // basic types would be: string, int, bool, etc. + basicTypes := map[string]bool{ + "string": true, "int": true, "int8": true, "int16": true, "int32": true, "int64": true, + "uint": true, "uint8": true, "uint16": true, "uint32": true, "uint64": true, + "float32": true, "float64": true, "bool": true, "byte": true, "rune": true, + } + return !basicTypes[t.Name] + case *ast.StarExpr: + // pointer types are not nested structs for our purposes + return false + case *ast.ArrayType, *ast.MapType: + return false + default: + return false + } +} diff --git a/internal/capabilities/generate/discover_app_config_test.go b/internal/capabilities/generate/discover_app_config_test.go new file mode 100644 index 000000000..2c8bc3704 --- /dev/null +++ b/internal/capabilities/generate/discover_app_config_test.go @@ -0,0 +1,413 @@ +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/capabilities/internal" +) + +// TestAppConfigFieldsHaveDescriptions ensures that all application config fields discovered from the +// options package have descriptions, which are required for user-facing documentation. +func TestAppConfigFieldsHaveDescriptions(t *testing.T) { + checkCompletenessTestsEnabled(t) + + repoRoot, err := internal.RepoRoot() + require.NoError(t, err) + + configs, err := DiscoverAppConfigs(repoRoot) + require.NoError(t, err) + + // verify that all configs have descriptions + var missingDescriptions []string + for _, cfg := range configs { + if cfg.Description == "" { + missingDescriptions = append(missingDescriptions, cfg.Key) + } + } + + require.Empty(t, missingDescriptions, "the following configs are missing descriptions: %v", missingDescriptions) +} + +// TestAppConfigKeyFormat validates that all application config keys follow the expected naming convention +// of "ecosystem.field-name" using kebab-case (lowercase with hyphens, no underscores or spaces). +func TestAppConfigKeyFormat(t *testing.T) { + checkCompletenessTestsEnabled(t) + + repoRoot, err := internal.RepoRoot() + require.NoError(t, err) + + configs, err := DiscoverAppConfigs(repoRoot) + require.NoError(t, err) + + // verify that all config keys follow the expected format + for _, cfg := range configs { + // keys should be in format "ecosystem.field-name" or "ecosystem.nested.field-name" + require.Contains(t, cfg.Key, ".", "config key should contain at least one dot: %s", cfg.Key) + + // keys should use kebab-case (all lowercase with hyphens) + require.NotContains(t, cfg.Key, "_", "config key should not contain underscores: %s", cfg.Key) + require.NotContains(t, cfg.Key, " ", "config key should not contain spaces: %s", cfg.Key) + } +} + +func TestCleanDescription(t *testing.T) { + tests := []struct { + name string + desc string + want string + }{ + { + name: "single line no extra whitespace", + desc: "this is a description", + want: "this is a description", + }, + { + name: "multiple spaces collapsed", + desc: "this has multiple spaces", + want: "this has multiple spaces", + }, + { + name: "multi-line description", + desc: "this is a\nmulti-line\ndescription", + want: "this is a multi-line description", + }, + { + name: "leading and trailing whitespace", + desc: " \t description with spaces \t ", + want: "description with spaces", + }, + { + name: "tabs and newlines", + desc: "description\t\twith\n\ttabs", + want: "description with tabs", + }, + { + name: "empty string", + desc: "", + want: "", + }, + { + name: "only whitespace", + desc: " \n\t ", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := cleanDescription(tt.desc) + require.Equal(t, tt.want, got) + }) + } +} + +func TestExtractYAMLTag(t *testing.T) { + tests := []struct { + name string + tagStr string + want string + }{ + { + name: "simple yaml tag", + tagStr: "`yaml:\"field-name\"`", + want: "field-name", + }, + { + name: "yaml tag with omitempty", + tagStr: "`yaml:\"field-name,omitempty\"`", + want: "field-name", + }, + { + name: "yaml tag with multiple options", + tagStr: "`yaml:\"field-name,omitempty,inline\"`", + want: "field-name", + }, + { + name: "yaml tag dash means skip", + tagStr: "`yaml:\"-\"`", + want: "-", + }, + { + name: "no yaml tag", + tagStr: "`json:\"field-name\"`", + want: "", + }, + { + name: "empty tag", + tagStr: "", + want: "", + }, + { + name: "yaml tag with json tag", + tagStr: "`yaml:\"yaml-name\" json:\"json-name\"`", + want: "yaml-name", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // construct a minimal ast.Field with the tag + field := &ast.Field{} + if tt.tagStr != "" { + field.Tag = &ast.BasicLit{ + Kind: token.STRING, + Value: tt.tagStr, + } + } + + got := extractYAMLTag(field) + require.Equal(t, tt.want, got) + }) + } +} + +func TestIsNestedStruct(t *testing.T) { + tests := []struct { + name string + expr ast.Expr + want bool + }{ + { + name: "custom struct type", + expr: &ast.Ident{Name: "MainModuleVersion"}, + want: true, + }, + { + name: "string type", + expr: &ast.Ident{Name: "string"}, + want: false, + }, + { + name: "int type", + expr: &ast.Ident{Name: "int"}, + want: false, + }, + { + name: "bool type", + expr: &ast.Ident{Name: "bool"}, + want: false, + }, + { + name: "pointer type", + expr: &ast.StarExpr{X: &ast.Ident{Name: "Config"}}, + want: false, + }, + { + name: "array type", + expr: &ast.ArrayType{Elt: &ast.Ident{Name: "string"}}, + want: false, + }, + { + name: "map type", + expr: &ast.MapType{ + Key: &ast.Ident{Name: "string"}, + Value: &ast.Ident{Name: "string"}, + }, + want: false, + }, + { + name: "int32 type", + expr: &ast.Ident{Name: "int32"}, + want: false, + }, + { + name: "uint64 type", + expr: &ast.Ident{Name: "uint64"}, + want: false, + }, + { + name: "float64 type", + expr: &ast.Ident{Name: "float64"}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isNestedStruct(tt.expr) + require.Equal(t, tt.want, got) + }) + } +} + +func TestExtractStringLiteral(t *testing.T) { + tests := []struct { + name string + expr ast.Expr + want string + }{ + { + name: "double quoted string", + expr: &ast.BasicLit{ + Kind: token.STRING, + Value: `"hello world"`, + }, + want: "hello world", + }, + { + name: "backtick string", + expr: &ast.BasicLit{ + Kind: token.STRING, + Value: "`hello world`", + }, + want: "hello world", + }, + { + name: "empty string", + expr: &ast.BasicLit{ + Kind: token.STRING, + Value: `""`, + }, + want: "", + }, + { + name: "string with spaces", + expr: &ast.BasicLit{ + Kind: token.STRING, + Value: `" spaces "`, + }, + want: " spaces ", + }, + { + name: "not a string literal (int)", + expr: &ast.BasicLit{ + Kind: token.INT, + Value: "42", + }, + want: "", + }, + { + name: "not a basic lit", + expr: &ast.Ident{Name: "someVar"}, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractStringLiteral(tt.expr) + require.Equal(t, tt.want, got) + }) + } +} + +func TestExtractFieldPathFromRef(t *testing.T) { + tests := []struct { + name string + src string + want string + }{ + { + name: "simple field reference", + src: "&o.Field", + want: "Field", + }, + { + name: "nested field reference", + src: "&o.Parent.Field", + want: "Parent.Field", + }, + { + name: "deeply nested field reference", + src: "&o.MainModuleVersion.FromLDFlags", + want: "MainModuleVersion.FromLDFlags", + }, + { + name: "three levels deep", + src: "&o.Level1.Level2.Level3", + want: "Level1.Level2.Level3", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // parse the expression + expr, err := parser.ParseExpr(tt.src) + require.NoError(t, err) + + got := extractFieldPathFromRef(expr) + require.Equal(t, tt.want, got) + }) + } +} + +func TestExtractAppValue(t *testing.T) { + tests := []struct { + name string + src string + want interface{} + }{ + { + name: "string literal", + src: `"hello"`, + want: "hello", + }, + { + name: "int literal", + src: "42", + want: "42", + }, + { + name: "float literal", + src: "3.14", + want: "3.14", + }, + { + name: "bool true", + src: "true", + want: true, + }, + { + name: "bool false", + src: "false", + want: false, + }, + { + name: "nil value", + src: "nil", + want: nil, + }, + { + name: "empty string", + src: `""`, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // parse the expression + expr, err := parser.ParseExpr(tt.src) + require.NoError(t, err) + + got := extractAppValue(expr) + require.Equal(t, tt.want, got) + }) + } +} + +func TestExtractAppValue_NestedStruct(t *testing.T) { + // test nested struct separately since it returns a map + src := `struct{Field1 string; Field2 bool}{Field1: "value", Field2: true}` + + // parse as a composite literal + expr, err := parser.ParseExpr(src) + require.NoError(t, err) + + // extract the composite literal + compositeLit, ok := expr.(*ast.CompositeLit) + require.True(t, ok) + + got := extractAppValue(compositeLit) + + // verify it's a map with the expected values + gotMap, ok := got.(map[string]interface{}) + require.True(t, ok) + require.Equal(t, "value", gotMap["Field1"]) + require.Equal(t, true, gotMap["Field2"]) +} diff --git a/internal/capabilities/generate/discover_cataloger_configs.go b/internal/capabilities/generate/discover_cataloger_configs.go new file mode 100644 index 000000000..a6d1b7e34 --- /dev/null +++ b/internal/capabilities/generate/discover_cataloger_configs.go @@ -0,0 +1,425 @@ +// this file discovers cataloger configuration structs using AST parsing to find Config structs and extract fields with app-config annotations. +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/anchore/syft/internal/capabilities/internal" +) + +// ConfigField represents a single field in a configuration struct +type ConfigField struct { + Name string // e.g., "SearchLocalModCacheLicenses" + Type string // e.g., "bool", "string", "[]string", etc. + Description string // extracted from doc comment (1-2 sentences) + AppKey string // from "// app-config: golang.search-local-mod-cache-licenses" +} + +// ConfigInfo represents a discovered configuration struct +type ConfigInfo struct { + PackageName string // e.g., "golang", "python", "dotnet" + StructName string // e.g., "CatalogerConfig", "MainModuleVersionConfig" + Fields []ConfigField // all fields with their metadata +} + +var appConfigAnnotationPattern = regexp.MustCompile(`^//\s*app-config:\s*(.+)$`) + +// DiscoverConfigs walks the cataloger directory and discovers all configuration structs +// Returns map where key is "packageName.StructName" (e.g., "golang.CatalogerConfig") +func DiscoverConfigs(repoRoot string) (map[string]ConfigInfo, error) { + catalogerRoot := filepath.Join(repoRoot, "syft", "pkg", "cataloger") + return DiscoverConfigsFromPath(catalogerRoot) +} + +// DiscoverConfigsFromPath walks the given directory and discovers all configuration structs +// Returns map where key is "packageName.StructName" (e.g., "golang.CatalogerConfig") +func DiscoverConfigsFromPath(catalogerRoot string) (map[string]ConfigInfo, error) { + // find all .go files under the directory recursively + var files []string + err := filepath.Walk(catalogerRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, "_test.go") { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk cataloger directory: %w", err) + } + + discovered := make(map[string]ConfigInfo) + + for _, file := range files { + configs, err := discoverConfigsInFile(file) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", file, err) + } + + for key, config := range configs { + if _, ok := discovered[key]; ok { + return nil, fmt.Errorf("duplicate config struct %q found in %s", key, file) + } + discovered[key] = config + } + } + + return discovered, nil +} + +func discoverConfigsInFile(path string) (map[string]ConfigInfo, error) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + return nil, err + } + + // extract package name from file path (use absolute path, not relative) + packageName := extractPackageNameFromPath(path) + if packageName == "" { + return nil, nil + } + + discovered := make(map[string]ConfigInfo) + + // find all type declarations + for _, decl := range f.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.TYPE { + continue + } + + for _, spec := range genDecl.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + + // check if this is a struct type that looks like a config + structType, ok := typeSpec.Type.(*ast.StructType) + if !ok { + continue + } + + // filter for config-related struct names + structName := typeSpec.Name.Name + if !isConfigStruct(structName) { + continue + } + + // extract fields from the struct + fields := extractCatalogerConfigFields(structType) + if len(fields) == 0 { + // skip structs with no documented config fields + continue + } + + config := ConfigInfo{ + PackageName: packageName, + StructName: structName, + Fields: fields, + } + + key := packageName + "." + structName + discovered[key] = config + } + } + + return discovered, nil +} + +// isConfigStruct determines if a struct name looks like a configuration struct +func isConfigStruct(name string) bool { + // check for common config patterns + return strings.Contains(name, "Config") +} + +// extractCatalogerConfigFields parses struct fields and extracts their metadata +func extractCatalogerConfigFields(structType *ast.StructType) []ConfigField { + return extractCatalogerConfigFieldsRecursive(structType, make(map[string]bool), false) +} + +// extractCatalogerConfigFieldsRecursive parses struct fields and extracts their metadata, +// handling embedded structs recursively with cycle detection. +// requireAppConfig controls whether to require app-config annotations: +// - false for top-level struct (only include fields with app-config) +// - true for embedded structs (include all exported fields) +func extractCatalogerConfigFieldsRecursive(structType *ast.StructType, visitedTypes map[string]bool, fromEmbedded bool) []ConfigField { + var fields []ConfigField + + for _, field := range structType.Fields.List { + // handle embedded fields with no names + if len(field.Names) == 0 { + // this is an embedded field - resolve and extract its fields + embeddedFields := resolveEmbeddedStructFields(field.Type, visitedTypes) + fields = append(fields, embeddedFields...) + continue + } + + // extract field name + fieldName := field.Names[0].Name + + // skip unexported fields + if !ast.IsExported(fieldName) { + continue + } + + // extract field type as string + fieldType := formatFieldType(field.Type) + + // extract doc comment and app-config annotation + description, appKey := extractFieldComments(field.Doc) + + // for top-level fields, only include fields that have an app-config annotation + // for embedded struct fields, include all exported fields + if !fromEmbedded && appKey == "" { + continue + } + + fields = append(fields, ConfigField{ + Name: fieldName, + Type: fieldType, + Description: description, + AppKey: appKey, + }) + } + + return fields +} + +// resolveEmbeddedStructFields resolves an embedded struct type and extracts its fields recursively +func resolveEmbeddedStructFields(fieldType ast.Expr, visitedTypes map[string]bool) []ConfigField { + // extract the type name from the expression + typeName := formatFieldType(fieldType) + + // check for cycles + if visitedTypes[typeName] { + return nil // avoid infinite recursion + } + visitedTypes[typeName] = true + + // parse the type to get package and struct name + // e.g., "cataloging.ArchiveSearchConfig" -> package="cataloging", struct="ArchiveSearchConfig" + var packageName, structName string + if strings.Contains(typeName, ".") { + parts := strings.Split(typeName, ".") + if len(parts) == 2 { + packageName = parts[0] + structName = parts[1] + } + } else { + // embedded type in the same package - would need same-file resolution + // for now, we'll skip these as they're less common + return nil + } + + if packageName == "" || structName == "" { + return nil + } + + // find the file containing this struct + // we need to search in the syft codebase for this package + repoRoot, err := internal.RepoRoot() + if err != nil { + return nil + } + + // try common locations for the package + searchPaths := []string{ + filepath.Join(repoRoot, "syft", packageName), + filepath.Join(repoRoot, "syft", "pkg", packageName), + filepath.Join(repoRoot, "syft", "cataloging", packageName), + } + + // add the direct path if packageName is a subpackage indicator + if strings.Contains(packageName, "/") || !strings.Contains(packageName, ".") { + searchPaths = append(searchPaths, filepath.Join(repoRoot, "syft", packageName)) + } + + for _, searchPath := range searchPaths { + // try to find a .go file in this directory that contains the struct + matches, err := filepath.Glob(filepath.Join(searchPath, "*.go")) + if err != nil { + continue + } + + for _, file := range matches { + // skip test files + if strings.HasSuffix(file, "_test.go") { + continue + } + + // parse the file and look for the struct + structType := findStructInFile(file, structName) + if structType != nil { + // found it! recursively extract fields (fromEmbedded=true means include all exported fields) + return extractCatalogerConfigFieldsRecursive(structType, visitedTypes, true) + } + } + } + + return nil +} + +// findStructInFile parses a Go file and returns the struct type with the given name, or nil if not found +func findStructInFile(filePath, structName string) *ast.StructType { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + return nil + } + + // find the struct declaration + for _, decl := range f.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.TYPE { + continue + } + + for _, spec := range genDecl.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + + if typeSpec.Name.Name == structName { + structType, ok := typeSpec.Type.(*ast.StructType) + if ok { + return structType + } + } + } + } + + return nil +} + +// extractFieldComments parses field comments to extract description and app-config annotation +func extractFieldComments(commentGroup *ast.CommentGroup) (description string, appKey string) { + if commentGroup == nil { + return "", "" + } + + var descLines []string + + for _, comment := range commentGroup.List { + text := strings.TrimPrefix(comment.Text, "//") + text = strings.TrimSpace(text) + + // check if this is an app-config annotation + if matches := appConfigAnnotationPattern.FindStringSubmatch(comment.Text); len(matches) > 1 { + appKey = strings.TrimSpace(matches[1]) + continue + } + + // accumulate description lines + if text != "" { + descLines = append(descLines, text) + } + } + + // join description lines + if len(descLines) > 0 { + description = strings.Join(descLines, " ") + } + + return description, appKey +} + +// formatFieldType converts an ast.Expr type to a readable string representation +func formatFieldType(expr ast.Expr) string { + switch t := expr.(type) { + case *ast.Ident: + return t.Name + case *ast.SelectorExpr: + // handle package.Type (e.g., cataloging.ArchiveSearchConfig) + if x, ok := t.X.(*ast.Ident); ok { + return x.Name + "." + t.Sel.Name + } + return t.Sel.Name + case *ast.ArrayType: + // handle []Type + return "[]" + formatFieldType(t.Elt) + case *ast.MapType: + // handle map[K]V + return "map[" + formatFieldType(t.Key) + "]" + formatFieldType(t.Value) + case *ast.StarExpr: + // handle *Type + return "*" + formatFieldType(t.X) + case *ast.InterfaceType: + return "interface{}" + default: + // fallback for complex types + return fmt.Sprintf("%T", expr) + } +} + +// DiscoverAllowedConfigStructs parses the pkgcataloging.Config struct and returns +// a set of allowed config struct names (e.g., "golang.CatalogerConfig"). +// This is used to filter discovered configs to only include top-level cataloger configs +// that are actually referenced in the main Config struct. +func DiscoverAllowedConfigStructs(repoRoot string) (map[string]bool, error) { + configFilePath := filepath.Join(repoRoot, "syft", "cataloging", "pkgcataloging", "config.go") + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, configFilePath, nil, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + + allowedConfigs := make(map[string]bool) + + // find the Config struct declaration + for _, decl := range f.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.TYPE { + continue + } + + for _, spec := range genDecl.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if !ok { + continue + } + + // we're looking for the "Config" struct specifically + if typeSpec.Name.Name != "Config" { + continue + } + + structType, ok := typeSpec.Type.(*ast.StructType) + if !ok { + continue + } + + // extract field types from the Config struct + for _, field := range structType.Fields.List { + // skip embedded fields with no names + if len(field.Names) == 0 { + continue + } + + // extract field type as "package.StructName" + fieldType := formatFieldType(field.Type) + + // only include types that look like config structs (contain a dot for package.Type) + if strings.Contains(fieldType, ".") { + allowedConfigs[fieldType] = true + } + } + + // we found the Config struct, no need to continue + return allowedConfigs, nil + } + } + + return nil, fmt.Errorf("config struct not found in %s", configFilePath) +} diff --git a/internal/capabilities/generate/discover_cataloger_configs_test.go b/internal/capabilities/generate/discover_cataloger_configs_test.go new file mode 100644 index 000000000..f22bf7b9f --- /dev/null +++ b/internal/capabilities/generate/discover_cataloger_configs_test.go @@ -0,0 +1,459 @@ +package main + +import ( + "go/ast" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/capabilities/internal" +) + +func TestDiscoverConfigs(t *testing.T) { + tests := []struct { + name string + fixturePath string + expectedConfigs []string + verifyConfig func(t *testing.T, configs map[string]ConfigInfo) + }{ + { + name: "simple config with annotations", + fixturePath: "simple-config", + expectedConfigs: []string{ + "golang.CatalogerConfig", + }, + verifyConfig: func(t *testing.T, configs map[string]ConfigInfo) { + golangConfig := configs["golang.CatalogerConfig"] + require.Equal(t, "golang", golangConfig.PackageName) + require.Equal(t, "CatalogerConfig", golangConfig.StructName) + require.Len(t, golangConfig.Fields, 3, "should have 3 annotated fields") + + // verify specific field + var foundSearchLocalModCache bool + for _, field := range golangConfig.Fields { + if field.Name == "SearchLocalModCacheLicenses" { + foundSearchLocalModCache = true + require.Equal(t, "bool", field.Type) + require.Equal(t, "golang.search-local-mod-cache-licenses", field.AppKey) + require.Contains(t, field.Description, "searching for go package licenses") + } + } + require.True(t, foundSearchLocalModCache, "should find SearchLocalModCacheLicenses field") + }, + }, + { + name: "nested config struct", + fixturePath: "nested-config", + expectedConfigs: []string{ + "golang.CatalogerConfig", + "golang.MainModuleVersionConfig", + }, + verifyConfig: func(t *testing.T, configs map[string]ConfigInfo) { + // verify main config + golangConfig := configs["golang.CatalogerConfig"] + require.Equal(t, "golang", golangConfig.PackageName) + require.Equal(t, "CatalogerConfig", golangConfig.StructName) + + // verify nested config + mainModuleConfig := configs["golang.MainModuleVersionConfig"] + require.Equal(t, "golang", mainModuleConfig.PackageName) + require.Equal(t, "MainModuleVersionConfig", mainModuleConfig.StructName) + require.Len(t, mainModuleConfig.Fields, 2, "should have 2 annotated fields") + + // check for specific nested field + var foundFromLDFlags bool + for _, field := range mainModuleConfig.Fields { + if field.Name == "FromLDFlags" { + foundFromLDFlags = true + require.Equal(t, "bool", field.Type) + require.Equal(t, "golang.main-module-version.from-ld-flags", field.AppKey) + require.Contains(t, field.Description, "extract version from LD flags") + } + } + require.True(t, foundFromLDFlags, "should find FromLDFlags field") + }, + }, + { + name: "multiple configs in different packages", + fixturePath: "multiple-configs", + expectedConfigs: []string{ + "python.CatalogerConfig", + "java.ArchiveCatalogerConfig", + }, + verifyConfig: func(t *testing.T, configs map[string]ConfigInfo) { + // verify python config + pythonConfig := configs["python.CatalogerConfig"] + require.Equal(t, "python", pythonConfig.PackageName) + require.Len(t, pythonConfig.Fields, 1) + + // verify java config + javaConfig := configs["java.ArchiveCatalogerConfig"] + require.Equal(t, "java", javaConfig.PackageName) + require.Len(t, javaConfig.Fields, 1) + }, + }, + { + name: "config without annotations", + fixturePath: "no-annotations", + expectedConfigs: []string{}, + verifyConfig: func(t *testing.T, configs map[string]ConfigInfo) { + // should not discover any configs without annotations + require.Empty(t, configs, "should not discover configs without app-config annotations") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixtureDir := filepath.Join("testdata", "config-discovery", tt.fixturePath, "cataloger") + configs, err := DiscoverConfigsFromPath(fixtureDir) + require.NoError(t, err) + + // Debug: log what was discovered + t.Logf("Discovered %d configs:", len(configs)) + for key, config := range configs { + t.Logf(" %s: %d fields", key, len(config.Fields)) + } + + // verify expected configs were discovered + for _, expected := range tt.expectedConfigs { + config, ok := configs[expected] + require.True(t, ok, "should discover config: %s", expected) + require.NotEmpty(t, config.Fields, "config %s should have fields", expected) + require.Equal(t, expected, config.PackageName+"."+config.StructName) + } + + // run custom verification + if tt.verifyConfig != nil { + tt.verifyConfig(t, configs) + } + }) + } +} + +func TestExtractPackageNameFromPath(t *testing.T) { + tests := []struct { + name string + filePath string + want string + }{ + { + name: "golang package", + filePath: "syft/pkg/cataloger/golang/config.go", + want: "golang", + }, + { + name: "java package", + filePath: "syft/pkg/cataloger/java/config.go", + want: "java", + }, + { + name: "python cataloger", + filePath: "syft/pkg/cataloger/python/cataloger.go", + want: "python", + }, + { + name: "kernel cataloger", + filePath: "syft/pkg/cataloger/kernel/cataloger.go", + want: "kernel", + }, + { + name: "binary classifier", + filePath: "syft/pkg/cataloger/binary/classifier_cataloger.go", + want: "binary", + }, + { + name: "not a cataloger path", + filePath: "syft/pkg/other/file.go", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractPackageNameFromPath(tt.filePath) + require.Equal(t, tt.want, got) + }) + } +} + +func TestFormatFieldType(t *testing.T) { + tests := []struct { + name string + expr ast.Expr + want string + }{ + { + name: "basic identifier - string", + expr: &ast.Ident{Name: "string"}, + want: "string", + }, + { + name: "basic identifier - bool", + expr: &ast.Ident{Name: "bool"}, + want: "bool", + }, + { + name: "basic identifier - int", + expr: &ast.Ident{Name: "int"}, + want: "int", + }, + { + name: "selector expression - package.Type", + expr: &ast.SelectorExpr{ + X: &ast.Ident{Name: "time"}, + Sel: &ast.Ident{Name: "Time"}, + }, + want: "time.Time", + }, + { + name: "selector expression - cataloging.Config", + expr: &ast.SelectorExpr{ + X: &ast.Ident{Name: "cataloging"}, + Sel: &ast.Ident{Name: "ArchiveSearchConfig"}, + }, + want: "cataloging.ArchiveSearchConfig", + }, + { + name: "array of strings", + expr: &ast.ArrayType{ + Elt: &ast.Ident{Name: "string"}, + }, + want: "[]string", + }, + { + name: "array of ints", + expr: &ast.ArrayType{ + Elt: &ast.Ident{Name: "int"}, + }, + want: "[]int", + }, + { + name: "map[string]bool", + expr: &ast.MapType{ + Key: &ast.Ident{Name: "string"}, + Value: &ast.Ident{Name: "bool"}, + }, + want: "map[string]bool", + }, + { + name: "map[string]int", + expr: &ast.MapType{ + Key: &ast.Ident{Name: "string"}, + Value: &ast.Ident{Name: "int"}, + }, + want: "map[string]int", + }, + { + name: "pointer to type", + expr: &ast.StarExpr{ + X: &ast.Ident{Name: "Config"}, + }, + want: "*Config", + }, + { + name: "pointer to selector", + expr: &ast.StarExpr{ + X: &ast.SelectorExpr{ + X: &ast.Ident{Name: "time"}, + Sel: &ast.Ident{Name: "Time"}, + }, + }, + want: "*time.Time", + }, + { + name: "interface{}", + expr: &ast.InterfaceType{ + Methods: &ast.FieldList{}, + }, + want: "interface{}", + }, + { + name: "nested array of arrays", + expr: &ast.ArrayType{ + Elt: &ast.ArrayType{ + Elt: &ast.Ident{Name: "string"}, + }, + }, + want: "[][]string", + }, + { + name: "map with array value", + expr: &ast.MapType{ + Key: &ast.Ident{Name: "string"}, + Value: &ast.ArrayType{ + Elt: &ast.Ident{Name: "int"}, + }, + }, + want: "map[string][]int", + }, + { + name: "pointer to array", + expr: &ast.StarExpr{ + X: &ast.ArrayType{ + Elt: &ast.Ident{Name: "string"}, + }, + }, + want: "*[]string", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := formatFieldType(tt.expr) + require.Equal(t, tt.want, got) + }) + } +} + +func TestExtractFieldComments(t *testing.T) { + tests := []struct { + name string + commentGroup *ast.CommentGroup + wantDescription string + wantAppKey string + }{ + { + name: "nil comment group", + commentGroup: nil, + wantDescription: "", + wantAppKey: "", + }, + { + name: "empty comment group", + commentGroup: &ast.CommentGroup{ + List: []*ast.Comment{}, + }, + wantDescription: "", + wantAppKey: "", + }, + { + name: "app-config annotation only", + commentGroup: &ast.CommentGroup{ + List: []*ast.Comment{ + {Text: "// app-config: golang.search-local-mod-cache-licenses"}, + }, + }, + wantDescription: "", + wantAppKey: "golang.search-local-mod-cache-licenses", + }, + { + name: "description only", + commentGroup: &ast.CommentGroup{ + List: []*ast.Comment{ + {Text: "// enable searching for go package licenses in the local mod cache"}, + }, + }, + wantDescription: "enable searching for go package licenses in the local mod cache", + wantAppKey: "", + }, + { + name: "description and app-config", + commentGroup: &ast.CommentGroup{ + List: []*ast.Comment{ + {Text: "// enable searching for go package licenses in the local mod cache"}, + {Text: "// app-config: golang.search-local-mod-cache-licenses"}, + }, + }, + wantDescription: "enable searching for go package licenses in the local mod cache", + wantAppKey: "golang.search-local-mod-cache-licenses", + }, + { + name: "app-config before description", + commentGroup: &ast.CommentGroup{ + List: []*ast.Comment{ + {Text: "// app-config: golang.search-local-mod-cache-licenses"}, + {Text: "// enable searching for go package licenses in the local mod cache"}, + }, + }, + wantDescription: "enable searching for go package licenses in the local mod cache", + wantAppKey: "golang.search-local-mod-cache-licenses", + }, + { + name: "multi-line description", + commentGroup: &ast.CommentGroup{ + List: []*ast.Comment{ + {Text: "// this is the first line of the description."}, + {Text: "// this is the second line of the description."}, + {Text: "// app-config: test.multi-line"}, + }, + }, + wantDescription: "this is the first line of the description. this is the second line of the description.", + wantAppKey: "test.multi-line", + }, + { + name: "app-config with extra whitespace", + commentGroup: &ast.CommentGroup{ + List: []*ast.Comment{ + {Text: "// app-config: golang.test-key "}, + }, + }, + wantDescription: "", + wantAppKey: "golang.test-key", + }, + { + name: "description with special characters", + commentGroup: &ast.CommentGroup{ + List: []*ast.Comment{ + {Text: "// enable searching for Go's package licenses (*.mod files)"}, + {Text: "// app-config: golang.search"}, + }, + }, + wantDescription: "enable searching for Go's package licenses (*.mod files)", + wantAppKey: "golang.search", + }, + { + name: "comment with empty lines", + commentGroup: &ast.CommentGroup{ + List: []*ast.Comment{ + {Text: "// first line"}, + {Text: "//"}, + {Text: "// second line"}, + {Text: "// app-config: test.key"}, + }, + }, + wantDescription: "first line second line", + wantAppKey: "test.key", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotDescription, gotAppKey := extractFieldComments(tt.commentGroup) + require.Equal(t, tt.wantDescription, gotDescription) + require.Equal(t, tt.wantAppKey, gotAppKey) + }) + } +} + +func TestDiscoverAllowedConfigStructs(t *testing.T) { + repoRoot, err := internal.RepoRoot() + require.NoError(t, err) + + allowedConfigs, err := DiscoverAllowedConfigStructs(repoRoot) + require.NoError(t, err) + + // verify we found multiple config types + require.NotEmpty(t, allowedConfigs, "should discover at least one allowed config type") + + // verify specific config types that should be in pkgcataloging.Config + expectedConfigs := []string{ + "golang.CatalogerConfig", + "java.ArchiveCatalogerConfig", + "python.CatalogerConfig", + "dotnet.CatalogerConfig", + "kernel.LinuxKernelCatalogerConfig", + "javascript.CatalogerConfig", + } + + for _, expected := range expectedConfigs { + require.True(t, allowedConfigs[expected], "should find %s in allowed configs", expected) + } + + // log all discovered configs for manual inspection + t.Logf("Discovered %d allowed config types:", len(allowedConfigs)) + for configType := range allowedConfigs { + t.Logf(" - %s", configType) + } +} diff --git a/internal/capabilities/generate/discover_catalogers.go b/internal/capabilities/generate/discover_catalogers.go new file mode 100644 index 000000000..b6f020c84 --- /dev/null +++ b/internal/capabilities/generate/discover_catalogers.go @@ -0,0 +1,651 @@ +// this file discovers generic catalogers from source code by walking syft/pkg/cataloger/ and using AST parsing to find generic.NewCataloger() calls and extract parser information. +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + + "github.com/anchore/syft/internal/capabilities" + "github.com/anchore/syft/syft/pkg/cataloger/binary" +) + +// DiscoveredCataloger represents a cataloger discovered by parsing source code +type DiscoveredCataloger struct { + Name string + Type string // "generic" or "custom" + PackageName string // e.g., "python", "swift" - extracted from source file path + SourceFile string + SourceFunction string + Parsers []DiscoveredParser // only for generic catalogers +} + +// DiscoveredParser represents a parser function and its detection criteria discovered from source +type DiscoveredParser struct { + ParserFunction string + Method capabilities.ArtifactDetectionMethod + Criteria []string + MetadataTypes []string // populated from cataloger-type-uses.json files + PackageTypes []string // populated from cataloger-type-uses.json files + JSONSchemaTypes []string // populated from MetadataTypes +} + +// discoverGenericCatalogers finds all uses of generic.NewCataloger() in the codebase +// Returns map[catalogerName]DiscoveredCataloger +func discoverGenericCatalogers(repoRoot string) (map[string]DiscoveredCataloger, error) { + catalogerRoot := filepath.Join(repoRoot, "syft", "pkg", "cataloger") + + // find all .go files under syft/pkg/cataloger/ recursively + var files []string + err := filepath.Walk(catalogerRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, "_test.go") { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk cataloger directory: %w", err) + } + + discovered := make(map[string]DiscoveredCataloger) + + for _, file := range files { + catalogers, err := discoverGenericCatalogersInFile(file, repoRoot) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", file, err) + } + + for name, cataloger := range catalogers { + if existing, ok := discovered[name]; ok { + return nil, fmt.Errorf("duplicate cataloger name %q found in %s and %s", name, existing.SourceFile, cataloger.SourceFile) + } + discovered[name] = cataloger + } + } + + return discovered, nil +} + +func discoverGenericCatalogersInFile(path, repoRoot string) (map[string]DiscoveredCataloger, error) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + return nil, err + } + + discovered := make(map[string]DiscoveredCataloger) + + // find all function declarations + for _, decl := range f.Decls { + funcDecl, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + + // check if function name matches New*Cataloger pattern + if !strings.HasPrefix(funcDecl.Name.Name, "New") || !strings.HasSuffix(funcDecl.Name.Name, "Cataloger") { + continue + } + + // check if return type is pkg.Cataloger + if !returnsPackageCataloger(funcDecl) { + continue + } + + // parse the function body to find generic.NewCataloger calls + cataloger, err := parseGenericCatalogerFunction(funcDecl, path, repoRoot) + if err != nil { + // not a generic cataloger, skip + continue + } + + if cataloger != nil { + discovered[cataloger.Name] = *cataloger + } + } + + return discovered, nil +} + +func returnsPackageCataloger(funcDecl *ast.FuncDecl) bool { + if funcDecl.Type.Results == nil || len(funcDecl.Type.Results.List) != 1 { + return false + } + + // check if the return type is pkg.Cataloger or just Cataloger + returnType := funcDecl.Type.Results.List[0].Type + selector, ok := returnType.(*ast.SelectorExpr) + if !ok { + // might be just "Cataloger" without package prefix + ident, ok := returnType.(*ast.Ident) + return ok && ident.Name == "Cataloger" + } + + pkg, ok := selector.X.(*ast.Ident) + if !ok { + return false + } + + return pkg.Name == "pkg" && selector.Sel.Name == "Cataloger" +} + +func parseGenericCatalogerFunction(funcDecl *ast.FuncDecl, filePath, repoRoot string) (*DiscoveredCataloger, error) { + if funcDecl.Body == nil { + return nil, fmt.Errorf("function has no body") + } + + // parse the file again to get imports and constants context + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments) + if err != nil { + return nil, err + } + + ctx := &parseContext{ + file: f, + filePath: filePath, + repoRoot: repoRoot, + } + + var catalogerName string + var parsers []DiscoveredParser + + // walk the function body to find generic.NewCataloger call + ast.Inspect(funcDecl.Body, func(n ast.Node) bool { + // look for call expressions + callExpr, ok := n.(*ast.CallExpr) + if !ok { + return true + } + + // check if this is a call to generic.NewCataloger + if isGenericNewCatalogerCall(callExpr) { + // extract the cataloger name from the first argument + if len(callExpr.Args) > 0 { + // handle string literals + if lit, ok := callExpr.Args[0].(*ast.BasicLit); ok && lit.Kind == token.STRING { + catalogerName = strings.Trim(lit.Value, `"`) + } + // handle constant identifiers + if ident, ok := callExpr.Args[0].(*ast.Ident); ok { + catalogerName = resolveLocalConstant(ident.Name, ctx) + } + } + } + + // check if this is a WithParserBy* call + if p := parseWithParserByCall(callExpr, ctx); p != nil { + parsers = append(parsers, *p) + } + + return true + }) + + if catalogerName == "" { + // not a generic cataloger + return nil, fmt.Errorf("no generic.NewCataloger call found") + } + + // make file path relative to repo root + relPath, err := filepath.Rel(repoRoot, filePath) + if err != nil { + relPath = filePath + } + + return &DiscoveredCataloger{ + Name: catalogerName, + Type: genericCatalogerType, + PackageName: extractPackageNameFromPath(relPath), + SourceFile: relPath, + SourceFunction: funcDecl.Name.Name, + Parsers: parsers, + }, nil +} + +// extractPackageNameFromPath extracts the package name from a cataloger source file path +// e.g., "syft/pkg/cataloger/swift/cataloger.go" -> "swift" +func extractPackageNameFromPath(filePath string) string { + parts := strings.Split(filePath, string(filepath.Separator)) + + // find the LAST occurrence of "cataloger" in the path + // (to handle test fixtures with multiple "cataloger" segments) + lastCatalogerIndex := -1 + for i, part := range parts { + if part == "cataloger" { + lastCatalogerIndex = i + } + } + + if lastCatalogerIndex != -1 && lastCatalogerIndex+1 < len(parts) { + // return the next segment after the last "cataloger" + return parts[lastCatalogerIndex+1] + } + + return "" +} + +func isGenericNewCatalogerCall(callExpr *ast.CallExpr) bool { + selector, ok := callExpr.Fun.(*ast.SelectorExpr) + if !ok { + return false + } + + pkg, ok := selector.X.(*ast.Ident) + if !ok { + return false + } + + return pkg.Name == "generic" && selector.Sel.Name == "NewCataloger" +} + +type parseContext struct { + file *ast.File + filePath string + repoRoot string +} + +func parseWithParserByCall(callExpr *ast.CallExpr, ctx *parseContext) *DiscoveredParser { + selector, ok := callExpr.Fun.(*ast.SelectorExpr) + if !ok { + return nil + } + + methodName := selector.Sel.Name + + var method capabilities.ArtifactDetectionMethod + switch { + case strings.HasPrefix(methodName, "WithParserByGlobs"): + method = capabilities.GlobDetection + case strings.HasPrefix(methodName, "WithParserByPath"): + method = capabilities.PathDetection + case strings.HasPrefix(methodName, "WithParserByMimeTypes"): + method = capabilities.MIMETypeDetection + default: + return nil + } + + if len(callExpr.Args) < 2 { + return nil + } + + // first argument is the parser function name + var parserFunction string + switch arg := callExpr.Args[0].(type) { + case *ast.Ident: + // simple identifier: parseFunc + parserFunction = arg.Name + case *ast.SelectorExpr: + // selector expression: adapter.parseFunc + parserFunction = arg.Sel.Name + default: + return nil + } + + // remaining arguments are detection criteria (can be string literals, constants, or method calls) + var criteria []string + for _, arg := range callExpr.Args[1:] { + values := resolveCriteriaValues(arg, ctx) + criteria = append(criteria, values...) + } + + if len(criteria) == 0 { + return nil + } + + return &DiscoveredParser{ + ParserFunction: parserFunction, + Method: method, + Criteria: criteria, + } +} + +// resolveCriteriaValues resolves criteria argument(s) to string value(s) +// handles string literals, constants, and method calls like .List() +func resolveCriteriaValues(arg ast.Expr, ctx *parseContext) []string { + switch expr := arg.(type) { + case *ast.BasicLit: + // direct string literal + if expr.Kind == token.STRING { + value := strings.Trim(expr.Value, `"`) + if value != "" { + return []string{value} + } + } + + case *ast.Ident: + // local constant reference + value := resolveLocalConstant(expr.Name, ctx) + if value != "" { + return []string{value} + } + + case *ast.SelectorExpr: + // imported constant reference (e.g., pkg.ApkDBGlob) + if pkgIdent, ok := expr.X.(*ast.Ident); ok { + value := resolveImportedConstant(pkgIdent.Name, expr.Sel.Name, ctx) + if value != "" { + return []string{value} + } + } + + case *ast.CallExpr: + // method call like mimetype.ExecutableMIMETypeSet.List() + return resolveMethodCallValues(expr, ctx) + } + + return nil +} + +// resolveMethodCallValues resolves method calls that return string slices +// specifically handles .List() calls on string sets +func resolveMethodCallValues(callExpr *ast.CallExpr, ctx *parseContext) []string { + // check if this is a .List() method call + selector, ok := callExpr.Fun.(*ast.SelectorExpr) + if !ok || selector.Sel.Name != "List" { + return nil + } + + // get the receiver (e.g., mimetype.ExecutableMIMETypeSet from mimetype.ExecutableMIMETypeSet.List()) + var pkgName, varName string + + switch recv := selector.X.(type) { + case *ast.SelectorExpr: + // format: package.Variable (e.g., mimetype.ExecutableMIMETypeSet) + if pkgIdent, ok := recv.X.(*ast.Ident); ok { + pkgName = pkgIdent.Name + varName = recv.Sel.Name + } + case *ast.Ident: + // format: Variable (local variable) + varName = recv.Name + } + + if varName == "" { + return nil + } + + // try to resolve the variable to its declaration and extract the string slice + if pkgName != "" { + // imported variable (e.g., mimetype.ExecutableMIMETypeSet) + return resolveImportedVariable(pkgName, varName, ctx) + } + + // local variable + return resolveLocalVariable(varName, ctx) +} + +// resolveImportedVariable finds a variable in an imported package and extracts its string slice values +func resolveImportedVariable(pkgName, varName string, ctx *parseContext) []string { + // find the import path for this package + var importPath string + for _, imp := range ctx.file.Imports { + path := strings.Trim(imp.Path.Value, `"`) + + if imp.Name != nil && imp.Name.Name == pkgName { + importPath = path + break + } + + parts := strings.Split(path, "/") + if len(parts) > 0 && parts[len(parts)-1] == pkgName { + importPath = path + break + } + } + + if importPath == "" { + return nil + } + + // resolve import path to file system path + pkgDir := resolveImportPath(importPath, ctx.repoRoot) + if pkgDir == "" { + return nil + } + + // find the variable in the package and extract string slice + return findVariableStringSlice(pkgDir, varName) +} + +// resolveLocalVariable finds a local variable and extracts its string slice values +func resolveLocalVariable(varName string, ctx *parseContext) []string { + return extractStringSliceFromFile(ctx.file, varName) +} + +// findVariableStringSlice searches for a variable in a package directory and extracts its string slice +func findVariableStringSlice(pkgDir, varName string) []string { + files, err := filepath.Glob(filepath.Join(pkgDir, "*.go")) + if err != nil { + return nil + } + + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, file, nil, 0) + if err != nil { + continue + } + + if values := extractStringSliceFromFile(f, varName); len(values) > 0 { + return values + } + } + + return nil +} + +// extractStringSliceFromFile extracts string values from a variable declaration like: +// var Foo = strset.New([]string{"a", "b", "c"}...) +func extractStringSliceFromFile(file *ast.File, varName string) []string { + for _, decl := range file.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.VAR { + continue + } + + for _, spec := range genDecl.Specs { + valueSpec, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + + for i, ident := range valueSpec.Names { + if ident.Name == varName && i < len(valueSpec.Values) { + return extractStringSliceFromExpr(valueSpec.Values[i]) + } + } + } + } + + return nil +} + +// extractStringSliceFromExpr extracts string literals from expressions like: +// strset.New([]string{"a", "b"}...) +func extractStringSliceFromExpr(expr ast.Expr) []string { + // handle strset.New(...) calls + callExpr, ok := expr.(*ast.CallExpr) + if !ok || len(callExpr.Args) == 0 { + return nil + } + + // get the first argument (should be a composite literal with strings) + arg := callExpr.Args[0] + + // handle []string{...} composite literals + compositeLit, ok := arg.(*ast.CompositeLit) + if !ok { + return nil + } + + var values []string + for _, elt := range compositeLit.Elts { + if lit, ok := elt.(*ast.BasicLit); ok && lit.Kind == token.STRING { + value := strings.Trim(lit.Value, `"`) + if value != "" { + values = append(values, value) + } + } + } + + return values +} + +// resolveLocalConstant finds a constant definition in the current file +func resolveLocalConstant(name string, ctx *parseContext) string { + for _, decl := range ctx.file.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.CONST { + continue + } + + for _, spec := range genDecl.Specs { + valueSpec, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + + for i, ident := range valueSpec.Names { + if ident.Name == name && i < len(valueSpec.Values) { + if lit, ok := valueSpec.Values[i].(*ast.BasicLit); ok && lit.Kind == token.STRING { + return strings.Trim(lit.Value, `"`) + } + } + } + } + } + + return "" +} + +// resolveImportedConstant finds a constant in an imported package +func resolveImportedConstant(pkgName, constName string, ctx *parseContext) string { + // find the import path for this package alias + var importPath string + for _, imp := range ctx.file.Imports { + path := strings.Trim(imp.Path.Value, `"`) + + // check if this import has the alias we're looking for + if imp.Name != nil && imp.Name.Name == pkgName { + importPath = path + break + } + + // check if the package name matches (last component of import path) + parts := strings.Split(path, "/") + if len(parts) > 0 && parts[len(parts)-1] == pkgName { + importPath = path + break + } + } + + if importPath == "" { + return "" + } + + // resolve import path to file system path + pkgDir := resolveImportPath(importPath, ctx.repoRoot) + if pkgDir == "" { + return "" + } + + // parse all .go files in the package directory to find the constant + return findConstantInPackage(pkgDir, constName) +} + +// resolveImportPath converts an import path to a file system path +func resolveImportPath(importPath, repoRoot string) string { + // for github.com/anchore/syft/... imports, convert to repo-relative path + if strings.HasPrefix(importPath, "github.com/anchore/syft/") { + relPath := strings.TrimPrefix(importPath, "github.com/anchore/syft/") + return filepath.Join(repoRoot, relPath) + } + + return "" +} + +// findConstantInPackage searches for a constant definition in a package directory +func findConstantInPackage(pkgDir, constName string) string { + files, err := filepath.Glob(filepath.Join(pkgDir, "*.go")) + if err != nil { + return "" + } + + for _, file := range files { + if strings.HasSuffix(file, "_test.go") { + continue + } + + if value := findConstantInFile(file, constName); value != "" { + return value + } + } + + return "" +} + +func findConstantInFile(filePath, constName string) string { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, filePath, nil, 0) + if err != nil { + return "" + } + + for _, decl := range f.Decls { + if value := searchConstInDecl(decl, constName); value != "" { + return value + } + } + + return "" +} + +func searchConstInDecl(decl ast.Decl, constName string) string { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.CONST { + return "" + } + + for _, spec := range genDecl.Specs { + valueSpec, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + + if value := getConstValue(valueSpec, constName); value != "" { + return value + } + } + + return "" +} + +func getConstValue(valueSpec *ast.ValueSpec, constName string) string { + for i, ident := range valueSpec.Names { + if ident.Name == constName && i < len(valueSpec.Values) { + if lit, ok := valueSpec.Values[i].(*ast.BasicLit); ok && lit.Kind == token.STRING { + return strings.Trim(lit.Value, `"`) + } + } + } + return "" +} + +// extractBinaryClassifiers extracts all binary classifiers with their full information +func extractBinaryClassifiers() []binary.Classifier { //nolint:staticcheck + classifiers := binary.DefaultClassifiers() + + // return all classifiers (already sorted by the default function) + return classifiers +} diff --git a/internal/capabilities/generate/discover_catalogers_test.go b/internal/capabilities/generate/discover_catalogers_test.go new file mode 100644 index 000000000..0672e9f6f --- /dev/null +++ b/internal/capabilities/generate/discover_catalogers_test.go @@ -0,0 +1,389 @@ +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReturnsPackageCataloger(t *testing.T) { + tests := []struct { + name string + code string + want bool + }{ + { + name: "returns pkg.Cataloger", + code: `func NewFoo() pkg.Cataloger { return nil }`, + want: true, + }, + { + name: "returns bare Cataloger", + code: `func NewFoo() Cataloger { return nil }`, + want: true, + }, + { + name: "returns multiple values", + code: `func NewFoo() (pkg.Cataloger, error) { return nil, nil }`, + want: false, + }, + { + name: "returns error", + code: `func NewFoo() error { return nil }`, + want: false, + }, + { + name: "returns pointer to Cataloger", + code: `func NewFoo() *pkg.Cataloger { return nil }`, + want: false, + }, + { + name: "returns string", + code: `func NewFoo() string { return "" }`, + want: false, + }, + { + name: "no return type", + code: `func NewFoo() { }`, + want: false, + }, + { + name: "returns wrong package Cataloger", + code: `func NewFoo() other.Cataloger { return nil }`, + want: false, + }, + { + name: "returns pkg.OtherType", + code: `func NewFoo() pkg.OtherType { return nil }`, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + funcDecl := parseFuncDecl(t, tt.code) + got := returnsPackageCataloger(funcDecl) + require.Equal(t, tt.want, got) + }) + } +} + +func TestIsGenericNewCatalogerCall(t *testing.T) { + tests := []struct { + name string + code string + want bool + }{ + { + name: "generic.NewCataloger call", + code: `generic.NewCataloger("foo")`, + want: true, + }, + { + name: "generic.NewCataloger with no args", + code: `generic.NewCataloger()`, + want: true, + }, + { + name: "other.NewCataloger call", + code: `other.NewCataloger("foo")`, + want: false, + }, + { + name: "generic.OtherMethod call", + code: `generic.OtherMethod("foo")`, + want: false, + }, + { + name: "bare NewCataloger call", + code: `NewCataloger("foo")`, + want: false, + }, + { + name: "nested call", + code: `foo(generic.NewCataloger("bar"))`, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + callExpr := parseCallExpr(t, tt.code) + got := isGenericNewCatalogerCall(callExpr) + require.Equal(t, tt.want, got) + }) + } +} + +func TestExtractStringSliceFromExpr(t *testing.T) { + tests := []struct { + name string + code string + want []string + }{ + { + name: "strset.New with strings", + code: `strset.New([]string{"foo", "bar", "baz"})`, + want: []string{"foo", "bar", "baz"}, + }, + { + name: "strset.New with single string", + code: `strset.New([]string{"single"})`, + want: []string{"single"}, + }, + { + name: "strset.New with empty slice", + code: `strset.New([]string{})`, + want: nil, + }, + { + name: "other.New with strings", + code: `other.New([]string{"x", "y"})`, + want: []string{"x", "y"}, + }, + { + name: "call with no args", + code: `strset.New()`, + want: nil, + }, + { + name: "call with non-composite-literal arg", + code: `strset.New("not a slice")`, + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + callExpr := parseCallExpr(t, tt.code) + got := extractStringSliceFromExpr(callExpr) + require.Equal(t, tt.want, got) + }) + } +} + +func TestSearchConstInDecl(t *testing.T) { + tests := []struct { + name string + code string + constName string + want string + }{ + { + name: "single const", + code: `const Foo = "bar"`, + constName: "Foo", + want: "bar", + }, + { + name: "grouped consts - first", + code: `const ( + Foo = "bar" + Baz = "qux" + )`, + constName: "Foo", + want: "bar", + }, + { + name: "grouped consts - second", + code: `const ( + Foo = "bar" + Baz = "qux" + )`, + constName: "Baz", + want: "qux", + }, + { + name: "const not found", + code: `const Foo = "bar"`, + constName: "Missing", + want: "", + }, + { + name: "var declaration instead of const", + code: `var Foo = "bar"`, + constName: "Foo", + want: "", + }, + { + name: "const with non-string value", + code: `const Foo = 42`, + constName: "Foo", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + genDecl := parseConstDecl(t, tt.code) + got := searchConstInDecl(genDecl, tt.constName) + require.Equal(t, tt.want, got) + }) + } +} + +func TestGetConstValue(t *testing.T) { + tests := []struct { + name string + code string + constName string + want string + }{ + { + name: "single const match", + code: `const Foo = "bar"`, + constName: "Foo", + want: "bar", + }, + { + name: "no match", + code: `const Foo = "bar"`, + constName: "NotFoo", + want: "", + }, + { + name: "non-string literal", + code: `const Foo = 123`, + constName: "Foo", + want: "", + }, + { + name: "const with complex value", + code: `const Foo = Bar + "suffix"`, + constName: "Foo", + want: "", + }, + { + name: "first of multiple in same spec", + code: `const Foo, Bar = "baz", "qux"`, + constName: "Foo", + want: "baz", + }, + { + name: "second of multiple in same spec", + code: `const Foo, Bar = "baz", "qux"`, + constName: "Bar", + want: "qux", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + genDecl := parseConstDecl(t, tt.code) + require.Equal(t, token.CONST, genDecl.Tok) + require.NotEmpty(t, genDecl.Specs) + + // getConstValue works on a single ValueSpec, so we need to find the right one + // in case of grouped constants, each const is its own spec + var got string + for _, spec := range genDecl.Specs { + valueSpec, ok := spec.(*ast.ValueSpec) + require.True(t, ok) + + got = getConstValue(valueSpec, tt.constName) + if got != "" { + break + } + } + + require.Equal(t, tt.want, got) + }) + } +} + +func TestResolveImportPath(t *testing.T) { + const testRepoRoot = "/repo/root" + + tests := []struct { + name string + importPath string + want string + }{ + { + name: "syft pkg cataloger golang", + importPath: "github.com/anchore/syft/syft/pkg/cataloger/golang", + want: "/repo/root/syft/pkg/cataloger/golang", + }, + { + name: "syft internal capabilities", + importPath: "github.com/anchore/syft/internal/capabilities", + want: "/repo/root/internal/capabilities", + }, + { + name: "syft root package", + importPath: "github.com/anchore/syft/syft", + want: "/repo/root/syft", + }, + { + name: "external package", + importPath: "github.com/other/repo/pkg", + want: "", + }, + { + name: "standard library", + importPath: "fmt", + want: "", + }, + { + name: "empty import path", + importPath: "", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := resolveImportPath(tt.importPath, testRepoRoot) + require.Equal(t, tt.want, got) + }) + } +} + +// test helper functions + +// parseFuncDecl parses a function declaration from a code string +func parseFuncDecl(t *testing.T, code string) *ast.FuncDecl { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "", "package test\n"+code, 0) + require.NoError(t, err) + require.Len(t, file.Decls, 1, "expected exactly one declaration") + funcDecl, ok := file.Decls[0].(*ast.FuncDecl) + require.True(t, ok, "expected declaration to be a function") + return funcDecl +} + +// parseCallExpr parses a call expression from a code string +func parseCallExpr(t *testing.T, code string) *ast.CallExpr { + t.Helper() + expr, err := parser.ParseExpr(code) + require.NoError(t, err) + callExpr, ok := expr.(*ast.CallExpr) + require.True(t, ok, "expected expression to be a call expression") + return callExpr +} + +// parseCompositeLit parses a composite literal from a code string +func parseCompositeLit(t *testing.T, code string) *ast.CompositeLit { + t.Helper() + expr, err := parser.ParseExpr(code) + require.NoError(t, err) + lit, ok := expr.(*ast.CompositeLit) + require.True(t, ok, "expected expression to be a composite literal") + return lit +} + +// parseConstDecl parses a const declaration from a code string and returns the GenDecl +func parseConstDecl(t *testing.T, code string) *ast.GenDecl { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "", "package test\n"+code, 0) + require.NoError(t, err) + require.Len(t, file.Decls, 1, "expected exactly one declaration") + genDecl, ok := file.Decls[0].(*ast.GenDecl) + require.True(t, ok, "expected declaration to be a general declaration") + return genDecl +} diff --git a/internal/capabilities/generate/discover_metadata.go b/internal/capabilities/generate/discover_metadata.go new file mode 100644 index 000000000..603ad8258 --- /dev/null +++ b/internal/capabilities/generate/discover_metadata.go @@ -0,0 +1,323 @@ +// this file discovers metadata and package types by reading test-observations.json files generated by pkgtest helpers during test execution. +package main + +import ( + "fmt" + "os" + "path/filepath" + "sort" + + "github.com/anchore/syft/internal/capabilities/internal" + "github.com/anchore/syft/internal/capabilities/pkgtestobservation" + "github.com/anchore/syft/internal/packagemetadata" +) + +// TestObservationIndex indexes all test observations for efficient lookup and application. +// parser-level observations are indexed by package name (from test file) + parser function, +// while cataloger-level observations are indexed by cataloger name. +type TestObservationIndex struct { + // parser-level observations: package -> parser function -> types + parsersByPackage map[string]map[string]*TypeObservation + // cataloger-level observations: cataloger name -> types + catalogers map[string]*TypeObservation +} + +// TypeObservation combines metadata types and package types +type TypeObservation struct { + MetadataTypes []string + PackageTypes []string + JSONSchemaTypes []string +} + +// newTestObservationIndex creates a new empty index +func newTestObservationIndex() *TestObservationIndex { + return &TestObservationIndex{ + parsersByPackage: make(map[string]map[string]*TypeObservation), + catalogers: make(map[string]*TypeObservation), + } +} + +// getParserObservations retrieves parser-level observations by package name and parser function +func (idx *TestObservationIndex) getParserObservations(packageName, parserFunction string) *TypeObservation { + if parsers, ok := idx.parsersByPackage[packageName]; ok { + return parsers[parserFunction] + } + return nil +} + +// getCatalogerObservations retrieves cataloger-level observations by cataloger name +func (idx *TestObservationIndex) getCatalogerObservations(catalogerName string) *TypeObservation { + return idx.catalogers[catalogerName] +} + +// setParserObservations stores parser-level observations +func (idx *TestObservationIndex) setParserObservations(packageName, parserFunction string, obs *TypeObservation) { + if idx.parsersByPackage[packageName] == nil { + idx.parsersByPackage[packageName] = make(map[string]*TypeObservation) + } + idx.parsersByPackage[packageName][parserFunction] = obs +} + +// setCatalogerObservations stores cataloger-level observations +func (idx *TestObservationIndex) setCatalogerObservations(catalogerName string, obs *TypeObservation) { + idx.catalogers[catalogerName] = obs +} + +// extractCustomCatalogerData extracts cataloger-level metadata and package types for custom catalogers +func (idx *TestObservationIndex) extractCustomCatalogerData() (map[string][]string, map[string][]string) { + metadata := make(map[string][]string) + packageTypes := make(map[string][]string) + + for catalogerName, obs := range idx.catalogers { + if len(obs.MetadataTypes) > 0 { + metadata[catalogerName] = obs.MetadataTypes + } + if len(obs.PackageTypes) > 0 { + packageTypes[catalogerName] = obs.PackageTypes + } + } + + return metadata, packageTypes +} + +// discoverMetadataTypes searches for test-observations.json files and merges metadata type information +// into the discovered catalogers. Returns maps of custom cataloger metadata types and package types. +func discoverMetadataTypes(repoRoot string, discovered map[string]DiscoveredCataloger) (map[string][]string, map[string][]string, error) { + testFixtureDirs, err := internal.FindTestFixtureDirs(repoRoot) + if err != nil { + return nil, nil, err + } + + // create index to aggregate all observations + index := newTestObservationIndex() + + // read all test-observations files and merge into index + for _, dir := range testFixtureDirs { + observationsFile := filepath.Join(dir, "test-observations.json") + if observations, err := internal.ReadTestObservations(observationsFile); err == nil { + mergeTestObservations(observations, index) + } else if !os.IsNotExist(err) { + fmt.Printf(" Warning: failed to read %s: %v\n", observationsFile, err) + } + } + + // check if any observations were found + if len(index.parsersByPackage) == 0 && len(index.catalogers) == 0 { + // no metadata found, this is not an error + return nil, nil, nil + } + + // apply observations to discovered catalogers + applyTypesToCatalogers(discovered, index) + + // extract custom cataloger data for return + customMetadata, customPackageTypes := index.extractCustomCatalogerData() + return customMetadata, customPackageTypes, nil +} + +// mergeAndDeduplicateStrings merges two string slices, removes duplicates, and returns a sorted slice +func mergeAndDeduplicateStrings(existing, additional []string) []string { + set := make(map[string]bool) + for _, s := range existing { + set[s] = true + } + for _, s := range additional { + set[s] = true + } + + result := make([]string, 0, len(set)) + for s := range set { + result = append(result, s) + } + sort.Strings(result) + return result +} + +// convertToJSONSchemaTypes converts Go struct names to UpperCamelCase JSON schema names +func convertToJSONSchemaTypes(metadataTypes []string) []string { + if len(metadataTypes) == 0 { + return nil + } + + result := make([]string, 0, len(metadataTypes)) + for _, typeName := range metadataTypes { + jsonName := packagemetadata.JSONNameFromString(typeName) + if jsonName != "" { + camelCase := packagemetadata.ToUpperCamelCase(jsonName) + result = append(result, camelCase) + } + } + return result +} + +// mergeTestObservations merges metadata and package type data from a test-observations.json file +// into the observation index +func mergeTestObservations(observations *pkgtestobservation.Test, index *TestObservationIndex) { + pkg := observations.Package + + // merge parser-level observations + for parserName, parserObs := range observations.Parsers { + if len(parserObs.MetadataTypes) == 0 && len(parserObs.PackageTypes) == 0 { + continue + } + + existing := index.getParserObservations(pkg, parserName) + if existing == nil { + existing = &TypeObservation{} + } + + // merge the types + existing.MetadataTypes = mergeAndDeduplicateStrings(existing.MetadataTypes, parserObs.MetadataTypes) + existing.PackageTypes = mergeAndDeduplicateStrings(existing.PackageTypes, parserObs.PackageTypes) + // generate JSON schema types from metadata types + existing.JSONSchemaTypes = convertToJSONSchemaTypes(existing.MetadataTypes) + + index.setParserObservations(pkg, parserName, existing) + } + + // merge cataloger-level observations + for catalogerName, catalogerObs := range observations.Catalogers { + if len(catalogerObs.MetadataTypes) == 0 && len(catalogerObs.PackageTypes) == 0 { + continue + } + + existing := index.getCatalogerObservations(catalogerName) + if existing == nil { + existing = &TypeObservation{} + } + + // merge the types + existing.MetadataTypes = mergeAndDeduplicateStrings(existing.MetadataTypes, catalogerObs.MetadataTypes) + existing.PackageTypes = mergeAndDeduplicateStrings(existing.PackageTypes, catalogerObs.PackageTypes) + // generate JSON schema types from metadata types + existing.JSONSchemaTypes = convertToJSONSchemaTypes(existing.MetadataTypes) + + index.setCatalogerObservations(catalogerName, existing) + } +} + +// applyParserObservations applies parser-level observations to a cataloger's parsers +func applyParserObservations(cataloger *DiscoveredCataloger, index *TestObservationIndex) bool { + foundData := false + + // apply parser-level observations by matching package name + parser function + for i, parser := range cataloger.Parsers { + if obs := index.getParserObservations(cataloger.PackageName, parser.ParserFunction); obs != nil { + if len(obs.MetadataTypes) > 0 { + cataloger.Parsers[i].MetadataTypes = obs.MetadataTypes + cataloger.Parsers[i].JSONSchemaTypes = obs.JSONSchemaTypes + foundData = true + } + if len(obs.PackageTypes) > 0 { + cataloger.Parsers[i].PackageTypes = obs.PackageTypes + foundData = true + } + } + } + + return foundData +} + +// applySingleParserCatalogerObservations applies cataloger-level observations to a single-parser cataloger +// by merging them with any existing parser-level observations +func applySingleParserCatalogerObservations(cataloger *DiscoveredCataloger, catalogerObs *TypeObservation) bool { + foundData := false + + if len(catalogerObs.MetadataTypes) > 0 { + cataloger.Parsers[0].MetadataTypes = mergeAndDeduplicateStrings( + cataloger.Parsers[0].MetadataTypes, + catalogerObs.MetadataTypes, + ) + cataloger.Parsers[0].JSONSchemaTypes = convertToJSONSchemaTypes(cataloger.Parsers[0].MetadataTypes) + foundData = true + } + + if len(catalogerObs.PackageTypes) > 0 { + cataloger.Parsers[0].PackageTypes = mergeAndDeduplicateStrings( + cataloger.Parsers[0].PackageTypes, + catalogerObs.PackageTypes, + ) + foundData = true + } + + return foundData +} + +// applyMultiParserCatalogerObservations applies cataloger-level observations to a multi-parser cataloger +// only applies to parsers that don't already have parser-level data +func applyMultiParserCatalogerObservations(cataloger *DiscoveredCataloger, catalogerObs *TypeObservation) bool { + foundData := false + + // count parsers without any data + parsersWithoutData := 0 + for _, parser := range cataloger.Parsers { + if len(parser.MetadataTypes) == 0 && len(parser.PackageTypes) == 0 { + parsersWithoutData++ + } + } + + // if all parsers lack data, apply to all and warn + if parsersWithoutData == len(cataloger.Parsers) { + fmt.Printf(" Warning: cataloger %q has %d parsers but only cataloger-level observations. Consider adding parser-level tests for better granularity.\n", + cataloger.Name, len(cataloger.Parsers)) + + for i := range cataloger.Parsers { + if len(catalogerObs.MetadataTypes) > 0 { + cataloger.Parsers[i].MetadataTypes = catalogerObs.MetadataTypes + cataloger.Parsers[i].JSONSchemaTypes = catalogerObs.JSONSchemaTypes + foundData = true + } + if len(catalogerObs.PackageTypes) > 0 { + cataloger.Parsers[i].PackageTypes = catalogerObs.PackageTypes + foundData = true + } + } + } else if parsersWithoutData > 0 { + // only apply to parsers without data + for i, parser := range cataloger.Parsers { + if len(parser.MetadataTypes) == 0 && len(catalogerObs.MetadataTypes) > 0 { + cataloger.Parsers[i].MetadataTypes = catalogerObs.MetadataTypes + cataloger.Parsers[i].JSONSchemaTypes = catalogerObs.JSONSchemaTypes + foundData = true + } + if len(parser.PackageTypes) == 0 && len(catalogerObs.PackageTypes) > 0 { + cataloger.Parsers[i].PackageTypes = catalogerObs.PackageTypes + foundData = true + } + } + } + + return foundData +} + +// applyTypesToCatalogers applies the aggregated metadata and package type data to the discovered catalogers. +// it updates the cataloger's parser entries with the appropriate metadata and package types. +func applyTypesToCatalogers(discovered map[string]DiscoveredCataloger, index *TestObservationIndex) { + for catalogerName, cataloger := range discovered { + var foundData bool + + // first, apply parser-level observations + if applyParserObservations(&cataloger, index) { + foundData = true + } + + // then, apply cataloger-level observations if they exist + if catalogerObs := index.getCatalogerObservations(catalogerName); catalogerObs != nil && len(cataloger.Parsers) > 0 { + if len(cataloger.Parsers) == 1 { + // single parser: merge cataloger-level with parser-level observations + if applySingleParserCatalogerObservations(&cataloger, catalogerObs) { + foundData = true + } + } else { + // multiple parsers: only apply to parsers without parser-level data + if applyMultiParserCatalogerObservations(&cataloger, catalogerObs) { + foundData = true + } + } + } + + if foundData { + discovered[catalogerName] = cataloger + } + } +} diff --git a/internal/capabilities/generate/discover_metadata_test.go b/internal/capabilities/generate/discover_metadata_test.go new file mode 100644 index 000000000..26805943c --- /dev/null +++ b/internal/capabilities/generate/discover_metadata_test.go @@ -0,0 +1,320 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestApplyParserObservations(t *testing.T) { + tests := []struct { + name string + cataloger DiscoveredCataloger + index *TestObservationIndex + wantFoundData bool + wantMetadataType string + wantPackageType string + }{ + { + name: "parser observations applied to matching parser", + cataloger: DiscoveredCataloger{ + Name: "test-cataloger", + PackageName: "testpkg", + Parsers: []DiscoveredParser{ + {ParserFunction: "parseTestFile"}, + }, + }, + index: func() *TestObservationIndex { + idx := newTestObservationIndex() + idx.setParserObservations("testpkg", "parseTestFile", &TypeObservation{ + MetadataTypes: []string{"pkg.TestMetadata"}, + PackageTypes: []string{"test-type"}, + }) + return idx + }(), + wantFoundData: true, + wantMetadataType: "pkg.TestMetadata", + wantPackageType: "test-type", + }, + { + name: "no observations found for parser", + cataloger: DiscoveredCataloger{ + Name: "test-cataloger", + PackageName: "testpkg", + Parsers: []DiscoveredParser{ + {ParserFunction: "parseOtherFile"}, + }, + }, + index: func() *TestObservationIndex { + idx := newTestObservationIndex() + idx.setParserObservations("testpkg", "parseTestFile", &TypeObservation{ + MetadataTypes: []string{"pkg.TestMetadata"}, + }) + return idx + }(), + wantFoundData: false, + }, + { + name: "multiple parsers with mixed observations", + cataloger: DiscoveredCataloger{ + Name: "test-cataloger", + PackageName: "testpkg", + Parsers: []DiscoveredParser{ + {ParserFunction: "parseFirst"}, + {ParserFunction: "parseSecond"}, + }, + }, + index: func() *TestObservationIndex { + idx := newTestObservationIndex() + idx.setParserObservations("testpkg", "parseFirst", &TypeObservation{ + MetadataTypes: []string{"pkg.FirstMetadata"}, + }) + // parseSecond has no observations + return idx + }(), + wantFoundData: true, + wantMetadataType: "pkg.FirstMetadata", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotFoundData := applyParserObservations(&tt.cataloger, tt.index) + require.Equal(t, tt.wantFoundData, gotFoundData) + + if tt.wantFoundData && tt.wantMetadataType != "" { + require.Contains(t, tt.cataloger.Parsers[0].MetadataTypes, tt.wantMetadataType) + } + + if tt.wantFoundData && tt.wantPackageType != "" { + require.Contains(t, tt.cataloger.Parsers[0].PackageTypes, tt.wantPackageType) + } + }) + } +} + +func TestApplySingleParserCatalogerObservations(t *testing.T) { + tests := []struct { + name string + cataloger DiscoveredCataloger + catalogerObs *TypeObservation + wantFoundData bool + wantMetadataType []string + wantPackageType []string + }{ + { + name: "cataloger-level observations applied to single parser", + cataloger: DiscoveredCataloger{ + Name: "single-parser-cataloger", + Parsers: []DiscoveredParser{ + {ParserFunction: "parseSingle"}, + }, + }, + catalogerObs: &TypeObservation{ + MetadataTypes: []string{"pkg.CatalogerMetadata"}, + PackageTypes: []string{"cataloger-type"}, + }, + wantFoundData: true, + wantMetadataType: []string{"pkg.CatalogerMetadata"}, + wantPackageType: []string{"cataloger-type"}, + }, + { + name: "cataloger-level merges with existing parser-level observations", + cataloger: DiscoveredCataloger{ + Name: "single-parser-cataloger", + Parsers: []DiscoveredParser{ + { + ParserFunction: "parseSingle", + MetadataTypes: []string{"pkg.ParserMetadata"}, + PackageTypes: []string{"parser-type"}, + }, + }, + }, + catalogerObs: &TypeObservation{ + MetadataTypes: []string{"pkg.CatalogerMetadata"}, + PackageTypes: []string{"cataloger-type"}, + }, + wantFoundData: true, + wantMetadataType: []string{"pkg.CatalogerMetadata", "pkg.ParserMetadata"}, + wantPackageType: []string{"cataloger-type", "parser-type"}, + }, + { + name: "empty cataloger observations", + cataloger: DiscoveredCataloger{ + Name: "single-parser-cataloger", + Parsers: []DiscoveredParser{ + {ParserFunction: "parseSingle"}, + }, + }, + catalogerObs: &TypeObservation{}, + wantFoundData: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotFoundData := applySingleParserCatalogerObservations(&tt.cataloger, tt.catalogerObs) + require.Equal(t, tt.wantFoundData, gotFoundData) + + if tt.wantFoundData { + if len(tt.wantMetadataType) > 0 { + require.ElementsMatch(t, tt.wantMetadataType, tt.cataloger.Parsers[0].MetadataTypes) + } + if len(tt.wantPackageType) > 0 { + require.ElementsMatch(t, tt.wantPackageType, tt.cataloger.Parsers[0].PackageTypes) + } + } + }) + } +} + +func TestApplyMultiParserCatalogerObservations(t *testing.T) { + tests := []struct { + name string + cataloger DiscoveredCataloger + catalogerObs *TypeObservation + wantFoundData bool + // expectations for each parser by index + wantParser0HasMetadata bool + wantParser1HasMetadata bool + }{ + { + name: "all parsers without data - cataloger-level applied to all", + cataloger: DiscoveredCataloger{ + Name: "multi-parser-cataloger", + Parsers: []DiscoveredParser{ + {ParserFunction: "parseFirst"}, + {ParserFunction: "parseSecond"}, + }, + }, + catalogerObs: &TypeObservation{ + MetadataTypes: []string{"pkg.SharedMetadata"}, + PackageTypes: []string{"shared-type"}, + }, + wantFoundData: true, + wantParser0HasMetadata: true, + wantParser1HasMetadata: true, + }, + { + name: "some parsers have data - cataloger-level only fills gaps", + cataloger: DiscoveredCataloger{ + Name: "multi-parser-cataloger", + Parsers: []DiscoveredParser{ + { + ParserFunction: "parseFirst", + MetadataTypes: []string{"pkg.FirstMetadata"}, + }, + {ParserFunction: "parseSecond"}, // no data + }, + }, + catalogerObs: &TypeObservation{ + MetadataTypes: []string{"pkg.SharedMetadata"}, + }, + wantFoundData: true, + wantParser0HasMetadata: false, // already has data, not overwritten + wantParser1HasMetadata: true, // gets cataloger-level data + }, + { + name: "all parsers have data - cataloger-level not applied", + cataloger: DiscoveredCataloger{ + Name: "multi-parser-cataloger", + Parsers: []DiscoveredParser{ + { + ParserFunction: "parseFirst", + MetadataTypes: []string{"pkg.FirstMetadata"}, + }, + { + ParserFunction: "parseSecond", + MetadataTypes: []string{"pkg.SecondMetadata"}, + }, + }, + }, + catalogerObs: &TypeObservation{ + MetadataTypes: []string{"pkg.SharedMetadata"}, + }, + wantFoundData: false, + wantParser0HasMetadata: false, // should not have shared metadata + wantParser1HasMetadata: false, // should not have shared metadata + }, + { + name: "empty cataloger observations", + cataloger: DiscoveredCataloger{ + Name: "multi-parser-cataloger", + Parsers: []DiscoveredParser{ + {ParserFunction: "parseFirst"}, + {ParserFunction: "parseSecond"}, + }, + }, + catalogerObs: &TypeObservation{}, + wantFoundData: false, + wantParser0HasMetadata: false, + wantParser1HasMetadata: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotFoundData := applyMultiParserCatalogerObservations(&tt.cataloger, tt.catalogerObs) + require.Equal(t, tt.wantFoundData, gotFoundData) + + if tt.wantParser0HasMetadata { + require.Contains(t, tt.cataloger.Parsers[0].MetadataTypes, "pkg.SharedMetadata", + "parser 0 should have shared metadata") + } else if len(tt.catalogerObs.MetadataTypes) > 0 { + // if cataloger has metadata but we don't expect it in parser 0, verify it's not there + require.NotContains(t, tt.cataloger.Parsers[0].MetadataTypes, "pkg.SharedMetadata", + "parser 0 should not have shared metadata") + } + + if tt.wantParser1HasMetadata { + require.Contains(t, tt.cataloger.Parsers[1].MetadataTypes, "pkg.SharedMetadata", + "parser 1 should have shared metadata") + } else if len(tt.catalogerObs.MetadataTypes) > 0 { + // if cataloger has metadata but we don't expect it in parser 1, verify it's not there + require.NotContains(t, tt.cataloger.Parsers[1].MetadataTypes, "pkg.SharedMetadata", + "parser 1 should not have shared metadata") + } + }) + } +} + +func TestMergeAndDeduplicateStrings(t *testing.T) { + tests := []struct { + name string + existing []string + additional []string + want []string + }{ + { + name: "merge with duplicates", + existing: []string{"a", "b"}, + additional: []string{"b", "c"}, + want: []string{"a", "b", "c"}, + }, + { + name: "empty existing", + existing: []string{}, + additional: []string{"a", "b"}, + want: []string{"a", "b"}, + }, + { + name: "empty additional", + existing: []string{"a", "b"}, + additional: []string{}, + want: []string{"a", "b"}, + }, + { + name: "both empty", + existing: []string{}, + additional: []string{}, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := mergeAndDeduplicateStrings(tt.existing, tt.additional) + require.ElementsMatch(t, tt.want, got) + }) + } +} diff --git a/internal/capabilities/generate/io.go b/internal/capabilities/generate/io.go new file mode 100644 index 000000000..aa7b12afa --- /dev/null +++ b/internal/capabilities/generate/io.go @@ -0,0 +1,402 @@ +// this file handles YAML file reading and writing with comment preservation, using gopkg.in/yaml.v3's node tree to maintain all existing comments during regeneration. +package main + +import ( + "fmt" + "os" + "regexp" + + "gopkg.in/yaml.v3" + + "github.com/anchore/syft/internal/capabilities" + "github.com/anchore/syft/internal/capabilities/internal" +) + +const autoGeneratedComment = "AUTO-GENERATED" + +// writeYAMLToFile writes a YAML node to a file with proper encoding +func writeYAMLToFile(path string, rootNode *yaml.Node) error { + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer f.Close() + + encoder := yaml.NewEncoder(f) + encoder.SetIndent(2) + if err := encoder.Encode(rootNode); err != nil { + return fmt.Errorf("failed to write YAML: %w", err) + } + if err := encoder.Close(); err != nil { + return fmt.Errorf("failed to close encoder: %w", err) + } + + return nil +} + +// addFieldComments adds comments to top-level fields in the YAML node +func addFieldComments(rootNode *yaml.Node) { + // navigate to the mapping node (handle both DocumentNode and MappingNode) + var mappingNode *yaml.Node + if rootNode.Kind == yaml.DocumentNode && len(rootNode.Content) > 0 { + mappingNode = rootNode.Content[0] + } else { + mappingNode = rootNode + } + + if mappingNode != nil && len(mappingNode.Content) > 0 { + // iterate through top-level keys + for i := 0; i < len(mappingNode.Content); i += 2 { + keyNode := mappingNode.Content[i] + valueNode := mappingNode.Content[i+1] + + switch keyNode.Value { + case "configs": + // configs section is AUTO-GENERATED + if keyNode.LineComment == "" { + keyNode.LineComment = "AUTO-GENERATED - config structs and their fields" + } + case "application": + // application section is AUTO-GENERATED + if keyNode.LineComment == "" { + keyNode.LineComment = "AUTO-GENERATED - application-level config keys" + } + case "catalogers": + // clear any HeadComment on catalogers key and value nodes to prevent duplicate headers + // (the only header should be on the root DocumentNode) + keyNode.HeadComment = "" + valueNode.HeadComment = "" + addCatalogerComments(valueNode) + } + } + } +} + +// saveCapabilities saves the capabilities document to distributed YAML files with comments. +// Groups catalogers by ecosystem and writes each to catalogerDir/{ecosystem}/capabilities.yaml. +// Also saves appconfig.yaml separately to internal/capabilities/. +func saveCapabilities(catalogerDir, repoRoot string, doc *capabilities.Document, existingNodes map[string]*yaml.Node) error { + catalogersByEcosystem, configsByEcosystem := groupCatalogersByEcosystem(doc) + + // write each ecosystem file + for ecosystem, catalogers := range catalogersByEcosystem { + if err := writeEcosystemFile(catalogerDir, ecosystem, catalogers, configsByEcosystem[ecosystem], existingNodes); err != nil { + return err + } + } + + // save appconfig.yaml to internal/capabilities/ + if len(doc.ApplicationConfig) > 0 { + if err := writeAppconfigFile(repoRoot, doc.ApplicationConfig, existingNodes); err != nil { + return err + } + } + + return nil +} + +// groupCatalogersByEcosystem groups catalogers and their configs by ecosystem +func groupCatalogersByEcosystem(doc *capabilities.Document) (map[string][]capabilities.CatalogerEntry, map[string]map[string]capabilities.CatalogerConfigEntry) { + catalogersByEcosystem := make(map[string][]capabilities.CatalogerEntry) + configsByEcosystem := make(map[string]map[string]capabilities.CatalogerConfigEntry) + + for _, cat := range doc.Catalogers { + ecosystem := mapCatalogerToEcosystem(cat) + catalogersByEcosystem[ecosystem] = append(catalogersByEcosystem[ecosystem], cat) + + // also group configs for this ecosystem + if cat.Config != "" { + if configEntry, exists := doc.Configs[cat.Config]; exists { + if configsByEcosystem[ecosystem] == nil { + configsByEcosystem[ecosystem] = make(map[string]capabilities.CatalogerConfigEntry) + } + configsByEcosystem[ecosystem][cat.Config] = configEntry + } + } + } + + return catalogersByEcosystem, configsByEcosystem +} + +// writeEcosystemFile writes a single ecosystem's catalogers and configs to a YAML file +func writeEcosystemFile(catalogerDir, ecosystem string, catalogers []capabilities.CatalogerEntry, configs map[string]capabilities.CatalogerConfigEntry, existingNodes map[string]*yaml.Node) error { + ecosystemDoc := struct { + Configs map[string]capabilities.CatalogerConfigEntry `yaml:"configs,omitempty"` + Catalogers []capabilities.CatalogerEntry `yaml:"catalogers"` + }{ + Configs: configs, + Catalogers: catalogers, + } + + var rootNode yaml.Node + existingNode, hasExisting := existingNodes[ecosystem] + + if hasExisting && existingNode != nil { + // update existing node tree + rootNode = *existingNode + rootNode.HeadComment = "# Cataloger capabilities. See ../README.md for documentation.\n" + if err := updateNodeTreeEcosystem(&rootNode, &ecosystemDoc); err != nil { + return fmt.Errorf("failed to update node tree for %s: %w", ecosystem, err) + } + } else { + // create new node tree + if err := rootNode.Encode(&ecosystemDoc); err != nil { + return fmt.Errorf("failed to encode %s: %w", ecosystem, err) + } + rootNode.HeadComment = "# Cataloger capabilities. See ../README.md for documentation.\n" + } + + // add field comments + addFieldComments(&rootNode) + + // write file to catalogerDir/{ecosystem}/capabilities.yaml + ecosystemPath := internal.EcosystemCapabilitiesPath(catalogerDir, ecosystem) + if err := writeYAMLToFile(ecosystemPath, &rootNode); err != nil { + return fmt.Errorf("failed to write %s: %w", ecosystem, err) + } + + return nil +} + +// writeAppconfigFile writes the application config to appconfig.yaml +func writeAppconfigFile(repoRoot string, appConfig []capabilities.ApplicationConfigField, existingNodes map[string]*yaml.Node) error { + appconfigDoc := struct { + Application []capabilities.ApplicationConfigField `yaml:"application"` + }{ + Application: appConfig, + } + + var appNode yaml.Node + existingAppNode, hasExisting := existingNodes["appconfig"] + + if hasExisting && existingAppNode != nil { + appNode = *existingAppNode + if err := updateNodeTreeAppConfig(&appNode, &appconfigDoc); err != nil { + return fmt.Errorf("failed to update appconfig node tree: %w", err) + } + } else { + if err := appNode.Encode(&appconfigDoc); err != nil { + return fmt.Errorf("failed to encode appconfig: %w", err) + } + appNode.HeadComment = "# Application-level configuration. See README.md for documentation.\n# This file is partially auto-generated. Run 'go generate ./internal/capabilities' to regenerate.\n" + } + + addFieldComments(&appNode) + + appconfigPath := internal.AppconfigPath(repoRoot) + if err := writeYAMLToFile(appconfigPath, &appNode); err != nil { + return fmt.Errorf("failed to write appconfig: %w", err) + } + + return nil +} + +// mapCatalogerToEcosystem determines which ecosystem file a cataloger belongs to +func mapCatalogerToEcosystem(cat capabilities.CatalogerEntry) string { + // first try using the source file path + if cat.Source.File != "" { + // extract directory from path like "syft/pkg/cataloger/golang/cataloger.go" + re := regexp.MustCompile(`syft/pkg/cataloger/([^/]+)/`) + if matches := re.FindStringSubmatch(cat.Source.File); len(matches) >= 2 { + return matches[1] + } + } + + // fallback to inferring from cataloger name (from merge.go) + return inferEcosystem(cat.Name) +} + +// updateNodeTreeEcosystem updates an existing ecosystem YAML node tree +func updateNodeTreeEcosystem(rootNode *yaml.Node, doc interface{}) error { + var newNode yaml.Node + if err := newNode.Encode(doc); err != nil { + return err + } + + var existingMapping *yaml.Node + if rootNode.Kind == yaml.DocumentNode && len(rootNode.Content) > 0 { + existingMapping = rootNode.Content[0] + } else { + existingMapping = rootNode + } + + var newMapping *yaml.Node + if newNode.Kind == yaml.DocumentNode && len(newNode.Content) > 0 { + newMapping = newNode.Content[0] + } else { + newMapping = &newNode + } + + if existingMapping.Kind == yaml.MappingNode && newMapping.Kind == yaml.MappingNode { + existingMapping.Content = newMapping.Content + } + + return nil +} + +// updateNodeTreeAppConfig updates appconfig YAML node tree +func updateNodeTreeAppConfig(rootNode *yaml.Node, doc interface{}) error { + return updateNodeTreeEcosystem(rootNode, doc) +} + +// addCatalogerFieldComment adds appropriate comment to a single cataloger field +func addCatalogerFieldComment(keyNode, valueNode *yaml.Node, catalogerName string) { + switch keyNode.Value { + case "ecosystem": + // ecosystem is MANUAL + if keyNode.LineComment == "" { + keyNode.LineComment = "MANUAL" + } + case "name", "type": + // add AUTO-GENERATED comment to these fields + if keyNode.LineComment == "" { + keyNode.LineComment = autoGeneratedComment + } + case "source": + // add AUTO-GENERATED comment to source field + if keyNode.LineComment == "" { + keyNode.LineComment = autoGeneratedComment + } + case "config": + // add AUTO-GENERATED comment to config field + if keyNode.LineComment == "" { + keyNode.LineComment = autoGeneratedComment + } + case "selectors": + // selectors are AUTO-GENERATED + if keyNode.LineComment == "" { + keyNode.LineComment = autoGeneratedComment + } + case "parsers": + // parsers is AUTO-GENERATED structure + if keyNode.LineComment == "" { + keyNode.LineComment = "AUTO-GENERATED structure" + } + addParserComments(valueNode) + case "detectors": + // detectors are AUTO-GENERATED for binary-classifier-cataloger, MANUAL for others + if catalogerName == "binary-classifier-cataloger" { + keyNode.LineComment = autoGeneratedComment + } else if keyNode.LineComment == "" { + keyNode.LineComment = "MANUAL - edit detectors here" + } + case "metadata_types": + // cataloger-level metadata_types (for custom catalogers) are AUTO-GENERATED + if keyNode.LineComment == "" { + keyNode.LineComment = autoGeneratedComment + } + case "package_types": + // cataloger-level package_types (for custom catalogers) are AUTO-GENERATED + if keyNode.LineComment == "" { + keyNode.LineComment = autoGeneratedComment + } + case "json_schema_types": + // json_schema_types are AUTO-GENERATED + if keyNode.LineComment == "" { + keyNode.LineComment = autoGeneratedComment + } + case "capabilities": + // capabilities are MANUAL + if keyNode.LineComment == "" { + keyNode.LineComment = "MANUAL - edit capabilities here" + } + } +} + +// findFieldValue finds a field in a YAML mapping node and returns its value +func findFieldValue(node *yaml.Node, fieldName string) string { + if node.Kind != yaml.MappingNode { + return "" + } + + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + valueNode := node.Content[i+1] + if keyNode.Value == fieldName { + return valueNode.Value + } + } + + return "" +} + +func addCatalogerComments(catalogersNode *yaml.Node) { + // catalogersNode should be a sequence of cataloger entries + if catalogersNode.Kind != yaml.SequenceNode { + return + } + + for _, catalogerNode := range catalogersNode.Content { + if catalogerNode.Kind != yaml.MappingNode { + continue + } + + // get the cataloger name for special handling + catalogerName := findFieldValue(catalogerNode, "name") + + // iterate through the fields of this cataloger entry + for i := 0; i < len(catalogerNode.Content); i += 2 { + keyNode := catalogerNode.Content[i] + valueNode := catalogerNode.Content[i+1] + addCatalogerFieldComment(keyNode, valueNode, catalogerName) + } + } +} + +func addParserComments(parsersNode *yaml.Node) { + // parsersNode should be a sequence of parser entries + if parsersNode.Kind != yaml.SequenceNode { + return + } + + for _, parserNode := range parsersNode.Content { + if parserNode.Kind != yaml.MappingNode { + continue + } + + // iterate through the fields of this parser entry + for i := 0; i < len(parserNode.Content); i += 2 { + keyNode := parserNode.Content[i] + valueNode := parserNode.Content[i+1] + + switch keyNode.Value { + case "parser_function", "metadata_types", "package_types", "json_schema_types": + // add AUTO-GENERATED comment to these fields + if keyNode.LineComment == "" { + keyNode.LineComment = autoGeneratedComment + } + case "detector": + // detector is AUTO-GENERATED + if keyNode.LineComment == "" { + keyNode.LineComment = autoGeneratedComment + } + addDetectorComments(valueNode) + case "capabilities": + // capabilities are MANUAL + if keyNode.LineComment == "" { + keyNode.LineComment = "MANUAL - preserved across regeneration" + } + } + } + } +} + +func addDetectorComments(detectorNode *yaml.Node) { + // detectorNode should be a mapping node with method and criteria fields + if detectorNode.Kind != yaml.MappingNode { + return + } + + // iterate through the fields of the detector + for i := 0; i < len(detectorNode.Content); i += 2 { + keyNode := detectorNode.Content[i] + + switch keyNode.Value { + case "method", "criteria": + // add AUTO-GENERATED comment to these fields + if keyNode.LineComment == "" { + keyNode.LineComment = autoGeneratedComment + } + } + } +} diff --git a/internal/capabilities/generate/io_test.go b/internal/capabilities/generate/io_test.go new file mode 100644 index 000000000..0d6e790b4 --- /dev/null +++ b/internal/capabilities/generate/io_test.go @@ -0,0 +1,185 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestFindFieldValue(t *testing.T) { + tests := []struct { + name string + yamlContent string + fieldName string + want string + }{ + { + name: "finds simple string field", + yamlContent: ` +name: test-cataloger +type: custom +`, + fieldName: "name", + want: "test-cataloger", + }, + { + name: "finds type field", + yamlContent: ` +name: test-cataloger +type: generic +`, + fieldName: "type", + want: "generic", + }, + { + name: "returns empty for non-existent field", + yamlContent: ` +name: test-cataloger +`, + fieldName: "nonexistent", + want: "", + }, + { + name: "finds parser_function field", + yamlContent: ` +parser_function: parseGoMod +metadata_types: [GoModMetadata] +`, + fieldName: "parser_function", + want: "parseGoMod", + }, + { + name: "handles empty mapping", + yamlContent: `{}`, + fieldName: "any", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var rootNode yaml.Node + err := yaml.Unmarshal([]byte(tt.yamlContent), &rootNode) + require.NoError(t, err) + + // get the mapping node + var mappingNode *yaml.Node + if rootNode.Kind == yaml.DocumentNode && len(rootNode.Content) > 0 { + mappingNode = rootNode.Content[0] + } else { + mappingNode = &rootNode + } + + got := findFieldValue(mappingNode, tt.fieldName) + require.Equal(t, tt.want, got) + }) + } +} + +func TestAddCatalogerFieldComment(t *testing.T) { + tests := []struct { + name string + fieldName string + fieldValue string + catalogerName string + wantLineComment string + }{ + { + name: "ecosystem is MANUAL", + fieldName: "ecosystem", + catalogerName: "test-cataloger", + wantLineComment: "MANUAL", + }, + { + name: "name is AUTO-GENERATED", + fieldName: "name", + catalogerName: "test-cataloger", + wantLineComment: autoGeneratedComment, + }, + { + name: "type is AUTO-GENERATED", + fieldName: "type", + catalogerName: "test-cataloger", + wantLineComment: autoGeneratedComment, + }, + { + name: "source is AUTO-GENERATED", + fieldName: "source", + catalogerName: "test-cataloger", + wantLineComment: autoGeneratedComment, + }, + { + name: "config is AUTO-GENERATED", + fieldName: "config", + catalogerName: "test-cataloger", + wantLineComment: autoGeneratedComment, + }, + { + name: "selectors is AUTO-GENERATED", + fieldName: "selectors", + catalogerName: "test-cataloger", + wantLineComment: autoGeneratedComment, + }, + { + name: "parsers is AUTO-GENERATED structure", + fieldName: "parsers", + catalogerName: "test-cataloger", + wantLineComment: "AUTO-GENERATED structure", + }, + { + name: "detectors for binary-classifier-cataloger is AUTO-GENERATED", + fieldName: "detectors", + catalogerName: "binary-classifier-cataloger", + wantLineComment: autoGeneratedComment, + }, + { + name: "detectors for other catalogers is MANUAL", + fieldName: "detectors", + catalogerName: "java-archive-cataloger", + wantLineComment: "MANUAL - edit detectors here", + }, + { + name: "metadata_types is AUTO-GENERATED", + fieldName: "metadata_types", + catalogerName: "test-cataloger", + wantLineComment: autoGeneratedComment, + }, + { + name: "package_types is AUTO-GENERATED", + fieldName: "package_types", + catalogerName: "test-cataloger", + wantLineComment: autoGeneratedComment, + }, + { + name: "json_schema_types is AUTO-GENERATED", + fieldName: "json_schema_types", + catalogerName: "test-cataloger", + wantLineComment: autoGeneratedComment, + }, + { + name: "capabilities is MANUAL", + fieldName: "capabilities", + catalogerName: "test-cataloger", + wantLineComment: "MANUAL - edit capabilities here", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // create key and value nodes + keyNode := &yaml.Node{ + Kind: yaml.ScalarNode, + Value: tt.fieldName, + } + valueNode := &yaml.Node{ + Kind: yaml.ScalarNode, + Value: tt.fieldValue, + } + + addCatalogerFieldComment(keyNode, valueNode, tt.catalogerName) + + require.Equal(t, tt.wantLineComment, keyNode.LineComment) + }) + } +} diff --git a/internal/capabilities/generate/main.go b/internal/capabilities/generate/main.go new file mode 100644 index 000000000..8e858143b --- /dev/null +++ b/internal/capabilities/generate/main.go @@ -0,0 +1,150 @@ +// this is the entry point for regenerating the cataloger/*/capabilities.yaml files, which orchestrates discovery, merging, and validation of cataloger capabilities. +package main + +import ( + "fmt" + "log" + "os" + + "github.com/charmbracelet/lipgloss" + + "github.com/anchore/syft/internal/capabilities" + "github.com/anchore/syft/internal/capabilities/internal" +) + +var ( + successStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("10")).Bold(true) // green + warningStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("11")).Bold(true) // yellow + errorStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("9")).Bold(true) // red + infoStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("14")) // cyan + dimStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("245")) // lighter grey (256-color) +) + +func main() { + repoRoot, err := internal.RepoRoot() + if err != nil { + log.Fatalf("failed to find repo root: %v", err) + } + + catalogerDir := internal.CatalogerDir(repoRoot) + + fmt.Println("Regenerating capabilities files...") + fmt.Println() + stats, err := RegenerateCapabilities(catalogerDir, repoRoot) + if err != nil { + log.Fatalf("failed to regenerate capabilities: %v", err) + } + + printSummary(stats) + checkIncompleteCapabilities(catalogerDir, repoRoot) + printMetadataTypeCoverageWarning(catalogerDir, repoRoot) + printPackageTypeCoverageWarning(catalogerDir, repoRoot) +} + +func printSummary(stats *Statistics) { + fmt.Println() + fmt.Println(infoStyle.Bold(true).Render("Summary:")) + fmt.Printf(" • Total catalogers: %s (%s generic, %s custom)\n", + infoStyle.Render(fmt.Sprintf("%d", stats.TotalGenericCatalogers+stats.TotalCustomCatalogers)), + infoStyle.Render(fmt.Sprintf("%d", stats.TotalGenericCatalogers)), + infoStyle.Render(fmt.Sprintf("%d", stats.TotalCustomCatalogers))) + fmt.Printf(" • Total parser functions: %s\n", infoStyle.Render(fmt.Sprintf("%d", stats.TotalParserFunctions))) + + if len(stats.NewCatalogers) > 0 { + fmt.Printf(" • New catalogers: %s\n", successStyle.Render(fmt.Sprintf("%d", len(stats.NewCatalogers)))) + for _, name := range stats.NewCatalogers { + fmt.Printf(" - %s\n", successStyle.Render(name)) + } + } + + if len(stats.NewParserFunctions) > 0 { + fmt.Printf(" • New parser functions: %s\n", successStyle.Render(fmt.Sprintf("%d", len(stats.NewParserFunctions)))) + for _, name := range stats.NewParserFunctions { + fmt.Printf(" - %s\n", successStyle.Render(name)) + } + } + + if len(stats.UpdatedCatalogers) > 0 { + fmt.Printf(" • Updated catalogers: %s\n", infoStyle.Render(fmt.Sprintf("%d", len(stats.UpdatedCatalogers)))) + } + + fmt.Println() + fmt.Println(successStyle.Render("✓ Updated capabilities files successfully")) +} + +func checkIncompleteCapabilities(catalogerDir, repoRoot string) { + doc, _, err := internal.LoadCapabilities(catalogerDir, repoRoot) + if err != nil { + log.Fatalf("failed to load updated capabilities: %v", err) + } + + var needsAttentionGeneric []string + var needsAttentionCustom []string + for _, cataloger := range doc.Catalogers { + switch cataloger.Type { + case genericCatalogerType: + for _, parser := range cataloger.Parsers { + if hasEmptyCapabilities(parser.Capabilities) { + needsAttentionGeneric = append(needsAttentionGeneric, fmt.Sprintf("%s/%s", cataloger.Name, parser.ParserFunction)) + } + } + case "custom": + if hasEmptyCapabilities(cataloger.Capabilities) { + needsAttentionCustom = append(needsAttentionCustom, cataloger.Name) + } + } + } + + if len(needsAttentionGeneric) > 0 || len(needsAttentionCustom) > 0 { + fmt.Println() + printFailureASCII() + fmt.Println(warningStyle.Render("⚠ WARNING:") + " The following entries have incomplete capabilities:") + + if len(needsAttentionGeneric) > 0 { + fmt.Printf(" • %s generic cataloger parser functions need capabilities:\n", errorStyle.Render(fmt.Sprintf("%d", len(needsAttentionGeneric)))) + for _, name := range needsAttentionGeneric { + fmt.Printf(" - %s\n", dimStyle.Render(name)) + } + } + + if len(needsAttentionCustom) > 0 { + fmt.Printf(" • %s custom catalogers need capabilities:\n", errorStyle.Render(fmt.Sprintf("%d", len(needsAttentionCustom)))) + for _, name := range needsAttentionCustom { + fmt.Printf(" - %s\n", dimStyle.Render(name)) + } + } + + fmt.Println() + fmt.Println(dimStyle.Render("Please update these entries in the capabilities files before running tests.")) + fmt.Println() + fmt.Println(dimStyle.Render("Exit code: 1")) + os.Exit(1) + } + // show success ASCII art when all validations pass + printSuccessASCII() +} + +func hasEmptyCapabilities(caps capabilities.CapabilitySet) bool { + // only flag if capabilities are completely missing (empty array) + // if someone filled out the capabilities section (even with all false/empty values), that's intentional + return len(caps) == 0 +} + +func printSuccessASCII() { + fmt.Println() + fmt.Println(successStyle.Render("✓ All validations passed!") + " 🎉") + fmt.Println() + fmt.Println(successStyle.Render(" ░█▀▀░█░█░█▀▀░█▀▀░█▀▀░█▀▀░█▀▀")) + fmt.Println(successStyle.Render(" ░▀▀█░█░█░█░░░█░░░█▀▀░▀▀█░▀▀█")) + fmt.Println(successStyle.Render(" ░▀▀▀░▀▀▀░▀▀▀░▀▀▀░▀▀▀░▀▀▀░▀▀▀")) + fmt.Println() +} + +func printFailureASCII() { + fmt.Println(errorStyle.Render("✗ Validation failed") + " 😢") + fmt.Println() + fmt.Println(errorStyle.Render(" ░█▀▀░█▀█░▀█▀░█░░░█▀▀░█▀▄")) + fmt.Println(errorStyle.Render(" ░█▀▀░█▀█░░█░░█░░░█▀▀░█░█")) + fmt.Println(errorStyle.Render(" ░▀░░░▀░▀░▀▀▀░▀▀▀░▀▀▀░▀▀░")) + fmt.Println() +} diff --git a/internal/capabilities/generate/merge.go b/internal/capabilities/generate/merge.go new file mode 100644 index 000000000..c42260e89 --- /dev/null +++ b/internal/capabilities/generate/merge.go @@ -0,0 +1,897 @@ +// this file contains the core merging logic that combines discovered cataloger data with existing cataloger/*/capabilities.yaml files, preserving all manual sections while updating auto-generated fields. +package main + +import ( + "fmt" + "strings" + + "github.com/scylladb/go-set/strset" + + "github.com/anchore/syft/internal/capabilities" + "github.com/anchore/syft/internal/capabilities/internal" + "github.com/anchore/syft/internal/packagemetadata" + "github.com/anchore/syft/syft/pkg/cataloger/binary" +) + +const genericCatalogerType = "generic" + +// stripPURLVersion removes the @version suffix from a PURL string +// e.g., "pkg:generic/python@version" -> "pkg:generic/python" +func stripPURLVersion(purl string) string { + if idx := strings.LastIndex(purl, "@"); idx != -1 { + return purl[:idx] + } + return purl +} + +// catalogerTypeOverrides specifies catalogers that should have their type manually controlled +// rather than determined from the discovered cataloger structure. +// This is useful when a cataloger is discovered as "generic" but should be treated as "custom" +// in the YAML (or vice versa). +var catalogerTypeOverrides = map[string]string{ + // the java cataloger is technically generic (it has parsers), but we want it to be treated as custom since + // these nuances can't automatically be detected, and it requires manual source info + "java-archive-cataloger": "custom", +} + +// catalogerConfigExceptions specifies catalogers that should NOT have config fields +// auto-generated even if a config mapping is discovered via AST parsing. +// This is useful when a cataloger uses a config struct internally but it shouldn't +// be exposed in the capabilities document. +var catalogerConfigExceptions = strset.New( + "binary-classifier-cataloger", +) + +// catalogerConfigOverrides specifies manual mappings from cataloger names to config struct names. +// Use this when the AST parser cannot automatically discover the config linkage, or when you want +// to explicitly override the discovered mapping. +// Format: cataloger-name -> "package.ConfigStructName" +var catalogerConfigOverrides = map[string]string{ + "dotnet-portable-executable-cataloger": "dotnet.CatalogerConfig", + "nix-store-cataloger": "nix.Config", +} + +// ecosystemMapping maps patterns in cataloger names to ecosystem names. +// order matters - more specific patterns should come first. +type ecosystemMapping struct { + patterns []string // patterns to match in the cataloger name + ecosystem string // ecosystem to return if any pattern matches +} + +// ecosystemMappings defines the pattern-to-ecosystem mappings. +// note: order matters - check more specific patterns first +var ecosystemMappings = []ecosystemMapping{ + // language-based ecosystems + {[]string{"rust", "cargo"}, "rust"}, + {[]string{"javascript", "node", "npm"}, "javascript"}, + {[]string{"python"}, "python"}, + {[]string{"java", "graalvm"}, "java"}, + {[]string{"go-module", "golang"}, "go"}, + {[]string{"ruby", "gem"}, "ruby"}, + {[]string{"php", "composer", "pear", "pecl"}, "php"}, + {[]string{"dotnet", ".net", "csharp"}, "dotnet"}, + {[]string{"swift", "cocoapods"}, "swift"}, + {[]string{"dart", "pubspec"}, "dart"}, + {[]string{"elixir", "mix"}, "elixir"}, + {[]string{"erlang", "rebar"}, "erlang"}, + {[]string{"haskell", "cabal", "stack"}, "haskell"}, + {[]string{"lua"}, "lua"}, + {[]string{"ocaml", "opam"}, "ocaml"}, + {[]string{"r-package"}, "r"}, + {[]string{"swipl", "prolog"}, "prolog"}, + {[]string{"cpp", "conan"}, "c++"}, + {[]string{"kotlin"}, "kotlin"}, + + // os/distro-based ecosystems + {[]string{"apk", "alpine"}, "alpine"}, + {[]string{"dpkg", "deb", "debian"}, "debian"}, + {[]string{"rpm", "redhat"}, "rpm"}, + {[]string{"alpm", "arch"}, "arch"}, + {[]string{"portage", "gentoo"}, "gentoo"}, + {[]string{"homebrew"}, "homebrew"}, + {[]string{"snap"}, "snap"}, + + // other ecosystems + {[]string{"binary", "elf", "pe-binary"}, "binary"}, + {[]string{"conda"}, "conda"}, + {[]string{"nix"}, "nix"}, + {[]string{"kernel"}, "kernel"}, + {[]string{"bitnami"}, "bitnami"}, + {[]string{"terraform"}, "terraform"}, + {[]string{"github"}, "github-actions"}, + {[]string{"wordpress"}, "wordpress"}, + {[]string{"sbom"}, "sbom"}, +} + +// inferEcosystem attempts to determine the ecosystem from a cataloger name +func inferEcosystem(catalogerName string) string { + name := strings.ToLower(catalogerName) + + for _, mapping := range ecosystemMappings { + for _, pattern := range mapping.patterns { + if strings.Contains(name, pattern) { + return mapping.ecosystem + } + } + } + + // default + return "other" +} + +// Statistics contains information about the regeneration process +type Statistics struct { + TotalGenericCatalogers int + TotalCustomCatalogers int + TotalParserFunctions int + NewCatalogers []string + NewParserFunctions []string + UpdatedCatalogers []string +} + +// RegenerateCapabilities updates the distributed YAML files with discovered catalogers +// while preserving manually-edited capability information. +// This is exported for use by the generator in generate/main.go +func RegenerateCapabilities(catalogerDir string, repoRoot string) (*Statistics, error) { + stats := &Statistics{} + + // 1-2. Discover all cataloger data + discovered, customCatalogerMetadata, customCatalogerPackageTypes, binaryClassifiers, allCatalogers, err := discoverAllCatalogerData(repoRoot, stats) + if err != nil { + return nil, err + } + + // 3. Load existing YAML files - now returns both document and node trees + fmt.Print(" → Loading existing capabilities files...") + existing, existingNodes, err := internal.LoadCapabilities(catalogerDir, repoRoot) + if err != nil { + return nil, fmt.Errorf("failed to load existing capabilities: %w", err) + } + fmt.Printf(" loaded %d entries\n", len(existing.Catalogers)) + + // 3a-3c. Discover and process all config-related information + discoveredConfigs, err := discoverAndFilterConfigs(repoRoot) + if err != nil { + return nil, err + } + + discoveredAppConfigs, err := discoverAndConvertAppConfigs(repoRoot) + if err != nil { + return nil, err + } + + catalogerConfigMappings, err := LinkCatalogersToConfigs(repoRoot) + if err != nil { + return nil, fmt.Errorf("failed to link catalogers to configs: %w", err) + } + fmt.Printf(" → Linking catalogers to config structs... found %d mappings\n", len(catalogerConfigMappings)) + + filteredCatalogerConfigMappings := applyConfigMappingFilters(catalogerConfigMappings) + + // 4. Build updated catalogers list + fmt.Println(" → Merging discovered catalogers with existing entries...") + updated, orphans, mergeStats := mergeDiscoveredWithExisting( + discovered, + customCatalogerMetadata, + customCatalogerPackageTypes, + binaryClassifiers, + allCatalogers, + existing, + discoveredConfigs, + discoveredAppConfigs, + filteredCatalogerConfigMappings, + ) + stats.NewCatalogers = mergeStats.NewCatalogers + stats.NewParserFunctions = mergeStats.NewParserFunctions + stats.UpdatedCatalogers = mergeStats.UpdatedCatalogers + stats.TotalCustomCatalogers = len(allCatalogers) - stats.TotalGenericCatalogers + + // 5. Check for orphaned parsers (parser functions that were renamed/deleted) + if len(orphans) > 0 { + return nil, fmt.Errorf("orphaned parsers detected (parser functions renamed or deleted):\n%s\n\nPlease manually remove these from the capabilities files or restore the parser functions in the code", + formatOrphans(orphans)) + } + + // 6. Write back to YAML files with comments, preserving existing node trees + fmt.Print(" → Writing updated capabilities files...") + if err := saveCapabilities(catalogerDir, repoRoot, updated, existingNodes); err != nil { + return nil, fmt.Errorf("failed to save capabilities: %w", err) + } + fmt.Println(" done") + + return stats, nil +} + +// discoverAllCatalogerData discovers all cataloger-related data including generic catalogers, metadata, binary classifiers, and all catalogers +func discoverAllCatalogerData(repoRoot string, stats *Statistics) ( + map[string]DiscoveredCataloger, + map[string][]string, + map[string][]string, + []binary.Classifier, //nolint:staticcheck + []capabilities.CatalogerInfo, + error, +) { + // discover generic catalogers + fmt.Print(" → Scanning source code for generic catalogers...") + discovered, err := discoverGenericCatalogers(repoRoot) + if err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("failed to discover catalogers: %w", err) + } + stats.TotalGenericCatalogers = len(discovered) + fmt.Printf(" found %d\n", stats.TotalGenericCatalogers) + + // discover metadata types + fmt.Print(" → Searching for metadata type and package type information...") + customCatalogerMetadata, customCatalogerPackageTypes, err := discoverMetadataTypes(repoRoot, discovered) + if err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("failed to discover metadata types: %w", err) + } + fmt.Println(" done") + + // extract binary classifiers + fmt.Print(" → Extracting binary classifiers...") + binaryClassifiers := extractBinaryClassifiers() + fmt.Printf(" found %d classifiers\n", len(binaryClassifiers)) + + // count parser functions + for _, disc := range discovered { + stats.TotalParserFunctions += len(disc.Parsers) + } + + // get all cataloger info + fmt.Print(" → Fetching all cataloger info from syft...") + allCatalogers, err := internal.AllPackageCatalogerInfo() + if err != nil { + return nil, nil, nil, nil, nil, fmt.Errorf("failed to get cataloger info: %w", err) + } + fmt.Printf(" found %d total\n", len(allCatalogers)) + + return discovered, customCatalogerMetadata, customCatalogerPackageTypes, binaryClassifiers, allCatalogers, nil +} + +// discoverAndFilterConfigs discovers cataloger config structs, filters by whitelist, and converts to capabilities format +func discoverAndFilterConfigs(repoRoot string) (map[string]capabilities.CatalogerConfigEntry, error) { + fmt.Print(" → Discovering cataloger config structs...") + configInfoMap, err := DiscoverConfigs(repoRoot) + if err != nil { + return nil, fmt.Errorf("failed to discover configs: %w", err) + } + fmt.Printf(" found %d\n", len(configInfoMap)) + + fmt.Print(" → Filtering configs by pkgcataloging.Config whitelist...") + allowedConfigs, err := DiscoverAllowedConfigStructs(repoRoot) + if err != nil { + return nil, fmt.Errorf("failed to discover allowed config structs: %w", err) + } + + // filter discovered configs to only include allowed ones + filteredConfigInfoMap := make(map[string]ConfigInfo) + for key, configInfo := range configInfoMap { + if allowedConfigs[key] { + filteredConfigInfoMap[key] = configInfo + } + } + fmt.Printf(" %d allowed (filtered %d)\n", len(filteredConfigInfoMap), len(configInfoMap)-len(filteredConfigInfoMap)) + + // convert ConfigInfo to CatalogerConfigEntry format for ecosystem YAML files + discoveredConfigs := make(map[string]capabilities.CatalogerConfigEntry) + for key, configInfo := range filteredConfigInfoMap { + fields := make([]capabilities.CatalogerConfigFieldEntry, len(configInfo.Fields)) + for i, field := range configInfo.Fields { + fields[i] = capabilities.CatalogerConfigFieldEntry{ + Key: field.Name, + Description: field.Description, + AppKey: field.AppKey, + } + } + discoveredConfigs[key] = capabilities.CatalogerConfigEntry{ + Fields: fields, + } + } + + return discoveredConfigs, nil +} + +// discoverAndConvertAppConfigs discovers app-level config fields and converts them to capabilities format +func discoverAndConvertAppConfigs(repoRoot string) ([]capabilities.ApplicationConfigField, error) { + fmt.Print(" → Discovering app-level config fields...") + appConfigFields, err := DiscoverAppConfigs(repoRoot) + if err != nil { + return nil, fmt.Errorf("failed to discover app configs: %w", err) + } + fmt.Printf(" found %d\n", len(appConfigFields)) + + // convert to ApplicationConfigField format + discoveredAppConfigs := make([]capabilities.ApplicationConfigField, len(appConfigFields)) + for i, field := range appConfigFields { + discoveredAppConfigs[i] = capabilities.ApplicationConfigField{ + Key: field.Key, + Description: field.Description, + DefaultValue: field.DefaultValue, + } + } + + return discoveredAppConfigs, nil +} + +// applyConfigMappingFilters applies exceptions and manual overrides to cataloger config mappings +func applyConfigMappingFilters(catalogerConfigMappings map[string]string) map[string]string { + // filter by exceptions + fmt.Print(" → Filtering cataloger config mappings by exceptions...") + filteredCatalogerConfigMappings := make(map[string]string) + filteredCount := 0 + for catalogerName, configName := range catalogerConfigMappings { + if catalogerConfigExceptions.Has(catalogerName) { + filteredCount++ + continue + } + filteredCatalogerConfigMappings[catalogerName] = configName + } + if filteredCount > 0 { + fmt.Printf(" filtered %d\n", filteredCount) + } else { + fmt.Println(" none") + } + + // merge manual overrides + fmt.Print(" → Merging manual config overrides...") + overrideCount := 0 + for catalogerName, configName := range catalogerConfigOverrides { + if catalogerConfigExceptions.Has(catalogerName) { + continue + } + filteredCatalogerConfigMappings[catalogerName] = configName + overrideCount++ + } + if overrideCount > 0 { + fmt.Printf(" added %d\n", overrideCount) + } else { + fmt.Println(" none") + } + + return filteredCatalogerConfigMappings +} + +type orphanInfo struct { + catalogerName string + parserFunction string +} + +type mergeStatistics struct { + NewCatalogers []string + NewParserFunctions []string + UpdatedCatalogers []string +} + +// CatalogerRegistry encapsulates cataloger lookup data and provides methods for querying cataloger information +type CatalogerRegistry struct { + discovered map[string]DiscoveredCataloger + all []capabilities.CatalogerInfo + infoByName map[string]*capabilities.CatalogerInfo +} + +// NewCatalogerRegistry creates a new registry with the given discovered and all catalogers +func NewCatalogerRegistry(discovered map[string]DiscoveredCataloger, all []capabilities.CatalogerInfo) *CatalogerRegistry { + infoByName := make(map[string]*capabilities.CatalogerInfo) + for i := range all { + infoByName[all[i].Name] = &all[i] + } + + return &CatalogerRegistry{ + discovered: discovered, + all: all, + infoByName: infoByName, + } +} + +// IsGeneric checks if a cataloger is a discovered generic cataloger and returns it if found +func (r *CatalogerRegistry) IsGeneric(name string) (DiscoveredCataloger, bool) { + disc, ok := r.discovered[name] + return disc, ok +} + +// GetInfo returns the cataloger info for the given name, or nil if not found +func (r *CatalogerRegistry) GetInfo(name string) *capabilities.CatalogerInfo { + return r.infoByName[name] +} + +// DiscoveredCatalogers returns all discovered generic catalogers +func (r *CatalogerRegistry) DiscoveredCatalogers() map[string]DiscoveredCataloger { + return r.discovered +} + +// AllCatalogers returns all catalogers from the syft cataloger list +func (r *CatalogerRegistry) AllCatalogers() []capabilities.CatalogerInfo { + return r.all +} + +// EnrichmentData encapsulates metadata enrichment information (metadata types, package types, binary classifiers) +type EnrichmentData struct { + metadata map[string][]string + packageTypes map[string][]string + binaryClassifiers []binary.Classifier //nolint:staticcheck +} + +// NewEnrichmentData creates a new enrichment data container +func NewEnrichmentData(metadata, packageTypes map[string][]string, binaryClassifiers []binary.Classifier) *EnrichmentData { //nolint:staticcheck + return &EnrichmentData{ + metadata: metadata, + packageTypes: packageTypes, + binaryClassifiers: binaryClassifiers, + } +} + +// GetMetadataTypes returns the metadata types for the given cataloger name +func (e *EnrichmentData) GetMetadataTypes(catalogerName string) ([]string, bool) { + types, ok := e.metadata[catalogerName] + return types, ok +} + +// GetPackageTypes returns the package types for the given cataloger name +func (e *EnrichmentData) GetPackageTypes(catalogerName string) ([]string, bool) { + types, ok := e.packageTypes[catalogerName] + return types, ok +} + +// EnrichEntry enriches a cataloger entry with metadata types and package types if available +func (e *EnrichmentData) EnrichEntry(catalogerName string, entry *capabilities.CatalogerEntry) { + // update metadata types if available + if types, ok := e.GetMetadataTypes(catalogerName); ok { + entry.MetadataTypes = types + // generate JSON schema types from metadata types + entry.JSONSchemaTypes = convertToJSONSchemaTypesFromMetadata(types) + } + // update package types if available + if types, ok := e.GetPackageTypes(catalogerName); ok { + entry.PackageTypes = types + } +} + +// convertToJSONSchemaTypesFromMetadata converts Go struct names to UpperCamelCase JSON schema names +func convertToJSONSchemaTypesFromMetadata(metadataTypes []string) []string { + if len(metadataTypes) == 0 { + return nil + } + + result := make([]string, 0, len(metadataTypes)) + for _, typeName := range metadataTypes { + jsonName := packagemetadata.JSONNameFromString(typeName) + if jsonName != "" { + camelCase := packagemetadata.ToUpperCamelCase(jsonName) + result = append(result, camelCase) + } + } + return result +} + +// EnrichWithBinaryClassifier enriches an entry with binary classifier detectors if it's the binary-classifier-cataloger +func (e *EnrichmentData) EnrichWithBinaryClassifier(catalogerName string, entry *capabilities.CatalogerEntry) { + // special handling for binary-classifier-cataloger: auto-generate one detector per classifier + if catalogerName == "binary-classifier-cataloger" && len(e.binaryClassifiers) > 0 { + var detectors []capabilities.Detector + for _, classifier := range e.binaryClassifiers { + // convert CPEs to strings + cpeStrings := make([]string, len(classifier.CPEs)) + for i, c := range classifier.CPEs { + cpeStrings[i] = c.Attributes.BindToFmtString() + } + + // strip @version from PURL + purlStr := stripPURLVersion(classifier.PURL.String()) + + packages := []capabilities.DetectorPackageInfo{ + { + Class: classifier.Class, + Name: classifier.Package, + PURL: purlStr, + CPEs: cpeStrings, + Type: "BinaryPkg", + }, + } + + for _, o := range binaryClassifierOverrides[classifier.Class] { + packages = append(packages, capabilities.DetectorPackageInfo{ + Class: o.Class, + Name: o.Package, + PURL: stripPURLVersion(o.PURL), + CPEs: o.CPEs, + Type: "BinaryPkg", + }) + } + + detectors = append(detectors, capabilities.Detector{ + Method: "glob", + Criteria: []string{classifier.FileGlob}, + Packages: packages, + }) + } + entry.Detectors = detectors + } +} + +// CatalogerMerger orchestrates the merging of discovered catalogers with existing capabilities +type CatalogerMerger struct { + registry *CatalogerRegistry + enrichment *EnrichmentData + existing *capabilities.Document + catalogerConfigMappings map[string]string // catalogerName -> config struct name + + // internal merge state + updated *capabilities.Document + processedCatalogers map[string]bool + orphans []orphanInfo + stats *mergeStatistics +} + +// NewCatalogerMerger creates a new merger with the given registry, enrichment data, and existing document +func NewCatalogerMerger(registry *CatalogerRegistry, enrichment *EnrichmentData, existing *capabilities.Document, catalogerConfigMappings map[string]string) *CatalogerMerger { + return &CatalogerMerger{ + registry: registry, + enrichment: enrichment, + existing: existing, + catalogerConfigMappings: catalogerConfigMappings, + updated: &capabilities.Document{}, + processedCatalogers: make(map[string]bool), + stats: &mergeStatistics{}, + } +} + +// Merge performs the merge operation and returns the updated document, orphans, and statistics. +// Note: Configs and ApplicationConfig must be set on the merger's updated document before or after calling Merge. +// They are AUTO-GENERATED sections that are completely replaced during regeneration. +func (m *CatalogerMerger) Merge() (*capabilities.Document, []orphanInfo, *mergeStatistics) { + // process catalogers + m.processExistingCatalogers() + m.addNewGenericCatalogers() + m.addNewCustomCatalogers() + return m.updated, m.orphans, m.stats +} + +// processExistingCatalogers processes all existing catalogers in their original order +func (m *CatalogerMerger) processExistingCatalogers() { + for i := range m.existing.Catalogers { + existingEntry := &m.existing.Catalogers[i] + catalogerName := existingEntry.Name + + disc, isGeneric := m.registry.IsGeneric(catalogerName) + info := m.registry.GetInfo(catalogerName) + + switch { + case isGeneric: + // existing generic cataloger - update auto-gen fields, preserve manual, check for orphans + m.processGenericCataloger(existingEntry, disc, info) + + case info != nil: + // existing custom cataloger - preserve but update ecosystem, selectors, metadata types, and package types + m.processCustomCataloger(existingEntry, info) + + default: + // cataloger no longer exists in syft - keep it as-is (user may have added manually) + m.updated.Catalogers = append(m.updated.Catalogers, *existingEntry) + } + + m.processedCatalogers[catalogerName] = true + } +} + +// addNewGenericCatalogers appends any new generic catalogers that weren't in existing +func (m *CatalogerMerger) addNewGenericCatalogers() { + for catalogerName, disc := range m.registry.DiscoveredCatalogers() { + if m.processedCatalogers[catalogerName] { + continue + } + + info := m.registry.GetInfo(catalogerName) + // new generic cataloger - create with template + entry := createTemplateEntry(disc, info) + // update config field from discovered mappings + if configName, hasConfig := m.catalogerConfigMappings[catalogerName]; hasConfig { + entry.Config = configName + } + m.updated.Catalogers = append(m.updated.Catalogers, entry) + m.stats.NewCatalogers = append(m.stats.NewCatalogers, catalogerName) + for _, parser := range disc.Parsers { + m.stats.NewParserFunctions = append(m.stats.NewParserFunctions, fmt.Sprintf("%s/%s", catalogerName, parser.ParserFunction)) + } + + m.processedCatalogers[catalogerName] = true + } +} + +// addNewCustomCatalogers appends any new custom catalogers from syft cataloger list +func (m *CatalogerMerger) addNewCustomCatalogers() { + for _, catalogerInfo := range m.registry.AllCatalogers() { + catalogerName := catalogerInfo.Name + if m.processedCatalogers[catalogerName] { + continue + } + + // new custom cataloger - create template entry + entry := capabilities.CatalogerEntry{ + Ecosystem: inferEcosystem(catalogerName), + Name: catalogerName, + Type: "custom", + Source: capabilities.Source{ + File: "", // must be filled manually + Function: "", // must be filled manually + }, + Selectors: catalogerInfo.Selectors, + Capabilities: capabilities.CapabilitySet{}, // empty array - must be filled manually + } + + // update config field from discovered mappings + if configName, hasConfig := m.catalogerConfigMappings[catalogerName]; hasConfig { + entry.Config = configName + } + + // enrich with metadata and package types + m.enrichment.EnrichEntry(catalogerName, &entry) + + // fallback: if we have metadata_types but no json_schema_types, convert them + // this handles cases where metadata_types exist in YAML but no enrichment data + if len(entry.MetadataTypes) > 0 && len(entry.JSONSchemaTypes) == 0 { + entry.JSONSchemaTypes = convertToJSONSchemaTypesFromMetadata(entry.MetadataTypes) + } + + // enrich with binary classifier globs + m.enrichment.EnrichWithBinaryClassifier(catalogerName, &entry) + + m.updated.Catalogers = append(m.updated.Catalogers, entry) + m.stats.NewCatalogers = append(m.stats.NewCatalogers, catalogerName) + + m.processedCatalogers[catalogerName] = true + } +} + +// processGenericCataloger processes an existing generic cataloger entry +func (m *CatalogerMerger) processGenericCataloger(existingEntry *capabilities.CatalogerEntry, disc DiscoveredCataloger, info *capabilities.CatalogerInfo) { + entry, catalogerOrphans, newParsers := updateEntry(existingEntry, disc, info, m.catalogerConfigMappings) + + // fallback for catalogers with type override to custom but processed as generic + // these may have cataloger-level metadata_types that need json_schema_types + if len(entry.MetadataTypes) > 0 && len(entry.JSONSchemaTypes) == 0 { + entry.JSONSchemaTypes = convertToJSONSchemaTypesFromMetadata(entry.MetadataTypes) + } + + m.updated.Catalogers = append(m.updated.Catalogers, entry) + m.orphans = append(m.orphans, catalogerOrphans...) + if len(newParsers) > 0 || len(catalogerOrphans) > 0 { + m.stats.UpdatedCatalogers = append(m.stats.UpdatedCatalogers, existingEntry.Name) + } + for _, parser := range newParsers { + m.stats.NewParserFunctions = append(m.stats.NewParserFunctions, fmt.Sprintf("%s/%s", existingEntry.Name, parser)) + } +} + +// processCustomCataloger processes an existing custom cataloger entry +func (m *CatalogerMerger) processCustomCataloger(existingEntry *capabilities.CatalogerEntry, info *capabilities.CatalogerInfo) { + entry := *existingEntry + // only infer ecosystem if not manually set (ecosystem is MANUAL) + if existingEntry.Ecosystem == "" { + entry.Ecosystem = inferEcosystem(existingEntry.Name) + } + entry.Selectors = info.Selectors + + // update config field from discovered mappings (AUTO-GENERATED) + if configName, hasConfig := m.catalogerConfigMappings[existingEntry.Name]; hasConfig { + entry.Config = configName + } + + // enrich with metadata and package types + m.enrichment.EnrichEntry(existingEntry.Name, &entry) + + // fallback: if we have metadata_types but no json_schema_types, convert them + // this handles cases where metadata_types exist in YAML but no enrichment data + if len(entry.MetadataTypes) > 0 && len(entry.JSONSchemaTypes) == 0 { + entry.JSONSchemaTypes = convertToJSONSchemaTypesFromMetadata(entry.MetadataTypes) + } + + // enrich with binary classifier globs + m.enrichment.EnrichWithBinaryClassifier(existingEntry.Name, &entry) + + m.updated.Catalogers = append(m.updated.Catalogers, entry) +} + +// mergeDiscoveredWithExisting combines discovered cataloger information with existing capabilities, +// preserving manual sections (capabilities) while updating AUTO-GENERATED sections. +// +// The configs and appConfigs parameters are AUTO-GENERATED sections that completely replace +// any existing configs/app-config data in the ecosystem YAML files. +// +// The catalogerConfigMappings parameter maps cataloger names to their config struct names +// (e.g., "go-module-binary-cataloger" -> "golang.CatalogerConfig"). +func mergeDiscoveredWithExisting( + discovered map[string]DiscoveredCataloger, + customMetadata map[string][]string, + customPackageTypes map[string][]string, + binaryClassifiers []binary.Classifier, //nolint:staticcheck + allCatalogers []capabilities.CatalogerInfo, + existing *capabilities.Document, + configs map[string]capabilities.CatalogerConfigEntry, + appConfigs []capabilities.ApplicationConfigField, + catalogerConfigMappings map[string]string, +) (*capabilities.Document, []orphanInfo, *mergeStatistics) { + registry := NewCatalogerRegistry(discovered, allCatalogers) + enrichment := NewEnrichmentData(customMetadata, customPackageTypes, binaryClassifiers) + merger := NewCatalogerMerger(registry, enrichment, existing, catalogerConfigMappings) + + // set the AUTO-GENERATED config sections + // these completely replace any existing data (not merged) + merger.updated.Configs = configs + merger.updated.ApplicationConfig = appConfigs + + return merger.Merge() +} + +func updateEntry(existing *capabilities.CatalogerEntry, discovered DiscoveredCataloger, info *capabilities.CatalogerInfo, catalogerConfigMappings map[string]string) (capabilities.CatalogerEntry, []orphanInfo, []string) { + updated := *existing + + // update AUTO-GENERATED fields + updated.Name = discovered.Name + + // check if there's a type override for this cataloger + if overrideType, hasOverride := catalogerTypeOverrides[discovered.Name]; hasOverride { + updated.Type = overrideType + } else { + updated.Type = discovered.Type + } + + updated.Source = capabilities.Source{ + File: discovered.SourceFile, + Function: discovered.SourceFunction, + } + + // update selectors from cataloger info + if info != nil { + updated.Selectors = info.Selectors + } + + // update config field from discovered mappings (AUTO-GENERATED) + if configName, hasConfig := catalogerConfigMappings[discovered.Name]; hasConfig { + updated.Config = configName + } else { + // clear config if no mapping exists (it may have been removed) + updated.Config = "" + } + + // only infer ecosystem if not manually set (ecosystem is MANUAL) + if existing.Ecosystem == "" { + updated.Ecosystem = inferEcosystem(discovered.Name) + } + + var orphans []orphanInfo + var newParsers []string + + // update parsers only if the final type is generic (not overridden to custom) + // if a cataloger is overridden from generic to custom, we don't update parsers + if discovered.Type == genericCatalogerType && updated.Type == genericCatalogerType { + updatedParsers, parserOrphans, newParserFuncs := updateParsers(existing.Parsers, discovered.Parsers, discovered.Name) + updated.Parsers = updatedParsers + orphans = append(orphans, parserOrphans...) + newParsers = newParserFuncs + } + + return updated, orphans, newParsers +} + +func updateParsers(existingParsers []capabilities.Parser, discoveredParsers []DiscoveredParser, catalogerName string) ([]capabilities.Parser, []orphanInfo, []string) { + var updated []capabilities.Parser + var orphans []orphanInfo + var newParserFuncs []string + + // create lookup for discovered parsers by parser function + discoveredByParserFunc := make(map[string]*DiscoveredParser) + for i := range discoveredParsers { + discoveredByParserFunc[discoveredParsers[i].ParserFunction] = &discoveredParsers[i] + } + + // create lookup for existing parsers by parser function + existingByParserFunc := make(map[string]*capabilities.Parser) + for i := range existingParsers { + existingByParserFunc[existingParsers[i].ParserFunction] = &existingParsers[i] + } + + // process all discovered parsers + for _, discParser := range discoveredParsers { + existingParser := existingByParserFunc[discParser.ParserFunction] + + if existingParser == nil { + // new parser - create with empty capabilities + updated = append(updated, createTemplateParser(discParser)) + newParserFuncs = append(newParserFuncs, discParser.ParserFunction) + } else { + // update auto-gen fields, preserve capabilities + p := *existingParser + p.Detector.Method = discParser.Method + p.Detector.Criteria = discParser.Criteria + + // only update metadata/package types if discovered parser has them + // this preserves existing YAML values when no test observations exist + if len(discParser.MetadataTypes) > 0 { + p.MetadataTypes = discParser.MetadataTypes + p.JSONSchemaTypes = discParser.JSONSchemaTypes + } else if len(p.MetadataTypes) > 0 && len(p.JSONSchemaTypes) == 0 { + // fallback: if parser has metadata_types but no json_schema_types, convert them + p.JSONSchemaTypes = convertToJSONSchemaTypesFromMetadata(p.MetadataTypes) + } + + if len(discParser.PackageTypes) > 0 { + p.PackageTypes = discParser.PackageTypes + } + + // p.Capabilities is preserved from existing + updated = append(updated, p) + } + + // mark this parser as processed + delete(existingByParserFunc, discParser.ParserFunction) + } + + // any remaining existing parsers are orphans (parser function was renamed/deleted) + for parserFunc := range existingByParserFunc { + orphans = append(orphans, orphanInfo{ + catalogerName: catalogerName, + parserFunction: parserFunc, + }) + } + + return updated, orphans, newParserFuncs +} + +func createTemplateEntry(disc DiscoveredCataloger, info *capabilities.CatalogerInfo) capabilities.CatalogerEntry { + // determine type, checking for overrides first + catalogerType := disc.Type + if overrideType, hasOverride := catalogerTypeOverrides[disc.Name]; hasOverride { + catalogerType = overrideType + } + + entry := capabilities.CatalogerEntry{ + Ecosystem: inferEcosystem(disc.Name), + Name: disc.Name, + Type: catalogerType, + Source: capabilities.Source{ + File: disc.SourceFile, + Function: disc.SourceFunction, + }, + } + + // add selectors from cataloger info + if info != nil { + entry.Selectors = info.Selectors + } + + // use the determined catalogerType (which may be overridden) to structure the entry + switch catalogerType { + case genericCatalogerType: + for _, discParser := range disc.Parsers { + entry.Parsers = append(entry.Parsers, createTemplateParser(discParser)) + } + case "custom": + // custom cataloger with empty capabilities (must be filled manually) + entry.Capabilities = capabilities.CapabilitySet{} + } + + return entry +} + +func createTemplateParser(disc DiscoveredParser) capabilities.Parser { + return capabilities.Parser{ + ParserFunction: disc.ParserFunction, + Detector: capabilities.Detector{ + Method: disc.Method, + Criteria: disc.Criteria, + }, + MetadataTypes: disc.MetadataTypes, + PackageTypes: disc.PackageTypes, + JSONSchemaTypes: disc.JSONSchemaTypes, + Capabilities: capabilities.CapabilitySet{}, // empty array - must be filled manually + } +} + +func formatOrphans(orphans []orphanInfo) string { + var lines []string + for _, o := range orphans { + lines = append(lines, fmt.Sprintf(" - cataloger: %s, parser function: %s", o.catalogerName, o.parserFunction)) + } + return strings.Join(lines, "\n") +} diff --git a/internal/capabilities/generate/merge_test.go b/internal/capabilities/generate/merge_test.go new file mode 100644 index 000000000..229218e5d --- /dev/null +++ b/internal/capabilities/generate/merge_test.go @@ -0,0 +1,554 @@ +package main + +import ( + "os" + "os/exec" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/capabilities" + "github.com/anchore/syft/internal/capabilities/internal" + "github.com/anchore/syft/syft/pkg/cataloger/binary" +) + +// TestCapabilitiesAreUpToDate verifies that cataloger/*/capabilities.yaml files are up to date by running regeneration and checking +// for uncommitted changes. This test only runs in CI to catch cases where code changed but capabilities weren't regenerated. +func TestCapabilitiesAreUpToDate(t *testing.T) { + checkCompletenessTestsEnabled(t) + + if os.Getenv("CI") == "" { + t.Skip("skipping regeneration test in local environment") + } + + repoRoot, err := internal.RepoRoot() + require.NoError(t, err) + + catalogerDir := internal.CatalogerDir(repoRoot) + + // regenerate should not fail + _, err = RegenerateCapabilities(catalogerDir, repoRoot) + require.NoError(t, err) + + // verify files haven't changed (i.e., they were already up to date) + cmd := exec.Command("git", "diff", "--exit-code", catalogerDir) + cmd.Dir = repoRoot + err = cmd.Run() + require.NoError(t, err, "cataloger/*/capabilities.yaml files have uncommitted changes after regeneration. Run 'go generate ./internal/capabilities' locally and commit the changes.") +} + +func TestMergeConfigSections(t *testing.T) { + tests := []struct { + name string + existingDoc *capabilities.Document + newConfigs map[string]capabilities.CatalogerConfigEntry + newAppConfigs []capabilities.ApplicationConfigField + expectedConfigs map[string]capabilities.CatalogerConfigEntry + expectedAppConfigs []capabilities.ApplicationConfigField + description string + }{ + { + name: "new configs replace existing configs", + description: "configs and app-config are AUTO-GENERATED, so new data should completely replace old", + existingDoc: &capabilities.Document{ + Configs: map[string]capabilities.CatalogerConfigEntry{ + "golang.OldConfig": { + Fields: []capabilities.CatalogerConfigFieldEntry{ + {Key: "OldField", Description: "old field"}, + }, + }, + }, + ApplicationConfig: []capabilities.ApplicationConfigField{ + {Key: "golang.old-config", Description: "old config"}, + }, + Catalogers: []capabilities.CatalogerEntry{}, + }, + newConfigs: map[string]capabilities.CatalogerConfigEntry{ + "golang.CatalogerConfig": { + Fields: []capabilities.CatalogerConfigFieldEntry{ + {Key: "SearchLocalModCacheLicenses", Description: "search local mod cache", AppKey: "golang.search-local-mod-cache-licenses"}, + }, + }, + }, + newAppConfigs: []capabilities.ApplicationConfigField{ + {Key: "golang.search-local-mod-cache-licenses", Description: "Search licenses in local mod cache", DefaultValue: false}, + }, + expectedConfigs: map[string]capabilities.CatalogerConfigEntry{ + "golang.CatalogerConfig": { + Fields: []capabilities.CatalogerConfigFieldEntry{ + {Key: "SearchLocalModCacheLicenses", Description: "search local mod cache", AppKey: "golang.search-local-mod-cache-licenses"}, + }, + }, + }, + expectedAppConfigs: []capabilities.ApplicationConfigField{ + {Key: "golang.search-local-mod-cache-licenses", Description: "Search licenses in local mod cache", DefaultValue: false}, + }, + }, + { + name: "empty new configs clears existing configs", + description: "if no configs are discovered, the sections should be empty (not nil)", + existingDoc: &capabilities.Document{ + Configs: map[string]capabilities.CatalogerConfigEntry{ + "golang.OldConfig": { + Fields: []capabilities.CatalogerConfigFieldEntry{ + {Key: "OldField", Description: "old field"}, + }, + }, + }, + ApplicationConfig: []capabilities.ApplicationConfigField{ + {Key: "golang.old-config", Description: "old config"}, + }, + Catalogers: []capabilities.CatalogerEntry{}, + }, + newConfigs: map[string]capabilities.CatalogerConfigEntry{}, + newAppConfigs: []capabilities.ApplicationConfigField{}, + expectedConfigs: map[string]capabilities.CatalogerConfigEntry{}, + expectedAppConfigs: []capabilities.ApplicationConfigField{}, + }, + { + name: "nil existing configs are replaced with new configs", + description: "first-time generation should populate configs", + existingDoc: &capabilities.Document{ + Catalogers: []capabilities.CatalogerEntry{}, + }, + newConfigs: map[string]capabilities.CatalogerConfigEntry{ + "python.CatalogerConfig": { + Fields: []capabilities.CatalogerConfigFieldEntry{ + {Key: "GuessUnpinnedRequirements", Description: "guess unpinned reqs", AppKey: "python.guess-unpinned-requirements"}, + }, + }, + }, + newAppConfigs: []capabilities.ApplicationConfigField{ + {Key: "python.guess-unpinned-requirements", Description: "Guess unpinned requirements", DefaultValue: false}, + }, + expectedConfigs: map[string]capabilities.CatalogerConfigEntry{ + "python.CatalogerConfig": { + Fields: []capabilities.CatalogerConfigFieldEntry{ + {Key: "GuessUnpinnedRequirements", Description: "guess unpinned reqs", AppKey: "python.guess-unpinned-requirements"}, + }, + }, + }, + expectedAppConfigs: []capabilities.ApplicationConfigField{ + {Key: "python.guess-unpinned-requirements", Description: "Guess unpinned requirements", DefaultValue: false}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // use mergeDiscoveredWithExisting to properly test the integration + updated, _, _ := mergeDiscoveredWithExisting( + map[string]DiscoveredCataloger{}, + map[string][]string{}, + map[string][]string{}, + []binary.Classifier{}, + []capabilities.CatalogerInfo{}, + tt.existingDoc, + tt.newConfigs, + tt.newAppConfigs, + map[string]string{}, + ) + + // verify configs were replaced (not merged) + if diff := cmp.Diff(tt.expectedConfigs, updated.Configs); diff != "" { + t.Errorf("Configs mismatch (-want +got):\n%s", diff) + } + + // verify app-configs were replaced (not merged) + if diff := cmp.Diff(tt.expectedAppConfigs, updated.ApplicationConfig); diff != "" { + t.Errorf("ApplicationConfig mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestMergeCatalogerConfigField(t *testing.T) { + tests := []struct { + name string + existingEntry capabilities.CatalogerEntry + discoveredInfo DiscoveredCataloger + catalogerConfigMappings map[string]string + expectedConfig string + }{ + { + name: "config field is updated from discovered data", + existingEntry: capabilities.CatalogerEntry{ + Name: "go-module-binary-cataloger", + Config: "", // was empty + }, + discoveredInfo: DiscoveredCataloger{ + Name: "go-module-binary-cataloger", + Type: "generic", + }, + catalogerConfigMappings: map[string]string{ + "go-module-binary-cataloger": "golang.CatalogerConfig", + }, + expectedConfig: "golang.CatalogerConfig", + }, + { + name: "config field is replaced if different", + existingEntry: capabilities.CatalogerEntry{ + Name: "go-module-binary-cataloger", + Config: "golang.OldConfig", + }, + discoveredInfo: DiscoveredCataloger{ + Name: "go-module-binary-cataloger", + Type: "generic", + }, + catalogerConfigMappings: map[string]string{ + "go-module-binary-cataloger": "golang.NewConfig", + }, + expectedConfig: "golang.NewConfig", + }, + { + name: "config field is cleared if no mapping exists", + existingEntry: capabilities.CatalogerEntry{ + Name: "go-module-binary-cataloger", + Config: "golang.OldConfig", + }, + discoveredInfo: DiscoveredCataloger{ + Name: "go-module-binary-cataloger", + Type: "generic", + }, + catalogerConfigMappings: map[string]string{}, + expectedConfig: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // simulate updateEntry which should update the Config field + updated, _, _ := updateEntry(&tt.existingEntry, tt.discoveredInfo, nil, tt.catalogerConfigMappings) + + require.Equal(t, tt.expectedConfig, updated.Config) + }) + } +} + +func TestMergePreservesManualCapabilities(t *testing.T) { + // ensure that while we update configs (AUTO-GENERATED), + // we still preserve capabilities (MANUAL) + existingDoc := &capabilities.Document{ + Configs: map[string]capabilities.CatalogerConfigEntry{ + "golang.OldConfig": { + Fields: []capabilities.CatalogerConfigFieldEntry{ + {Key: "OldField", Description: "old"}, + }, + }, + }, + Catalogers: []capabilities.CatalogerEntry{ + { + Name: "test-cataloger", + Type: "generic", + Parsers: []capabilities.Parser{ + { + ParserFunction: "parseTest", + Capabilities: capabilities.CapabilitySet{ + {Name: "license", Default: true}, // manual value + }, + }, + }, + }, + }, + } + + discovered := map[string]DiscoveredCataloger{ + "test-cataloger": { + Name: "test-cataloger", + Type: "generic", + Parsers: []DiscoveredParser{ + { + ParserFunction: "parseTest", + Method: "glob", + Criteria: []string{"**/*.test"}, + }, + }, + }, + } + + newConfigs := map[string]capabilities.CatalogerConfigEntry{ + "golang.NewConfig": { + Fields: []capabilities.CatalogerConfigFieldEntry{ + {Key: "NewField", Description: "new"}, + }, + }, + } + + updated, _, _ := mergeDiscoveredWithExisting( + discovered, + map[string][]string{}, + map[string][]string{}, + []binary.Classifier{}, + []capabilities.CatalogerInfo{ + {Name: "test-cataloger", Selectors: []string{"test"}}, + }, + existingDoc, + newConfigs, + []capabilities.ApplicationConfigField{}, + map[string]string{}, + ) + + // verify configs were replaced + require.Len(t, updated.Configs, 1) + _, hasOld := updated.Configs["golang.OldConfig"] + require.False(t, hasOld, "old config should be removed") + _, hasNew := updated.Configs["golang.NewConfig"] + require.True(t, hasNew, "new config should be present") + + // verify capabilities were preserved + require.Len(t, updated.Catalogers, 1) + require.Len(t, updated.Catalogers[0].Parsers, 1) + parser := updated.Catalogers[0].Parsers[0] + require.Len(t, parser.Capabilities, 1) + require.Equal(t, "license", parser.Capabilities[0].Name, "manual capability field should be preserved") + require.Equal(t, true, parser.Capabilities[0].Default, "manual capability value should be preserved") + + // verify AUTO-GENERATED parser fields were updated + require.Equal(t, "glob", string(parser.Detector.Method)) + require.Equal(t, []string{"**/*.test"}, parser.Detector.Criteria) +} + +func TestCatalogerConfigFieldUpdatedForNewCatalogers(t *testing.T) { + tests := []struct { + name string + catalogerName string + catalogerType string + catalogerConfigMappings map[string]string + expectedConfig string + }{ + { + name: "new generic cataloger gets config from mapping", + catalogerName: "go-module-binary-cataloger", + catalogerType: "generic", + catalogerConfigMappings: map[string]string{ + "go-module-binary-cataloger": "golang.CatalogerConfig", + }, + expectedConfig: "golang.CatalogerConfig", + }, + { + name: "new custom cataloger gets config from mapping", + catalogerName: "java-archive-cataloger", + catalogerType: "custom", + catalogerConfigMappings: map[string]string{ + "java-archive-cataloger": "java.CatalogerConfig", + }, + expectedConfig: "java.CatalogerConfig", + }, + { + name: "new cataloger without mapping has empty config", + catalogerName: "python-cataloger", + catalogerType: "generic", + catalogerConfigMappings: map[string]string{}, + expectedConfig: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // test for generic catalogers + if tt.catalogerType == "generic" { + discovered := map[string]DiscoveredCataloger{ + tt.catalogerName: { + Name: tt.catalogerName, + Type: "generic", + Parsers: []DiscoveredParser{ + { + ParserFunction: "parseTest", + Method: "glob", + Criteria: []string{"**/*.test"}, + }, + }, + }, + } + + updated, _, _ := mergeDiscoveredWithExisting( + discovered, + map[string][]string{}, + map[string][]string{}, + []binary.Classifier{}, + []capabilities.CatalogerInfo{ + {Name: tt.catalogerName, Selectors: []string{"test"}}, + }, + &capabilities.Document{Catalogers: []capabilities.CatalogerEntry{}}, + map[string]capabilities.CatalogerConfigEntry{}, + []capabilities.ApplicationConfigField{}, + tt.catalogerConfigMappings, + ) + + require.Len(t, updated.Catalogers, 1) + require.Equal(t, tt.expectedConfig, updated.Catalogers[0].Config) + } + + // test for custom catalogers + if tt.catalogerType == "custom" { + updated, _, _ := mergeDiscoveredWithExisting( + map[string]DiscoveredCataloger{}, + map[string][]string{}, + map[string][]string{}, + []binary.Classifier{}, + []capabilities.CatalogerInfo{ + {Name: tt.catalogerName, Selectors: []string{"test"}}, + }, + &capabilities.Document{Catalogers: []capabilities.CatalogerEntry{}}, + map[string]capabilities.CatalogerConfigEntry{}, + []capabilities.ApplicationConfigField{}, + tt.catalogerConfigMappings, + ) + + require.Len(t, updated.Catalogers, 1) + require.Equal(t, tt.expectedConfig, updated.Catalogers[0].Config) + } + }) + } +} + +func TestStripPURLVersion(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "purl with version", + input: "pkg:generic/python@1.0.0", + want: "pkg:generic/python", + }, + { + name: "purl without version", + input: "pkg:generic/python", + want: "pkg:generic/python", + }, + { + name: "purl with multiple @ signs", + input: "pkg:generic/py@thon@1.0.0", + want: "pkg:generic/py@thon", + }, + { + name: "empty string", + input: "", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := stripPURLVersion(tt.input) + require.Equal(t, tt.want, got) + }) + } +} + +func TestInferEcosystem(t *testing.T) { + tests := []struct { + name string + catalogerName string + want string + }{ + { + name: "go module cataloger", + catalogerName: "go-module-binary-cataloger", + want: "go", + }, + { + name: "python cataloger", + catalogerName: "python-package-cataloger", + want: "python", + }, + { + name: "java archive cataloger", + catalogerName: "java-archive-cataloger", + want: "java", + }, + { + name: "rust cargo cataloger", + catalogerName: "rust-cargo-lock-cataloger", + want: "rust", + }, + { + name: "javascript npm cataloger", + catalogerName: "javascript-package-cataloger", + want: "javascript", + }, + { + name: "ruby gem cataloger", + catalogerName: "ruby-gemspec-cataloger", + want: "ruby", + }, + { + name: "debian dpkg cataloger", + catalogerName: "dpkg-db-cataloger", + want: "debian", + }, + { + name: "alpine apk cataloger", + catalogerName: "apk-db-cataloger", + want: "alpine", + }, + { + name: "linux kernel cataloger", + catalogerName: "linux-kernel-cataloger", + want: "kernel", + }, + { + name: "binary classifier cataloger", + catalogerName: "binary-classifier-cataloger", + want: "binary", + }, + { + name: "github actions cataloger", + catalogerName: "github-actions-usage-cataloger", + want: "github-actions", + }, + { + name: "unknown cataloger defaults to other", + catalogerName: "unknown-custom-cataloger", + want: "other", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := inferEcosystem(tt.catalogerName) + require.Equal(t, tt.want, got) + }) + } +} + +func TestConvertToJSONSchemaTypesFromMetadata(t *testing.T) { + tests := []struct { + name string + metadataTypes []string + want []string + }{ + { + name: "empty slice returns nil", + metadataTypes: []string{}, + want: nil, + }, + { + name: "nil slice returns nil", + metadataTypes: nil, + want: nil, + }, + { + name: "single metadata type", + metadataTypes: []string{"pkg.AlpmDBEntry"}, + want: []string{"AlpmDbEntry"}, + }, + { + name: "multiple metadata types", + metadataTypes: []string{"pkg.ApkDBEntry", "pkg.BinarySignature"}, + want: []string{"ApkDbEntry", "BinarySignature"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := convertToJSONSchemaTypesFromMetadata(tt.metadataTypes) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("convertToJSONSchemaTypesFromMetadata() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/internal/capabilities/generate/metadata_check.go b/internal/capabilities/generate/metadata_check.go new file mode 100644 index 000000000..2a606a974 --- /dev/null +++ b/internal/capabilities/generate/metadata_check.go @@ -0,0 +1,373 @@ +// this file validates that all known metadata and package types are documented in cataloger/*/capabilities.yaml by checking coverage and reporting any missing types. +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "sort" + "strings" + + "github.com/charmbracelet/lipgloss" + + "github.com/anchore/syft/internal/capabilities" + "github.com/anchore/syft/internal/capabilities/internal" +) + +var ( + warningStyleMeta = lipgloss.NewStyle().Foreground(lipgloss.Color("11")).Bold(true) // yellow + dimStyleMeta = lipgloss.NewStyle().Foreground(lipgloss.Color("245")) // lighter grey (256-color) + + // exceptions for metadata types that are intentionally not referenced in cataloger/*/capabilities.yaml + metadataTypeExceptions = map[string]bool{ + "pkg.MicrosoftKbPatch": true, + } + + // exceptions for package types that are intentionally not referenced in cataloger/*/capabilities.yaml + packageTypeExceptions = map[string]bool{ + "jenkins-plugin": true, + "msrc-kb": true, + } +) + +// parsePackageMetadataTypes parses packagemetadata/generated.go and extracts all metadata type names +// from the AllTypes() function (e.g., "pkg.AlpmDBEntry", "pkg.ApkDBEntry", etc.) +func parsePackageMetadataTypes(repoRoot string) ([]string, error) { + metadataFile := filepath.Join(repoRoot, "internal", "packagemetadata", "generated.go") + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, metadataFile, nil, 0) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", metadataFile, err) + } + + var types []string + + // find the AllTypes function + for _, decl := range f.Decls { + funcDecl, ok := decl.(*ast.FuncDecl) + if !ok || funcDecl.Name.Name != "AllTypes" { + continue + } + + // walk the function body to find return statement + ast.Inspect(funcDecl.Body, func(n ast.Node) bool { + returnStmt, ok := n.(*ast.ReturnStmt) + if !ok { + return true + } + + // should have one return value: []any{...} + if len(returnStmt.Results) != 1 { + return true + } + + // parse the composite literal (slice) + compositeLit, ok := returnStmt.Results[0].(*ast.CompositeLit) + if !ok { + return true + } + + // extract each element (should be pkg.TypeName{}) + for _, elt := range compositeLit.Elts { + if typeExpr, ok := elt.(*ast.CompositeLit); ok { + typeName := extractTypeName(typeExpr.Type) + if typeName != "" { + types = append(types, typeName) + } + } + } + + return false + }) + } + + return types, nil +} + +// extractTypeName extracts the full type name from an AST type expression +// e.g., pkg.AlpmDBEntry -> "pkg.AlpmDBEntry" +func extractTypeName(expr ast.Expr) string { + switch t := expr.(type) { + case *ast.SelectorExpr: + // pkg.TypeName + if pkgIdent, ok := t.X.(*ast.Ident); ok { + return fmt.Sprintf("%s.%s", pkgIdent.Name, t.Sel.Name) + } + case *ast.Ident: + // just TypeName + return t.Name + } + return "" +} + +// collectReferencedMetadataTypes walks through all catalogers and collects +// all metadata types referenced in parser and cataloger-level metadata_types fields +func collectReferencedMetadataTypes(doc *capabilities.Document) []string { + typeSet := make(map[string]bool) + + for _, cataloger := range doc.Catalogers { + // collect from parsers (for generic catalogers) + for _, parser := range cataloger.Parsers { + for _, metadataType := range parser.MetadataTypes { + typeSet[metadataType] = true + } + } + + // collect from cataloger-level metadata_types (for custom catalogers) + for _, metadataType := range cataloger.MetadataTypes { + typeSet[metadataType] = true + } + } + + // convert set to sorted slice + var types []string + for typeName := range typeSet { + types = append(types, typeName) + } + sort.Strings(types) + + return types +} + +// checkMetadataTypeCoverage compares metadata types from packagemetadata/generated.go +// with types referenced in cataloger/*/capabilities.yaml and returns unreferenced types +func checkMetadataTypeCoverage(capabilitiesDir string, repoRoot string) ([]string, error) { + // parse packagemetadata/generated.go to get all types + allTypes, err := parsePackageMetadataTypes(repoRoot) + if err != nil { + return nil, fmt.Errorf("failed to parse package metadata types: %w", err) + } + + // load capabilities files to get referenced types + doc, _, err := internal.LoadCapabilities(capabilitiesDir, repoRoot) + if err != nil { + return nil, fmt.Errorf("failed to load capabilities files: %w", err) + } + + referencedTypes := collectReferencedMetadataTypes(doc) + + // create a set of referenced types for quick lookup + referencedSet := make(map[string]bool) + for _, typeName := range referencedTypes { + referencedSet[typeName] = true + } + + // find unreferenced types (excluding exceptions) + var unreferenced []string + for _, typeName := range allTypes { + if !referencedSet[typeName] && !metadataTypeExceptions[typeName] { + unreferenced = append(unreferenced, typeName) + } + } + + return unreferenced, nil +} + +// printMetadataTypeCoverageWarning prints a warning if there are metadata types +// from packagemetadata/generated.go that aren't referenced in cataloger/*/capabilities.yaml +func printMetadataTypeCoverageWarning(capabilitiesDir string, repoRoot string) { + unreferenced, err := checkMetadataTypeCoverage(capabilitiesDir, repoRoot) + if err != nil { + // don't fail generation, just skip the check + fmt.Printf("%s Could not check metadata type coverage: %v\n", warningStyleMeta.Render("⚠"), err) + return + } + + if len(unreferenced) > 0 { + fmt.Println() + fmt.Printf("%s %s metadata types from packagemetadata are not referenced in cataloger/*/capabilities.yaml:\n", + warningStyleMeta.Render("⚠ INFO:"), + warningStyleMeta.Render(fmt.Sprintf("%d", len(unreferenced)))) + for _, typeName := range unreferenced { + // extract just the type name without "pkg." prefix for cleaner output + shortName := strings.TrimPrefix(typeName, "pkg.") + fmt.Printf(" - %s\n", dimStyleMeta.Render(shortName)) + } + fmt.Println() + fmt.Println(dimStyleMeta.Render("These types may be:")) + fmt.Println(dimStyleMeta.Render(" • Used in custom catalogers (which don't have metadata_types)")) + fmt.Println(dimStyleMeta.Render(" • Deprecated or unused")) + fmt.Println(dimStyleMeta.Render(" • Missing from cataloger pattern documentation")) + } +} + +// parseConstValues extracts constant names to their string values from an AST file +func parseConstValues(f *ast.File) map[string]string { + constValues := make(map[string]string) + for _, decl := range f.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.CONST { + continue + } + + for _, spec := range genDecl.Specs { + valueSpec, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + + for i, ident := range valueSpec.Names { + if i < len(valueSpec.Values) { + if lit, ok := valueSpec.Values[i].(*ast.BasicLit); ok && lit.Kind == token.STRING { + constValues[ident.Name] = strings.Trim(lit.Value, `"`) + } + } + } + } + } + return constValues +} + +// extractTypesFromCompositeLit extracts package type names from a composite literal +func extractTypesFromCompositeLit(compositeLit *ast.CompositeLit, constValues map[string]string) []string { + var types []string + for _, elt := range compositeLit.Elts { + if ident, ok := elt.(*ast.Ident); ok { + // look up the string value for this constant + if typeName, ok := constValues[ident.Name]; ok && typeName != "UnknownPackage" { + types = append(types, typeName) + } + } + } + return types +} + +// extractAllPkgsTypes finds the AllPkgs variable and extracts package type names +func extractAllPkgsTypes(f *ast.File, constValues map[string]string) []string { + for _, decl := range f.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.VAR { + continue + } + + for _, spec := range genDecl.Specs { + valueSpec, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + + for i, ident := range valueSpec.Names { + if ident.Name == "AllPkgs" && i < len(valueSpec.Values) { + // found AllPkgs, extract the slice elements + compositeLit, ok := valueSpec.Values[i].(*ast.CompositeLit) + if !ok { + continue + } + return extractTypesFromCompositeLit(compositeLit, constValues) + } + } + } + } + + return []string{} +} + +// parseAllPackageTypes parses syft/pkg/type.go and extracts all package type names +// from the AllPkgs variable by looking up their const string values +func parseAllPackageTypes(repoRoot string) ([]string, error) { + typeFile := filepath.Join(repoRoot, "syft", "pkg", "type.go") + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, typeFile, nil, 0) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", typeFile, err) + } + + // first, build a map of constant names to their string values + constValues := parseConstValues(f) + + // find the AllPkgs variable and extract types + types := extractAllPkgsTypes(f, constValues) + + return types, nil +} + +// collectReferencedPackageTypes walks through all catalogers and collects +// all package types referenced in parser and cataloger-level package_types fields +func collectReferencedPackageTypes(doc *capabilities.Document) []string { + typeSet := make(map[string]bool) + + for _, cataloger := range doc.Catalogers { + // collect from parsers (for generic catalogers) + for _, parser := range cataloger.Parsers { + for _, pkgType := range parser.PackageTypes { + typeSet[pkgType] = true + } + } + + // collect from cataloger-level package_types (for custom catalogers) + for _, pkgType := range cataloger.PackageTypes { + typeSet[pkgType] = true + } + } + + // convert set to sorted slice + var types []string + for typeName := range typeSet { + types = append(types, typeName) + } + sort.Strings(types) + + return types +} + +// checkPackageTypeCoverage compares package types from pkg.AllPkgs +// with types referenced in cataloger/*/capabilities.yaml and returns unreferenced types +func checkPackageTypeCoverage(capabilitiesDir string, repoRoot string) ([]string, error) { + // parse pkg/type.go to get all package types + allTypes, err := parseAllPackageTypes(repoRoot) + if err != nil { + return nil, fmt.Errorf("failed to parse package types: %w", err) + } + + // load capabilities files to get referenced types + doc, _, err := internal.LoadCapabilities(capabilitiesDir, repoRoot) + if err != nil { + return nil, fmt.Errorf("failed to load capabilities files: %w", err) + } + + referencedTypes := collectReferencedPackageTypes(doc) + + // create a set of referenced types for quick lookup + referencedSet := make(map[string]bool) + for _, typeName := range referencedTypes { + referencedSet[typeName] = true + } + + // find unreferenced types (excluding exceptions) + var unreferenced []string + for _, typeName := range allTypes { + if !referencedSet[typeName] && !packageTypeExceptions[typeName] { + unreferenced = append(unreferenced, typeName) + } + } + + return unreferenced, nil +} + +// printPackageTypeCoverageWarning prints a warning if there are package types +// from pkg.AllPkgs that aren't referenced in cataloger/*/capabilities.yaml +func printPackageTypeCoverageWarning(capabilitiesDir string, repoRoot string) { + unreferenced, err := checkPackageTypeCoverage(capabilitiesDir, repoRoot) + if err != nil { + // don't fail generation, just skip the check + fmt.Printf("%s Could not check package type coverage: %v\n", warningStyleMeta.Render("⚠"), err) + return + } + + if len(unreferenced) > 0 { + fmt.Println() + fmt.Printf("%s %s package types from pkg.AllPkgs are not referenced in cataloger/*/capabilities.yaml:\n", + warningStyleMeta.Render("⚠ WARNING:"), + warningStyleMeta.Render(fmt.Sprintf("%d", len(unreferenced)))) + for _, typeName := range unreferenced { + fmt.Printf(" - %s\n", dimStyleMeta.Render(typeName)) + } + fmt.Println() + fmt.Println(dimStyleMeta.Render("These package types should be documented in cataloger/*/capabilities.yaml.")) + fmt.Println(dimStyleMeta.Render("If a package type is not emitted by any cataloger, it may be deprecated or unused.")) + } +} diff --git a/internal/capabilities/generate/overrides.go b/internal/capabilities/generate/overrides.go new file mode 100644 index 000000000..bf1afa47d --- /dev/null +++ b/internal/capabilities/generate/overrides.go @@ -0,0 +1,107 @@ +package main + +import ( + "github.com/anchore/syft/syft/cpe" +) + +// this is a hack to get some information in the output that is otherwise difficult to extract. +// it should be removed after we figure out how to extract it properly from the classifiers +type binaryClassifierOverride struct { + Class string + Package string + PURL string + CPEs []string +} + +var binaryClassifierOverrides = map[string][]binaryClassifierOverride{ + "java-binary": { + { + Class: "java-binary-graalvm", + Package: "graalvm", + PURL: mustPURL("pkg:generic/oracle/graalvm@version"), + CPEs: singleCPE("cpe:2.3:a:oracle:graalvm:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + { + Class: "java-binary-openjdk-zulu", + Package: "zulu", + PURL: mustPURL("pkg:generic/azul/zulu@version"), + CPEs: singleCPE("cpe:2.3:a:azul:zulu:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + { + Class: "java-binary-openjdk-with-update", + Package: "openjdk", + PURL: mustPURL("pkg:generic/oracle/openjdk@version"), + CPEs: singleCPE("cpe:2.3:a:oracle:openjdk:{{.primary}}:update{{.update}}:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + { + Class: "java-binary-openjdk", + Package: "openjdk", + PURL: mustPURL("pkg:generic/oracle/openjdk@version"), + CPEs: singleCPE("cpe:2.3:a:oracle:openjdk:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + { + Class: "java-binary-ibm", + Package: "java", + PURL: mustPURL("pkg:generic/ibm/java@version"), + CPEs: singleCPE("cpe:2.3:a:ibm:java:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + { + Class: "java-binary-openjdk-fallthrough", + Package: "jre", + PURL: mustPURL("pkg:generic/oracle/jre@version"), + CPEs: singleCPE("cpe:2.3:a:oracle:jre:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + { + Class: "java-binary-oracle", + Package: "jre", + PURL: mustPURL("pkg:generic/oracle/jre@version"), + CPEs: singleCPE("cpe:2.3:a:oracle:jre:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + }, + "java-jdb-binary": { + { + Class: "java-binary-graalvm", + Package: "graalvm", + PURL: mustPURL("pkg:generic/oracle/graalvm@version"), + CPEs: singleCPE("cpe:2.3:a:oracle:graalvm_for_jdk:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + { + Class: "jdb-binary-openjdk-zulu", + Package: "zulu", + PURL: mustPURL("pkg:generic/azul/zulu@version"), + CPEs: singleCPE("cpe:2.3:a:azul:zulu:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + { + Class: "java-jdb-binary-openjdk", + Package: "openjdk", + PURL: mustPURL("pkg:generic/oracle/openjdk@version"), + CPEs: singleCPE("cpe:2.3:a:oracle:openjdk:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + { + Class: "java-sdk-binary-ibm", + Package: "java_sdk", + PURL: mustPURL("pkg:generic/ibm/java_sdk@version"), + CPEs: singleCPE("cpe:2.3:a:ibm:java_sdk:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + { + Class: "java-binary-openjdk-fallthrough", + Package: "openjdk", + PURL: mustPURL("pkg:generic/oracle/openjdk@version"), + CPEs: singleCPE("cpe:2.3:a:oracle:openjdk:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + { + Class: "java-binary-jdk", + Package: "jdk", + PURL: mustPURL("pkg:generic/oracle/jdk@version"), + CPEs: singleCPE("cpe:2.3:a:oracle:jdk:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), + }, + }, +} + +func mustPURL(purl string) string { + return purl +} + +func singleCPE(cpeString string, _ ...any) []string { + return []string{cpeString} +} diff --git a/internal/capabilities/generate/testdata/cataloger/cataloger-with-constant/cataloger/python/cataloger.go b/internal/capabilities/generate/testdata/cataloger/cataloger-with-constant/cataloger/python/cataloger.go new file mode 100644 index 000000000..ff98a2653 --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/cataloger-with-constant/cataloger/python/cataloger.go @@ -0,0 +1,25 @@ +package python + +import ( + "context" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +const catalogerName = "python-package-cataloger" + +type CatalogerConfig struct { + Setting string +} + +func NewPythonCataloger(_ CatalogerConfig) pkg.Cataloger { + return generic.NewCataloger(catalogerName). + WithParserByGlobs(parse, "**/*.py") +} + +func parse(_ context.Context, _ file.Resolver, _ *generic.Environment, _ file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + return nil, nil, nil +} diff --git a/internal/capabilities/generate/testdata/cataloger/conflicting-names/cataloger/duplicate1/cataloger.go b/internal/capabilities/generate/testdata/cataloger/conflicting-names/cataloger/duplicate1/cataloger.go new file mode 100644 index 000000000..0ebcb3272 --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/conflicting-names/cataloger/duplicate1/cataloger.go @@ -0,0 +1,23 @@ +package duplicate1 + +import ( + "context" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +type Config struct { + Option1 bool +} + +func NewDuplicateCataloger(_ Config) pkg.Cataloger { + return generic.NewCataloger("duplicate-cataloger"). + WithParserByGlobs(parse, "**/*.txt") +} + +func parse(_ context.Context, _ file.Resolver, _ *generic.Environment, _ file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + return nil, nil, nil +} diff --git a/internal/capabilities/generate/testdata/cataloger/conflicting-names/cataloger/duplicate2/cataloger.go b/internal/capabilities/generate/testdata/cataloger/conflicting-names/cataloger/duplicate2/cataloger.go new file mode 100644 index 000000000..370ec2025 --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/conflicting-names/cataloger/duplicate2/cataloger.go @@ -0,0 +1,23 @@ +package duplicate2 + +import ( + "context" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +type Config struct { + Option2 string +} + +func NewDuplicateCataloger(_ Config) pkg.Cataloger { + return generic.NewCataloger("duplicate-cataloger"). + WithParserByGlobs(parse, "**/*.json") +} + +func parse(_ context.Context, _ file.Resolver, _ *generic.Environment, _ file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + return nil, nil, nil +} diff --git a/internal/capabilities/generate/testdata/cataloger/custom-cataloger-different-file/cataloger/dotnet/cataloger.go b/internal/capabilities/generate/testdata/cataloger/custom-cataloger-different-file/cataloger/dotnet/cataloger.go new file mode 100644 index 000000000..78d934313 --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/custom-cataloger-different-file/cataloger/dotnet/cataloger.go @@ -0,0 +1,9 @@ +package dotnet + +import ( + "github.com/anchore/syft/syft/pkg" +) + +func NewDotnetCataloger(cfg CatalogerConfig) pkg.Cataloger { + return dotnetCataloger{cfg: cfg} +} diff --git a/internal/capabilities/generate/testdata/cataloger/custom-cataloger-different-file/cataloger/dotnet/types.go b/internal/capabilities/generate/testdata/cataloger/custom-cataloger-different-file/cataloger/dotnet/types.go new file mode 100644 index 000000000..931f65742 --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/custom-cataloger-different-file/cataloger/dotnet/types.go @@ -0,0 +1,27 @@ +package dotnet + +import ( + "context" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" +) + +const catalogerName = "dotnet-cataloger" + +type CatalogerConfig struct { + Option bool +} + +type dotnetCataloger struct { + cfg CatalogerConfig +} + +func (d dotnetCataloger) Name() string { + return catalogerName +} + +func (d dotnetCataloger) Catalog(_ context.Context, _ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + return nil, nil, nil +} diff --git a/internal/capabilities/generate/testdata/cataloger/custom-cataloger-same-file/cataloger/java/cataloger.go b/internal/capabilities/generate/testdata/cataloger/custom-cataloger-same-file/cataloger/java/cataloger.go new file mode 100644 index 000000000..06bc16107 --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/custom-cataloger-same-file/cataloger/java/cataloger.go @@ -0,0 +1,31 @@ +package java + +import ( + "context" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" +) + +const pomCatalogerName = "java-pom-cataloger" + +type ArchiveCatalogerConfig struct { + IncludeArchives bool +} + +type pomXMLCataloger struct { + cfg ArchiveCatalogerConfig +} + +func (p pomXMLCataloger) Name() string { + return pomCatalogerName +} + +func (p pomXMLCataloger) Catalog(_ context.Context, _ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + return nil, nil, nil +} + +func NewPomCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger { + return pomXMLCataloger{cfg: cfg} +} diff --git a/internal/capabilities/generate/testdata/cataloger/imported-config-type/cataloger/kernel/cataloger.go b/internal/capabilities/generate/testdata/cataloger/imported-config-type/cataloger/kernel/cataloger.go new file mode 100644 index 000000000..71fca9477 --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/imported-config-type/cataloger/kernel/cataloger.go @@ -0,0 +1,19 @@ +package kernel + +import ( + "context" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +func NewLinuxKernelCataloger(_ LinuxKernelCatalogerConfig) pkg.Cataloger { + return generic.NewCataloger("linux-kernel-cataloger"). + WithParserByGlobs(parse, "**/vmlinuz") +} + +func parse(_ context.Context, _ file.Resolver, _ *generic.Environment, _ file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + return nil, nil, nil +} diff --git a/internal/capabilities/generate/testdata/cataloger/imported-config-type/cataloger/kernel/config.go b/internal/capabilities/generate/testdata/cataloger/imported-config-type/cataloger/kernel/config.go new file mode 100644 index 000000000..c77d49bff --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/imported-config-type/cataloger/kernel/config.go @@ -0,0 +1,5 @@ +package kernel + +type LinuxKernelCatalogerConfig struct { + KernelVersion string +} diff --git a/internal/capabilities/generate/testdata/cataloger/mixed-naming-patterns/cataloger/ruby/cataloger.go b/internal/capabilities/generate/testdata/cataloger/mixed-naming-patterns/cataloger/ruby/cataloger.go new file mode 100644 index 000000000..c6ffa1aee --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/mixed-naming-patterns/cataloger/ruby/cataloger.go @@ -0,0 +1,23 @@ +package ruby + +import ( + "context" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +type Config struct { + Setting bool +} + +func NewRubyCataloger(_ Config) pkg.Cataloger { + return generic.NewCataloger("ruby-cataloger"). + WithParserByGlobs(parse, "**/Gemfile") +} + +func parse(_ context.Context, _ file.Resolver, _ *generic.Environment, _ file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + return nil, nil, nil +} diff --git a/internal/capabilities/generate/testdata/cataloger/no-config-cataloger/cataloger/javascript/cataloger.go b/internal/capabilities/generate/testdata/cataloger/no-config-cataloger/cataloger/javascript/cataloger.go new file mode 100644 index 000000000..c450f5ebb --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/no-config-cataloger/cataloger/javascript/cataloger.go @@ -0,0 +1,19 @@ +package javascript + +import ( + "context" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +func NewJavaScriptCataloger() pkg.Cataloger { + return generic.NewCataloger("javascript-cataloger"). + WithParserByGlobs(parse, "**/*.js") +} + +func parse(_ context.Context, _ file.Resolver, _ *generic.Environment, _ file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + return nil, nil, nil +} diff --git a/internal/capabilities/generate/testdata/cataloger/non-config-first-param/cataloger/binary/cataloger.go b/internal/capabilities/generate/testdata/cataloger/non-config-first-param/cataloger/binary/cataloger.go new file mode 100644 index 000000000..f2affe4d0 --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/non-config-first-param/cataloger/binary/cataloger.go @@ -0,0 +1,21 @@ +package binary + +import ( + "context" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +type Parser struct{} + +func NewBinaryCataloger(_ Parser) pkg.Cataloger { + return generic.NewCataloger("binary-cataloger"). + WithParserByGlobs(parse, "**/*") +} + +func parse(_ context.Context, _ file.Resolver, _ *generic.Environment, _ file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + return nil, nil, nil +} diff --git a/internal/capabilities/generate/testdata/cataloger/selector-expression-config/cataloger/rust/cataloger.go b/internal/capabilities/generate/testdata/cataloger/selector-expression-config/cataloger/rust/cataloger.go new file mode 100644 index 000000000..ed04ec546 --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/selector-expression-config/cataloger/rust/cataloger.go @@ -0,0 +1,24 @@ +package rust + +import ( + "context" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +// CatalogerConfig is imported from a selector expression in the real code +type CatalogerConfig struct { + SomeOption bool +} + +func NewRustCataloger(_ CatalogerConfig) pkg.Cataloger { + return generic.NewCataloger("rust-cataloger"). + WithParserByGlobs(parse, "**/Cargo.toml") +} + +func parse(_ context.Context, _ file.Resolver, _ *generic.Environment, _ file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + return nil, nil, nil +} diff --git a/internal/capabilities/generate/testdata/cataloger/simple-generic-cataloger/cataloger/golang/cataloger.go b/internal/capabilities/generate/testdata/cataloger/simple-generic-cataloger/cataloger/golang/cataloger.go new file mode 100644 index 000000000..8792d8be9 --- /dev/null +++ b/internal/capabilities/generate/testdata/cataloger/simple-generic-cataloger/cataloger/golang/cataloger.go @@ -0,0 +1,23 @@ +package golang + +import ( + "context" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" +) + +type CatalogerConfig struct { + SomeOption bool +} + +func NewGoModuleCataloger(_ CatalogerConfig) pkg.Cataloger { + return generic.NewCataloger("go-module-cataloger"). + WithParserByGlobs(parseGoMod, "**/go.mod") +} + +func parseGoMod(_ context.Context, _ file.Resolver, _ *generic.Environment, _ file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + return nil, nil, nil +} diff --git a/internal/capabilities/generate/testdata/config-discovery/multiple-configs/cataloger/java/config.go b/internal/capabilities/generate/testdata/config-discovery/multiple-configs/cataloger/java/config.go new file mode 100644 index 000000000..bfe140ad6 --- /dev/null +++ b/internal/capabilities/generate/testdata/config-discovery/multiple-configs/cataloger/java/config.go @@ -0,0 +1,8 @@ +package java + +// ArchiveCatalogerConfig contains configuration for the java archive cataloger +type ArchiveCatalogerConfig struct { + // include archive contents in catalog + // app-config: java.use-network + UseNetwork bool +} diff --git a/internal/capabilities/generate/testdata/config-discovery/multiple-configs/cataloger/python/config.go b/internal/capabilities/generate/testdata/config-discovery/multiple-configs/cataloger/python/config.go new file mode 100644 index 000000000..b760edc8a --- /dev/null +++ b/internal/capabilities/generate/testdata/config-discovery/multiple-configs/cataloger/python/config.go @@ -0,0 +1,8 @@ +package python + +// CatalogerConfig contains configuration for the python cataloger +type CatalogerConfig struct { + // guess unpinned python package requirements + // app-config: python.guess-unpinned-requirements + GuessUnpinnedRequirements bool +} diff --git a/internal/capabilities/generate/testdata/config-discovery/nested-config/cataloger/golang/config.go b/internal/capabilities/generate/testdata/config-discovery/nested-config/cataloger/golang/config.go new file mode 100644 index 000000000..c62bcb9d7 --- /dev/null +++ b/internal/capabilities/generate/testdata/config-discovery/nested-config/cataloger/golang/config.go @@ -0,0 +1,22 @@ +package golang + +// MainModuleVersionConfig contains nested configuration for main module version detection +type MainModuleVersionConfig struct { + // extract version from LD flags + // app-config: golang.main-module-version.from-ld-flags + FromLDFlags bool + + // extract version from build info + // app-config: golang.main-module-version.from-build-info + FromBuildInfo bool +} + +// CatalogerConfig contains configuration for the golang cataloger +type CatalogerConfig struct { + // enable searching for go package licenses in the local mod cache + // app-config: golang.search-local-mod-cache-licenses + SearchLocalModCacheLicenses bool + + // main module version configuration + MainModuleVersion MainModuleVersionConfig +} diff --git a/internal/capabilities/generate/testdata/config-discovery/no-annotations/cataloger/javascript/config.go b/internal/capabilities/generate/testdata/config-discovery/no-annotations/cataloger/javascript/config.go new file mode 100644 index 000000000..c80e799a4 --- /dev/null +++ b/internal/capabilities/generate/testdata/config-discovery/no-annotations/cataloger/javascript/config.go @@ -0,0 +1,10 @@ +package javascript + +// CatalogerConfig contains configuration for the javascript cataloger (no annotations) +type CatalogerConfig struct { + // this field has no app-config annotation + SearchRemoteLicenses bool + + // this field also has no annotation + IncludeDevDependencies bool +} diff --git a/internal/capabilities/generate/testdata/config-discovery/simple-config/cataloger/golang/config.go b/internal/capabilities/generate/testdata/config-discovery/simple-config/cataloger/golang/config.go new file mode 100644 index 000000000..34f522836 --- /dev/null +++ b/internal/capabilities/generate/testdata/config-discovery/simple-config/cataloger/golang/config.go @@ -0,0 +1,16 @@ +package golang + +// CatalogerConfig contains configuration for the golang cataloger +type CatalogerConfig struct { + // enable searching for go package licenses in the local mod cache + // app-config: golang.search-local-mod-cache-licenses + SearchLocalModCacheLicenses bool + + // base URL for npm registry + // app-config: golang.npm-base-url + NpmBaseURL string + + // list of globs to search for go.mod files + // app-config: golang.search-patterns + SearchPatterns []string +} diff --git a/internal/capabilities/generate/util_test.go b/internal/capabilities/generate/util_test.go new file mode 100644 index 000000000..aea3bd5d2 --- /dev/null +++ b/internal/capabilities/generate/util_test.go @@ -0,0 +1,20 @@ +package main + +import ( + "os" + "testing" +) + +// checkCompletenessTestsEnabled skips the test if completeness tests are not enabled via environment variable. +// Why do this at all? Can't we just run these tests all the time? Short answer: No. +// These tests are coupled with unit tests under ./syft/pkg/..., which means that these tests must be run not only +// after those unit tests, but also after code generation that reads observations from test results. +// This means that we should not really consider these tests as part of normal unit test runs, but rather as a separate +// self-consistency check during generation, and it's really static analysis that should be checking +// if the generated code has drifted (not a unit test). +func checkCompletenessTestsEnabled(t *testing.T) { + enabled := os.Getenv("SYFT_ENABLE_COMPLETENESS_TESTS") == "true" + if !enabled { + t.Skip("skipping completeness tests (SYFT_ENABLE_COMPLETENESS_TESTS is not set to 'true')") + } +} diff --git a/internal/capabilities/internal/cataloger_names.go b/internal/capabilities/internal/cataloger_names.go new file mode 100644 index 000000000..a9506f4b5 --- /dev/null +++ b/internal/capabilities/internal/cataloger_names.go @@ -0,0 +1,28 @@ +// this file retrieves the canonical list of cataloger names and their selectors from syft's task factories. +package internal + +import ( + "fmt" + "sort" + + "github.com/anchore/syft/internal/capabilities" + "github.com/anchore/syft/internal/task" +) + +// AllPackageCatalogerInfo gets all package cataloger info (names and selectors) from task factories +func AllPackageCatalogerInfo() ([]capabilities.CatalogerInfo, error) { + pkgTaskFactories := task.DefaultPackageTaskFactories() + allPkgTasks, err := pkgTaskFactories.Tasks(task.DefaultCatalogingFactoryConfig()) + if err != nil { + return nil, fmt.Errorf("unable to create pkg cataloger tasks: %w", err) + } + + infos := capabilities.ExtractCatalogerInfo(allPkgTasks) + + // sort by name for consistency + sort.Slice(infos, func(i, j int) bool { + return infos[i].Name < infos[j].Name + }) + + return infos, nil +} diff --git a/internal/capabilities/internal/fixtures.go b/internal/capabilities/internal/fixtures.go new file mode 100644 index 000000000..b6b7a677a --- /dev/null +++ b/internal/capabilities/internal/fixtures.go @@ -0,0 +1,46 @@ +package internal + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/anchore/syft/internal/capabilities/pkgtestobservation" +) + +// FindTestFixtureDirs walks the cataloger directory tree and returns all test-fixtures directories +func FindTestFixtureDirs(repoRoot string) ([]string, error) { + catalogerRoot := filepath.Join(repoRoot, "syft", "pkg", "cataloger") + var testFixtureDirs []string + + err := filepath.Walk(catalogerRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() && info.Name() == "test-fixtures" { + testFixtureDirs = append(testFixtureDirs, path) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk cataloger directory: %w", err) + } + + return testFixtureDirs, nil +} + +// ReadTestObservations reads and parses a test-observations.json file +func ReadTestObservations(path string) (*pkgtestobservation.Test, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var observations pkgtestobservation.Test + if err := json.Unmarshal(data, &observations); err != nil { + return nil, fmt.Errorf("failed to parse JSON: %w", err) + } + + return &observations, nil +} diff --git a/internal/capabilities/internal/load_capabilities.go b/internal/capabilities/internal/load_capabilities.go new file mode 100644 index 000000000..583e49165 --- /dev/null +++ b/internal/capabilities/internal/load_capabilities.go @@ -0,0 +1,91 @@ +package internal + +import ( + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" + + "github.com/anchore/syft/internal/capabilities" +) + +// LoadCapabilities loads the capabilities document from a YAML file. +// Returns both the parsed document and the original YAML node tree to preserve comments. +// Exported for use by the generator in generate/main.go +func LoadCapabilities(catalogerDir, repoRoot string) (*capabilities.Document, map[string]*yaml.Node, error) { + // load all cataloger/*/capabilities.yaml files + files, err := filepath.Glob(filepath.Join(catalogerDir, "*", CapabilitiesFilename)) + if err != nil { + return nil, nil, fmt.Errorf("failed to glob capabilities files: %w", err) + } + + mergedDoc := &capabilities.Document{ + Configs: make(map[string]capabilities.CatalogerConfigEntry), + Catalogers: []capabilities.CatalogerEntry{}, + } + nodeMap := make(map[string]*yaml.Node) + + // load each package file + for _, file := range files { + data, err := os.ReadFile(file) + if err != nil { + return nil, nil, fmt.Errorf("failed to read %s: %w", file, err) + } + + // parse into node tree to preserve comments + var rootNode yaml.Node + if err := yaml.Unmarshal(data, &rootNode); err != nil { + return nil, nil, fmt.Errorf("failed to parse %s into node tree: %w", file, err) + } + + // parse into struct + var doc struct { + Configs map[string]capabilities.CatalogerConfigEntry `yaml:"configs"` + Catalogers []capabilities.CatalogerEntry `yaml:"catalogers"` + } + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, nil, fmt.Errorf("failed to parse %s into struct: %w", file, err) + } + + // merge configs + for k, v := range doc.Configs { + mergedDoc.Configs[k] = v + } + + // merge catalogers + mergedDoc.Catalogers = append(mergedDoc.Catalogers, doc.Catalogers...) + + // store node tree by ecosystem directory name + // path is like "/path/to/syft/pkg/cataloger/alpine/capabilities.yaml" + ecosystem := filepath.Base(filepath.Dir(file)) + nodeMap[ecosystem] = &rootNode + } + + // load appconfig.yaml separately (from internal/capabilities/) + appconfigPath := AppconfigPath(repoRoot) + if _, err := os.Stat(appconfigPath); err == nil { + data, err := os.ReadFile(appconfigPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to read appconfig.yaml: %w", err) + } + + var appDoc struct { + Application []capabilities.ApplicationConfigField `yaml:"application"` + } + if err := yaml.Unmarshal(data, &appDoc); err != nil { + return nil, nil, fmt.Errorf("failed to parse appconfig.yaml: %w", err) + } + + mergedDoc.ApplicationConfig = appDoc.Application + + // load node tree for appconfig + var appNode yaml.Node + if err := yaml.Unmarshal(data, &appNode); err != nil { + return nil, nil, fmt.Errorf("failed to parse appconfig.yaml into node tree: %w", err) + } + nodeMap["appconfig"] = &appNode + } + + return mergedDoc, nodeMap, nil +} diff --git a/internal/capabilities/internal/load_capabilities_test.go b/internal/capabilities/internal/load_capabilities_test.go new file mode 100644 index 000000000..a7c49a038 --- /dev/null +++ b/internal/capabilities/internal/load_capabilities_test.go @@ -0,0 +1,1355 @@ +// this file verifies the claims made in cataloger/*/capabilities.yaml against test observations and source code, ensuring cataloger capabilities are accurate and complete. +package internal + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + + "github.com/scylladb/go-set/strset" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/capabilities" + "github.com/anchore/syft/internal/packagemetadata" + "github.com/anchore/syft/syft/pkg" + _ "github.com/anchore/syft/syft/pkg/cataloger" +) + +// requireParserObservations controls whether TestAllCatalogersHaveObservations enforces that all parsers have observations +// - true: fail test if any parser is missing observations (strict mode) +// - false: only check custom catalogers, skip parser checks (lenient mode, not all parsers are observable) +const requireParserObservations = false + +// metadataTypeCoverageExceptions lists metadata types that are allowed to not be represented in any cataloger +var metadataTypeCoverageExceptions = strset.New( + reflect.TypeOf(pkg.MicrosoftKbPatch{}).Name(), +) + +// packageTypeCoverageExceptions lists package types that are allowed to not be represented in any cataloger +var packageTypeCoverageExceptions = strset.New( + string(pkg.JenkinsPluginPkg), // TODO: this should probably be covered by a cataloger test one day + string(pkg.KbPkg), +) + +// observationExceptions maps cataloger/parser names to observation types that should be ignored during validation +// +// TestAllCatalogersHaveObservations: +// - always checks custom catalogers +// - checks parsers only if requireParserObservations=true +// - nil or non-nil value: skip existence check for this cataloger/parser +// +// examples: +// +// "graalvm-native-image-cataloger": nil, // custom cataloger: skip existence check +// "linux-kernel-cataloger": strset.New("relationships"), // custom cataloger: skip only relationships validation +// "conan-cataloger/parseConanLock": nil, // parser: skip all observation validation +// "cataloger-name/parser-function": strset.New("file_digests"), // parser: skip only file_digests validation +var observationExceptions = map[string]*strset.Set{ + // for the graalvm-native-image-cataloger, we don't have a really reliable test fixture yet + "graalvm-native-image-cataloger": nil, + // the linux-kernel-cataloger produces relationships but aren't really indicative of dependency information in the way the user might expect + "linux-kernel-cataloger": strset.New("relationships"), +} + +func TestLoadDocument(t *testing.T) { + doc, err := capabilities.LoadDocument() + require.NoError(t, err) + require.NotNil(t, doc) + + // validate application config is loaded + assert.NotEmpty(t, doc.ApplicationConfig, "should have application config") + + // validate catalogers are loaded and merged from all cataloger/*/capabilities.yaml files + assert.NotEmpty(t, doc.Catalogers, "should have catalogers") + assert.Greater(t, len(doc.Catalogers), 50, "should have at least 50 catalogers") + + // validate configs are loaded + assert.NotEmpty(t, doc.Configs, "should have configs") + + // check that catalogers are sorted by name + for i := 1; i < len(doc.Catalogers); i++ { + assert.LessOrEqual(t, doc.Catalogers[i-1].Name, doc.Catalogers[i].Name, + "catalogers should be sorted by name") + } +} + +func TestPackages(t *testing.T) { + catalogers, err := capabilities.Packages() + require.NoError(t, err) + require.NotNil(t, catalogers) + + assert.Greater(t, len(catalogers), 50, "should have at least 50 catalogers") +} + +// TestConfigCompleteness validates the integrity of config references in cataloger/*/capabilities.yaml, ensuring that all +// configs in the configs section are referenced by at least one cataloger, all cataloger config references exist, +// and all app-key references in config fields exist in the application section. +func TestConfigCompleteness(t *testing.T) { + checkCompletenessTestsEnabled(t) + + repoRoot, err := RepoRoot() + require.NoError(t, err) + + // load the cataloger/*/capabilities.yaml files + doc, _, err := LoadCapabilities(CatalogerDir(repoRoot), repoRoot) + require.NoError(t, err) + + // collect all validation errors before failing + var errors []string + + // validation 1: all entries in configs section are referenced by at least one cataloger + configsReferenced := make(map[string]bool) + for _, cataloger := range doc.Catalogers { + if cataloger.Config != "" { + configsReferenced[cataloger.Config] = true + } + } + + for configKey := range doc.Configs { + if !configsReferenced[configKey] { + errors = append(errors, fmt.Sprintf("Config %q is not referenced by any cataloger", configKey)) + } + } + + // validation 2: all catalogers with non-empty config field have entry in configs + for _, cataloger := range doc.Catalogers { + if cataloger.Config != "" { + if _, exists := doc.Configs[cataloger.Config]; !exists { + errors = append(errors, fmt.Sprintf("Cataloger %q references config %q which doesn't exist in configs section", cataloger.Name, cataloger.Config)) + } + } + } + + // validation 3: all app-key references in configs exist in app-config section + // build a set of all app-config keys for quick lookup + appConfigKeys := make(map[string]bool) + for _, appConfig := range doc.ApplicationConfig { + appConfigKeys[appConfig.Key] = true + } + + for configName, configEntry := range doc.Configs { + for _, field := range configEntry.Fields { + if field.AppKey != "" { + if !appConfigKeys[field.AppKey] { + errors = append(errors, fmt.Sprintf("Config field %q.%s references app-key %q which doesn't exist in app-config section", configName, field.Key, field.AppKey)) + } + } + } + } + + // report all errors at once + if len(errors) > 0 { + require.Fail(t, "Config completeness validation failed", strings.Join(errors, "\n")) + } +} + +// TestCapabilityConfigFieldReferences validates that config field names referenced in capability conditions +// actually exist in the cataloger's config struct, preventing typos and ensuring capability conditions can +// be properly evaluated at runtime. +func TestCapabilityConfigFieldReferences(t *testing.T) { + checkCompletenessTestsEnabled(t) + + repoRoot, err := RepoRoot() + require.NoError(t, err) + + // load the packages.yaml + doc, _, err := LoadCapabilities(CatalogerDir(repoRoot), repoRoot) + require.NoError(t, err) + + // collect all validation errors before failing + var errors []string + + // for each cataloger with CapabilitiesV2 + for _, cataloger := range doc.Catalogers { + // check cataloger-level CapabilitiesV2 (for custom catalogers) + if cataloger.Type == "custom" && len(cataloger.Capabilities) > 0 { + // load the cataloger's config struct if it has one + if cataloger.Config != "" { + configEntry, exists := doc.Configs[cataloger.Config] + if !exists { + errors = append(errors, fmt.Sprintf("Cataloger %q references config %q which doesn't exist", cataloger.Name, cataloger.Config)) + continue + } + + // build a set of valid config field names + validFields := make(map[string]bool) + for _, field := range configEntry.Fields { + validFields[field.Key] = true + } + + // validate each capability field + for _, capField := range cataloger.Capabilities { + // check conditions for config field references + for _, condition := range capField.Conditions { + for fieldName := range condition.When { + if !validFields[fieldName] { + errors = append(errors, + fmt.Sprintf("Cataloger %q capability field %q references config field %q which doesn't exist in config struct %q", + cataloger.Name, capField.Name, fieldName, cataloger.Config)) + } + } + } + } + } else if len(cataloger.Capabilities) > 0 { + // cataloger has CapabilitiesV2 with conditions but no config - check if any conditions reference fields + for _, capField := range cataloger.Capabilities { + if len(capField.Conditions) > 0 { + for _, condition := range capField.Conditions { + if len(condition.When) > 0 { + errors = append(errors, + fmt.Sprintf("Cataloger %q capability field %q has conditions but cataloger has no config struct", + cataloger.Name, capField.Name)) + break + } + } + } + } + } + } + + // check parser-level CapabilitiesV2 (for generic catalogers) + if cataloger.Type == "generic" { + for _, p := range cataloger.Parsers { + if len(p.Capabilities) > 0 { + // load the cataloger's config struct if it has one + if cataloger.Config != "" { + configEntry, exists := doc.Configs[cataloger.Config] + if !exists { + errors = append(errors, fmt.Sprintf("Cataloger %q references config %q which doesn't exist", cataloger.Name, cataloger.Config)) + continue + } + + // build a set of valid config field names + validFields := make(map[string]bool) + for _, field := range configEntry.Fields { + validFields[field.Key] = true + } + + // validate each capability field + for _, capField := range p.Capabilities { + // check conditions for config field references + for _, condition := range capField.Conditions { + for fieldName := range condition.When { + if !validFields[fieldName] { + errors = append(errors, + fmt.Sprintf("Parser %q/%s capability field %q references config field %q which doesn't exist in config struct %q", + cataloger.Name, p.ParserFunction, capField.Name, fieldName, cataloger.Config)) + } + } + } + } + } else { + // parser has CapabilitiesV2 with conditions but cataloger has no config + for _, capField := range p.Capabilities { + if len(capField.Conditions) > 0 { + for _, condition := range capField.Conditions { + if len(condition.When) > 0 { + errors = append(errors, + fmt.Sprintf("Parser %q/%s capability field %q has conditions but cataloger has no config struct", + cataloger.Name, p.ParserFunction, capField.Name)) + break + } + } + } + } + } + } + } + } + } + + // report all errors at once + if len(errors) > 0 { + require.Fail(t, "CapabilityV2 config field reference validation failed", strings.Join(errors, "\n")) + } +} + +// TestCapabilityFieldNaming validates that all capability field names follow known patterns +// (e.g., "license", "dependency.depth", "package_manager.files.listing"), catching typos and ensuring +// consistency across catalogers. +func TestCapabilityFieldNaming(t *testing.T) { + checkCompletenessTestsEnabled(t) + + repoRoot, err := RepoRoot() + require.NoError(t, err) + + // load the packages.yaml + doc, _, err := LoadCapabilities(CatalogerDir(repoRoot), repoRoot) + require.NoError(t, err) + + // define known capability field paths + knownFields := strset.New( + "license", + "dependency.depth", + "dependency.edges", + "dependency.kinds", + "package_manager.files.listing", + "package_manager.files.digests", + "package_manager.package_integrity_hash", + ) + + // collect all validation errors/warnings + var errors []string + + // check cataloger-level CapabilitiesV2 + for _, cataloger := range doc.Catalogers { + if cataloger.Type == "custom" && len(cataloger.Capabilities) > 0 { + for _, capField := range cataloger.Capabilities { + if !knownFields.Has(capField.Name) { + errors = append(errors, + fmt.Sprintf("Cataloger %q uses unknown capability field %q - may be a typo or new field not in known list", + cataloger.Name, capField.Name)) + } + } + } + + // check parser-level CapabilitiesV2 + if cataloger.Type == "generic" { + for _, parser := range cataloger.Parsers { + if len(parser.Capabilities) > 0 { + for _, capField := range parser.Capabilities { + if !knownFields.Has(capField.Name) { + errors = append(errors, + fmt.Sprintf("Parser %q/%s uses unknown capability field %q - may be a typo or new field not in known list", + cataloger.Name, parser.ParserFunction, capField.Name)) + } + } + } + } + } + } + + // report all errors at once + if len(errors) > 0 { + require.Fail(t, "Capability field naming validation failed", strings.Join(errors, "\n")) + } +} + +// TestCapabilityValueTypes validates that capability field values match their expected types based on the +// field name (e.g., boolean fields like "license" must have bool values, array fields like "dependency.depth" +// must have []string values), preventing type mismatches that would cause runtime errors. +func TestCapabilityValueTypes(t *testing.T) { + checkCompletenessTestsEnabled(t) + + repoRoot, err := RepoRoot() + require.NoError(t, err) + + // load the packages.yaml + doc, _, err := LoadCapabilities(CatalogerDir(repoRoot), repoRoot) + require.NoError(t, err) + + // collect all validation errors + var errors []string + + // validate cataloger-level CapabilitiesV2 + for _, cataloger := range doc.Catalogers { + if cataloger.Type == "custom" && len(cataloger.Capabilities) > 0 { + for _, capField := range cataloger.Capabilities { + // validate default value type + err := validateCapabilityValueType(capField.Name, capField.Default) + if err != nil { + errors = append(errors, + fmt.Sprintf("Cataloger %q capability field %q default value: %v", + cataloger.Name, capField.Name, err)) + } + + // validate condition value types + for i, condition := range capField.Conditions { + err := validateCapabilityValueType(capField.Name, condition.Value) + if err != nil { + errors = append(errors, + fmt.Sprintf("Cataloger %q capability field %q condition %d value: %v", + cataloger.Name, capField.Name, i, err)) + } + } + } + } + + // validate parser-level CapabilitiesV2 + if cataloger.Type == "generic" { + for _, parser := range cataloger.Parsers { + if len(parser.Capabilities) > 0 { + for _, capField := range parser.Capabilities { + // validate default value type + err := validateCapabilityValueType(capField.Name, capField.Default) + if err != nil { + errors = append(errors, + fmt.Sprintf("Parser %q/%s capability field %q default value: %v", + cataloger.Name, parser.ParserFunction, capField.Name, err)) + } + + // validate condition value types + for i, condition := range capField.Conditions { + err := validateCapabilityValueType(capField.Name, condition.Value) + if err != nil { + errors = append(errors, + fmt.Sprintf("Parser %q/%s capability field %q condition %d value: %v", + cataloger.Name, parser.ParserFunction, capField.Name, i, err)) + } + } + } + } + } + } + } + + // report all errors at once + if len(errors) > 0 { + require.Fail(t, "Capability value type validation failed", strings.Join(errors, "\n")) + } +} + +// validateCapabilityValueType checks if a value matches the expected type for a capability field +func validateCapabilityValueType(fieldPath string, value interface{}) error { + if value == nil { + return nil // nil is acceptable + } + + switch fieldPath { + case "license", + "package_manager.files.listing", + "package_manager.files.digests", + "package_manager.package_integrity_hash": + // expect bool + if _, ok := value.(bool); !ok { + return fmt.Errorf("expected bool, got %T", value) + } + + case "dependency.depth", "dependency.kinds": + // expect []string or []interface{} that can be converted to []string + switch v := value.(type) { + case []string: + // ok + case []interface{}: + // check each element is a string + for i, elem := range v { + if _, ok := elem.(string); !ok { + return fmt.Errorf("expected []string, but element %d is %T", i, elem) + } + } + default: + return fmt.Errorf("expected []string, got %T", value) + } + + case "dependency.edges": + // expect string + if _, ok := value.(string); !ok { + return fmt.Errorf("expected string, got %T", value) + } + + default: + // unknown field, skip validation + return nil + } + + return nil +} + +// TestMetadataTypesHaveJSONSchemaTypes validates that metadata_types and json_schema_types arrays are synchronized +// in packages.yaml, ensuring every metadata type (e.g., "pkg.AlpmDBEntry") has a corresponding json_schema_type +// (e.g., "AlpmDbEntry") with correct conversion, which is required for JSON schema generation. +func TestMetadataTypesHaveJSONSchemaTypes(t *testing.T) { + checkCompletenessTestsEnabled(t) + + repoRoot, err := RepoRoot() + require.NoError(t, err) + + // load the packages.yaml + doc, _, err := LoadCapabilities(CatalogerDir(repoRoot), repoRoot) + require.NoError(t, err) + + // collect all validation errors + var errors []string + + // validate cataloger-level types (custom catalogers) + for _, c := range doc.Catalogers { + if c.Type == "custom" { + if len(c.MetadataTypes) > 0 { + // verify counts match + if len(c.MetadataTypes) != len(c.JSONSchemaTypes) { + errors = append(errors, + fmt.Sprintf("Cataloger %q has %d metadata_types but %d json_schema_types (counts must match)", + c.Name, len(c.MetadataTypes), len(c.JSONSchemaTypes))) + continue + } + + // verify each metadata_type converts to its corresponding json_schema_type + for i, metadataType := range c.MetadataTypes { + expectedJSONSchemaType := convertMetadataTypeToJSONSchemaType(metadataType) + if expectedJSONSchemaType == "" { + errors = append(errors, + fmt.Sprintf("Cataloger %q metadata_type[%d] %q could not be converted to json_schema_type (not found in packagemetadata registry)", + c.Name, i, metadataType)) + continue + } + + actualJSONSchemaType := c.JSONSchemaTypes[i] + if expectedJSONSchemaType != actualJSONSchemaType { + errors = append(errors, + fmt.Sprintf("Cataloger %q metadata_type[%d] %q should convert to json_schema_type %q but found %q", + c.Name, i, metadataType, expectedJSONSchemaType, actualJSONSchemaType)) + } + } + } + } + + // validate parser-level types (generic catalogers) + if c.Type == "generic" { + for _, p := range c.Parsers { + if len(p.MetadataTypes) > 0 { + // verify counts match + if len(p.MetadataTypes) != len(p.JSONSchemaTypes) { + errors = append(errors, + fmt.Sprintf("Parser %q/%s has %d metadata_types but %d json_schema_types (counts must match)", + c.Name, p.ParserFunction, len(p.MetadataTypes), len(p.JSONSchemaTypes))) + continue + } + + // verify each metadata_type converts to its corresponding json_schema_type + for i, metadataType := range p.MetadataTypes { + expectedJSONSchemaType := convertMetadataTypeToJSONSchemaType(metadataType) + if expectedJSONSchemaType == "" { + errors = append(errors, + fmt.Sprintf("Parser %q/%s metadata_type[%d] %q could not be converted to json_schema_type (not found in packagemetadata registry)", + c.Name, p.ParserFunction, i, metadataType)) + continue + } + + actualJSONSchemaType := p.JSONSchemaTypes[i] + if expectedJSONSchemaType != actualJSONSchemaType { + errors = append(errors, + fmt.Sprintf("Parser %q/%s metadata_type[%d] %q should convert to json_schema_type %q but found %q", + c.Name, p.ParserFunction, i, metadataType, expectedJSONSchemaType, actualJSONSchemaType)) + } + } + } + } + } + } + + // report all errors at once + if len(errors) > 0 { + require.Fail(t, "Metadata types and JSON schema types validation failed", strings.Join(errors, "\n")) + } +} + +// convertMetadataTypeToJSONSchemaType converts a metadata type (e.g., "pkg.AlpmDBEntry") to its JSON schema type (e.g., "AlpmDbEntry") +func convertMetadataTypeToJSONSchemaType(metadataType string) string { + jsonName := packagemetadata.JSONNameFromString(metadataType) + if jsonName == "" { + return "" + } + return packagemetadata.ToUpperCamelCase(jsonName) +} + +// findStructFields extracts field names and types from a struct definition +func findStructFields(file *ast.File, structName string) map[string]string { + fields := make(map[string]string) + + ast.Inspect(file, func(n ast.Node) bool { + // look for type declarations + typeSpec, ok := n.(*ast.TypeSpec) + if !ok || typeSpec.Name.Name != structName { + return true + } + + // check if it's a struct type + structType, ok := typeSpec.Type.(*ast.StructType) + if !ok { + return false + } + + // extract field names and types + for _, field := range structType.Fields.List { + if len(field.Names) == 0 { + continue // embedded field + } + + fieldName := field.Names[0].Name + fieldType := getTypeName(field.Type) + fields[fieldName] = fieldType + } + + return false + }) + + return fields +} + +// getTypeName extracts a string representation of a type +func getTypeName(expr ast.Expr) string { + switch t := expr.(type) { + case *ast.Ident: + return t.Name + case *ast.SelectorExpr: + return fmt.Sprintf("%s.%s", getTypeName(t.X), t.Sel.Name) + case *ast.ArrayType: + return fmt.Sprintf("[]%s", getTypeName(t.Elt)) + case *ast.MapType: + return fmt.Sprintf("map[%s]%s", getTypeName(t.Key), getTypeName(t.Value)) + case *ast.StarExpr: + return fmt.Sprintf("*%s", getTypeName(t.X)) + default: + return "unknown" + } +} + +// findMetadataStructFile finds the Go file containing a metadata struct definition +// searches in syft/pkg/*.go for the given struct name +// also handles type aliases and returns the underlying struct name +func findMetadataStructFile(repoRoot, structName string) (filePath string, actualStructName string, err error) { + pkgDir := filepath.Join(repoRoot, "syft", "pkg") + files, err := filepath.Glob(filepath.Join(pkgDir, "*.go")) + if err != nil { + return "", "", err + } + + for _, fpath := range files { + if strings.HasSuffix(fpath, "_test.go") { + continue + } + + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, fpath, nil, 0) + if err != nil { + continue + } + + // check if this file contains the struct definition or type alias + found := false + var resolvedName string + ast.Inspect(file, func(n ast.Node) bool { + typeSpec, ok := n.(*ast.TypeSpec) + if !ok || typeSpec.Name.Name != structName { + return true + } + // check if it's a struct type + if _, ok := typeSpec.Type.(*ast.StructType); ok { + found = true + resolvedName = structName + return false + } + // check if it's a type alias (e.g., type DpkgArchiveEntry DpkgDBEntry) + if ident, ok := typeSpec.Type.(*ast.Ident); ok { + found = true + resolvedName = ident.Name + return false + } + return true + }) + + if found { + // if it's a type alias, recursively find the underlying struct + if resolvedName != structName { + return findMetadataStructFile(repoRoot, resolvedName) + } + return fpath, structName, nil + } + } + + return "", "", fmt.Errorf("struct %q not found in syft/pkg/", structName) +} + +// parseEvidenceReference parses an evidence string like "StructName.Field" or "StructName.Field1.Field2" +// into struct name and field path components +// examples: +// - "CondaMetaPackage.MD5" -> ("CondaMetaPackage", []string{"MD5"}) +// - "CondaMetaPackage.PathsData.Paths" -> ("CondaMetaPackage", []string{"PathsData", "Paths"}) +// - "AlpmDBEntry.Files[].Digest" -> ("AlpmDBEntry", []string{"Files", "[]", "Digest"}) +func parseEvidenceReference(evidence string) (structName string, fieldPath []string, err error) { + parts := strings.Split(evidence, ".") + if len(parts) < 2 { + return "", nil, fmt.Errorf("invalid evidence format: %q (expected at least StructName.Field)", evidence) + } + + structName = parts[0] + // process the remaining parts, splitting on [] for array notation + for _, part := range parts[1:] { + // check if this part contains array notation + if strings.Contains(part, "[]") { + // split on [] - e.g., "Files[]" becomes ["Files", ""] + subparts := strings.Split(part, "[]") + if len(subparts) > 0 && subparts[0] != "" { + fieldPath = append(fieldPath, subparts[0]) + } + fieldPath = append(fieldPath, "[]") + } else { + fieldPath = append(fieldPath, part) + } + } + + return structName, fieldPath, nil +} + +// validateFieldPath validates that a field path exists in a struct definition +// handles simple fields, nested fields, and array element fields +// fieldPath can contain "[]" to indicate array dereferencing +func validateFieldPath(repoRoot, structName string, fieldPath []string) error { + if len(fieldPath) == 0 { + return fmt.Errorf("empty field path") + } + + // find the file containing the struct (handles type aliases) + filePath, actualStructName, err := findMetadataStructFile(repoRoot, structName) + if err != nil { + return err + } + + // parse the file + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, filePath, nil, 0) + if err != nil { + return fmt.Errorf("failed to parse %s: %w", filePath, err) + } + + // find the struct fields using the actual struct name + fields := findStructFields(file, actualStructName) + if len(fields) == 0 { + return fmt.Errorf("struct %q not found in %s", actualStructName, filePath) + } + + // validate each component of the field path + currentFields := fields + currentStructName := actualStructName + for i, component := range fieldPath { + if component == "[]" { + // array dereference - this is a no-op for validation + // the previous component should have been an array type + continue + } + + fieldType, exists := currentFields[component] + if !exists { + return fmt.Errorf("field %q not found in struct %q (path: %s)", component, currentStructName, strings.Join(fieldPath[:i+1], ".")) + } + + // if there are more components, we need to navigate to the next struct + if i < len(fieldPath)-1 { + // extract the actual type name, removing pointer/array/slice markers + typeName := strings.TrimPrefix(fieldType, "*") + typeName = strings.TrimPrefix(typeName, "[]") + + // if it's not a simple type name (e.g., "CondaPathsData"), skip validation + // this handles primitive types that don't have further fields + if strings.Contains(typeName, ".") { + // qualified type like "pkg.Something" - extract just "Something" + parts := strings.Split(typeName, ".") + typeName = parts[len(parts)-1] + } + + // try to find the nested struct (handles type aliases) + nestedFilePath, nestedStructName, err := findMetadataStructFile(repoRoot, typeName) + if err != nil { + // if we can't find the struct, it might be a primitive or external type + // we'll allow this to pass + continue + } + + nestedFset := token.NewFileSet() + nestedFile, err := parser.ParseFile(nestedFset, nestedFilePath, nil, 0) + if err != nil { + continue + } + + currentFields = findStructFields(nestedFile, nestedStructName) + currentStructName = nestedStructName + if len(currentFields) == 0 { + // couldn't load the nested struct, but we found the field, so allow it + continue + } + } + } + + return nil +} + +// TestCapabilityEvidenceFieldReferences validates that evidence field references in capabilities +// (e.g., "AlpmDBEntry.Files[].Digests") actually exist on their corresponding metadata structs by using +// AST parsing to verify the field paths, preventing broken references when structs are refactored. +func TestCapabilityEvidenceFieldReferences(t *testing.T) { + checkCompletenessTestsEnabled(t) + + repoRoot, err := RepoRoot() + require.NoError(t, err) + + // load the cataloger/*/capabilities.yaml + doc, _, err := LoadCapabilities(CatalogerDir(repoRoot), repoRoot) + require.NoError(t, err) + + // collect all evidence field references + type evidenceRef struct { + catalogerName string + parserFunction string // empty for cataloger-level + capabilityName string + evidenceField string + } + + var allReferences []evidenceRef + + // collect from cataloger-level capabilities (custom catalogers) + for _, cataloger := range doc.Catalogers { + if cataloger.Type == "custom" && len(cataloger.Capabilities) > 0 { + for _, capField := range cataloger.Capabilities { + for _, evidence := range capField.Evidence { + allReferences = append(allReferences, evidenceRef{ + catalogerName: cataloger.Name, + capabilityName: capField.Name, + evidenceField: evidence, + }) + } + } + } + + // collect from parser-level capabilities (generic catalogers) + if cataloger.Type == "generic" { + for _, parser := range cataloger.Parsers { + if len(parser.Capabilities) > 0 { + for _, capField := range parser.Capabilities { + for _, evidence := range capField.Evidence { + allReferences = append(allReferences, evidenceRef{ + catalogerName: cataloger.Name, + parserFunction: parser.ParserFunction, + capabilityName: capField.Name, + evidenceField: evidence, + }) + } + } + } + } + } + } + + // validate each evidence reference + for _, ref := range allReferences { + ref := ref // capture for subtest + + // create test name + testName := ref.catalogerName + if ref.parserFunction != "" { + testName = fmt.Sprintf("%s/%s", ref.catalogerName, ref.parserFunction) + } + testName = fmt.Sprintf("%s/%s/%s", testName, ref.capabilityName, ref.evidenceField) + + t.Run(testName, func(t *testing.T) { + // parse the evidence reference + structName, fieldPath, err := parseEvidenceReference(ref.evidenceField) + require.NoError(t, err, "failed to parse evidence reference") + + // validate the field path exists + err = validateFieldPath(repoRoot, structName, fieldPath) + require.NoError(t, err, "evidence field reference is invalid") + }) + } +} + +// TestDetectorConfigFieldReferences validates that config field names referenced in detector conditions +// actually exist in the cataloger's config struct, ensuring that conditional detectors can properly +// evaluate their activation conditions based on configuration. +func TestDetectorConfigFieldReferences(t *testing.T) { + checkCompletenessTestsEnabled(t) + + repoRoot, err := RepoRoot() + require.NoError(t, err) + + // load the cataloger/*/capabilities.yaml + doc, _, err := LoadCapabilities(CatalogerDir(repoRoot), repoRoot) + require.NoError(t, err) + + // collect all validation errors before failing + var errors []string + + // check each cataloger's detectors + for _, cataloger := range doc.Catalogers { + if cataloger.Type != "custom" { + continue // only custom catalogers have detectors + } + + for detectorIdx, detector := range cataloger.Detectors { + // if detector has no conditions, skip validation + if len(detector.Conditions) == 0 { + continue + } + + // detector has conditions - cataloger must have a config + if cataloger.Config == "" { + errors = append(errors, + fmt.Sprintf("Cataloger %q detector %d has conditions but cataloger has no config struct", + cataloger.Name, detectorIdx)) + continue + } + + // load the cataloger's config struct + configEntry, exists := doc.Configs[cataloger.Config] + if !exists { + errors = append(errors, + fmt.Sprintf("Cataloger %q references config %q which doesn't exist", + cataloger.Name, cataloger.Config)) + continue + } + + // build a set of valid config field names + validFields := make(map[string]bool) + for _, field := range configEntry.Fields { + validFields[field.Key] = true + } + + // validate each condition + for condIdx, condition := range detector.Conditions { + for fieldName := range condition.When { + if !validFields[fieldName] { + errors = append(errors, + fmt.Sprintf("Cataloger %q detector %d condition %d references config field %q which doesn't exist in config struct %q", + cataloger.Name, detectorIdx, condIdx, fieldName, cataloger.Config)) + } + } + } + } + } + + // report all errors at once + if len(errors) > 0 { + require.Fail(t, "Detector config field reference validation failed", strings.Join(errors, "\n")) + } +} + +// TestCatalogersInSync ensures that all catalogers from the syft binary are documented in cataloger/*/capabilities.yaml +// and vice versa, and that all capability fields are properly filled without TODOs or null values. +func TestCatalogersInSync(t *testing.T) { + checkCompletenessTestsEnabled(t) + + // get canonical list from syft binary + catalogersInBinary := getCatalogerNamesFromBinary(t) + + // load catalogers from embedded YAML + catalogerEntries, err := capabilities.Packages() + require.NoError(t, err) + + yamlCatalogers := strset.New() + for _, c := range catalogerEntries { + yamlCatalogers.Add(c.Name) + } + + // test 1: All catalogers in binary must be in YAML + var missingFromYAML []string + for _, name := range catalogersInBinary { + if !yamlCatalogers.Has(name) { + missingFromYAML = append(missingFromYAML, name) + } + } + require.Empty(t, missingFromYAML, + "The following catalogers are in 'syft cataloger list' but missing from capabilities YAML: %v\n"+ + "Run 'go generate ./internal/capabilities' to auto-add generic catalogers, or manually add custom catalogers.", + missingFromYAML) + + // test 2: All catalogers in YAML must exist in binary + var orphanedInYAML []string + binarySet := strset.New() + for _, name := range catalogersInBinary { + binarySet.Add(name) + } + for _, name := range yamlCatalogers.List() { + if !binarySet.Has(name) { + orphanedInYAML = append(orphanedInYAML, name) + } + } + require.Empty(t, orphanedInYAML, + "The following catalogers are in capabilities YAML but not found in binary: %v\n"+ + "These catalogers may have been removed. Delete them from the YAML.", + orphanedInYAML) + + // test 3: All capabilities must be filled (no TODOs/nulls) + validateCapabilitiesFilled(t, catalogerEntries) +} + +func getCatalogerNamesFromBinary(t *testing.T) []string { + checkCompletenessTestsEnabled(t) + + // get cataloger names from task factories + infos, err := AllPackageCatalogerInfo() + require.NoError(t, err) + + var names []string + for _, info := range infos { + names = append(names, info.Name) + } + + sort.Strings(names) + return names +} + +func validateCapabilitiesFilled(t *testing.T, catalogers []capabilities.CatalogerEntry) { + checkCompletenessTestsEnabled(t) + + for _, c := range catalogers { + c := c // capture loop variable for subtest + + t.Run(c.Name, func(t *testing.T) { + if c.Type == "generic" { + // generic catalogers have parsers with capabilities + require.NotEmpty(t, c.Parsers, "generic cataloger must have at least one parser") + + for _, p := range c.Parsers { + p := p // capture loop variable for subtest + + t.Run(p.ParserFunction, func(t *testing.T) { + require.NotEmpty(t, p.Capabilities, "parser must have at least one capability field defined") + }) + } + } else if c.Type == "custom" { + // custom catalogers have cataloger-level capabilities + require.NotEmpty(t, c.Capabilities, "custom cataloger must have at least one capability field defined") + } + }) + } +} + +// TestPackageTypeCoverage ensures that every package type defined in pkg.AllPkgs is represented in at least +// one cataloger's capabilities, preventing orphaned package types that are defined but never documented. +func TestPackageTypeCoverage(t *testing.T) { + checkCompletenessTestsEnabled(t) + + // load catalogers from embedded YAML + catalogerEntries, err := capabilities.Packages() + require.NoError(t, err) + + // collect all package types mentioned in catalogers + foundPkgTypes := strset.New() + for _, cataloger := range catalogerEntries { + if cataloger.Type == "generic" { + for _, parser := range cataloger.Parsers { + for _, pkgType := range parser.PackageTypes { + foundPkgTypes.Add(pkgType) + } + } + } else if cataloger.Type == "custom" { + for _, pkgType := range cataloger.PackageTypes { + foundPkgTypes.Add(pkgType) + } + } + } + + // check that all known package types are represented + var missingTypes []pkg.Type + for _, pkgType := range pkg.AllPkgs { + if !foundPkgTypes.Has(string(pkgType)) { + missingTypes = append(missingTypes, pkgType) + } + } + + // filter out exceptions + var missingTypesWithoutExceptions []pkg.Type + for _, pkgType := range missingTypes { + if !packageTypeCoverageExceptions.Has(string(pkgType)) { + missingTypesWithoutExceptions = append(missingTypesWithoutExceptions, pkgType) + } + } + + require.Empty(t, missingTypesWithoutExceptions, + "The following package types are not represented in any cataloger: %v\n"+ + "Either add catalogers for these types or update pkg.AllPkgs if they're no longer supported.", + missingTypesWithoutExceptions) +} + +// TestMetadataTypeCoverage ensures that every metadata type defined in packagemetadata.AllTypes() is represented +// in at least one cataloger's capabilities, preventing orphaned metadata types that are defined but never produced. +func TestMetadataTypeCoverage(t *testing.T) { + checkCompletenessTestsEnabled(t) + + // load catalogers from embedded YAML + catalogerEntries, err := capabilities.Packages() + require.NoError(t, err) + + // collect all metadata types mentioned in catalogers + foundMetadataTypes := strset.New() + for _, cataloger := range catalogerEntries { + if cataloger.Type == "generic" { + for _, p := range cataloger.Parsers { + for _, metadataType := range p.MetadataTypes { + foundMetadataTypes.Add(strings.TrimPrefix(metadataType, "pkg.")) + } + } + } else if cataloger.Type == "custom" { + for _, metadataType := range cataloger.MetadataTypes { + foundMetadataTypes.Add(strings.TrimPrefix(metadataType, "pkg.")) + } + } + } + + // get all known metadata types + allMetadataTypes := packagemetadata.AllTypes() + + // check that all known metadata types are represented + var missingTypes []string + for _, metadataType := range allMetadataTypes { + typeName := reflect.TypeOf(metadataType).Name() + if !foundMetadataTypes.Has(typeName) { + missingTypes = append(missingTypes, typeName) + } + } + + // filter out exceptions + var missingTypesWithoutExceptions []string + for _, metadataType := range missingTypes { + if !metadataTypeCoverageExceptions.Has(metadataType) { + missingTypesWithoutExceptions = append(missingTypesWithoutExceptions, metadataType) + } + } + + require.Empty(t, missingTypesWithoutExceptions, + "The following metadata types are not represented in any cataloger: %v\n"+ + "Either add catalogers for these types or update packagemetadata.AllTypes() if they're no longer supported.", + missingTypesWithoutExceptions) +} + +// TestCatalogerStructure validates that catalogers follow structural conventions: generic catalogers must have +// parsers and parser-level capabilities, custom catalogers must have detectors and cataloger-level capabilities, +// and all catalogers must have an ecosystem set. +func TestCatalogerStructure(t *testing.T) { + checkCompletenessTestsEnabled(t) + + // load catalogers from embedded YAML + catalogerEntries, err := capabilities.Packages() + require.NoError(t, err) + + for _, c := range catalogerEntries { + c := c // capture loop variable for subtest + + t.Run(c.Name, func(t *testing.T) { + // ecosystem must always be set (it's MANUAL) + require.NotEmpty(t, c.Ecosystem, "ecosystem must be set for all catalogers") + + if c.Type == "generic" { + // generic catalogers must have parsers + require.NotEmpty(t, c.Parsers, "generic cataloger must have at least one parser") + + // generic catalogers should not have cataloger-level capabilities + require.Empty(t, c.Capabilities, "generic cataloger should not have cataloger-level capabilities (use parser-level instead)") + + // generic catalogers should not have cataloger-level metadata/package types + require.Empty(t, c.MetadataTypes, "generic cataloger should not have cataloger-level metadata types") + require.Empty(t, c.PackageTypes, "generic cataloger should not have cataloger-level package types") + } else if c.Type == "custom" { + // custom catalogers must have detectors + require.NotEmpty(t, c.Detectors, "custom cataloger must have at least one detector") + + // custom catalogers must have cataloger-level capabilities + require.NotEmpty(t, c.Capabilities, "custom cataloger must have cataloger-level capabilities") + + // custom catalogers should not have parsers + require.Empty(t, c.Parsers, "custom cataloger should not have parsers (those are for generic catalogers)") + } else { + t.Errorf("unknown cataloger type: %q (must be 'generic' or 'custom')", c.Type) + } + }) + } +} + +// TestCatalogerDataQuality checks for data integrity issues in cataloger/*/capabilities.yaml, including duplicate cataloger +// names, duplicate parser functions within catalogers, and validates that detector definitions are well-formed. +func TestCatalogerDataQuality(t *testing.T) { + checkCompletenessTestsEnabled(t) + + // load catalogers from embedded YAML + catalogerEntries, err := capabilities.Packages() + require.NoError(t, err) + + t.Run("no duplicate cataloger names", func(t *testing.T) { + nameCount := make(map[string]int) + for _, cataloger := range catalogerEntries { + nameCount[cataloger.Name]++ + } + + var duplicates []string + for name, count := range nameCount { + if count > 1 { + duplicates = append(duplicates, fmt.Sprintf("%s (appears %d times)", name, count)) + } + } + + require.Empty(t, duplicates, "Found duplicate cataloger names: %v", duplicates) + }) + + t.Run("detector validation for custom catalogers", func(t *testing.T) { + for _, c := range catalogerEntries { + if c.Type != "custom" { + continue + } + + c := c // capture loop variable + + t.Run(c.Name, func(t *testing.T) { + require.NotEmpty(t, c.Detectors, "custom cataloger must have at least one detector") + + for i, detector := range c.Detectors { + t.Run(fmt.Sprintf("detector-%d", i), func(t *testing.T) { + // detector criteria must not be empty + require.NotEmpty(t, detector.Criteria, "detector criteria must not be empty") + + // detector method must be valid + validMethods := map[capabilities.ArtifactDetectionMethod]bool{ + capabilities.GlobDetection: true, + capabilities.PathDetection: true, + capabilities.MIMETypeDetection: true, + } + require.True(t, validMethods[detector.Method], + "detector method must be one of: glob, path, mimetype (got %q)", detector.Method) + }) + } + }) + } + }) + + t.Run("no duplicate parser functions within cataloger", func(t *testing.T) { + for _, c := range catalogerEntries { + if c.Type != "generic" { + continue + } + + c := c // capture loop variable + + t.Run(c.Name, func(t *testing.T) { + parserFuncs := strset.New() + var duplicates []string + + for _, p := range c.Parsers { + if parserFuncs.Has(p.ParserFunction) { + duplicates = append(duplicates, p.ParserFunction) + } + parserFuncs.Add(p.ParserFunction) + } + + require.Empty(t, duplicates, "Found duplicate parser functions: %v", duplicates) + }) + } + }) +} + +// TestCatalogersHaveTestObservations ensures that all custom catalogers (and optionally parsers) have +// test observations recorded in test-fixtures/test-observations.json, which proves they are using the +// pkgtest.CatalogTester helpers and have test coverage. +func TestCatalogersHaveTestObservations(t *testing.T) { + checkCompletenessTestsEnabled(t) + + repoRoot, err := RepoRoot() + require.NoError(t, err) + + // load catalogers from YAML + catalogerEntries, err := capabilities.Packages() + require.NoError(t, err) + + // collect all catalogers and parsers from observations + observedCatalogers := strset.New() + observedParsers := make(map[string]*strset.Set) // package -> parser set + + // walk test-fixtures directories to find test-observations.json files + testFixtureDirs, err := FindTestFixtureDirs(repoRoot) + require.NoError(t, err) + + for _, dir := range testFixtureDirs { + observationsFile := filepath.Join(dir, "test-observations.json") + if _, err := os.Stat(observationsFile); os.IsNotExist(err) { + continue + } + + observations, err := ReadTestObservations(observationsFile) + if err != nil { + t.Logf("Warning: failed to read %s: %v", observationsFile, err) + continue + } + + // track observed catalogers + for catalogerName := range observations.Catalogers { + observedCatalogers.Add(catalogerName) + } + + // track observed parsers + p := observations.Package + if observedParsers[p] == nil { + observedParsers[p] = strset.New() + } + for parserName := range observations.Parsers { + observedParsers[p].Add(parserName) + } + } + + // infer parser observations for single-parser catalogers + // if a cataloger has only one parser and the cataloger was observed, assume the parser was evaluated + for _, cataloger := range catalogerEntries { + if cataloger.Type == "generic" && len(cataloger.Parsers) == 1 && observedCatalogers.Has(cataloger.Name) { + packageName := extractPackageName(cataloger.Name) + if observedParsers[packageName] == nil { + observedParsers[packageName] = strset.New() + } + observedParsers[packageName].Add(cataloger.Parsers[0].ParserFunction) + } + } + + // verify catalogers have observations + var missingCatalogers []string + var missingParsers []string + + for _, cataloger := range catalogerEntries { + if cataloger.Type == "custom" { + // custom catalogers should always have cataloger-level observations + // skip if this cataloger has an exception (nil or non-nil) + if _, hasException := observationExceptions[cataloger.Name]; hasException { + continue + } + if !observedCatalogers.Has(cataloger.Name) { + missingCatalogers = append(missingCatalogers, cataloger.Name) + } + } else if cataloger.Type == "generic" && requireParserObservations { + // generic catalogers have parser-level observations (only checked if requireParserObservations=true) + // skip if the cataloger itself has an exception (applies to all its parsers) + if _, hasException := observationExceptions[cataloger.Name]; hasException { + continue + } + + // extract package name from cataloger name + packageName := extractPackageName(cataloger.Name) + + for _, p := range cataloger.Parsers { + parserKey := fmt.Sprintf("%s/%s", cataloger.Name, p.ParserFunction) + // skip if this specific parser has an exception (nil or non-nil) + if _, hasException := observationExceptions[parserKey]; hasException { + continue + } + if observedParsers[packageName] == nil || !observedParsers[packageName].Has(p.ParserFunction) { + missingParsers = append(missingParsers, parserKey) + } + } + } + } + + require.Empty(t, missingCatalogers, + "The following custom catalogers have no test observations (not using pkgtest helpers): %v\n"+ + "Update tests to use CatalogTester.TestCataloger() from syft/pkg/cataloger/internal/pkgtest", + missingCatalogers) + + if requireParserObservations { + require.Empty(t, missingParsers, + "The following parsers have no test observations (not using pkgtest helpers): %v\n"+ + "Update tests to use CatalogTester.TestParser() from syft/pkg/cataloger/internal/pkgtest", + missingParsers) + } +} + +// extractPackageName extracts the package name from a cataloger name +// e.g., "javascript-lock-cataloger" -> "javascript" +func extractPackageName(catalogerName string) string { + // package name is the first segment before the first dash + for i, ch := range catalogerName { + if ch == '-' { + return catalogerName[:i] + } + } + return catalogerName +} diff --git a/internal/capabilities/internal/paths.go b/internal/capabilities/internal/paths.go new file mode 100644 index 000000000..68c0791c5 --- /dev/null +++ b/internal/capabilities/internal/paths.go @@ -0,0 +1,39 @@ +// this file provides a single source of truth for all capability file paths used in generation and testing. +package internal + +import "path/filepath" + +// path constants - single source of truth for all capability file locations +const ( + // CatalogerDirRel is the cataloger directory relative to repo root + CatalogerDirRel = "syft/pkg/cataloger" + + // InternalCapabilitiesDirRel is internal capabilities dir relative to repo root + InternalCapabilitiesDirRel = "internal/capabilities" + + // CapabilitiesFilename is the capabilities file name within each ecosystem dir + CapabilitiesFilename = "capabilities.yaml" + + // AppconfigFilename is the application config filename + AppconfigFilename = "appconfig.yaml" +) + +// CatalogerDir returns absolute path to cataloger directory +func CatalogerDir(repoRoot string) string { + return filepath.Join(repoRoot, CatalogerDirRel) +} + +// CapabilitiesDir returns absolute path to internal capabilities +func CapabilitiesDir(repoRoot string) string { + return filepath.Join(repoRoot, InternalCapabilitiesDirRel) +} + +// EcosystemCapabilitiesPath returns path to an ecosystem's capabilities.yaml +func EcosystemCapabilitiesPath(catalogerDir, ecosystem string) string { + return filepath.Join(catalogerDir, ecosystem, CapabilitiesFilename) +} + +// AppconfigPath returns path to appconfig.yaml (in internal/capabilities) +func AppconfigPath(repoRoot string) string { + return filepath.Join(CapabilitiesDir(repoRoot), AppconfigFilename) +} diff --git a/internal/capabilities/internal/repo_root.go b/internal/capabilities/internal/repo_root.go new file mode 100644 index 000000000..6fede8f35 --- /dev/null +++ b/internal/capabilities/internal/repo_root.go @@ -0,0 +1,22 @@ +package internal + +import ( + "fmt" + "os/exec" + "path/filepath" + "strings" +) + +// RepoRoot finds the git repository root directory. +// Exported for use by the generator in generate/main.go +func RepoRoot() (string, error) { + root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + return "", fmt.Errorf("unable to find repo root dir: %+v", err) + } + absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root))) + if err != nil { + return "", fmt.Errorf("unable to get abs path to repo root: %w", err) + } + return absRepoRoot, nil +} diff --git a/internal/capabilities/internal/util_test.go b/internal/capabilities/internal/util_test.go new file mode 100644 index 000000000..7182869ef --- /dev/null +++ b/internal/capabilities/internal/util_test.go @@ -0,0 +1,20 @@ +package internal + +import ( + "os" + "testing" +) + +// checkCompletenessTestsEnabled skips the test if completeness tests are not enabled via environment variable. +// Why do this at all? Can't we just run these tests all the time? Short answer: No. +// These tests are coupled with unit tests under ./syft/pkg/..., which means that these tests must be run not only +// after those unit tests, but also after code generation that reads observations from test results. +// This means that we should not really consider these tests as part of normal unit test runs, but rather as a separate +// self-consistency check during generation, and it's really static analysis that should be checking +// if the generated code has drifted (not a unit test). +func checkCompletenessTestsEnabled(t *testing.T) { + enabled := os.Getenv("SYFT_ENABLE_COMPLETENESS_TESTS") == "true" + if !enabled { + t.Skip("skipping completeness tests (SYFT_ENABLE_COMPLETENESS_TESTS is not set to 'true')") + } +} diff --git a/internal/capabilities/model.go b/internal/capabilities/model.go new file mode 100644 index 000000000..c969897b8 --- /dev/null +++ b/internal/capabilities/model.go @@ -0,0 +1,127 @@ +package capabilities + +// ArtifactDetectionMethod specifies the type of artifact detection mechanism +type ArtifactDetectionMethod string + +const ( + // GlobDetection matches artifacts using glob patterns (e.g., "**/*.jar", "*.pyc") + GlobDetection ArtifactDetectionMethod = "glob" + // PathDetection matches artifacts by exact file path (e.g., "/usr/bin/python", "package.json") + PathDetection ArtifactDetectionMethod = "path" + // MIMETypeDetection matches artifacts by MIME type (e.g., "application/x-executable", "text/x-python") + MIMETypeDetection ArtifactDetectionMethod = "mimetype" +) + +// Document represents the root structure of the capabilities YAML file +type Document struct { + Configs map[string]CatalogerConfigEntry `yaml:"configs,omitempty" json:"configs,omitempty"` // config structs with their fields + ApplicationConfig []ApplicationConfigField `yaml:"application,omitempty" json:"application,omitempty"` // application-level config keys + Catalogers []CatalogerEntry `yaml:"catalogers" json:"catalogers"` +} + +// CatalogerConfigFieldEntry represents a single field in a cataloger configuration struct +type CatalogerConfigFieldEntry struct { + Key string `yaml:"key" json:"key"` + Description string `yaml:"description" json:"description"` + AppKey string `yaml:"app_key,omitempty" json:"app_key,omitempty"` // maps to app-level config key +} + +// CatalogerConfigEntry represents a complete configuration struct (e.g., golang.CatalogerConfig) +type CatalogerConfigEntry struct { + Fields []CatalogerConfigFieldEntry `yaml:"fields" json:"fields"` +} + +// ApplicationConfigField represents an application-level configuration field +type ApplicationConfigField struct { + Key string `yaml:"key" json:"key"` + Description string `yaml:"description" json:"description"` + DefaultValue any `yaml:"default,omitempty" json:"default,omitempty"` +} + +// Source describes the source code location of a cataloger +type Source struct { + File string `yaml:"file" json:"file"` // AUTO-GENERATED for generic, MANUAL for custom + Function string `yaml:"function" json:"function"` // AUTO-GENERATED for generic, MANUAL for custom +} + +// Detector describes how artifacts are detected (method and criteria) +type Detector struct { + Method ArtifactDetectionMethod `yaml:"method" json:"method"` // AUTO-GENERATED + Criteria []string `yaml:"criteria" json:"criteria"` // AUTO-GENERATED + Conditions []DetectorCondition `yaml:"conditions,omitempty" json:"conditions,omitempty"` // MANUAL - when this detector should be active + Packages []DetectorPackageInfo `yaml:"packages,omitempty" json:"packages,omitempty"` // AUTO-GENERATED for binary-classifier-cataloger + Comment string `yaml:"comment,omitempty" json:"comment,omitempty"` // MANUAL - explanation of this detector +} + +// DetectorPackageInfo describes package information that a detector can produce +type DetectorPackageInfo struct { + Class string `yaml:"class" json:"class"` // classifier class (e.g., "python-binary-lib") + Name string `yaml:"name" json:"name"` // package name (e.g., "python") + PURL string `yaml:"purl" json:"purl"` // package URL without version (e.g., "pkg:generic/python") + CPEs []string `yaml:"cpes" json:"cpes"` // CPE strings + Type string `yaml:"type" json:"type"` // package type (e.g., "BinaryPkg") +} + +// DetectorCondition specifies when a detector should be active based on configuration +type DetectorCondition struct { + // When specifies config field names and their required values (all must match - AND logic) + When map[string]any `yaml:"when" json:"when"` + // Comment provides optional explanation of this condition + Comment string `yaml:"comment,omitempty" json:"comment,omitempty"` +} + +// CatalogerEntry represents a single cataloger's capabilities +type CatalogerEntry struct { + Ecosystem string `yaml:"ecosystem" json:"ecosystem"` // MANUAL - ecosystem categorization (e.g., "python", "java", "javascript") + Name string `yaml:"name" json:"name"` // AUTO-GENERATED for generic, MANUAL for custom + Type string `yaml:"type" json:"type"` // AUTO-GENERATED: "generic" or "custom" + Source Source `yaml:"source" json:"source"` // AUTO-GENERATED for generic, MANUAL for custom + Config string `yaml:"config,omitempty" json:"config,omitempty"` // e.g., "golang.CatalogerConfig" + Selectors []string `yaml:"selectors,omitempty" json:"selectors,omitempty"` // AUTO-GENERATED - cataloger name tags for selection + Parsers []Parser `yaml:"parsers,omitempty" json:"parsers,omitempty"` // AUTO-GENERATED structure, only for type=generic + Detectors []Detector `yaml:"detectors,omitempty" json:"detectors,omitempty"` // AUTO-GENERATED - detection methods (only for type=custom) + MetadataTypes []string `yaml:"metadata_types,omitempty" json:"metadata_types,omitempty"` // AUTO-GENERATED - pkg metadata types emitted (only for type=custom) + PackageTypes []string `yaml:"package_types,omitempty" json:"package_types,omitempty"` // AUTO-GENERATED - package types emitted (only for type=custom) + JSONSchemaTypes []string `yaml:"json_schema_types,omitempty" json:"json_schema_types,omitempty"` // AUTO-GENERATED - JSON schema type names (UpperCamelCase) + Capabilities CapabilitySet `yaml:"capabilities,omitempty" json:"capabilities,omitempty"` // MANUAL - config-driven capability definitions (only for type=custom) +} + +// Parser represents a parser function and its artifact detection criteria for generic catalogers +type Parser struct { + ParserFunction string `yaml:"function" json:"function"` // AUTO-GENERATED (used as preservation key) + Detector Detector `yaml:"detector" json:"detector"` // AUTO-GENERATED - how artifacts are detected + MetadataTypes []string `yaml:"metadata_types,omitempty" json:"metadata_types,omitempty"` // AUTO-GENERATED - pkg metadata types emitted by this parser + PackageTypes []string `yaml:"package_types,omitempty" json:"package_types,omitempty"` // AUTO-GENERATED - package types emitted by this parser + JSONSchemaTypes []string `yaml:"json_schema_types,omitempty" json:"json_schema_types,omitempty"` // AUTO-GENERATED - JSON schema type names (UpperCamelCase) + Capabilities CapabilitySet `yaml:"capabilities,omitempty" json:"capabilities,omitempty"` // MANUAL - config-driven capability definitions +} + +// CapabilityField represents a single capability field with optional conditional values based on configuration. +// This is the V2 capabilities format that replaces the mode-based approach. +type CapabilityField struct { + // Name is the dot-notation path to the capability (e.g., "license", "dependency.depth") + Name string `yaml:"name" json:"name"` + // Default is the value when no conditions match + Default any `yaml:"default" json:"default"` + // Conditions are optional conditional overrides evaluated in order (first match wins) + Conditions []CapabilityCondition `yaml:"conditions,omitempty" json:"conditions,omitempty"` + // Evidence provides optional references to source code that implements this capability + Evidence []string `yaml:"evidence,omitempty" json:"evidence,omitempty"` + // Comment provides optional human-readable explanation + Comment string `yaml:"comment,omitempty" json:"comment,omitempty"` +} + +// CapabilityCondition represents a conditional override for a capability field value. +// When the config fields specified in When match, the Value is used instead of Default. +type CapabilityCondition struct { + // When specifies config field names and their required values (all must match - AND logic) + // Example: {"SearchRemoteLicenses": true, "UseNetwork": true} + When map[string]any `yaml:"when" json:"when"` + // Value is the capability value when the condition matches + Value any `yaml:"value" json:"value"` + // Comment provides optional explanation of this condition + Comment string `yaml:"comment,omitempty" json:"comment,omitempty"` +} + +// CapabilitySet represents a collection of capability fields (V2 format) +type CapabilitySet []CapabilityField diff --git a/internal/capabilities/model_test.go b/internal/capabilities/model_test.go new file mode 100644 index 000000000..12ddb8994 --- /dev/null +++ b/internal/capabilities/model_test.go @@ -0,0 +1,148 @@ +package capabilities + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestConfigSchemaExtensions(t *testing.T) { + // create a document with configs and app-configs + doc := Document{ + Configs: map[string]CatalogerConfigEntry{ + "golang.CatalogerConfig": { + Fields: []CatalogerConfigFieldEntry{ + { + Key: "SearchLocalModCacheLicenses", + Description: "searchLocalModCacheLicenses enables searching for go package licenses in the local GOPATH mod cache.", + AppKey: "golang.search-local-mod-cache-licenses", + }, + { + Key: "LocalModCacheDir", + Description: "localModCacheDir specifies the location of the local go module cache directory.", + AppKey: "golang.local-mod-cache-dir", + }, + }, + }, + "python.CatalogerConfig": { + Fields: []CatalogerConfigFieldEntry{ + { + Key: "GuessUnpinnedRequirements", + Description: "guessUnpinnedRequirements attempts to infer package versions from version constraints...", + AppKey: "python.guess-unpinned-requirements", + }, + }, + }, + }, + ApplicationConfig: []ApplicationConfigField{ + { + Key: "golang.search-local-mod-cache-licenses", + Description: "search for go package licences in the GOPATH of the system running Syft", + DefaultValue: false, + }, + { + Key: "python.guess-unpinned-requirements", + Description: "attempt to guess what the version could be based on version requirements", + DefaultValue: false, + }, + }, + Catalogers: []CatalogerEntry{ + { + Name: "go-module-binary-cataloger", + Type: "generic", + Config: "golang.CatalogerConfig", + }, + { + Name: "python-package-cataloger", + Type: "generic", + Config: "python.CatalogerConfig", + }, + }, + } + + // marshal to YAML + yamlData, err := yaml.Marshal(&doc) + require.NoError(t, err) + + // verify YAML contains expected sections + yamlStr := string(yamlData) + require.Contains(t, yamlStr, "configs:") + require.Contains(t, yamlStr, "golang.CatalogerConfig:") + require.Contains(t, yamlStr, "SearchLocalModCacheLicenses") + require.Contains(t, yamlStr, "application:") + require.Contains(t, yamlStr, "golang.search-local-mod-cache-licenses") + + // unmarshal back + var unmarshaled Document + err = yaml.Unmarshal(yamlData, &unmarshaled) + require.NoError(t, err) + + // verify data integrity + if diff := cmp.Diff(doc, unmarshaled); diff != "" { + t.Errorf("document mismatch (-want +got):\n%s", diff) + } +} + +func TestConfigSchemaOmitEmpty(t *testing.T) { + // create a document with no configs or app-configs + doc := Document{ + Catalogers: []CatalogerEntry{ + { + Name: "test-cataloger", + Type: "generic", + }, + }, + } + + // marshal to YAML + yamlData, err := yaml.Marshal(&doc) + require.NoError(t, err) + + // verify configs and application are omitted when empty + yamlStr := string(yamlData) + require.NotContains(t, yamlStr, "configs:") + require.NotContains(t, yamlStr, "application:") +} + +func TestCatalogerConfigField(t *testing.T) { + // create a cataloger with config field + entry := CatalogerEntry{ + Name: "test-cataloger", + Type: "generic", + Config: "test.CatalogerConfig", + } + + // marshal to YAML + yamlData, err := yaml.Marshal(&entry) + require.NoError(t, err) + + // verify config field is present + yamlStr := string(yamlData) + require.Contains(t, yamlStr, "config: test.CatalogerConfig") + + // unmarshal back + var unmarshaled CatalogerEntry + err = yaml.Unmarshal(yamlData, &unmarshaled) + require.NoError(t, err) + + // verify data integrity + require.Equal(t, entry.Config, unmarshaled.Config) +} + +func TestCatalogerConfigFieldOmitEmpty(t *testing.T) { + // create a cataloger without config field + entry := CatalogerEntry{ + Name: "test-cataloger", + Type: "generic", + } + + // marshal to YAML + yamlData, err := yaml.Marshal(&entry) + require.NoError(t, err) + + // verify config field is omitted when empty + yamlStr := string(yamlData) + require.NotContains(t, yamlStr, "config:") +} diff --git a/internal/capabilities/util_test.go b/internal/capabilities/util_test.go new file mode 100644 index 000000000..731aa0cde --- /dev/null +++ b/internal/capabilities/util_test.go @@ -0,0 +1,20 @@ +package capabilities + +import ( + "os" + "testing" +) + +// checkCompletenessTestsEnabled skips the test if completeness tests are not enabled via environment variable. +// Why do this at all? Can't we just run these tests all the time? Short answer: No. +// These tests are coupled with unit tests under ./syft/pkg/..., which means that these tests must be run not only +// after those unit tests, but also after code generation that reads observations from test results. +// This means that we should not really consider these tests as part of normal unit test runs, but rather as a separate +// self-consistency check during generation, and it's really static analysis that should be checking +// if the generated code has drifted (not a unit test). +func checkCompletenessTestsEnabled(t *testing.T) { + enabled := os.Getenv("SYFT_ENABLE_COMPLETENESS_TESTS") == "true" + if !enabled { + t.Skip("skipping completeness tests (SYFT_ENABLE_COMPLETENESS_TESTS is not set to 'true')") + } +} diff --git a/internal/cmptest/common_options.go b/internal/cmptest/common_options.go index fed53a6f5..8e8c2c3ea 100644 --- a/internal/cmptest/common_options.go +++ b/internal/cmptest/common_options.go @@ -29,7 +29,8 @@ func BuildOptions(licenseCmp LicenseComparer, locationCmp LocationComparer) []cm } return []cmp.Option{ - cmpopts.IgnoreFields(pkg.Package{}, "id"), // note: ID is not deterministic for test purposes + cmpopts.IgnoreFields(pkg.Package{}, "id"), // note: ID is not deterministic for test purposes + cmpopts.IgnoreFields(pkg.Package{}, "FoundBy"), // not necessary in most tests cmpopts.SortSlices(pkg.Less), cmpopts.SortSlices(DefaultRelationshipComparer), cmp.Comparer(buildSetComparer[file.Location, file.LocationSet](locationCmp, locationSorter)), diff --git a/internal/packagemetadata/names.go b/internal/packagemetadata/names.go index 44e686190..5b91d8911 100644 --- a/internal/packagemetadata/names.go +++ b/internal/packagemetadata/names.go @@ -167,3 +167,30 @@ func ReflectTypeFromJSONName(name string) reflect.Type { name = strings.ToLower(name) return jsonTypes.nameToType[name] } + +// JSONNameFromString converts a Go struct name string (e.g., "pkg.AlpmDBEntry" or "AlpmDBEntry") +// to its JSON schema name (e.g., "alpm-db-entry"). Returns empty string if not found. +func JSONNameFromString(typeName string) string { + // strip "pkg." prefix if present + typeName = strings.TrimPrefix(typeName, "pkg.") + + // look through all types to find matching struct name + for typ, jsonName := range jsonTypes.typeToName { + if typ.Name() == typeName { + return jsonName + } + } + return "" +} + +// ToUpperCamelCase converts kebab-case to UpperCamelCase +// e.g., "alpm-db-entry" -> "AlpmDbEntry" +func ToUpperCamelCase(kebab string) string { + parts := strings.Split(kebab, "-") + for i, part := range parts { + if len(part) > 0 { + parts[i] = strings.ToUpper(part[0:1]) + part[1:] + } + } + return strings.Join(parts, "") +} diff --git a/internal/task/package_tasks.go b/internal/task/package_tasks.go index a50099593..12e12422b 100644 --- a/internal/task/package_tasks.go +++ b/internal/task/package_tasks.go @@ -62,7 +62,7 @@ const ( func DefaultPackageTaskFactories() Factories { return []factory{ // OS package installed catalogers /////////////////////////////////////////////////////////////////////////// - newSimplePackageTaskFactory(arch.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "alpm", "archlinux"), + newSimplePackageTaskFactory(arch.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "alpm", "archlinux", "pacman"), newSimplePackageTaskFactory(alpine.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "apk", "alpine"), newSimplePackageTaskFactory(debian.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "dpkg", "debian"), newSimplePackageTaskFactory(gentoo.NewPortageCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "portage", "gentoo"), diff --git a/syft/cataloging/archive_search.go b/syft/cataloging/archive_search.go index bfaf0a1b2..56dff6cc6 100644 --- a/syft/cataloging/archive_search.go +++ b/syft/cataloging/archive_search.go @@ -1,7 +1,10 @@ package cataloging type ArchiveSearchConfig struct { - IncludeIndexedArchives bool `yaml:"include-indexed-archives" json:"include-indexed-archives" mapstructure:"include-indexed-archives"` + // IncludeIndexedArchives indicates whether to search within indexed archive files (e.g., .zip). + IncludeIndexedArchives bool `yaml:"include-indexed-archives" json:"include-indexed-archives" mapstructure:"include-indexed-archives"` + + // IncludeUnindexedArchives indicates whether to search within unindexed archive files (e.g., .tar*). IncludeUnindexedArchives bool `yaml:"include-unindexed-archives" json:"include-unindexed-archives" mapstructure:"include-unindexed-archives"` } diff --git a/syft/pkg/cataloger/ai/capabilities.yaml b/syft/pkg/cataloger/ai/capabilities.yaml new file mode 100644 index 000000000..fff9fa3a7 --- /dev/null +++ b/syft/pkg/cataloger/ai/capabilities.yaml @@ -0,0 +1,46 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: ai # MANUAL + name: gguf-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/ai/cataloger.go + function: NewGGUFCataloger + selectors: # AUTO-GENERATED + - ai + - directory + - gguf + - image + - ml + - model + - package + parsers: # AUTO-GENERATED structure + - function: parseGGUFModel + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/*.gguf' + metadata_types: # AUTO-GENERATED + - pkg.GGUFFileHeader + package_types: # AUTO-GENERATED + - model + json_schema_types: # AUTO-GENERATED + - GgufFileHeader + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - GGUFFileHeader.MetadataKeyValuesHash diff --git a/syft/pkg/cataloger/alpine/capabilities.yaml b/syft/pkg/cataloger/alpine/capabilities.yaml new file mode 100644 index 000000000..8f0efe66f --- /dev/null +++ b/syft/pkg/cataloger/alpine/capabilities.yaml @@ -0,0 +1,54 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: alpine # MANUAL + name: apk-db-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/alpine/cataloger.go + function: NewDBCataloger + selectors: # AUTO-GENERATED + - alpine + - apk + - directory + - image + - installed + - linux + - os + - package + parsers: # AUTO-GENERATED structure + - function: parseApkDB + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/lib/apk/db/installed' + metadata_types: # AUTO-GENERATED + - pkg.ApkDBEntry + package_types: # AUTO-GENERATED + - apk + json_schema_types: # AUTO-GENERATED + - ApkDbEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: true + evidence: + - ApkDBEntry.Files + - name: package_manager.files.digests + default: true + evidence: + - ApkDBEntry.Files[].Digest + - name: package_manager.package_integrity_hash + default: true + evidence: + - ApkDBEntry.Checksum diff --git a/syft/pkg/cataloger/arch/capabilities.yaml b/syft/pkg/cataloger/arch/capabilities.yaml new file mode 100644 index 000000000..1c8c7057b --- /dev/null +++ b/syft/pkg/cataloger/arch/capabilities.yaml @@ -0,0 +1,53 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: alpm # MANUAL + name: alpm-db-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/arch/cataloger.go + function: NewDBCataloger + selectors: # AUTO-GENERATED + - alpm + - archlinux + - directory + - image + - installed + - linux + - os + - package + - pacman + parsers: # AUTO-GENERATED structure + - function: parseAlpmDB + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/var/lib/pacman/local/**/desc' + metadata_types: # AUTO-GENERATED + - pkg.AlpmDBEntry + package_types: # AUTO-GENERATED + - alpm + json_schema_types: # AUTO-GENERATED + - AlpmDbEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: true + evidence: + - AlpmDBEntry.Files + - name: package_manager.files.digests + default: true + evidence: + - AlpmDBEntry.Files[].Digests + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/binary/capabilities.yaml b/syft/pkg/cataloger/binary/capabilities.yaml new file mode 100644 index 000000000..d2d37adee --- /dev/null +++ b/syft/pkg/cataloger/binary/capabilities.yaml @@ -0,0 +1,801 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: binary # MANUAL + name: binary-classifier-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/binary/classifier_cataloger.go + function: NewClassifierCataloger + selectors: # AUTO-GENERATED + - binary + - declared + - directory + - image + - installed + - package + detectors: # AUTO-GENERATED + - method: glob + criteria: + - '**/python*' + packages: + - class: python-binary + name: python + purl: pkg:generic/python + cpes: + - cpe:2.3:a:python_software_foundation:python:*:*:*:*:*:*:*:* + - cpe:2.3:a:python:python:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/libpython*.so*' + packages: + - class: python-binary-lib + name: python + purl: pkg:generic/python + cpes: + - cpe:2.3:a:python_software_foundation:python:*:*:*:*:*:*:*:* + - cpe:2.3:a:python:python:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/libpypy*.so*' + packages: + - class: pypy-binary-lib + name: pypy + purl: pkg:generic/pypy + cpes: [] + type: BinaryPkg + - method: glob + criteria: + - '**/go' + packages: + - class: go-binary + name: go + purl: pkg:generic/go + cpes: + - cpe:2.3:a:golang:go:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/libjulia-internal.so' + packages: + - class: julia-binary + name: julia + purl: pkg:generic/julia + cpes: + - cpe:2.3:a:julialang:julia:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/helm' + packages: + - class: helm + name: helm + purl: pkg:golang/helm.sh/helm + cpes: + - cpe:2.3:a:helm:helm:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/redis-server' + packages: + - class: redis-binary + name: redis + purl: pkg:generic/redis + cpes: + - cpe:2.3:a:redislabs:redis:*:*:*:*:*:*:*:* + - cpe:2.3:a:redis:redis:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/node' + packages: + - class: nodejs-binary + name: node + purl: pkg:generic/node + cpes: + - cpe:2.3:a:nodejs:node.js:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/VERSION*' + packages: + - class: go-binary-hint + name: go + purl: pkg:generic/go + cpes: + - cpe:2.3:a:golang:go:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/busybox' + packages: + - class: busybox-binary + name: busybox + purl: pkg:generic/busybox + cpes: + - cpe:2.3:a:busybox:busybox:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/getopt' + packages: + - class: util-linux-binary + name: util-linux + purl: pkg:generic/util-linux + cpes: + - cpe:2.3:a:kernel:util-linux:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/haproxy' + packages: + - class: haproxy-binary + name: haproxy + purl: pkg:generic/haproxy + cpes: + - cpe:2.3:a:haproxy:haproxy:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/perl' + packages: + - class: perl-binary + name: perl + purl: pkg:generic/perl + cpes: + - cpe:2.3:a:perl:perl:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/composer*' + packages: + - class: php-composer-binary + name: composer + purl: pkg:generic/composer + cpes: + - cpe:2.3:a:getcomposer:composer:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/httpd' + packages: + - class: httpd-binary + name: httpd + purl: pkg:generic/httpd + cpes: + - cpe:2.3:a:apache:http_server:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/memcached' + packages: + - class: memcached-binary + name: memcached + purl: pkg:generic/memcached + cpes: + - cpe:2.3:a:memcached:memcached:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/traefik' + packages: + - class: traefik-binary + name: traefik + purl: pkg:generic/traefik + cpes: + - cpe:2.3:a:traefik:traefik:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/arangosh' + packages: + - class: arangodb-binary + name: arangodb + purl: pkg:generic/arangodb + cpes: + - cpe:2.3:a:arangodb:arangodb:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/postgres' + packages: + - class: postgresql-binary + name: postgresql + purl: pkg:generic/postgresql + cpes: + - cpe:2.3:a:postgresql:postgresql:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/mysql' + packages: + - class: mysql-binary + name: mysql + purl: pkg:generic/mysql + cpes: + - cpe:2.3:a:oracle:mysql:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/mysql' + packages: + - class: mysql-binary + name: percona-server + purl: pkg:generic/percona-server + cpes: + - cpe:2.3:a:oracle:mysql:*:*:*:*:*:*:*:* + - cpe:2.3:a:percona:percona_server:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/mysql' + packages: + - class: mysql-binary + name: percona-xtradb-cluster + purl: pkg:generic/percona-xtradb-cluster + cpes: + - cpe:2.3:a:oracle:mysql:*:*:*:*:*:*:*:* + - cpe:2.3:a:percona:percona_server:*:*:*:*:*:*:*:* + - cpe:2.3:a:percona:xtradb_cluster:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/xtrabackup' + packages: + - class: xtrabackup-binary + name: percona-xtrabackup + purl: pkg:generic/percona-xtrabackup + cpes: + - cpe:2.3:a:percona:xtrabackup:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/{mariadb,mysql}' + packages: + - class: mariadb-binary + name: mariadb + purl: pkg:generic/mariadb + cpes: + - cpe:2.3:a:mariadb:mariadb:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/libstd-????????????????.so' + packages: + - class: rust-standard-library-linux + name: rust + purl: pkg:generic/rust + cpes: + - cpe:2.3:a:rust-lang:rust:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/libstd-????????????????.dylib' + packages: + - class: rust-standard-library-macos + name: rust + purl: pkg:generic/rust + cpes: + - cpe:2.3:a:rust-lang:rust:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/ruby' + packages: + - class: ruby-binary + name: ruby + purl: pkg:generic/ruby + cpes: + - cpe:2.3:a:ruby-lang:ruby:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/erlexec' + packages: + - class: erlang-binary + name: erlang + purl: pkg:generic/erlang + cpes: + - cpe:2.3:a:erlang:erlang/otp:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/beam.smp' + packages: + - class: erlang-alpine-binary + name: erlang + purl: pkg:generic/erlang + cpes: + - cpe:2.3:a:erlang:erlang/otp:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/liberts_internal.a' + packages: + - class: erlang-library + name: erlang + purl: pkg:generic/erlang + cpes: + - cpe:2.3:a:erlang:erlang/otp:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/swipl' + packages: + - class: swipl-binary + name: swipl + purl: pkg:generic/swipl + cpes: + - cpe:2.3:a:erlang:erlang/otp:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/dart' + packages: + - class: dart-binary + name: dart + purl: pkg:generic/dart + cpes: + - cpe:2.3:a:dart:dart_software_development_kit:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/ghc*' + packages: + - class: haskell-ghc-binary + name: haskell/ghc + purl: pkg:generic/haskell/ghc + cpes: + - cpe:2.3:a:haskell:ghc:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/cabal' + packages: + - class: haskell-cabal-binary + name: haskell/cabal + purl: pkg:generic/haskell/cabal + cpes: + - cpe:2.3:a:haskell:cabal:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/stack' + packages: + - class: haskell-stack-binary + name: haskell/stack + purl: pkg:generic/haskell/stack + cpes: + - cpe:2.3:a:haskell:stack:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/consul' + packages: + - class: consul-binary + name: consul + purl: pkg:golang/github.com/hashicorp/consul + cpes: + - cpe:2.3:a:hashicorp:consul:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/vault' + packages: + - class: hashicorp-vault-binary + name: github.com/hashicorp/vault + purl: pkg:golang/github.com/hashicorp/vault + cpes: + - cpe:2.3:a:hashicorp:vault:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/nginx' + packages: + - class: nginx-binary + name: nginx + purl: pkg:generic/nginx + cpes: + - cpe:2.3:a:f5:nginx:*:*:*:*:*:*:*:* + - cpe:2.3:a:nginx:nginx:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/bash' + packages: + - class: bash-binary + name: bash + purl: pkg:generic/bash + cpes: + - cpe:2.3:a:gnu:bash:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/openssl' + packages: + - class: openssl-binary + name: openssl + purl: pkg:generic/openssl + cpes: + - cpe:2.3:a:openssl:openssl:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/gcc' + packages: + - class: gcc-binary + name: gcc + purl: pkg:generic/gcc + cpes: + - cpe:2.3:a:gnu:gcc:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/fluent-bit' + packages: + - class: fluent-bit-binary + name: fluent-bit + purl: pkg:github/fluent/fluent-bit + cpes: + - cpe:2.3:a:treasuredata:fluent_bit:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/wp' + packages: + - class: wordpress-cli-binary + name: wp-cli + purl: pkg:generic/wp-cli + cpes: + - cpe:2.3:a:wp-cli:wp-cli:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/curl' + packages: + - class: curl-binary + name: curl + purl: pkg:generic/curl + cpes: + - cpe:2.3:a:haxx:curl:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/lighttpd' + packages: + - class: lighttpd-binary + name: lighttpd + purl: pkg:generic/lighttpd + cpes: + - cpe:2.3:a:lighttpd:lighttpd:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/proftpd' + packages: + - class: proftpd-binary + name: proftpd + purl: pkg:generic/proftpd + cpes: + - cpe:2.3:a:proftpd:proftpd:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/zstd' + packages: + - class: zstd-binary + name: zstd + purl: pkg:generic/zstd + cpes: + - cpe:2.3:a:facebook:zstandard:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/xz' + packages: + - class: xz-binary + name: xz + purl: pkg:generic/xz + cpes: + - cpe:2.3:a:tukaani:xz:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/gzip' + packages: + - class: gzip-binary + name: gzip + purl: pkg:generic/gzip + cpes: + - cpe:2.3:a:gnu:gzip:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/sqlcipher' + packages: + - class: sqlcipher-binary + name: sqlcipher + purl: pkg:generic/sqlcipher + cpes: + - cpe:2.3:a:zetetic:sqlcipher:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/jq' + packages: + - class: jq-binary + name: jq + purl: pkg:generic/jq + cpes: + - cpe:2.3:a:jqlang:jq:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/chrome' + packages: + - class: chrome-binary + name: chrome + purl: pkg:generic/chrome + cpes: + - cpe:2.3:a:google:chrome:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/ffmpeg' + packages: + - class: ffmpeg-binary + name: ffmpeg + purl: pkg:generic/ffmpeg + cpes: + - cpe:2.3:a:ffmpeg:ffmpeg:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/libav*' + packages: + - class: ffmpeg-library + name: ffmpeg + purl: pkg:generic/ffmpeg + cpes: + - cpe:2.3:a:ffmpeg:ffmpeg:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/libswresample*' + packages: + - class: ffmpeg-library + name: ffmpeg + purl: pkg:generic/ffmpeg + cpes: + - cpe:2.3:a:ffmpeg:ffmpeg:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/elixir' + packages: + - class: elixir-binary + name: elixir + purl: pkg:generic/elixir + cpes: + - cpe:2.3:a:elixir-lang:elixir:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/elixir/ebin/elixir.app' + packages: + - class: elixir-library + name: elixir + purl: pkg:generic/elixir + cpes: + - cpe:2.3:a:elixir-lang:elixir:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/java' + packages: + - class: java-binary + name: "" + purl: pkg:/ + cpes: [] + type: BinaryPkg + - class: java-binary-graalvm + name: graalvm + purl: pkg:generic/oracle/graalvm + cpes: + - cpe:2.3:a:oracle:graalvm:*:*:*:*:*:*:*:* + type: BinaryPkg + - class: java-binary-openjdk-zulu + name: zulu + purl: pkg:generic/azul/zulu + cpes: + - cpe:2.3:a:azul:zulu:*:*:*:*:*:*:*:* + type: BinaryPkg + - class: java-binary-openjdk-with-update + name: openjdk + purl: pkg:generic/oracle/openjdk + cpes: + - cpe:2.3:a:oracle:openjdk:{{.primary}}:update{{.update}}:*:*:*:*:*:* + type: BinaryPkg + - class: java-binary-openjdk + name: openjdk + purl: pkg:generic/oracle/openjdk + cpes: + - cpe:2.3:a:oracle:openjdk:*:*:*:*:*:*:*:* + type: BinaryPkg + - class: java-binary-ibm + name: java + purl: pkg:generic/ibm/java + cpes: + - cpe:2.3:a:ibm:java:*:*:*:*:*:*:*:* + type: BinaryPkg + - class: java-binary-openjdk-fallthrough + name: jre + purl: pkg:generic/oracle/jre + cpes: + - cpe:2.3:a:oracle:jre:*:*:*:*:*:*:*:* + type: BinaryPkg + - class: java-binary-oracle + name: jre + purl: pkg:generic/oracle/jre + cpes: + - cpe:2.3:a:oracle:jre:*:*:*:*:*:*:*:* + type: BinaryPkg + - method: glob + criteria: + - '**/jdb' + packages: + - class: java-jdb-binary + name: "" + purl: pkg:/ + cpes: [] + type: BinaryPkg + - class: java-binary-graalvm + name: graalvm + purl: pkg:generic/oracle/graalvm + cpes: + - cpe:2.3:a:oracle:graalvm_for_jdk:*:*:*:*:*:*:*:* + type: BinaryPkg + - class: jdb-binary-openjdk-zulu + name: zulu + purl: pkg:generic/azul/zulu + cpes: + - cpe:2.3:a:azul:zulu:*:*:*:*:*:*:*:* + type: BinaryPkg + - class: java-jdb-binary-openjdk + name: openjdk + purl: pkg:generic/oracle/openjdk + cpes: + - cpe:2.3:a:oracle:openjdk:*:*:*:*:*:*:*:* + type: BinaryPkg + - class: java-sdk-binary-ibm + name: java_sdk + purl: pkg:generic/ibm/java_sdk + cpes: + - cpe:2.3:a:ibm:java_sdk:*:*:*:*:*:*:*:* + type: BinaryPkg + - class: java-binary-openjdk-fallthrough + name: openjdk + purl: pkg:generic/oracle/openjdk + cpes: + - cpe:2.3:a:oracle:openjdk:*:*:*:*:*:*:*:* + type: BinaryPkg + - class: java-binary-jdk + name: jdk + purl: pkg:generic/oracle/jdk + cpes: + - cpe:2.3:a:oracle:jdk:*:*:*:*:*:*:*:* + type: BinaryPkg + metadata_types: # AUTO-GENERATED + - pkg.BinarySignature + package_types: # AUTO-GENERATED + - binary + json_schema_types: # AUTO-GENERATED + - BinarySignature + capabilities: # MANUAL - edit capabilities here + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: binary # MANUAL + name: elf-binary-package-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: "" + function: "" + selectors: # AUTO-GENERATED + - binary + - declared + - directory + - elf + - elf-package + - image + - installed + - package + detectors: # MANUAL - edit detectors here + - method: mimetype + criteria: + - application/x-executable + - application/x-mach-binary + - application/x-elf + - application/x-sharedlib + - application/vnd.microsoft.portable-executable + metadata_types: # AUTO-GENERATED + - pkg.ELFBinaryPackageNoteJSONPayload + package_types: # AUTO-GENERATED + - apk + - binary + - rpm + json_schema_types: # AUTO-GENERATED + - ElfBinaryPackageNoteJsonPayload + capabilities: # MANUAL - edit capabilities here + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: binary # MANUAL + name: pe-binary-package-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/binary/pe_package_cataloger.go + function: NewPEPackageCataloger + selectors: # AUTO-GENERATED + - binary + - declared + - directory + - dll + - exe + - image + - installed + - package + - pe + - pe-package + parsers: # AUTO-GENERATED structure + - function: parsePE + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/*.dll' + - '**/*.exe' + metadata_types: # AUTO-GENERATED + - pkg.PEBinary + package_types: # AUTO-GENERATED + - binary + json_schema_types: # AUTO-GENERATED + - PeBinary + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/bitnami/capabilities.yaml b/syft/pkg/cataloger/bitnami/capabilities.yaml new file mode 100644 index 000000000..69ff615fe --- /dev/null +++ b/syft/pkg/cataloger/bitnami/capabilities.yaml @@ -0,0 +1,46 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: bitnami # MANUAL + name: bitnami-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/bitnami/cataloger.go + function: NewCataloger + selectors: # AUTO-GENERATED + - bitnami + - image + - installed + - package + parsers: # AUTO-GENERATED structure + - function: parseSBOM + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - /opt/bitnami/**/.spdx-*.spdx + metadata_types: # AUTO-GENERATED + - pkg.BitnamiSBOMEntry + package_types: # AUTO-GENERATED + - bitnami + json_schema_types: # AUTO-GENERATED + - BitnamiSbomEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: true + evidence: + - BitnamiSBOMEntry.Files + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/capabilities.go b/syft/pkg/cataloger/capabilities.go new file mode 100644 index 000000000..af8f46493 --- /dev/null +++ b/syft/pkg/cataloger/capabilities.go @@ -0,0 +1,14 @@ +package cataloger + +import ( + "embed" + + "github.com/anchore/syft/internal/capabilities" +) + +//go:embed */capabilities.yaml +var catalogerFiles embed.FS + +func init() { + capabilities.RegisterCatalogerFiles(catalogerFiles) +} diff --git a/syft/pkg/cataloger/conda/capabilities.yaml b/syft/pkg/cataloger/conda/capabilities.yaml new file mode 100644 index 000000000..7c306c660 --- /dev/null +++ b/syft/pkg/cataloger/conda/capabilities.yaml @@ -0,0 +1,51 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: conda # MANUAL + name: conda-meta-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/conda/cataloger.go + function: NewCondaMetaCataloger + selectors: # AUTO-GENERATED + - conda + - directory + - installed + - package + parsers: # AUTO-GENERATED structure + - function: parseCondaMetaJSON + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/conda-meta/*.json' + metadata_types: # AUTO-GENERATED + - pkg.CondaMetaPackage + package_types: # AUTO-GENERATED + - conda + json_schema_types: # AUTO-GENERATED + - CondaMetadataEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: true + evidence: + - CondaMetaPackage.Files + - CondaMetaPackage.PathsData.Paths + - name: package_manager.files.digests + default: true + evidence: + - CondaMetaPackage.PathsData.Paths.SHA256 + - name: package_manager.package_integrity_hash + default: true + evidence: + - CondaMetaPackage.MD5 + - CondaMetaPackage.SHA256 diff --git a/syft/pkg/cataloger/cpp/capabilities.yaml b/syft/pkg/cataloger/cpp/capabilities.yaml new file mode 100644 index 000000000..e96431121 --- /dev/null +++ b/syft/pkg/cataloger/cpp/capabilities.yaml @@ -0,0 +1,122 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: c++ # MANUAL + name: conan-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/cpp/cataloger.go + function: NewConanCataloger + selectors: # AUTO-GENERATED + - conan + - cpp + - declared + - directory + - language + - package + parsers: # AUTO-GENERATED structure + - function: parseConanLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/conan.lock' + metadata_types: # AUTO-GENERATED + - pkg.ConanV1LockEntry + - pkg.ConanV2LockEntry + package_types: # AUTO-GENERATED + - conan + json_schema_types: # AUTO-GENERATED + - CConanLockEntry + - CConanLockV2Entry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - build + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - ConanV1LockEntry.Ref + - ConanV2LockEntry.RecipeRevision + - function: parseConanfile + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/conanfile.txt' + metadata_types: # AUTO-GENERATED + - pkg.ConanfileEntry + package_types: # AUTO-GENERATED + - conan + json_schema_types: # AUTO-GENERATED + - CConanFileEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: c++ # MANUAL + name: conan-info-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/cpp/cataloger.go + function: NewConanInfoCataloger + selectors: # AUTO-GENERATED + - conan + - cpp + - image + - installed + - language + - package + parsers: # AUTO-GENERATED structure + - function: parseConaninfo + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/conaninfo.txt' + metadata_types: # AUTO-GENERATED + - pkg.ConaninfoEntry + package_types: # AUTO-GENERATED + - conan + json_schema_types: # AUTO-GENERATED + - CConanInfoEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: flat + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/dart/capabilities.yaml b/syft/pkg/cataloger/dart/capabilities.yaml new file mode 100644 index 000000000..cfcf8a0e8 --- /dev/null +++ b/syft/pkg/cataloger/dart/capabilities.yaml @@ -0,0 +1,88 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: dart # MANUAL + name: dart-pubspec-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/dart/cataloger.go + function: NewPubspecCataloger + selectors: # AUTO-GENERATED + - dart + - declared + - directory + - language + - package + parsers: # AUTO-GENERATED structure + - function: parsePubspec + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/pubspec.yml' + - '**/pubspec.yaml' + metadata_types: # AUTO-GENERATED + - pkg.DartPubspec + package_types: # AUTO-GENERATED + - dart-pub + json_schema_types: # AUTO-GENERATED + - DartPubspec + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: dart # MANUAL + name: dart-pubspec-lock-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/dart/cataloger.go + function: NewPubspecLockCataloger + selectors: # AUTO-GENERATED + - dart + - declared + - directory + - language + - package + parsers: # AUTO-GENERATED structure + - function: parsePubspecLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/pubspec.lock' + metadata_types: # AUTO-GENERATED + - pkg.DartPubspecLockEntry + package_types: # AUTO-GENERATED + - dart-pub + json_schema_types: # AUTO-GENERATED + - DartPubspecLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/debian/capabilities.yaml b/syft/pkg/cataloger/debian/capabilities.yaml new file mode 100644 index 000000000..248812217 --- /dev/null +++ b/syft/pkg/cataloger/debian/capabilities.yaml @@ -0,0 +1,100 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: dpkg # MANUAL + name: dpkg-db-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/debian/cataloger.go + function: NewDBCataloger + selectors: # AUTO-GENERATED + - debian + - directory + - dpkg + - image + - installed + - linux + - os + - package + parsers: # AUTO-GENERATED structure + - function: parseDpkgDB + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/lib/dpkg/status' + - '**/lib/dpkg/status.d/*' + - '**/lib/opkg/info/*.control' + - '**/lib/opkg/status' + metadata_types: # AUTO-GENERATED + - pkg.DpkgDBEntry + package_types: # AUTO-GENERATED + - deb + json_schema_types: # AUTO-GENERATED + - DpkgDbEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: true + evidence: + - DpkgDBEntry.Files + - name: package_manager.files.digests + default: true + evidence: + - DpkgDBEntry.Files[].Digest + - name: package_manager.package_integrity_hash + default: false + - ecosystem: dpkg # MANUAL + name: deb-archive-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/debian/cataloger.go + function: NewArchiveCataloger + selectors: # AUTO-GENERATED + - deb + - debian + - declared + - directory + - linux + - os + - package + parsers: # AUTO-GENERATED structure + - function: parseDebArchive + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/*.deb' + metadata_types: # AUTO-GENERATED + - pkg.DpkgArchiveEntry + package_types: # AUTO-GENERATED + - deb + json_schema_types: # AUTO-GENERATED + - DpkgArchiveEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: true + evidence: + - DpkgArchiveEntry.Files + - name: package_manager.files.digests + default: true + evidence: + - DpkgArchiveEntry.Files[].Digest + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/dotnet/capabilities.yaml b/syft/pkg/cataloger/dotnet/capabilities.yaml new file mode 100644 index 000000000..11e537c7c --- /dev/null +++ b/syft/pkg/cataloger/dotnet/capabilities.yaml @@ -0,0 +1,189 @@ +# Cataloger capabilities. See ../README.md for documentation. + +configs: # AUTO-GENERATED - config structs and their fields + dotnet.CatalogerConfig: + fields: + - key: DepPackagesMustHaveDLL + description: DepPackagesMustHaveDLL allows for deps.json packages to be included only if there is a DLL on disk for that package. + app_key: dotnet.dep-packages-must-have-dll + - key: DepPackagesMustClaimDLL + description: DepPackagesMustClaimDLL allows for deps.json packages to be included only if there is a runtime/resource DLL claimed in the deps.json targets section. This does not require such claimed DLLs to exist on disk. The behavior of this + app_key: dotnet.dep-packages-must-claim-dll + - key: PropagateDLLClaimsToParents + description: PropagateDLLClaimsToParents allows for deps.json packages to be included if any child (transitive) package claims a DLL. This applies to both the claims configuration and evidence-on-disk configurations. + app_key: dotnet.propagate-dll-claims-to-parents + - key: RelaxDLLClaimsWhenBundlingDetected + description: RelaxDLLClaimsWhenBundlingDetected will look for indications of IL bundle tooling via deps.json package names and, if found (and this config option is enabled), will relax the DepPackagesMustClaimDLL value to `false` only in those cases. + app_key: dotnet.relax-dll-claims-when-bundling-detected + - key: ExcludeProjectReferences + description: ExcludeProjectReferences excludes packages with type "project" from deps.json output. These are internal project references, not external NuGet packages. + app_key: dotnet.exclude-project-references +catalogers: + - ecosystem: dotnet # MANUAL + name: dotnet-deps-binary-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: "" + function: "" + selectors: # AUTO-GENERATED + - c# + - directory + - dotnet + - image + - installed + - language + - package + detectors: # MANUAL - edit detectors here + - method: glob + criteria: + - '**/*.deps.json' + - '**/*.dll' + - '**/*.exe' + metadata_types: # AUTO-GENERATED + - pkg.DotnetDepsEntry + - pkg.DotnetPortableExecutableEntry + package_types: # AUTO-GENERATED + - dotnet + - npm + json_schema_types: # AUTO-GENERATED + - DotnetDepsEntry + - DotnetPortableExecutableEntry + capabilities: # MANUAL - edit capabilities here + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: dotnet # MANUAL + name: dotnet-deps-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: "" + function: "" + selectors: # AUTO-GENERATED + - deprecated + - package + detectors: # MANUAL - edit detectors here + - method: glob + criteria: + - '**/*.deps.json' + metadata_types: # AUTO-GENERATED + - pkg.DotnetDepsEntry + package_types: # AUTO-GENERATED + - dotnet + json_schema_types: # AUTO-GENERATED + - DotnetDepsEntry + capabilities: # MANUAL - edit capabilities here + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: dotnet # MANUAL + name: dotnet-packages-lock-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/dotnet/cataloger.go + function: NewDotnetPackagesLockCataloger + selectors: # AUTO-GENERATED + - c# + - declared + - directory + - dotnet + - image + - language + - package + parsers: # AUTO-GENERATED structure + - function: parseDotnetPackagesLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/packages.lock.json' + metadata_types: # AUTO-GENERATED + - pkg.DotnetPackagesLockEntry + package_types: # AUTO-GENERATED + - dotnet + json_schema_types: # AUTO-GENERATED + - DotnetPackagesLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - dev + - build + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - DotnetPackagesLockEntry.ContentHash + - ecosystem: dotnet # MANUAL + name: dotnet-portable-executable-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: "" + function: "" + config: dotnet.CatalogerConfig # AUTO-GENERATED + selectors: # AUTO-GENERATED + - deprecated + - package + detectors: # MANUAL - edit detectors here + - method: glob + criteria: + - '**/*.dll' + - '**/*.exe' + metadata_types: # AUTO-GENERATED + - pkg.DotnetPortableExecutableEntry + package_types: # AUTO-GENERATED + - dotnet + json_schema_types: # AUTO-GENERATED + - DotnetPortableExecutableEntry + capabilities: # MANUAL - edit capabilities here + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/elixir/capabilities.yaml b/syft/pkg/cataloger/elixir/capabilities.yaml new file mode 100644 index 000000000..4ce07a862 --- /dev/null +++ b/syft/pkg/cataloger/elixir/capabilities.yaml @@ -0,0 +1,49 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: elixir # MANUAL + name: elixir-mix-lock-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/elixir/cataloger.go + function: NewMixLockCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - elixir + - language + - package + parsers: # AUTO-GENERATED structure + - function: parseMixLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/mix.lock' + metadata_types: # AUTO-GENERATED + - pkg.ElixirMixLockEntry + package_types: # AUTO-GENERATED + - hex + json_schema_types: # AUTO-GENERATED + - ElixirMixLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - ElixirMixLockEntry.PkgHash + - ElixirMixLockEntry.PkgHashExt diff --git a/syft/pkg/cataloger/erlang/capabilities.yaml b/syft/pkg/cataloger/erlang/capabilities.yaml new file mode 100644 index 000000000..b7d490909 --- /dev/null +++ b/syft/pkg/cataloger/erlang/capabilities.yaml @@ -0,0 +1,88 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: erlang # MANUAL + name: erlang-otp-application-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/erlang/cataloger.go + function: NewOTPCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - erlang + - language + - otp + - package + parsers: # AUTO-GENERATED structure + - function: parseOTPApp + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/*.app' + package_types: # AUTO-GENERATED + - erlang-otp + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: erlang # MANUAL + name: erlang-rebar-lock-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/erlang/cataloger.go + function: NewRebarLockCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - erlang + - language + - package + parsers: # AUTO-GENERATED structure + - function: parseRebarLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/rebar.lock' + metadata_types: # AUTO-GENERATED + - pkg.ErlangRebarLockEntry + package_types: # AUTO-GENERATED + - hex + json_schema_types: # AUTO-GENERATED + - ErlangRebarLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - ErlangRebarLockEntry.PkgHash + - ErlangRebarLockEntry.PkgHashExt diff --git a/syft/pkg/cataloger/gentoo/capabilities.yaml b/syft/pkg/cataloger/gentoo/capabilities.yaml new file mode 100644 index 000000000..5b8745c36 --- /dev/null +++ b/syft/pkg/cataloger/gentoo/capabilities.yaml @@ -0,0 +1,49 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: portage # MANUAL + name: portage-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/gentoo/cataloger.go + function: NewPortageCataloger + selectors: # AUTO-GENERATED + - directory + - gentoo + - image + - installed + - linux + - os + - package + - portage + parsers: # AUTO-GENERATED structure + - function: parsePortageContents + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/var/db/pkg/*/*/CONTENTS' + metadata_types: # AUTO-GENERATED + - pkg.PortageEntry + package_types: # AUTO-GENERATED + - portage + json_schema_types: # AUTO-GENERATED + - PortageDbEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: true + evidence: + - PortageEntry.Files + - name: package_manager.files.digests + default: true + evidence: + - PortageEntry.Files[].Digest + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/githubactions/capabilities.yaml b/syft/pkg/cataloger/githubactions/capabilities.yaml new file mode 100644 index 000000000..4ff9d032d --- /dev/null +++ b/syft/pkg/cataloger/githubactions/capabilities.yaml @@ -0,0 +1,110 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: github-actions # MANUAL + name: github-action-workflow-usage-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/githubactions/cataloger.go + function: NewWorkflowUsageCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - github + - github-actions + - package + parsers: # AUTO-GENERATED structure + - function: parseWorkflowForWorkflowUsage + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/.github/workflows/*.yaml' + - '**/.github/workflows/*.yml' + metadata_types: # AUTO-GENERATED + - pkg.GitHubActionsUseStatement + package_types: # AUTO-GENERATED + - github-action-workflow + json_schema_types: # AUTO-GENERATED + - GithubActionsUseStatement + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: github-actions # MANUAL + name: github-actions-usage-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/githubactions/cataloger.go + function: NewActionUsageCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - github + - github-actions + - package + parsers: # AUTO-GENERATED structure + - function: parseCompositeActionForActionUsage + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/.github/actions/*/action.yml' + - '**/.github/actions/*/action.yaml' + metadata_types: # AUTO-GENERATED + - pkg.GitHubActionsUseStatement + package_types: # AUTO-GENERATED + - github-action + json_schema_types: # AUTO-GENERATED + - GithubActionsUseStatement + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - function: parseWorkflowForActionUsage + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/.github/workflows/*.yaml' + - '**/.github/workflows/*.yml' + metadata_types: # AUTO-GENERATED + - pkg.GitHubActionsUseStatement + package_types: # AUTO-GENERATED + - github-action + json_schema_types: # AUTO-GENERATED + - GithubActionsUseStatement + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/golang/capabilities.yaml b/syft/pkg/cataloger/golang/capabilities.yaml new file mode 100644 index 000000000..49e87f1a3 --- /dev/null +++ b/syft/pkg/cataloger/golang/capabilities.yaml @@ -0,0 +1,147 @@ +# Cataloger capabilities. See ../README.md for documentation. + +configs: # AUTO-GENERATED - config structs and their fields + golang.CatalogerConfig: + fields: + - key: SearchLocalModCacheLicenses + description: SearchLocalModCacheLicenses enables searching for go package licenses in the local GOPATH mod cache. + app_key: golang.search-local-mod-cache-licenses + - key: LocalModCacheDir + description: LocalModCacheDir specifies the location of the local go module cache directory. When not set, syft will attempt to discover the GOPATH env or default to $HOME/go. + app_key: golang.local-mod-cache-dir + - key: SearchLocalVendorLicenses + description: SearchLocalVendorLicenses enables searching for go package licenses in the local vendor directory relative to the go.mod file. + app_key: golang.search-local-vendor-licenses + - key: LocalVendorDir + description: LocalVendorDir specifies the location of the local vendor directory. When not set, syft will search for a vendor directory relative to the go.mod file. + app_key: golang.local-vendor-dir + - key: SearchRemoteLicenses + description: SearchRemoteLicenses enables downloading go package licenses from the upstream go proxy (typically proxy.golang.org). + app_key: golang.search-remote-licenses + - key: Proxies + description: Proxies is a list of go module proxies to use when fetching go module metadata and licenses. When not set, syft will use the GOPROXY env or default to https://proxy.golang.org,direct. + app_key: golang.proxy + - key: NoProxy + description: NoProxy is a list of glob patterns that match go module names that should not be fetched from the go proxy. When not set, syft will use the GOPRIVATE and GONOPROXY env vars. + app_key: golang.no-proxy +catalogers: + - ecosystem: go # MANUAL + name: go-module-binary-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/golang/cataloger.go + function: NewGoModuleBinaryCataloger + config: golang.CatalogerConfig # AUTO-GENERATED + selectors: # AUTO-GENERATED + - binary + - directory + - go + - golang + - gomod + - image + - installed + - language + - package + parsers: # AUTO-GENERATED structure + - function: parseGoBinary + detector: # AUTO-GENERATED + method: mimetype # AUTO-GENERATED + criteria: # AUTO-GENERATED + - application/x-executable + - application/x-mach-binary + - application/x-elf + - application/x-sharedlib + - application/vnd.microsoft.portable-executable + - application/x-executable + metadata_types: # AUTO-GENERATED + - pkg.GolangBinaryBuildinfoEntry + package_types: # AUTO-GENERATED + - go-module + json_schema_types: # AUTO-GENERATED + - GoModuleBuildinfoEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + conditions: + - when: + SearchLocalModCacheLicenses: true + value: true + - when: + SearchRemoteLicenses: true + value: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: flat + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - GolangBinaryBuildinfoEntry.H1Digest + - ecosystem: go # MANUAL + name: go-module-file-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/golang/cataloger.go + function: NewGoModuleFileCataloger + config: golang.CatalogerConfig # AUTO-GENERATED + selectors: # AUTO-GENERATED + - declared + - directory + - go + - golang + - gomod + - language + - package + parsers: # AUTO-GENERATED structure + - function: parseGoModFile + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/go.mod' + metadata_types: # AUTO-GENERATED + - pkg.GolangModuleEntry + - pkg.GolangSourceEntry + package_types: # AUTO-GENERATED + - go-module + json_schema_types: # AUTO-GENERATED + - GoModuleEntry + - GoSourceEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + conditions: + - when: + SearchLocalModCacheLicenses: true + value: true + - when: + SearchRemoteLicenses: true + value: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: flat + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - GolangModuleEntry.H1Digest + - GolangSourceEntry.H1Digest diff --git a/syft/pkg/cataloger/haskell/capabilities.yaml b/syft/pkg/cataloger/haskell/capabilities.yaml new file mode 100644 index 000000000..0c1de7b1b --- /dev/null +++ b/syft/pkg/cataloger/haskell/capabilities.yaml @@ -0,0 +1,107 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: haskell # MANUAL + name: haskell-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/haskell/cataloger.go + function: NewHackageCataloger + selectors: # AUTO-GENERATED + - cabal + - declared + - directory + - hackage + - haskell + - language + - package + parsers: # AUTO-GENERATED structure + - function: parseCabalFreeze + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/cabal.project.freeze' + package_types: # AUTO-GENERATED + - hackage + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - function: parseStackLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/stack.yaml.lock' + metadata_types: # AUTO-GENERATED + - pkg.HackageStackYamlLockEntry + package_types: # AUTO-GENERATED + - hackage + json_schema_types: # AUTO-GENERATED + - HaskellHackageStackLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - HackageStackYamlLockEntry.PkgHash + - function: parseStackYaml + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/stack.yaml' + metadata_types: # AUTO-GENERATED + - pkg.HackageStackYamlEntry + package_types: # AUTO-GENERATED + - hackage + json_schema_types: # AUTO-GENERATED + - HaskellHackageStackEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - HackageStackYamlEntry.PkgHash diff --git a/syft/pkg/cataloger/homebrew/capabilities.yaml b/syft/pkg/cataloger/homebrew/capabilities.yaml new file mode 100644 index 000000000..fb7ac85f8 --- /dev/null +++ b/syft/pkg/cataloger/homebrew/capabilities.yaml @@ -0,0 +1,43 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: homebrew # MANUAL + name: homebrew-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/homebrew/cataloger.go + function: NewCataloger + selectors: # AUTO-GENERATED + - directory + - homebrew + - image + - installed + - package + parsers: # AUTO-GENERATED structure + - function: parseHomebrewFormula + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/Cellar/*/*/.brew/*.rb' + - '**/Library/Taps/*/*/Formula/*.rb' + metadata_types: # AUTO-GENERATED + - pkg.HomebrewFormula + package_types: # AUTO-GENERATED + - homebrew + json_schema_types: # AUTO-GENERATED + - HomebrewFormula + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/java/capabilities.yaml b/syft/pkg/cataloger/java/capabilities.yaml new file mode 100644 index 000000000..06c5526b9 --- /dev/null +++ b/syft/pkg/cataloger/java/capabilities.yaml @@ -0,0 +1,289 @@ +# Cataloger capabilities. See ../README.md for documentation. + +configs: # AUTO-GENERATED - config structs and their fields + java.ArchiveCatalogerConfig: + fields: + - key: IncludeIndexedArchives + description: IncludeIndexedArchives indicates whether to search within indexed archive files (e.g., .zip). + - key: IncludeUnindexedArchives + description: IncludeUnindexedArchives indicates whether to search within unindexed archive files (e.g., .tar*). + - key: UseNetwork + description: UseNetwork enables network operations for java package metadata enrichment, such as fetching parent POMs and license information. + app_key: java.use-network + - key: UseMavenLocalRepository + description: UseMavenLocalRepository enables searching the local maven repository (`~/.m2/repository` by default) for parent POMs and other metadata. + app_key: java.use-maven-local-repository + - key: MavenLocalRepositoryDir + description: MavenLocalRepositoryDir specifies the location of the local maven repository. When not set, defaults to `~/.m2/repository`. + app_key: java.maven-local-repository-dir + - key: MavenBaseURL + description: MavenBaseURL specifies the base URL(s) to use for fetching POMs and metadata from maven central or other repositories. When not set, defaults to `https://repo1.maven.org/maven2`. + app_key: java.maven-url + - key: MaxParentRecursiveDepth + description: MaxParentRecursiveDepth limits how many parent POMs will be fetched recursively before stopping. This prevents fetching excessively deep parent graphs. + app_key: java.max-parent-recursive-depth + - key: ResolveTransitiveDependencies + description: ResolveTransitiveDependencies enables resolving transitive dependencies for java packages found within archives. + app_key: java.resolve-transitive-dependencies +catalogers: + - ecosystem: java # MANUAL + name: java-archive-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/java/cataloger.go + function: NewArchiveCataloger + config: java.ArchiveCatalogerConfig # AUTO-GENERATED + selectors: # AUTO-GENERATED + - directory + - image + - installed + - java + - language + - maven + - package + detectors: # MANUAL - edit detectors here + - method: glob + criteria: + - '**/*.jar' + - '**/*.war' + - '**/*.ear' + - '**/*.par' + - '**/*.sar' + - '**/*.nar' + - '**/*.jpi' + - '**/*.hpi' + - '**/*.kar' + - '**/*.lpkg' + comment: JAR-based archives - always active + - method: glob + criteria: + - '**/*.zip' + conditions: + - when: + IncludeIndexedArchives: true + comment: ZIP archives require indexed archive support + - method: glob + criteria: + - '**/*.tar' + - '**/*.tar.gz' + - '**/*.tgz' + - '**/*.tar.bz' + - '**/*.tar.bz2' + - '**/*.tbz' + - '**/*.tbz2' + - '**/*.tar.br' + - '**/*.tbr' + - '**/*.tar.lz4' + - '**/*.tlz4' + - '**/*.tar.sz' + - '**/*.tsz' + - '**/*.tar.xz' + - '**/*.txz' + - '**/*.tar.zst' + - '**/*.tzst' + - '**/*.tar.zstd' + - '**/*.tzstd' + conditions: + - when: + IncludeUnindexedArchives: true + comment: TAR archives require unindexed archive support + metadata_types: # AUTO-GENERATED + - pkg.JavaArchive + package_types: # AUTO-GENERATED + - java-archive + json_schema_types: # AUTO-GENERATED + - JavaArchive + capabilities: # MANUAL - edit capabilities here + - name: license + default: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - JavaArchive.ArchiveDigests + - ecosystem: java # MANUAL + name: java-gradle-lockfile-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/java/cataloger.go + function: NewGradleLockfileCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - gradle + - java + - language + - package + parsers: # AUTO-GENERATED structure + - function: parseGradleLockfile + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/gradle.lockfile*' + metadata_types: # AUTO-GENERATED + - pkg.JavaArchive + package_types: # AUTO-GENERATED + - java-archive + json_schema_types: # AUTO-GENERATED + - JavaArchive + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: java # MANUAL + name: java-pom-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: "" + function: "" + selectors: # AUTO-GENERATED + - declared + - directory + - java + - language + - maven + - package + detectors: # MANUAL - edit detectors here + - method: glob + criteria: + - '*pom.xml' + metadata_types: # AUTO-GENERATED + - pkg.JavaArchive + package_types: # AUTO-GENERATED + - java-archive + json_schema_types: # AUTO-GENERATED + - JavaArchive + capabilities: # MANUAL - edit capabilities here + - name: license + default: true + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: java # MANUAL + name: java-jvm-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/java/cataloger.go + function: NewJvmDistributionCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - image + - installed + - java + - jdk + - jre + - jvm + - package + parsers: # AUTO-GENERATED structure + - function: parseJVMRelease + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/release' + metadata_types: # AUTO-GENERATED + - pkg.JavaVMInstallation + package_types: # AUTO-GENERATED + - binary + json_schema_types: # AUTO-GENERATED + - JavaJvmInstallation + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: null + - name: dependency.edges + default: "" + - name: dependency.kinds + default: null + - name: package_manager.files.listing + default: true + evidence: + - JavaVMInstallation.Files + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: java # MANUAL + name: graalvm-native-image-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: "" + function: "" + selectors: # AUTO-GENERATED + - directory + - image + - installed + - java + - language + - package + detectors: # MANUAL - edit detectors here + - method: mimetype + criteria: + - application/x-executable + - application/x-mach-binary + - application/x-elf + - application/x-sharedlib + - application/vnd.microsoft.portable-executable + package_types: # AUTO-GENERATED + - graalvm-native-image + capabilities: # MANUAL - edit capabilities here + - name: license + default: true + - name: dependency.depth + default: + - direct + - indirect + comment: the dependencies ultimately depends on the quality of the embedded SBOM + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/java/config.go b/syft/pkg/cataloger/java/config.go index cea7a009d..652ca827b 100644 --- a/syft/pkg/cataloger/java/config.go +++ b/syft/pkg/cataloger/java/config.go @@ -14,19 +14,19 @@ type ArchiveCatalogerConfig struct { // app-config: java.use-network UseNetwork bool `yaml:"use-network" json:"use-network" mapstructure:"use-network"` - // UseMavenLocalRepository enables searching the local maven repository (~/.m2/repository by default) for parent POMs and other metadata. + // UseMavenLocalRepository enables searching the local maven repository (`~/.m2/repository` by default) for parent POMs and other metadata. // app-config: java.use-maven-local-repository UseMavenLocalRepository bool `yaml:"use-maven-localrepository" json:"use-maven-localrepository" mapstructure:"use-maven-localrepository"` - // MavenLocalRepositoryDir specifies the location of the local maven repository. When not set, defaults to ~/.m2/repository. + // MavenLocalRepositoryDir specifies the location of the local maven repository. When not set, defaults to `~/.m2/repository`. // app-config: java.maven-local-repository-dir MavenLocalRepositoryDir string `yaml:"maven-localrepository-dir" json:"maven-localrepository-dir" mapstructure:"maven-localrepository-dir"` - // MavenBaseURL specifies the base URL(s) to use for fetching POMs and metadata from maven central or other repositories. When not set, defaults to https://repo1.maven.org/maven2. + // MavenBaseURL specifies the base URL(s) to use for fetching POMs and metadata from maven central or other repositories. When not set, defaults to `https://repo1.maven.org/maven2`. // app-config: java.maven-url MavenBaseURL string `yaml:"maven-base-url" json:"maven-base-url" mapstructure:"maven-base-url"` - // MaxParentRecursiveDepth limits how many parent POMs will be fetched recursively before stopping. This prevents infinite loops or excessively deep parent chains. + // MaxParentRecursiveDepth limits how many parent POMs will be fetched recursively before stopping. This prevents fetching excessively deep parent graphs. // app-config: java.max-parent-recursive-depth MaxParentRecursiveDepth int `yaml:"max-parent-recursive-depth" json:"max-parent-recursive-depth" mapstructure:"max-parent-recursive-depth"` diff --git a/syft/pkg/cataloger/javascript/capabilities.yaml b/syft/pkg/cataloger/javascript/capabilities.yaml new file mode 100644 index 000000000..0bfa48c01 --- /dev/null +++ b/syft/pkg/cataloger/javascript/capabilities.yaml @@ -0,0 +1,167 @@ +# Cataloger capabilities. See ../README.md for documentation. + +configs: # AUTO-GENERATED - config structs and their fields + javascript.CatalogerConfig: + fields: + - key: SearchRemoteLicenses + description: SearchRemoteLicenses enables querying the NPM registry API to retrieve license information for packages that are missing license data in their local metadata. + app_key: javascript.search-remote-licenses + - key: NPMBaseURL + description: NPMBaseURL specifies the base URL for the NPM registry API used when searching for remote license information. + app_key: javascript.npm-base-url + - key: IncludeDevDependencies + description: IncludeDevDependencies controls whether development dependencies should be included in the catalog results, in addition to production dependencies. + app_key: javascript.include-dev-dependencies +catalogers: + - ecosystem: javascript # MANUAL + name: javascript-lock-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/javascript/cataloger.go + function: NewLockCataloger + config: javascript.CatalogerConfig # AUTO-GENERATED + selectors: # AUTO-GENERATED + - declared + - directory + - javascript + - language + - node + - npm + - package + parsers: # AUTO-GENERATED structure + - function: parsePnpmLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/pnpm-lock.yaml' + metadata_types: # AUTO-GENERATED + - pkg.PnpmLockEntry + package_types: # AUTO-GENERATED + - npm + json_schema_types: # AUTO-GENERATED + - JavascriptPnpmLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + - function: parseYarnLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/yarn.lock' + metadata_types: # AUTO-GENERATED + - pkg.YarnLockEntry + package_types: # AUTO-GENERATED + - npm + json_schema_types: # AUTO-GENERATED + - JavascriptYarnLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - YarnLockEntry.Integrity + - function: parsePackageLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/package-lock.json' + metadata_types: # AUTO-GENERATED + - pkg.NpmPackageLockEntry + package_types: # AUTO-GENERATED + - npm + json_schema_types: # AUTO-GENERATED + - JavascriptNpmPackageLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - NpmPackageLockEntry.Integrity + - ecosystem: javascript # MANUAL + name: javascript-package-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/javascript/cataloger.go + function: NewPackageCataloger + selectors: # AUTO-GENERATED + - image + - installed + - javascript + - language + - node + - package + parsers: # AUTO-GENERATED structure + - function: parsePackageJSON + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/package.json' + metadata_types: # AUTO-GENERATED + - pkg.NpmPackage + package_types: # AUTO-GENERATED + - npm + json_schema_types: # AUTO-GENERATED + - JavascriptNpmPackage + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/kernel/capabilities.yaml b/syft/pkg/cataloger/kernel/capabilities.yaml new file mode 100644 index 000000000..e3c92abd5 --- /dev/null +++ b/syft/pkg/cataloger/kernel/capabilities.yaml @@ -0,0 +1,58 @@ +# Cataloger capabilities. See ../README.md for documentation. + +configs: # AUTO-GENERATED - config structs and their fields + kernel.LinuxKernelCatalogerConfig: + fields: + - key: CatalogModules + description: CatalogModules enables cataloging linux kernel modules (`*.ko` files) in addition to the kernel itself. + app_key: linux-kernel.catalog-modules +catalogers: + - ecosystem: linux # MANUAL + name: linux-kernel-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: "" + function: "" + config: kernel.LinuxKernelCatalogerConfig # AUTO-GENERATED + selectors: # AUTO-GENERATED + - declared + - directory + - image + - installed + - kernel + - linux + - package + detectors: # MANUAL - edit detectors here + - method: glob + criteria: + - '**/kernel' + - '**/kernel-*' + - '**/vmlinux' + - '**/vmlinux-*' + - '**/vmlinuz' + - '**/vmlinuz-*' + - '**/lib/modules/**/*.ko' + metadata_types: # AUTO-GENERATED + - pkg.LinuxKernel + - pkg.LinuxKernelModule + package_types: # AUTO-GENERATED + - linux-kernel + - linux-kernel-module + json_schema_types: # AUTO-GENERATED + - LinuxKernelArchive + - LinuxKernelModule + capabilities: # MANUAL - edit capabilities here + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/kernel/cataloger.go b/syft/pkg/cataloger/kernel/cataloger.go index 389ccf3d7..6e76518e9 100644 --- a/syft/pkg/cataloger/kernel/cataloger.go +++ b/syft/pkg/cataloger/kernel/cataloger.go @@ -17,7 +17,7 @@ import ( var _ pkg.Cataloger = (*linuxKernelCataloger)(nil) type LinuxKernelCatalogerConfig struct { - // CatalogModules enables cataloging linux kernel modules (*.ko files) in addition to the kernel itself. + // CatalogModules enables cataloging linux kernel modules (`*.ko` files) in addition to the kernel itself. // app-config: linux-kernel.catalog-modules CatalogModules bool `yaml:"catalog-modules" json:"catalog-modules" mapstructure:"catalog-modules"` } diff --git a/syft/pkg/cataloger/lua/capabilities.yaml b/syft/pkg/cataloger/lua/capabilities.yaml new file mode 100644 index 000000000..edbdcf894 --- /dev/null +++ b/syft/pkg/cataloger/lua/capabilities.yaml @@ -0,0 +1,43 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: lua # MANUAL + name: lua-rock-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/lua/cataloger.go + function: NewPackageCataloger + selectors: # AUTO-GENERATED + - directory + - image + - installed + - language + - lua + - package + parsers: # AUTO-GENERATED structure + - function: parseRockspec + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/*.rockspec' + metadata_types: # AUTO-GENERATED + - pkg.LuaRocksPackage + package_types: # AUTO-GENERATED + - lua-rocks + json_schema_types: # AUTO-GENERATED + - LuarocksPackage + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/nix/capabilities.yaml b/syft/pkg/cataloger/nix/capabilities.yaml new file mode 100644 index 000000000..6df5d58f4 --- /dev/null +++ b/syft/pkg/cataloger/nix/capabilities.yaml @@ -0,0 +1,101 @@ +# Cataloger capabilities. See ../README.md for documentation. + +configs: # AUTO-GENERATED - config structs and their fields + nix.Config: + fields: + - key: CaptureOwnedFiles + description: CaptureOwnedFiles determines whether to record the list of files owned by each Nix package discovered in the store. Recording owned files provides more detailed information but increases processing time and memory usage. + app_key: nix.capture-owned-files +catalogers: + - ecosystem: nix # MANUAL + name: nix-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: "" + function: "" + selectors: # AUTO-GENERATED + - directory + - image + - installed + - language + - nix + - package + detectors: # MANUAL - edit detectors here + - method: glob + criteria: + - '**/nix/var/nix/db/db.sqlite' + - '**/nix/store/*' + - '**/nix/store/*.drv' + metadata_types: # AUTO-GENERATED + - pkg.NixStoreEntry + package_types: # AUTO-GENERATED + - nix + json_schema_types: # AUTO-GENERATED + - NixStoreEntry + capabilities: # MANUAL - edit capabilities here + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: true + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - NixStoreEntry.OutputHash + - ecosystem: nix # MANUAL + name: nix-store-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: "" + function: "" + config: nix.Config # AUTO-GENERATED + selectors: # AUTO-GENERATED + - deprecated + - package + detectors: # MANUAL - edit detectors here + - method: glob + criteria: + - '**/nix/store/*' + - '**/nix/store/*.drv' + metadata_types: # AUTO-GENERATED + - pkg.NixStoreEntry + package_types: # AUTO-GENERATED + - nix + json_schema_types: # AUTO-GENERATED + - NixStoreEntry + capabilities: # MANUAL - edit capabilities here + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + conditions: + - when: + CaptureOwnedFiles: true + value: true + evidence: + - NixStoreEntry.Files + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - NixStoreEntry.OutputHash diff --git a/syft/pkg/cataloger/ocaml/capabilities.yaml b/syft/pkg/cataloger/ocaml/capabilities.yaml new file mode 100644 index 000000000..b49a98a14 --- /dev/null +++ b/syft/pkg/cataloger/ocaml/capabilities.yaml @@ -0,0 +1,43 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: ocaml # MANUAL + name: opam-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/ocaml/cataloger.go + function: NewOpamPackageManagerCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - language + - ocaml + - opam + - package + parsers: # AUTO-GENERATED structure + - function: parseOpamPackage + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/*opam' + metadata_types: # AUTO-GENERATED + - pkg.OpamPackage + package_types: # AUTO-GENERATED + - opam + json_schema_types: # AUTO-GENERATED + - OpamPackage + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/php/capabilities.yaml b/syft/pkg/cataloger/php/capabilities.yaml new file mode 100644 index 000000000..7fe336c05 --- /dev/null +++ b/syft/pkg/cataloger/php/capabilities.yaml @@ -0,0 +1,215 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: php # MANUAL + name: php-composer-installed-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/php/cataloger.go + function: NewComposerInstalledCataloger + selectors: # AUTO-GENERATED + - composer + - image + - installed + - language + - package + - php + parsers: # AUTO-GENERATED structure + - function: parseInstalledJSON + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/installed.json' + metadata_types: # AUTO-GENERATED + - pkg.PhpComposerInstalledEntry + package_types: # AUTO-GENERATED + - php-composer + json_schema_types: # AUTO-GENERATED + - PhpComposerInstalledEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: php # MANUAL + name: php-composer-lock-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/php/cataloger.go + function: NewComposerLockCataloger + selectors: # AUTO-GENERATED + - composer + - declared + - directory + - language + - package + - php + parsers: # AUTO-GENERATED structure + - function: parseComposerLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/composer.lock' + metadata_types: # AUTO-GENERATED + - pkg.PhpComposerLockEntry + package_types: # AUTO-GENERATED + - php-composer + json_schema_types: # AUTO-GENERATED + - PhpComposerLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + - ecosystem: php # MANUAL + name: php-interpreter-cataloger # AUTO-GENERATED + type: custom # AUTO-GENERATED + source: # AUTO-GENERATED + file: "" + function: "" + selectors: # AUTO-GENERATED + - binary + - declared + - directory + - image + - installed + - package + - php + detectors: # MANUAL - edit detectors here + - method: glob + criteria: + - '**/php*' + - '**/php*/**/*.so' + - '**/php-fpm*' + - '**/apache*/**/libphp*.so' + metadata_types: # AUTO-GENERATED + - pkg.BinarySignature + package_types: # AUTO-GENERATED + - binary + json_schema_types: # AUTO-GENERATED + - BinarySignature + capabilities: # MANUAL - edit capabilities here + - name: license + default: false + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: flat + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: php # MANUAL + name: php-pear-serialized-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/php/cataloger.go + function: NewPearCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - image + - language + - package + - pear + - php + parsers: # AUTO-GENERATED structure + - function: parsePear + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/php/.registry/**/*.reg' + metadata_types: # AUTO-GENERATED + - pkg.PhpPearEntry + package_types: # AUTO-GENERATED + - php-pear + json_schema_types: # AUTO-GENERATED + - PhpPearEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: true + - name: package_manager.files.digests + default: true + - name: package_manager.package_integrity_hash + default: false + - ecosystem: php # MANUAL + name: php-pecl-serialized-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/php/cataloger.go + function: NewPeclCataloger + selectors: # AUTO-GENERATED + - deprecated + - package + parsers: # AUTO-GENERATED structure + - function: parsePecl + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/php/.registry/.channel.*/*.reg' + metadata_types: # AUTO-GENERATED + - pkg.PhpPeclEntry + package_types: # AUTO-GENERATED + - php-pecl + json_schema_types: # AUTO-GENERATED + - PhpPeclEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/python/capabilities.yaml b/syft/pkg/cataloger/python/capabilities.yaml new file mode 100644 index 000000000..30211eb71 --- /dev/null +++ b/syft/pkg/cataloger/python/capabilities.yaml @@ -0,0 +1,255 @@ +# Cataloger capabilities. See ../README.md for documentation. + +configs: # AUTO-GENERATED - config structs and their fields + python.CatalogerConfig: + fields: + - key: GuessUnpinnedRequirements + description: GuessUnpinnedRequirements attempts to infer package versions from version constraints when no explicit version is specified in requirements files. + app_key: python.guess-unpinned-requirements + - key: SearchRemoteLicenses + description: SearchRemoteLicenses enables querying the PyPI registry API to retrieve license information for packages that are missing license data in their local metadata. + app_key: python.search-remote-licenses + - key: PypiBaseURL + description: PypiBaseURL specifies the base URL for the Pypi registry API used when searching for remote license information. + app_key: python.pypi-base-url +catalogers: + - ecosystem: python # MANUAL + name: python-installed-package-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/python/cataloger.go + function: NewInstalledPackageCataloger + selectors: # AUTO-GENERATED + - directory + - image + - installed + - language + - package + - python + parsers: # AUTO-GENERATED structure + - function: parseWheelOrEgg + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/*.egg-info' + - '**/*dist-info/METADATA' + - '**/*egg-info/PKG-INFO' + - '**/*DIST-INFO/METADATA' + - '**/*EGG-INFO/PKG-INFO' + metadata_types: # AUTO-GENERATED + - pkg.PythonPackage + package_types: # AUTO-GENERATED + - python + json_schema_types: # AUTO-GENERATED + - PythonPackage + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: true + evidence: + - PythonPackage.Files + - name: package_manager.files.digests + default: true + evidence: + - PythonPackage.Files[].Digest + - name: package_manager.package_integrity_hash + default: false + - ecosystem: python # MANUAL + name: python-package-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/python/cataloger.go + function: NewPackageCataloger + config: python.CatalogerConfig # AUTO-GENERATED + selectors: # AUTO-GENERATED + - declared + - directory + - language + - package + - python + parsers: # AUTO-GENERATED structure + - function: parsePdmLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/pdm.lock' + metadata_types: # AUTO-GENERATED + - pkg.PythonPdmLockEntry + package_types: # AUTO-GENERATED + - python + json_schema_types: # AUTO-GENERATED + - PythonPdmLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - dev + - optional + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - function: parseUvLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/uv.lock' + metadata_types: # AUTO-GENERATED + - pkg.PythonUvLockEntry + package_types: # AUTO-GENERATED + - python + json_schema_types: # AUTO-GENERATED + - PythonUvLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - dev + - optional + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - function: parseSetupFile + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/setup.py' + package_types: # AUTO-GENERATED + - python + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - any + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - function: parsePipfileLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/Pipfile.lock' + metadata_types: # AUTO-GENERATED + - pkg.PythonPipfileLockEntry + package_types: # AUTO-GENERATED + - python + json_schema_types: # AUTO-GENERATED + - PythonPipfileLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: true + evidence: + - PythonPipfileLockEntry.Hashes + - name: package_manager.package_integrity_hash + default: false + - function: parsePoetryLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/poetry.lock' + metadata_types: # AUTO-GENERATED + - pkg.PythonPoetryLockEntry + package_types: # AUTO-GENERATED + - python + json_schema_types: # AUTO-GENERATED + - PythonPoetryLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - dev + - optional + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - function: parseRequirementsTxt + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/*requirements*.txt' + metadata_types: # AUTO-GENERATED + - pkg.PythonRequirementsEntry + package_types: # AUTO-GENERATED + - python + json_schema_types: # AUTO-GENERATED + - PythonPipRequirementsEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - any + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/python/cataloger_test.go b/syft/pkg/cataloger/python/cataloger_test.go index d64dbb56a..ab7e15b06 100644 --- a/syft/pkg/cataloger/python/cataloger_test.go +++ b/syft/pkg/cataloger/python/cataloger_test.go @@ -14,7 +14,7 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) -func Test_PackageCataloger(t *testing.T) { +func Test_InstalledPackageCataloger(t *testing.T) { ctx := context.TODO() tests := []struct { name string @@ -402,10 +402,57 @@ func Test_PackageCataloger(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - (pkgtest.NewCatalogTester(). + pkgtest.NewCatalogTester(). FromDirectory(t, test.fixture). Expects(test.expectedPackages, nil). - TestCataloger(t, NewInstalledPackageCataloger())) + TestCataloger(t, NewInstalledPackageCataloger()) + }) + } +} + +func Test_PackageCataloger(t *testing.T) { + + tests := []struct { + name string + fixture string + expectedPackages []string + expectedRelationships []string + }{ + { + name: "pdm", + fixture: "test-fixtures/pdm-lock", + expectedPackages: []string{ + "certifi @ 2025.1.31 (pdm.lock)", + "chardet @ 3.0.4 (pdm.lock)", + "charset-normalizer @ 2.0.12 (pdm.lock)", + "colorama @ 0.3.9 (pdm.lock)", + "idna @ 2.7 (pdm.lock)", + "py @ 1.4.34 (pdm.lock)", + "pytest @ 3.2.5 (pdm.lock)", + "requests @ 2.27.1 (pdm.lock)", + "setuptools @ 39.2.0 (pdm.lock)", + "urllib3 @ 1.26.20 (pdm.lock)", + }, + expectedRelationships: []string{ + "certifi @ 2025.1.31 (pdm.lock) [dependency-of] requests @ 2.27.1 (pdm.lock)", + "chardet @ 3.0.4 (pdm.lock) [dependency-of] requests @ 2.27.1 (pdm.lock)", + "charset-normalizer @ 2.0.12 (pdm.lock) [dependency-of] requests @ 2.27.1 (pdm.lock)", + "colorama @ 0.3.9 (pdm.lock) [dependency-of] pytest @ 3.2.5 (pdm.lock)", + "idna @ 2.7 (pdm.lock) [dependency-of] requests @ 2.27.1 (pdm.lock)", + "py @ 1.4.34 (pdm.lock) [dependency-of] pytest @ 3.2.5 (pdm.lock)", + "setuptools @ 39.2.0 (pdm.lock) [dependency-of] pytest @ 3.2.5 (pdm.lock)", + "urllib3 @ 1.26.20 (pdm.lock) [dependency-of] requests @ 2.27.1 (pdm.lock)", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pkgtest.NewCatalogTester(). + FromDirectory(t, tt.fixture). + ExpectsPackageStrings(tt.expectedPackages). + ExpectsRelationshipStrings(tt.expectedRelationships). + TestCataloger(t, NewPackageCataloger(DefaultCatalogerConfig())) }) } } diff --git a/syft/pkg/cataloger/python/config.go b/syft/pkg/cataloger/python/config.go index 0e5aba0c9..8e46c3730 100644 --- a/syft/pkg/cataloger/python/config.go +++ b/syft/pkg/cataloger/python/config.go @@ -6,7 +6,7 @@ type CatalogerConfig struct { // GuessUnpinnedRequirements attempts to infer package versions from version constraints when no explicit version is specified in requirements files. // app-config: python.guess-unpinned-requirements GuessUnpinnedRequirements bool `yaml:"guess-unpinned-requirements" json:"guess-unpinned-requirements" mapstructure:"guess-unpinned-requirements"` - // SearchRemoteLicenses enables querying the NPM registry API to retrieve license information for packages that are missing license data in their local metadata. + // SearchRemoteLicenses enables querying the PyPI registry API to retrieve license information for packages that are missing license data in their local metadata. // app-config: python.search-remote-licenses SearchRemoteLicenses bool `json:"search-remote-licenses" yaml:"search-remote-licenses" mapstructure:"search-remote-licenses"` // PypiBaseURL specifies the base URL for the Pypi registry API used when searching for remote license information. diff --git a/syft/pkg/cataloger/r/capabilities.yaml b/syft/pkg/cataloger/r/capabilities.yaml new file mode 100644 index 000000000..b890f082c --- /dev/null +++ b/syft/pkg/cataloger/r/capabilities.yaml @@ -0,0 +1,43 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: r # MANUAL + name: r-package-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/r/cataloger.go + function: NewPackageCataloger + selectors: # AUTO-GENERATED + - directory + - image + - installed + - language + - package + - r + parsers: # AUTO-GENERATED structure + - function: parseDescriptionFile + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/DESCRIPTION' + metadata_types: # AUTO-GENERATED + - pkg.RDescription + package_types: # AUTO-GENERATED + - R-package + json_schema_types: # AUTO-GENERATED + - RDescription + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/redhat/capabilities.yaml b/syft/pkg/cataloger/redhat/capabilities.yaml new file mode 100644 index 000000000..2480296a0 --- /dev/null +++ b/syft/pkg/cataloger/redhat/capabilities.yaml @@ -0,0 +1,126 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: rpm # MANUAL + name: rpm-archive-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/redhat/cataloger.go + function: NewArchiveCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - linux + - os + - package + - redhat + - rpm + parsers: # AUTO-GENERATED structure + - function: parseRpmArchive + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/*.rpm' + metadata_types: # AUTO-GENERATED + - pkg.RpmArchive + package_types: # AUTO-GENERATED + - rpm + json_schema_types: # AUTO-GENERATED + - RpmArchive + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: true + evidence: + - RpmArchive.Files + - name: package_manager.files.digests + default: true + evidence: + - RpmArchive.Files[].Digest + - name: package_manager.package_integrity_hash + default: false + - ecosystem: rpm # MANUAL + name: rpm-db-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/redhat/cataloger.go + function: NewDBCataloger + selectors: # AUTO-GENERATED + - directory + - image + - installed + - linux + - os + - package + - redhat + - rpm + parsers: # AUTO-GENERATED structure + - function: parseRpmManifest + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/var/lib/rpmmanifest/container-manifest-2' + metadata_types: # AUTO-GENERATED + - pkg.RpmDBEntry + package_types: # AUTO-GENERATED + - rpm + json_schema_types: # AUTO-GENERATED + - RpmDbEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - function: parseRpmDB + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/{var/lib,usr/share,usr/lib/sysimage}/rpm/{Packages,Packages.db,rpmdb.sqlite}' + metadata_types: # AUTO-GENERATED + - pkg.RpmDBEntry + package_types: # AUTO-GENERATED + - rpm + json_schema_types: # AUTO-GENERATED + - RpmDbEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: true + evidence: + - RpmDBEntry.Files + - name: package_manager.files.digests + default: true + evidence: + - RpmDBEntry.Files[].Digest + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/ruby/capabilities.yaml b/syft/pkg/cataloger/ruby/capabilities.yaml new file mode 100644 index 000000000..991596552 --- /dev/null +++ b/syft/pkg/cataloger/ruby/capabilities.yaml @@ -0,0 +1,133 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: ruby # MANUAL + name: ruby-gemfile-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/ruby/cataloger.go + function: NewGemFileLockCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - gem + - language + - package + - ruby + parsers: # AUTO-GENERATED structure + - function: parseGemFileLockEntries + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/Gemfile.lock' + - '**/Gemfile.next.lock' + package_types: # AUTO-GENERATED + - gem + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - dev + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: ruby # MANUAL + name: ruby-gemspec-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/ruby/cataloger.go + function: NewGemSpecCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - gem + - gemspec + - language + - package + - ruby + parsers: # AUTO-GENERATED structure + - function: parseGemSpecEntries + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/*.gemspec' + metadata_types: # AUTO-GENERATED + - pkg.RubyGemspec + package_types: # AUTO-GENERATED + - gem + json_schema_types: # AUTO-GENERATED + - RubyGemspec + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: true + evidence: + - RubyGemspec.Files + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: ruby # MANUAL + name: ruby-installed-gemspec-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/ruby/cataloger.go + function: NewInstalledGemSpecCataloger + selectors: # AUTO-GENERATED + - gem + - gemspec + - image + - installed + - language + - package + - ruby + parsers: # AUTO-GENERATED structure + - function: parseGemSpecEntries + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/specifications/**/*.gemspec' + metadata_types: # AUTO-GENERATED + - pkg.RubyGemspec + package_types: # AUTO-GENERATED + - gem + json_schema_types: # AUTO-GENERATED + - RubyGemspec + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: true + evidence: + - RubyGemspec.Files + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/rust/capabilities.yaml b/syft/pkg/cataloger/rust/capabilities.yaml new file mode 100644 index 000000000..82d211be6 --- /dev/null +++ b/syft/pkg/cataloger/rust/capabilities.yaml @@ -0,0 +1,99 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: rust # MANUAL + name: cargo-auditable-binary-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/rust/cataloger.go + function: NewAuditBinaryCataloger + selectors: # AUTO-GENERATED + - binary + - directory + - image + - installed + - language + - package + - rust + parsers: # AUTO-GENERATED structure + - function: parseAuditBinary + detector: # AUTO-GENERATED + method: mimetype # AUTO-GENERATED + criteria: # AUTO-GENERATED + - application/x-executable + - application/x-mach-binary + - application/x-elf + - application/x-sharedlib + - application/vnd.microsoft.portable-executable + - application/x-executable + metadata_types: # AUTO-GENERATED + - pkg.RustBinaryAuditEntry + package_types: # AUTO-GENERATED + - rust-crate + json_schema_types: # AUTO-GENERATED + - RustCargoAuditEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - ecosystem: rust # MANUAL + name: rust-cargo-lock-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/rust/cataloger.go + function: NewCargoLockCataloger + selectors: # AUTO-GENERATED + - cargo + - declared + - directory + - language + - package + - rust + parsers: # AUTO-GENERATED structure + - function: parseCargoLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/Cargo.lock' + metadata_types: # AUTO-GENERATED + - pkg.RustCargoLockEntry + package_types: # AUTO-GENERATED + - rust-crate + json_schema_types: # AUTO-GENERATED + - RustCargoLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: complete + - name: dependency.kinds + default: + - runtime + - dev + - build + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - RustCargoLockEntry.Checksum diff --git a/syft/pkg/cataloger/rust/cataloger_test.go b/syft/pkg/cataloger/rust/cataloger_test.go index 62b7bc099..c933d2fc7 100644 --- a/syft/pkg/cataloger/rust/cataloger_test.go +++ b/syft/pkg/cataloger/rust/cataloger_test.go @@ -16,7 +16,6 @@ func TestNewAuditBinaryCataloger(t *testing.T) { Name: "argh", Version: "0.1.12", PURL: "pkg:cargo/argh@0.1.12", - FoundBy: "cargo-auditable-binary-cataloger", Locations: locations, Language: pkg.Rust, Type: pkg.RustPkg, @@ -31,7 +30,6 @@ func TestNewAuditBinaryCataloger(t *testing.T) { Name: "argh_derive", Version: "0.1.12", PURL: "pkg:cargo/argh_derive@0.1.12", - FoundBy: "cargo-auditable-binary-cataloger", Locations: locations, Language: pkg.Rust, Type: pkg.RustPkg, @@ -46,7 +44,6 @@ func TestNewAuditBinaryCataloger(t *testing.T) { Name: "argh_shared", Version: "0.1.12", PURL: "pkg:cargo/argh_shared@0.1.12", - FoundBy: "cargo-auditable-binary-cataloger", Locations: locations, Language: pkg.Rust, Type: pkg.RustPkg, @@ -61,7 +58,6 @@ func TestNewAuditBinaryCataloger(t *testing.T) { Name: "hello_world", Version: "0.1.0", PURL: "pkg:cargo/hello_world@0.1.0", - FoundBy: "cargo-auditable-binary-cataloger", Locations: locations, Language: pkg.Rust, Type: pkg.RustPkg, @@ -76,7 +72,6 @@ func TestNewAuditBinaryCataloger(t *testing.T) { Name: "proc-macro2", Version: "1.0.92", PURL: "pkg:cargo/proc-macro2@1.0.92", - FoundBy: "cargo-auditable-binary-cataloger", Locations: locations, Language: pkg.Rust, Type: pkg.RustPkg, @@ -91,7 +86,6 @@ func TestNewAuditBinaryCataloger(t *testing.T) { Name: "quote", Version: "1.0.37", PURL: "pkg:cargo/quote@1.0.37", - FoundBy: "cargo-auditable-binary-cataloger", Locations: locations, Language: pkg.Rust, Type: pkg.RustPkg, @@ -106,7 +100,6 @@ func TestNewAuditBinaryCataloger(t *testing.T) { Name: "serde", Version: "1.0.215", PURL: "pkg:cargo/serde@1.0.215", - FoundBy: "cargo-auditable-binary-cataloger", Locations: locations, Language: pkg.Rust, Type: pkg.RustPkg, @@ -121,7 +114,6 @@ func TestNewAuditBinaryCataloger(t *testing.T) { Name: "serde_derive", Version: "1.0.215", PURL: "pkg:cargo/serde_derive@1.0.215", - FoundBy: "cargo-auditable-binary-cataloger", Locations: locations, Language: pkg.Rust, Type: pkg.RustPkg, @@ -136,7 +128,6 @@ func TestNewAuditBinaryCataloger(t *testing.T) { Name: "syn", Version: "2.0.90", PURL: "pkg:cargo/syn@2.0.90", - FoundBy: "cargo-auditable-binary-cataloger", Locations: locations, Language: pkg.Rust, Type: pkg.RustPkg, @@ -151,7 +142,6 @@ func TestNewAuditBinaryCataloger(t *testing.T) { Name: "unicode-ident", Version: "1.0.14", PURL: "pkg:cargo/unicode-ident@1.0.14", - FoundBy: "cargo-auditable-binary-cataloger", Locations: locations, Language: pkg.Rust, Type: pkg.RustPkg, diff --git a/syft/pkg/cataloger/sbom/capabilities.yaml b/syft/pkg/cataloger/sbom/capabilities.yaml new file mode 100644 index 000000000..5fe58eb18 --- /dev/null +++ b/syft/pkg/cataloger/sbom/capabilities.yaml @@ -0,0 +1,49 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: sbom # MANUAL + name: sbom-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/sbom/cataloger.go + function: NewCataloger + selectors: # AUTO-GENERATED + - package + - sbom + parsers: # AUTO-GENERATED structure + - function: parseSBOM + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/*.syft.json' + - '**/*.bom.*' + - '**/*.bom' + - '**/bom' + - '**/*.sbom.*' + - '**/*.sbom' + - '**/sbom' + - '**/*.cdx.*' + - '**/*.cdx' + - '**/*.spdx.*' + - '**/*.spdx' + metadata_types: # AUTO-GENERATED + - pkg.ApkDBEntry + package_types: # AUTO-GENERATED + - apk + json_schema_types: # AUTO-GENERATED + - ApkDbEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/snap/capabilities.yaml b/syft/pkg/cataloger/snap/capabilities.yaml new file mode 100644 index 000000000..e0258b86e --- /dev/null +++ b/syft/pkg/cataloger/snap/capabilities.yaml @@ -0,0 +1,146 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: snap # MANUAL + name: snap-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/snap/cataloger.go + function: NewCataloger + selectors: # AUTO-GENERATED + - directory + - image + - installed + - package + - snap + parsers: # AUTO-GENERATED structure + - function: parseSnapdSnapcraft + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/snap/snapcraft.yaml' + metadata_types: # AUTO-GENERATED + - pkg.SnapEntry + package_types: # AUTO-GENERATED + - deb + json_schema_types: # AUTO-GENERATED + - SnapEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - function: parseSystemManifest + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/snap/manifest.yaml' + metadata_types: # AUTO-GENERATED + - pkg.SnapEntry + package_types: # AUTO-GENERATED + - deb + json_schema_types: # AUTO-GENERATED + - SnapEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - function: parseKernelChangelog + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/doc/linux-modules-*/changelog.Debian.gz' + metadata_types: # AUTO-GENERATED + - pkg.SnapEntry + package_types: # AUTO-GENERATED + - deb + json_schema_types: # AUTO-GENERATED + - SnapEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - function: parseBaseDpkgYaml + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/usr/share/snappy/dpkg.yaml' + metadata_types: # AUTO-GENERATED + - pkg.SnapEntry + package_types: # AUTO-GENERATED + - deb + json_schema_types: # AUTO-GENERATED + - SnapEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false + - function: parseSnapYaml + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/meta/snap.yaml' + metadata_types: # AUTO-GENERATED + - pkg.SnapEntry + package_types: # AUTO-GENERATED + - deb + json_schema_types: # AUTO-GENERATED + - SnapEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/swift/capabilities.yaml b/syft/pkg/cataloger/swift/capabilities.yaml new file mode 100644 index 000000000..b03352c59 --- /dev/null +++ b/syft/pkg/cataloger/swift/capabilities.yaml @@ -0,0 +1,92 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: swift # MANUAL + name: cocoapods-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/swift/cataloger.go + function: NewCocoapodsCataloger + selectors: # AUTO-GENERATED + - cocoapods + - declared + - directory + - language + - package + - swift + parsers: # AUTO-GENERATED structure + - function: parsePodfileLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/Podfile.lock' + metadata_types: # AUTO-GENERATED + - pkg.CocoaPodfileLockEntry + package_types: # AUTO-GENERATED + - pod + json_schema_types: # AUTO-GENERATED + - CocoaPodfileLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - CocoaPodfileLockEntry.Checksum + - ecosystem: swift # MANUAL + name: swift-package-manager-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/swift/cataloger.go + function: NewSwiftPackageManagerCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - language + - package + - spm + - swift + parsers: # AUTO-GENERATED structure + - function: parsePackageResolved + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/Package.resolved' + - '**/.package.resolved' + metadata_types: # AUTO-GENERATED + - pkg.SwiftPackageManagerResolvedEntry + package_types: # AUTO-GENERATED + - swift + json_schema_types: # AUTO-GENERATED + - SwiftPackageManagerLockEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - indirect + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/swipl/capabilities.yaml b/syft/pkg/cataloger/swipl/capabilities.yaml new file mode 100644 index 000000000..f88328567 --- /dev/null +++ b/syft/pkg/cataloger/swipl/capabilities.yaml @@ -0,0 +1,43 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: prolog # MANUAL + name: swipl-pack-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/swipl/cataloger.go + function: NewSwiplPackCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - language + - pack + - package + - swipl + parsers: # AUTO-GENERATED structure + - function: parsePackPackage + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/pack.pl' + metadata_types: # AUTO-GENERATED + - pkg.SwiplPackEntry + package_types: # AUTO-GENERATED + - swiplpack + json_schema_types: # AUTO-GENERATED + - SwiplpackPackage + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/syft/pkg/cataloger/terraform/capabilities.yaml b/syft/pkg/cataloger/terraform/capabilities.yaml new file mode 100644 index 000000000..427cb5f5d --- /dev/null +++ b/syft/pkg/cataloger/terraform/capabilities.yaml @@ -0,0 +1,45 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: terraform # MANUAL + name: terraform-lock-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/terraform/cataloger.go + function: NewLockCataloger + selectors: # AUTO-GENERATED + - declared + - directory + - package + - terraform + parsers: # AUTO-GENERATED structure + - function: parseTerraformLock + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/.terraform.lock.hcl' + metadata_types: # AUTO-GENERATED + - pkg.TerraformLockProviderEntry + package_types: # AUTO-GENERATED + - terraform + json_schema_types: # AUTO-GENERATED + - TerraformLockProviderEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: false + - name: dependency.depth + default: + - direct + - name: dependency.edges + default: "" + - name: dependency.kinds + default: + - runtime + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: true + evidence: + - TerraformLockProviderEntry.Hashes diff --git a/syft/pkg/cataloger/wordpress/capabilities.yaml b/syft/pkg/cataloger/wordpress/capabilities.yaml new file mode 100644 index 000000000..9da6257a3 --- /dev/null +++ b/syft/pkg/cataloger/wordpress/capabilities.yaml @@ -0,0 +1,41 @@ +# Cataloger capabilities. See ../README.md for documentation. + +catalogers: + - ecosystem: wordpress # MANUAL + name: wordpress-plugins-cataloger # AUTO-GENERATED + type: generic # AUTO-GENERATED + source: # AUTO-GENERATED + file: syft/pkg/cataloger/wordpress/cataloger.go + function: NewWordpressPluginCataloger + selectors: # AUTO-GENERATED + - directory + - image + - package + - wordpress + parsers: # AUTO-GENERATED structure + - function: parseWordpressPluginFiles + detector: # AUTO-GENERATED + method: glob # AUTO-GENERATED + criteria: # AUTO-GENERATED + - '**/wp-content/plugins/*/*.php' + metadata_types: # AUTO-GENERATED + - pkg.WordpressPluginEntry + package_types: # AUTO-GENERATED + - wordpress-plugin + json_schema_types: # AUTO-GENERATED + - WordpressPluginEntry + capabilities: # MANUAL - preserved across regeneration + - name: license + default: true + - name: dependency.depth + default: [] + - name: dependency.edges + default: "" + - name: dependency.kinds + default: [] + - name: package_manager.files.listing + default: false + - name: package_manager.files.digests + default: false + - name: package_manager.package_integrity_hash + default: false diff --git a/test/cli/cataloger_info_test.go b/test/cli/cataloger_info_test.go new file mode 100644 index 000000000..893e4ab41 --- /dev/null +++ b/test/cli/cataloger_info_test.go @@ -0,0 +1,99 @@ +package cli + +import ( + "testing" +) + +func TestCatalogerCapsExperimentalFeatureGate(t *testing.T) { + // the "caps" subcommand description from cataloger_caps.go + const capsSubcommandText = "Show detailed capabilities of catalogers" + + // table header from the cataloger info output + const capsTableHeader = "ECOSYSTEM" + + tests := []struct { + name string + args []string + env map[string]string + assertions []traitAssertion + }{ + // tests without SYFT_EXP_CAPABILITIES set + { + name: "cataloger help does not show caps subcommand without env var", + args: []string{"cataloger", "--help"}, + assertions: []traitAssertion{ + assertNotInOutput(capsSubcommandText), + assertSuccessfulReturnCode, + }, + }, + { + name: "cataloger caps command shows help without env var", + args: []string{"cataloger", "info"}, + assertions: []traitAssertion{ + // when the subcommand is not registered, the parent help is shown + assertNotInOutput(capsTableHeader), + assertInOutput("Available Commands"), + assertSuccessfulReturnCode, + }, + }, + // tests with SYFT_EXP_CAPABILITIES=false + { + name: "cataloger help does not show caps subcommand with env var false", + args: []string{"cataloger", "--help"}, + env: map[string]string{ + "SYFT_EXP_CAPABILITIES": "false", + }, + assertions: []traitAssertion{ + assertNotInOutput(capsSubcommandText), + assertSuccessfulReturnCode, + }, + }, + { + name: "cataloger caps command shows help with env var false", + args: []string{"cataloger", "info"}, + env: map[string]string{ + "SYFT_EXP_CAPABILITIES": "false", + }, + assertions: []traitAssertion{ + // when the subcommand is not registered, the parent help is shown + assertNotInOutput(capsTableHeader), + assertInOutput("Available Commands"), + assertSuccessfulReturnCode, + }, + }, + // tests with SYFT_EXP_CAPABILITIES=true + { + name: "cataloger help shows caps subcommand with env var true", + args: []string{"cataloger", "--help"}, + env: map[string]string{ + "SYFT_EXP_CAPABILITIES": "true", + }, + assertions: []traitAssertion{ + assertInOutput(capsSubcommandText), + assertSuccessfulReturnCode, + }, + }, + { + name: "cataloger caps command shows capabilities table with env var true", + args: []string{"cataloger", "info"}, + env: map[string]string{ + "SYFT_EXP_CAPABILITIES": "true", + }, + assertions: []traitAssertion{ + // the info command shows the capabilities table + assertInOutput(capsTableHeader), + assertSuccessfulReturnCode, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd, stdout, stderr := runSyftSafe(t, test.env, test.args...) + for _, traitFn := range test.assertions { + traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode()) + } + logOutputOnFailure(t, cmd, stdout, stderr) + }) + } +}