mirror of
https://github.com/anchore/syft.git
synced 2026-11-03 07:56:36 +01:00
Add experimental cataloger capabilities command (#4317)
* add info command from generated capabilities Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * correct gentoo and arch ecosystems Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename os pkg types Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * better binary cataloger description Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * expose metadata and pacakge types in json Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * expose json schema types Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add completeness tests for metadata types Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * latest generation Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * improve testing a docs Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix tests and linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * restore goreleaser config Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * tweak diagram Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix pdm Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * chore: java binary data Signed-off-by: Keith Zantow <kzantow@gmail.com> * new capability descriptions for gguf and python Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * correct poetry lock integrity hash claim Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix compile error Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix: remove purl version from overrides Signed-off-by: Keith Zantow <kzantow@gmail.com> * fix lua deps ref Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * keep gguf as ai ecosystem Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * split packages.yaml to multiple files by go package Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * ensure tests do not use go test cache Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * sort json output for info command Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * docs: fix ocaml, php, and portage capabilities yaml Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> * chore: update erlang capabilities Signed-off-by: Keith Zantow <kzantow@gmail.com> * chore: update java capabilities Signed-off-by: Keith Zantow <kzantow@gmail.com> * chore: update javascript capabilities Signed-off-by: Keith Zantow <kzantow@gmail.com> * chore: update linux kernel capabilities Signed-off-by: Keith Zantow <kzantow@gmail.com> * remove missing tests Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix package.yaml references Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * revert license list change Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * check for drift in capability descriptions Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * regenerate capabilities Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * test cleanup Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * use fixture cache in static analysis Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * claim fixtures pre-req for cap generation Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update documentation with correct regeneration procedure Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * chore: ruby-gemspec-cataloger finds no dependencies Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> * chore: fix python docs and config comment Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> * chore: commit re-generated java yaml Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> * add cataloger selection to caps command Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * re-generate cap yamls Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix tests for cataloger selection Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix cli test Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add missing tests Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename cmd to `cataloger info` Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * [wip] change capability description locations Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * [wip] continued Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * [wip] adjust for import cycles Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * correct docs Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> Signed-off-by: Keith Zantow <kzantow@gmail.com> Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> Co-authored-by: Keith Zantow <kzantow@gmail.com> Co-authored-by: Will Murphy <willmurphyscode@users.noreply.github.com>
This commit is contained in:
Alex Goodman
co-authored by
Keith Zantow
Will Murphy
Keith Zantow
Will Murphy
parent
ae1a247f3d
commit
b3c70da3ea
@@ -0,0 +1,402 @@
|
||||
// this file handles YAML file reading and writing with comment preservation, using gopkg.in/yaml.v3's node tree to maintain all existing comments during regeneration.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/anchore/syft/internal/capabilities"
|
||||
"github.com/anchore/syft/internal/capabilities/internal"
|
||||
)
|
||||
|
||||
const autoGeneratedComment = "AUTO-GENERATED"
|
||||
|
||||
// writeYAMLToFile writes a YAML node to a file with proper encoding
|
||||
func writeYAMLToFile(path string, rootNode *yaml.Node) error {
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
encoder := yaml.NewEncoder(f)
|
||||
encoder.SetIndent(2)
|
||||
if err := encoder.Encode(rootNode); err != nil {
|
||||
return fmt.Errorf("failed to write YAML: %w", err)
|
||||
}
|
||||
if err := encoder.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close encoder: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addFieldComments adds comments to top-level fields in the YAML node
|
||||
func addFieldComments(rootNode *yaml.Node) {
|
||||
// navigate to the mapping node (handle both DocumentNode and MappingNode)
|
||||
var mappingNode *yaml.Node
|
||||
if rootNode.Kind == yaml.DocumentNode && len(rootNode.Content) > 0 {
|
||||
mappingNode = rootNode.Content[0]
|
||||
} else {
|
||||
mappingNode = rootNode
|
||||
}
|
||||
|
||||
if mappingNode != nil && len(mappingNode.Content) > 0 {
|
||||
// iterate through top-level keys
|
||||
for i := 0; i < len(mappingNode.Content); i += 2 {
|
||||
keyNode := mappingNode.Content[i]
|
||||
valueNode := mappingNode.Content[i+1]
|
||||
|
||||
switch keyNode.Value {
|
||||
case "configs":
|
||||
// configs section is AUTO-GENERATED
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = "AUTO-GENERATED - config structs and their fields"
|
||||
}
|
||||
case "application":
|
||||
// application section is AUTO-GENERATED
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = "AUTO-GENERATED - application-level config keys"
|
||||
}
|
||||
case "catalogers":
|
||||
// clear any HeadComment on catalogers key and value nodes to prevent duplicate headers
|
||||
// (the only header should be on the root DocumentNode)
|
||||
keyNode.HeadComment = ""
|
||||
valueNode.HeadComment = ""
|
||||
addCatalogerComments(valueNode)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// saveCapabilities saves the capabilities document to distributed YAML files with comments.
|
||||
// Groups catalogers by ecosystem and writes each to catalogerDir/{ecosystem}/capabilities.yaml.
|
||||
// Also saves appconfig.yaml separately to internal/capabilities/.
|
||||
func saveCapabilities(catalogerDir, repoRoot string, doc *capabilities.Document, existingNodes map[string]*yaml.Node) error {
|
||||
catalogersByEcosystem, configsByEcosystem := groupCatalogersByEcosystem(doc)
|
||||
|
||||
// write each ecosystem file
|
||||
for ecosystem, catalogers := range catalogersByEcosystem {
|
||||
if err := writeEcosystemFile(catalogerDir, ecosystem, catalogers, configsByEcosystem[ecosystem], existingNodes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// save appconfig.yaml to internal/capabilities/
|
||||
if len(doc.ApplicationConfig) > 0 {
|
||||
if err := writeAppconfigFile(repoRoot, doc.ApplicationConfig, existingNodes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// groupCatalogersByEcosystem groups catalogers and their configs by ecosystem
|
||||
func groupCatalogersByEcosystem(doc *capabilities.Document) (map[string][]capabilities.CatalogerEntry, map[string]map[string]capabilities.CatalogerConfigEntry) {
|
||||
catalogersByEcosystem := make(map[string][]capabilities.CatalogerEntry)
|
||||
configsByEcosystem := make(map[string]map[string]capabilities.CatalogerConfigEntry)
|
||||
|
||||
for _, cat := range doc.Catalogers {
|
||||
ecosystem := mapCatalogerToEcosystem(cat)
|
||||
catalogersByEcosystem[ecosystem] = append(catalogersByEcosystem[ecosystem], cat)
|
||||
|
||||
// also group configs for this ecosystem
|
||||
if cat.Config != "" {
|
||||
if configEntry, exists := doc.Configs[cat.Config]; exists {
|
||||
if configsByEcosystem[ecosystem] == nil {
|
||||
configsByEcosystem[ecosystem] = make(map[string]capabilities.CatalogerConfigEntry)
|
||||
}
|
||||
configsByEcosystem[ecosystem][cat.Config] = configEntry
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return catalogersByEcosystem, configsByEcosystem
|
||||
}
|
||||
|
||||
// writeEcosystemFile writes a single ecosystem's catalogers and configs to a YAML file
|
||||
func writeEcosystemFile(catalogerDir, ecosystem string, catalogers []capabilities.CatalogerEntry, configs map[string]capabilities.CatalogerConfigEntry, existingNodes map[string]*yaml.Node) error {
|
||||
ecosystemDoc := struct {
|
||||
Configs map[string]capabilities.CatalogerConfigEntry `yaml:"configs,omitempty"`
|
||||
Catalogers []capabilities.CatalogerEntry `yaml:"catalogers"`
|
||||
}{
|
||||
Configs: configs,
|
||||
Catalogers: catalogers,
|
||||
}
|
||||
|
||||
var rootNode yaml.Node
|
||||
existingNode, hasExisting := existingNodes[ecosystem]
|
||||
|
||||
if hasExisting && existingNode != nil {
|
||||
// update existing node tree
|
||||
rootNode = *existingNode
|
||||
rootNode.HeadComment = "# Cataloger capabilities. See ../README.md for documentation.\n"
|
||||
if err := updateNodeTreeEcosystem(&rootNode, &ecosystemDoc); err != nil {
|
||||
return fmt.Errorf("failed to update node tree for %s: %w", ecosystem, err)
|
||||
}
|
||||
} else {
|
||||
// create new node tree
|
||||
if err := rootNode.Encode(&ecosystemDoc); err != nil {
|
||||
return fmt.Errorf("failed to encode %s: %w", ecosystem, err)
|
||||
}
|
||||
rootNode.HeadComment = "# Cataloger capabilities. See ../README.md for documentation.\n"
|
||||
}
|
||||
|
||||
// add field comments
|
||||
addFieldComments(&rootNode)
|
||||
|
||||
// write file to catalogerDir/{ecosystem}/capabilities.yaml
|
||||
ecosystemPath := internal.EcosystemCapabilitiesPath(catalogerDir, ecosystem)
|
||||
if err := writeYAMLToFile(ecosystemPath, &rootNode); err != nil {
|
||||
return fmt.Errorf("failed to write %s: %w", ecosystem, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeAppconfigFile writes the application config to appconfig.yaml
|
||||
func writeAppconfigFile(repoRoot string, appConfig []capabilities.ApplicationConfigField, existingNodes map[string]*yaml.Node) error {
|
||||
appconfigDoc := struct {
|
||||
Application []capabilities.ApplicationConfigField `yaml:"application"`
|
||||
}{
|
||||
Application: appConfig,
|
||||
}
|
||||
|
||||
var appNode yaml.Node
|
||||
existingAppNode, hasExisting := existingNodes["appconfig"]
|
||||
|
||||
if hasExisting && existingAppNode != nil {
|
||||
appNode = *existingAppNode
|
||||
if err := updateNodeTreeAppConfig(&appNode, &appconfigDoc); err != nil {
|
||||
return fmt.Errorf("failed to update appconfig node tree: %w", err)
|
||||
}
|
||||
} else {
|
||||
if err := appNode.Encode(&appconfigDoc); err != nil {
|
||||
return fmt.Errorf("failed to encode appconfig: %w", err)
|
||||
}
|
||||
appNode.HeadComment = "# Application-level configuration. See README.md for documentation.\n# This file is partially auto-generated. Run 'go generate ./internal/capabilities' to regenerate.\n"
|
||||
}
|
||||
|
||||
addFieldComments(&appNode)
|
||||
|
||||
appconfigPath := internal.AppconfigPath(repoRoot)
|
||||
if err := writeYAMLToFile(appconfigPath, &appNode); err != nil {
|
||||
return fmt.Errorf("failed to write appconfig: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// mapCatalogerToEcosystem determines which ecosystem file a cataloger belongs to
|
||||
func mapCatalogerToEcosystem(cat capabilities.CatalogerEntry) string {
|
||||
// first try using the source file path
|
||||
if cat.Source.File != "" {
|
||||
// extract directory from path like "syft/pkg/cataloger/golang/cataloger.go"
|
||||
re := regexp.MustCompile(`syft/pkg/cataloger/([^/]+)/`)
|
||||
if matches := re.FindStringSubmatch(cat.Source.File); len(matches) >= 2 {
|
||||
return matches[1]
|
||||
}
|
||||
}
|
||||
|
||||
// fallback to inferring from cataloger name (from merge.go)
|
||||
return inferEcosystem(cat.Name)
|
||||
}
|
||||
|
||||
// updateNodeTreeEcosystem updates an existing ecosystem YAML node tree
|
||||
func updateNodeTreeEcosystem(rootNode *yaml.Node, doc interface{}) error {
|
||||
var newNode yaml.Node
|
||||
if err := newNode.Encode(doc); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var existingMapping *yaml.Node
|
||||
if rootNode.Kind == yaml.DocumentNode && len(rootNode.Content) > 0 {
|
||||
existingMapping = rootNode.Content[0]
|
||||
} else {
|
||||
existingMapping = rootNode
|
||||
}
|
||||
|
||||
var newMapping *yaml.Node
|
||||
if newNode.Kind == yaml.DocumentNode && len(newNode.Content) > 0 {
|
||||
newMapping = newNode.Content[0]
|
||||
} else {
|
||||
newMapping = &newNode
|
||||
}
|
||||
|
||||
if existingMapping.Kind == yaml.MappingNode && newMapping.Kind == yaml.MappingNode {
|
||||
existingMapping.Content = newMapping.Content
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateNodeTreeAppConfig updates appconfig YAML node tree
|
||||
func updateNodeTreeAppConfig(rootNode *yaml.Node, doc interface{}) error {
|
||||
return updateNodeTreeEcosystem(rootNode, doc)
|
||||
}
|
||||
|
||||
// addCatalogerFieldComment adds appropriate comment to a single cataloger field
|
||||
func addCatalogerFieldComment(keyNode, valueNode *yaml.Node, catalogerName string) {
|
||||
switch keyNode.Value {
|
||||
case "ecosystem":
|
||||
// ecosystem is MANUAL
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = "MANUAL"
|
||||
}
|
||||
case "name", "type":
|
||||
// add AUTO-GENERATED comment to these fields
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = autoGeneratedComment
|
||||
}
|
||||
case "source":
|
||||
// add AUTO-GENERATED comment to source field
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = autoGeneratedComment
|
||||
}
|
||||
case "config":
|
||||
// add AUTO-GENERATED comment to config field
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = autoGeneratedComment
|
||||
}
|
||||
case "selectors":
|
||||
// selectors are AUTO-GENERATED
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = autoGeneratedComment
|
||||
}
|
||||
case "parsers":
|
||||
// parsers is AUTO-GENERATED structure
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = "AUTO-GENERATED structure"
|
||||
}
|
||||
addParserComments(valueNode)
|
||||
case "detectors":
|
||||
// detectors are AUTO-GENERATED for binary-classifier-cataloger, MANUAL for others
|
||||
if catalogerName == "binary-classifier-cataloger" {
|
||||
keyNode.LineComment = autoGeneratedComment
|
||||
} else if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = "MANUAL - edit detectors here"
|
||||
}
|
||||
case "metadata_types":
|
||||
// cataloger-level metadata_types (for custom catalogers) are AUTO-GENERATED
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = autoGeneratedComment
|
||||
}
|
||||
case "package_types":
|
||||
// cataloger-level package_types (for custom catalogers) are AUTO-GENERATED
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = autoGeneratedComment
|
||||
}
|
||||
case "json_schema_types":
|
||||
// json_schema_types are AUTO-GENERATED
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = autoGeneratedComment
|
||||
}
|
||||
case "capabilities":
|
||||
// capabilities are MANUAL
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = "MANUAL - edit capabilities here"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// findFieldValue finds a field in a YAML mapping node and returns its value
|
||||
func findFieldValue(node *yaml.Node, fieldName string) string {
|
||||
if node.Kind != yaml.MappingNode {
|
||||
return ""
|
||||
}
|
||||
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
keyNode := node.Content[i]
|
||||
valueNode := node.Content[i+1]
|
||||
if keyNode.Value == fieldName {
|
||||
return valueNode.Value
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func addCatalogerComments(catalogersNode *yaml.Node) {
|
||||
// catalogersNode should be a sequence of cataloger entries
|
||||
if catalogersNode.Kind != yaml.SequenceNode {
|
||||
return
|
||||
}
|
||||
|
||||
for _, catalogerNode := range catalogersNode.Content {
|
||||
if catalogerNode.Kind != yaml.MappingNode {
|
||||
continue
|
||||
}
|
||||
|
||||
// get the cataloger name for special handling
|
||||
catalogerName := findFieldValue(catalogerNode, "name")
|
||||
|
||||
// iterate through the fields of this cataloger entry
|
||||
for i := 0; i < len(catalogerNode.Content); i += 2 {
|
||||
keyNode := catalogerNode.Content[i]
|
||||
valueNode := catalogerNode.Content[i+1]
|
||||
addCatalogerFieldComment(keyNode, valueNode, catalogerName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func addParserComments(parsersNode *yaml.Node) {
|
||||
// parsersNode should be a sequence of parser entries
|
||||
if parsersNode.Kind != yaml.SequenceNode {
|
||||
return
|
||||
}
|
||||
|
||||
for _, parserNode := range parsersNode.Content {
|
||||
if parserNode.Kind != yaml.MappingNode {
|
||||
continue
|
||||
}
|
||||
|
||||
// iterate through the fields of this parser entry
|
||||
for i := 0; i < len(parserNode.Content); i += 2 {
|
||||
keyNode := parserNode.Content[i]
|
||||
valueNode := parserNode.Content[i+1]
|
||||
|
||||
switch keyNode.Value {
|
||||
case "parser_function", "metadata_types", "package_types", "json_schema_types":
|
||||
// add AUTO-GENERATED comment to these fields
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = autoGeneratedComment
|
||||
}
|
||||
case "detector":
|
||||
// detector is AUTO-GENERATED
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = autoGeneratedComment
|
||||
}
|
||||
addDetectorComments(valueNode)
|
||||
case "capabilities":
|
||||
// capabilities are MANUAL
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = "MANUAL - preserved across regeneration"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func addDetectorComments(detectorNode *yaml.Node) {
|
||||
// detectorNode should be a mapping node with method and criteria fields
|
||||
if detectorNode.Kind != yaml.MappingNode {
|
||||
return
|
||||
}
|
||||
|
||||
// iterate through the fields of the detector
|
||||
for i := 0; i < len(detectorNode.Content); i += 2 {
|
||||
keyNode := detectorNode.Content[i]
|
||||
|
||||
switch keyNode.Value {
|
||||
case "method", "criteria":
|
||||
// add AUTO-GENERATED comment to these fields
|
||||
if keyNode.LineComment == "" {
|
||||
keyNode.LineComment = autoGeneratedComment
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user