Add experimental cataloger capabilities command (#4317)

* add info command from generated capabilities Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * correct gentoo and arch ecosystems Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename os pkg types Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * better binary cataloger description Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * expose metadata and pacakge types in json Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * expose json schema types Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add completeness tests for metadata types Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * latest generation Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * improve testing a docs Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix tests and linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * restore goreleaser config Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * tweak diagram Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix pdm Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * chore: java binary data Signed-off-by: Keith Zantow <kzantow@gmail.com> * new capability descriptions for gguf and python Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * correct poetry lock integrity hash claim Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix compile error Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix: remove purl version from overrides Signed-off-by: Keith Zantow <kzantow@gmail.com> * fix lua deps ref Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * keep gguf as ai ecosystem Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * split packages.yaml to multiple files by go package Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * ensure tests do not use go test cache Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * sort json output for info command Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * docs: fix ocaml, php, and portage capabilities yaml Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> * chore: update erlang capabilities Signed-off-by: Keith Zantow <kzantow@gmail.com> * chore: update java capabilities Signed-off-by: Keith Zantow <kzantow@gmail.com> * chore: update javascript capabilities Signed-off-by: Keith Zantow <kzantow@gmail.com> * chore: update linux kernel capabilities Signed-off-by: Keith Zantow <kzantow@gmail.com> * remove missing tests Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix package.yaml references Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * revert license list change Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * check for drift in capability descriptions Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * regenerate capabilities Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * test cleanup Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * use fixture cache in static analysis Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * claim fixtures pre-req for cap generation Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update documentation with correct regeneration procedure Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * chore: ruby-gemspec-cataloger finds no dependencies Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> * chore: fix python docs and config comment Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> * chore: commit re-generated java yaml Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> * add cataloger selection to caps command Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * re-generate cap yamls Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix tests for cataloger selection Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix cli test Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add missing tests Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename cmd to `cataloger info` Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * [wip] change capability description locations Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * [wip] continued Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * [wip] adjust for import cycles Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * correct docs Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix linting Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> Signed-off-by: Keith Zantow <kzantow@gmail.com> Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com> Co-authored-by: Keith Zantow <kzantow@gmail.com> Co-authored-by: Will Murphy <willmurphyscode@users.noreply.github.com>
2026-11-01 23:16:30 +01:00 · 2025-12-22 19:34:10 +00:00
parent ae1a247f3d
commit b3c70da3ea
116 changed files with 17277 additions and 139 deletions
@@ -0,0 +1,28 @@
+// this file retrieves the canonical list of cataloger names and their selectors from syft's task factories.
+package internal
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/anchore/syft/internal/capabilities"
+	"github.com/anchore/syft/internal/task"
+)
+
+// AllPackageCatalogerInfo gets all package cataloger info (names and selectors) from task factories
+func AllPackageCatalogerInfo() ([]capabilities.CatalogerInfo, error) {
+	pkgTaskFactories := task.DefaultPackageTaskFactories()
+	allPkgTasks, err := pkgTaskFactories.Tasks(task.DefaultCatalogingFactoryConfig())
+	if err != nil {
+		return nil, fmt.Errorf("unable to create pkg cataloger tasks: %w", err)
+	}
+
+	infos := capabilities.ExtractCatalogerInfo(allPkgTasks)
+
+	// sort by name for consistency
+	sort.Slice(infos, func(i, j int) bool {
+		return infos[i].Name < infos[j].Name
+	})
+
+	return infos, nil
+}
@@ -0,0 +1,46 @@
+package internal
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/anchore/syft/internal/capabilities/pkgtestobservation"
+)
+
+// FindTestFixtureDirs walks the cataloger directory tree and returns all test-fixtures directories
+func FindTestFixtureDirs(repoRoot string) ([]string, error) {
+	catalogerRoot := filepath.Join(repoRoot, "syft", "pkg", "cataloger")
+	var testFixtureDirs []string
+
+	err := filepath.Walk(catalogerRoot, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() && info.Name() == "test-fixtures" {
+			testFixtureDirs = append(testFixtureDirs, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to walk cataloger directory: %w", err)
+	}
+
+	return testFixtureDirs, nil
+}
+
+// ReadTestObservations reads and parses a test-observations.json file
+func ReadTestObservations(path string) (*pkgtestobservation.Test, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var observations pkgtestobservation.Test
+	if err := json.Unmarshal(data, &observations); err != nil {
+		return nil, fmt.Errorf("failed to parse JSON: %w", err)
+	}
+
+	return &observations, nil
+}
@@ -0,0 +1,91 @@
+package internal
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/anchore/syft/internal/capabilities"
+)
+
+// LoadCapabilities loads the capabilities document from a YAML file.
+// Returns both the parsed document and the original YAML node tree to preserve comments.
+// Exported for use by the generator in generate/main.go
+func LoadCapabilities(catalogerDir, repoRoot string) (*capabilities.Document, map[string]*yaml.Node, error) {
+	// load all cataloger/*/capabilities.yaml files
+	files, err := filepath.Glob(filepath.Join(catalogerDir, "*", CapabilitiesFilename))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to glob capabilities files: %w", err)
+	}
+
+	mergedDoc := &capabilities.Document{
+		Configs:    make(map[string]capabilities.CatalogerConfigEntry),
+		Catalogers: []capabilities.CatalogerEntry{},
+	}
+	nodeMap := make(map[string]*yaml.Node)
+
+	// load each package file
+	for _, file := range files {
+		data, err := os.ReadFile(file)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to read %s: %w", file, err)
+		}
+
+		// parse into node tree to preserve comments
+		var rootNode yaml.Node
+		if err := yaml.Unmarshal(data, &rootNode); err != nil {
+			return nil, nil, fmt.Errorf("failed to parse %s into node tree: %w", file, err)
+		}
+
+		// parse into struct
+		var doc struct {
+			Configs    map[string]capabilities.CatalogerConfigEntry `yaml:"configs"`
+			Catalogers []capabilities.CatalogerEntry                `yaml:"catalogers"`
+		}
+		if err := yaml.Unmarshal(data, &doc); err != nil {
+			return nil, nil, fmt.Errorf("failed to parse %s into struct: %w", file, err)
+		}
+
+		// merge configs
+		for k, v := range doc.Configs {
+			mergedDoc.Configs[k] = v
+		}
+
+		// merge catalogers
+		mergedDoc.Catalogers = append(mergedDoc.Catalogers, doc.Catalogers...)
+
+		// store node tree by ecosystem directory name
+		// path is like "/path/to/syft/pkg/cataloger/alpine/capabilities.yaml"
+		ecosystem := filepath.Base(filepath.Dir(file))
+		nodeMap[ecosystem] = &rootNode
+	}
+
+	// load appconfig.yaml separately (from internal/capabilities/)
+	appconfigPath := AppconfigPath(repoRoot)
+	if _, err := os.Stat(appconfigPath); err == nil {
+		data, err := os.ReadFile(appconfigPath)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to read appconfig.yaml: %w", err)
+		}
+
+		var appDoc struct {
+			Application []capabilities.ApplicationConfigField `yaml:"application"`
+		}
+		if err := yaml.Unmarshal(data, &appDoc); err != nil {
+			return nil, nil, fmt.Errorf("failed to parse appconfig.yaml: %w", err)
+		}
+
+		mergedDoc.ApplicationConfig = appDoc.Application
+
+		// load node tree for appconfig
+		var appNode yaml.Node
+		if err := yaml.Unmarshal(data, &appNode); err != nil {
+			return nil, nil, fmt.Errorf("failed to parse appconfig.yaml into node tree: %w", err)
+		}
+		nodeMap["appconfig"] = &appNode
+	}
+
+	return mergedDoc, nodeMap, nil
+}
@@ -0,0 +1,39 @@
+// this file provides a single source of truth for all capability file paths used in generation and testing.
+package internal
+
+import "path/filepath"
+
+// path constants - single source of truth for all capability file locations
+const (
+	// CatalogerDirRel is the cataloger directory relative to repo root
+	CatalogerDirRel = "syft/pkg/cataloger"
+
+	// InternalCapabilitiesDirRel is internal capabilities dir relative to repo root
+	InternalCapabilitiesDirRel = "internal/capabilities"
+
+	// CapabilitiesFilename is the capabilities file name within each ecosystem dir
+	CapabilitiesFilename = "capabilities.yaml"
+
+	// AppconfigFilename is the application config filename
+	AppconfigFilename = "appconfig.yaml"
+)
+
+// CatalogerDir returns absolute path to cataloger directory
+func CatalogerDir(repoRoot string) string {
+	return filepath.Join(repoRoot, CatalogerDirRel)
+}
+
+// CapabilitiesDir returns absolute path to internal capabilities
+func CapabilitiesDir(repoRoot string) string {
+	return filepath.Join(repoRoot, InternalCapabilitiesDirRel)
+}
+
+// EcosystemCapabilitiesPath returns path to an ecosystem's capabilities.yaml
+func EcosystemCapabilitiesPath(catalogerDir, ecosystem string) string {
+	return filepath.Join(catalogerDir, ecosystem, CapabilitiesFilename)
+}
+
+// AppconfigPath returns path to appconfig.yaml (in internal/capabilities)
+func AppconfigPath(repoRoot string) string {
+	return filepath.Join(CapabilitiesDir(repoRoot), AppconfigFilename)
+}
@@ -0,0 +1,22 @@
+package internal
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// RepoRoot finds the git repository root directory.
+// Exported for use by the generator in generate/main.go
+func RepoRoot() (string, error) {
+	root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
+	if err != nil {
+		return "", fmt.Errorf("unable to find repo root dir: %+v", err)
+	}
+	absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root)))
+	if err != nil {
+		return "", fmt.Errorf("unable to get abs path to repo root: %w", err)
+	}
+	return absRepoRoot, nil
+}
@@ -0,0 +1,20 @@
+package internal
+
+import (
+	"os"
+	"testing"
+)
+
+// checkCompletenessTestsEnabled skips the test if completeness tests are not enabled via environment variable.
+// Why do this at all? Can't we just run these tests all the time? Short answer: No.
+// These tests are coupled with unit tests under ./syft/pkg/..., which means that these tests must be run not only
+// after those unit tests, but also after code generation that reads observations from test results.
+// This means that we should not really consider these tests as part of normal unit test runs, but rather as a separate
+// self-consistency check during generation, and it's really static analysis that should be checking
+// if the generated code has drifted (not a unit test).
+func checkCompletenessTestsEnabled(t *testing.T) {
+	enabled := os.Getenv("SYFT_ENABLE_COMPLETENESS_TESTS") == "true"
+	if !enabled {
+		t.Skip("skipping completeness tests (SYFT_ENABLE_COMPLETENESS_TESTS is not set to 'true')")
+	}
+}