Refactor source API (#1846)

* refactor source API and syft json source block Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update source detection and format test utils Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * generate list of all source metadata types Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * extract base and root normalization into helper functions Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * preserve syftjson model package name import ref Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * alias should not be a pointer Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <alex.goodman@anchore.com> Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
2026-11-02 15:36:41 +01:00 · 2023-06-30 14:19:16 +00:00
parent 608dbded06
commit 4da3be864f
126 changed files with 7384 additions and 3190 deletions
@@ -0,0 +1,150 @@
+package packagemetadata
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"unicode"
+
+	"github.com/scylladb/go-set/strset"
+)
+
+var metadataExceptions = strset.New(
+	"FileMetadata",
+)
+
+func DiscoverTypeNames() ([]string, error) {
+	root, err := RepoRoot()
+	if err != nil {
+		return nil, err
+	}
+	files, err := filepath.Glob(filepath.Join(root, "syft/pkg/*.go"))
+	if err != nil {
+		return nil, err
+	}
+	return findMetadataDefinitionNames(files...)
+}
+
+func RepoRoot() (string, error) {
+	root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
+	if err != nil {
+		return "", fmt.Errorf("unable to find repo root dir: %+v", err)
+	}
+	absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root)))
+	if err != nil {
+		return "", fmt.Errorf("unable to get abs path to repo root: %w", err)
+	}
+	return absRepoRoot, nil
+}
+
+func findMetadataDefinitionNames(paths ...string) ([]string, error) {
+	names := strset.New()
+	usedNames := strset.New()
+	for _, path := range paths {
+		metadataDefinitions, usedTypeNames, err := findMetadataDefinitionNamesInFile(path)
+		if err != nil {
+			return nil, err
+		}
+
+		// useful for debugging...
+		// fmt.Println(path)
+		// fmt.Println("Defs:", metadataDefinitions)
+		// fmt.Println("Used Types:", usedTypeNames)
+		// fmt.Println()
+
+		names.Add(metadataDefinitions...)
+		usedNames.Add(usedTypeNames...)
+	}
+
+	// any definition that is used within another struct should not be considered a top-level metadata definition
+	names.Remove(usedNames.List()...)
+
+	strNames := names.List()
+	sort.Strings(strNames)
+
+	// note: 30 is a point-in-time gut check. This number could be updated if new metadata definitions are added, but is not required.
+	// it is really intended to catch any major issues with the generation process that would generate, say, 0 definitions.
+	if len(strNames) < 30 {
+		return nil, fmt.Errorf("not enough metadata definitions found (discovered: " + fmt.Sprintf("%d", len(strNames)) + ")")
+	}
+
+	return strNames, nil
+}
+
+func findMetadataDefinitionNamesInFile(path string) ([]string, []string, error) {
+	// set up the parser
+	fs := token.NewFileSet()
+	f, err := parser.ParseFile(fs, path, nil, parser.ParseComments)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var metadataDefinitions []string
+	var usedTypeNames []string
+	for _, decl := range f.Decls {
+		// check if the declaration is a type declaration
+		spec, ok := decl.(*ast.GenDecl)
+		if !ok || spec.Tok != token.TYPE {
+			continue
+		}
+
+		// loop over all types declared in the type declaration
+		for _, typ := range spec.Specs {
+			// check if the type is a struct type
+			spec, ok := typ.(*ast.TypeSpec)
+			if !ok || spec.Type == nil {
+				continue
+			}
+
+			structType, ok := spec.Type.(*ast.StructType)
+			if !ok {
+				continue
+			}
+
+			// check if the struct type ends with "Metadata"
+			name := spec.Name.String()
+
+			// only look for exported types that end with "Metadata"
+			if isMetadataTypeCandidate(name) {
+				// print the full declaration of the struct type
+				metadataDefinitions = append(metadataDefinitions, name)
+				usedTypeNames = append(usedTypeNames, typeNamesUsedInStruct(structType)...)
+			}
+		}
+	}
+	return metadataDefinitions, usedTypeNames, nil
+}
+
+func typeNamesUsedInStruct(structType *ast.StructType) []string {
+	// recursively find all type names used in the struct type
+	var names []string
+	for i := range structType.Fields.List {
+		// capture names of all of the types (not field names)
+		ast.Inspect(structType.Fields.List[i].Type, func(n ast.Node) bool {
+			ident, ok := n.(*ast.Ident)
+			if !ok {
+				return true
+			}
+
+			// add the type name to the list
+			names = append(names, ident.Name)
+
+			// continue inspecting
+			return true
+		})
+	}
+
+	return names
+}
+
+func isMetadataTypeCandidate(name string) bool {
+	return len(name) > 0 &&
+		strings.HasSuffix(name, "Metadata") &&
+		unicode.IsUpper(rune(name[0])) && // must be exported
+		!metadataExceptions.Has(name)
+}
@@ -0,0 +1,55 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/dave/jennifer/jen"
+
+	"github.com/anchore/syft/syft/internal/packagemetadata"
+)
+
+// This program is invoked from syft/internal and generates packagemetadata/generated.go
+
+const (
+	pkgImport = "github.com/anchore/syft/syft/pkg"
+	path      = "packagemetadata/generated.go"
+)
+
+func main() {
+	typeNames, err := packagemetadata.DiscoverTypeNames()
+	if err != nil {
+		panic(fmt.Errorf("unable to get all metadata type names: %w", err))
+	}
+
+	fmt.Printf("updating package metadata type list with %+v types\n", len(typeNames))
+
+	f := jen.NewFile("packagemetadata")
+	f.HeaderComment("DO NOT EDIT: generated by syft/internal/packagemetadata/generate/main.go")
+	f.ImportName(pkgImport, "pkg")
+	f.Comment("AllTypes returns a list of all pkg metadata types that syft supports (that are represented in the pkg.Package.Metadata field).")
+
+	f.Func().Id("AllTypes").Params().Index().Any().BlockFunc(func(g *jen.Group) {
+		g.ReturnFunc(func(g *jen.Group) {
+			g.Index().Any().ValuesFunc(func(g *jen.Group) {
+				for _, typeName := range typeNames {
+					g.Qual(pkgImport, typeName).Values()
+				}
+			})
+		})
+	})
+
+	rendered := fmt.Sprintf("%#v", f)
+
+	fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		panic(fmt.Errorf("unable to open file: %w", err))
+	}
+	_, err = fh.WriteString(rendered)
+	if err != nil {
+		panic(fmt.Errorf("unable to write file: %w", err))
+	}
+	if err := fh.Close(); err != nil {
+		panic(fmt.Errorf("unable to close file: %w", err))
+	}
+}
@@ -0,0 +1,10 @@
+// DO NOT EDIT: generated by syft/internal/packagemetadata/generate/main.go
+
+package packagemetadata
+
+import "github.com/anchore/syft/syft/pkg"
+
+// AllTypes returns a list of all pkg metadata types that syft supports (that are represented in the pkg.Package.Metadata field).
+func AllTypes() []any {
+	return []any{pkg.AlpmMetadata{}, pkg.ApkMetadata{}, pkg.BinaryMetadata{}, pkg.CargoPackageMetadata{}, pkg.CocoapodsMetadata{}, pkg.ConanLockMetadata{}, pkg.ConanMetadata{}, pkg.DartPubMetadata{}, pkg.DotnetDepsMetadata{}, pkg.DpkgMetadata{}, pkg.GemMetadata{}, pkg.GolangBinMetadata{}, pkg.GolangModMetadata{}, pkg.HackageMetadata{}, pkg.JavaMetadata{}, pkg.KbPackageMetadata{}, pkg.LinuxKernelMetadata{}, pkg.LinuxKernelModuleMetadata{}, pkg.MixLockMetadata{}, pkg.NixStoreMetadata{}, pkg.NpmPackageJSONMetadata{}, pkg.NpmPackageLockJSONMetadata{}, pkg.PhpComposerJSONMetadata{}, pkg.PortageMetadata{}, pkg.PythonPackageMetadata{}, pkg.PythonPipfileLockMetadata{}, pkg.PythonRequirementsMetadata{}, pkg.RDescriptionFileMetadata{}, pkg.RebarLockMetadata{}, pkg.RpmMetadata{}}
+}
@@ -0,0 +1,13 @@
+package packagemetadata
+
+import (
+	"reflect"
+)
+
+func AllNames() []string {
+	names := make([]string, 0)
+	for _, t := range AllTypes() {
+		names = append(names, reflect.TypeOf(t).Name())
+	}
+	return names
+}
@@ -0,0 +1,25 @@
+package packagemetadata
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAllNames(t *testing.T) {
+	// note: this is a form of completion testing relative to the current code base.
+
+	expected, err := DiscoverTypeNames()
+	require.NoError(t, err)
+
+	actual := AllNames()
+
+	// ensure that the codebase (from ast analysis) reflects the latest code generated state
+	if !assert.ElementsMatch(t, expected, actual) {
+		t.Errorf("metadata types not fully represented: \n%s", cmp.Diff(expected, actual))
+		t.Log("did you add a new pkg.*Metadata type without updating the JSON schema?")
+		t.Log("if so, you need to update the schema version and regenerate the JSON schema (make generate-json-schema)")
+	}
+}