mirror of
https://github.com/anchore/syft.git
synced 2026-10-11 21:57:22 +02:00
Refactor source API (#1846)
* refactor source API and syft json source block Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update source detection and format test utils Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * generate list of all source metadata types Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * extract base and root normalization into helper functions Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * preserve syftjson model package name import ref Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * alias should not be a pointer Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <alex.goodman@anchore.com> Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
@@ -3,8 +3,8 @@
|
||||
This is the JSON schema for output from the JSON presenters (`syft packages <img> -o json` and `syft power-user <img>`). The required inputs for defining the JSON schema are as follows:
|
||||
|
||||
- the value of `internal.JSONSchemaVersion` that governs the schema filename
|
||||
- the `Document` struct definition within `internal/presenters/poweruser/json_document.go` that governs the overall document shape
|
||||
- the `artifactMetadataContainer` struct definition within `schema/json/generate.go` that governs the allowable shapes of `pkg.Package.Metadata`
|
||||
- the `Document` struct definition within `github.com/anchore/syft/syft/formats/syftjson/model/document.go` that governs the overall document shape
|
||||
- generated `AllTypes()` helper function within the `syft/internal/sourcemetadata` and `syft/internal/packagemetadata` packages
|
||||
|
||||
With regard to testing the JSON schema, integration test cases provided by the developer are used as examples to validate that JSON output from Syft is always valid relative to the `schema/json/schema-$VERSION.json` file.
|
||||
|
||||
@@ -22,15 +22,13 @@ Given a version number format `MODEL.REVISION.ADDITION`:
|
||||
|
||||
## Adding a New `pkg.*Metadata` Type
|
||||
|
||||
When adding a new `pkg.*Metadata` that is assigned to the `pkg.Package.Metadata` struct field it is important that a few things
|
||||
are done:
|
||||
When adding a new `pkg.*Metadata` that is assigned to the `pkg.Package.Metadata` struct field you must add a test case to `test/integration/catalog_packages_cases_test.go` that exercises the new package type with the new metadata.
|
||||
|
||||
- a new integration test case is added to `test/integration/catalog_packages_cases_test.go` that exercises the new package type with the new metadata
|
||||
- the new metadata struct is added to the `artifactMetadataContainer` struct within `schema/json/generate.go`
|
||||
Additionally it is important to generate a new JSON schema since the `pkg.Package.Metadata` field is covered by the schema.
|
||||
|
||||
## Generating a New Schema
|
||||
|
||||
Create the new schema by running `cd schema/json && go run generate.go` (note you must be in the `schema/json` dir while running this):
|
||||
Create the new schema by running `make generate-json-schema` from the root of the repo:
|
||||
|
||||
- If there is **not** an existing schema for the given version, then the new schema file will be written to `schema/json/schema-$VERSION.json`
|
||||
- If there is an existing schema for the given version and the new schema matches the existing schema, no action is taken
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/dave/jennifer/jen"
|
||||
|
||||
"github.com/anchore/syft/schema/json/internal"
|
||||
)
|
||||
|
||||
// This program generates internal/generated.go.
|
||||
|
||||
const (
|
||||
pkgImport = "github.com/anchore/syft/syft/pkg"
|
||||
path = "internal/generated.go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
typeNames, err := internal.AllSyftMetadataTypeNames()
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("unable to get all metadata type names: %w", err))
|
||||
}
|
||||
|
||||
fmt.Printf("updating metadata container object with %+v types\n", len(typeNames))
|
||||
|
||||
f := jen.NewFile("internal")
|
||||
f.HeaderComment("DO NOT EDIT: generated by schema/json/generate/main.go")
|
||||
f.ImportName(pkgImport, "pkg")
|
||||
f.Comment("ArtifactMetadataContainer is a struct that contains all the metadata types for a package, as represented in the pkg.Package.Metadata field.")
|
||||
f.Type().Id("ArtifactMetadataContainer").StructFunc(func(g *jen.Group) {
|
||||
for _, typeName := range typeNames {
|
||||
g.Id(typeName).Qual(pkgImport, typeName)
|
||||
}
|
||||
})
|
||||
|
||||
rendered := fmt.Sprintf("%#v", f)
|
||||
|
||||
fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("unable to open file: %w", err))
|
||||
}
|
||||
_, err = fh.WriteString(rendered)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("unable to write file: %w", err))
|
||||
}
|
||||
if err := fh.Close(); err != nil {
|
||||
panic(fmt.Errorf("unable to close file: %w", err))
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
// DO NOT EDIT: generated by schema/json/generate/main.go
|
||||
|
||||
package internal
|
||||
|
||||
import "github.com/anchore/syft/syft/pkg"
|
||||
|
||||
// ArtifactMetadataContainer is a struct that contains all the metadata types for a package, as represented in the pkg.Package.Metadata field.
|
||||
type ArtifactMetadataContainer struct {
|
||||
AlpmMetadata pkg.AlpmMetadata
|
||||
ApkMetadata pkg.ApkMetadata
|
||||
BinaryMetadata pkg.BinaryMetadata
|
||||
CargoPackageMetadata pkg.CargoPackageMetadata
|
||||
CocoapodsMetadata pkg.CocoapodsMetadata
|
||||
ConanLockMetadata pkg.ConanLockMetadata
|
||||
ConanMetadata pkg.ConanMetadata
|
||||
DartPubMetadata pkg.DartPubMetadata
|
||||
DotnetDepsMetadata pkg.DotnetDepsMetadata
|
||||
DpkgMetadata pkg.DpkgMetadata
|
||||
GemMetadata pkg.GemMetadata
|
||||
GolangBinMetadata pkg.GolangBinMetadata
|
||||
GolangModMetadata pkg.GolangModMetadata
|
||||
HackageMetadata pkg.HackageMetadata
|
||||
JavaMetadata pkg.JavaMetadata
|
||||
KbPackageMetadata pkg.KbPackageMetadata
|
||||
LinuxKernelMetadata pkg.LinuxKernelMetadata
|
||||
LinuxKernelModuleMetadata pkg.LinuxKernelModuleMetadata
|
||||
MixLockMetadata pkg.MixLockMetadata
|
||||
NixStoreMetadata pkg.NixStoreMetadata
|
||||
NpmPackageJSONMetadata pkg.NpmPackageJSONMetadata
|
||||
NpmPackageLockJSONMetadata pkg.NpmPackageLockJSONMetadata
|
||||
PhpComposerJSONMetadata pkg.PhpComposerJSONMetadata
|
||||
PortageMetadata pkg.PortageMetadata
|
||||
PythonPackageMetadata pkg.PythonPackageMetadata
|
||||
PythonPipfileLockMetadata pkg.PythonPipfileLockMetadata
|
||||
PythonRequirementsMetadata pkg.PythonRequirementsMetadata
|
||||
RDescriptionFileMetadata pkg.RDescriptionFileMetadata
|
||||
RebarLockMetadata pkg.RebarLockMetadata
|
||||
RpmMetadata pkg.RpmMetadata
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/scylladb/go-set/strset"
|
||||
)
|
||||
|
||||
var metadataExceptions = strset.New(
|
||||
"FileMetadata",
|
||||
)
|
||||
|
||||
func AllSyftMetadataTypeNames() ([]string, error) {
|
||||
root, err := repoRoot()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files, err := filepath.Glob(filepath.Join(root, "syft/pkg/*.go"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return findMetadataDefinitionNames(files...)
|
||||
}
|
||||
|
||||
func repoRoot() (string, error) {
|
||||
root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to find repo root dir: %+v", err)
|
||||
}
|
||||
absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root)))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to get abs path to repo root: %w", err)
|
||||
}
|
||||
return absRepoRoot, nil
|
||||
}
|
||||
|
||||
func findMetadataDefinitionNames(paths ...string) ([]string, error) {
|
||||
names := strset.New()
|
||||
usedNames := strset.New()
|
||||
for _, path := range paths {
|
||||
metadataDefinitions, usedTypeNames, err := findMetadataDefinitionNamesInFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// useful for debugging...
|
||||
// fmt.Println(path)
|
||||
// fmt.Println("Defs:", metadataDefinitions)
|
||||
// fmt.Println("Used Types:", usedTypeNames)
|
||||
// fmt.Println()
|
||||
|
||||
names.Add(metadataDefinitions...)
|
||||
usedNames.Add(usedTypeNames...)
|
||||
}
|
||||
|
||||
// any definition that is used within another struct should not be considered a top-level metadata definition
|
||||
names.Remove(usedNames.List()...)
|
||||
|
||||
strNames := names.List()
|
||||
sort.Strings(strNames)
|
||||
|
||||
// note: 30 is a point-in-time gut check. This number could be updated if new metadata definitions are added, but is not required.
|
||||
// it is really intended to catch any major issues with the generation process that would generate, say, 0 definitions.
|
||||
if len(strNames) < 30 {
|
||||
return nil, fmt.Errorf("not enough metadata definitions found (discovered: " + fmt.Sprintf("%d", len(strNames)) + ")")
|
||||
}
|
||||
|
||||
return strNames, nil
|
||||
}
|
||||
|
||||
func findMetadataDefinitionNamesInFile(path string) ([]string, []string, error) {
|
||||
// set up the parser
|
||||
fs := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fs, path, nil, parser.ParseComments)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
var metadataDefinitions []string
|
||||
var usedTypeNames []string
|
||||
for _, decl := range f.Decls {
|
||||
// check if the declaration is a type declaration
|
||||
spec, ok := decl.(*ast.GenDecl)
|
||||
if !ok || spec.Tok != token.TYPE {
|
||||
continue
|
||||
}
|
||||
|
||||
// loop over all types declared in the type declaration
|
||||
for _, typ := range spec.Specs {
|
||||
// check if the type is a struct type
|
||||
spec, ok := typ.(*ast.TypeSpec)
|
||||
if !ok || spec.Type == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
structType, ok := spec.Type.(*ast.StructType)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// check if the struct type ends with "Metadata"
|
||||
name := spec.Name.String()
|
||||
|
||||
// only look for exported types that end with "Metadata"
|
||||
if isMetadataTypeCandidate(name) {
|
||||
// print the full declaration of the struct type
|
||||
metadataDefinitions = append(metadataDefinitions, name)
|
||||
usedTypeNames = append(usedTypeNames, typeNamesUsedInStruct(structType)...)
|
||||
}
|
||||
}
|
||||
}
|
||||
return metadataDefinitions, usedTypeNames, nil
|
||||
}
|
||||
|
||||
func typeNamesUsedInStruct(structType *ast.StructType) []string {
|
||||
// recursively find all type names used in the struct type
|
||||
var names []string
|
||||
for i := range structType.Fields.List {
|
||||
// capture names of all of the types (not field names)
|
||||
ast.Inspect(structType.Fields.List[i].Type, func(n ast.Node) bool {
|
||||
ident, ok := n.(*ast.Ident)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
|
||||
// add the type name to the list
|
||||
names = append(names, ident.Name)
|
||||
|
||||
// continue inspecting
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
return names
|
||||
}
|
||||
|
||||
func isMetadataTypeCandidate(name string) bool {
|
||||
return len(name) > 0 &&
|
||||
strings.HasSuffix(name, "Metadata") &&
|
||||
unicode.IsUpper(rune(name[0])) && // must be exported
|
||||
!metadataExceptions.Has(name)
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/invopop/jsonschema"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
genInt "github.com/anchore/syft/schema/json/internal"
|
||||
syftjsonModel "github.com/anchore/syft/syft/formats/syftjson/model"
|
||||
)
|
||||
|
||||
/*
|
||||
This method of creating the JSON schema only captures strongly typed fields for the purpose of integrations between syft
|
||||
JSON output and integrations. The downside to this approach is that any values and types used on weakly typed fields
|
||||
are not captured (empty interfaces). This means that pkg.Package.Metadata is not validated at this time. This approach
|
||||
can be extended to include specific package metadata struct shapes in the future.
|
||||
*/
|
||||
|
||||
//go:generate go run ./generate/main.go
|
||||
|
||||
const schemaVersion = internal.JSONSchemaVersion
|
||||
|
||||
func main() {
|
||||
write(encode(build()))
|
||||
}
|
||||
|
||||
func build() *jsonschema.Schema {
|
||||
reflector := &jsonschema.Reflector{
|
||||
AllowAdditionalProperties: true,
|
||||
Namer: func(r reflect.Type) string {
|
||||
return strings.TrimPrefix(r.Name(), "JSON")
|
||||
},
|
||||
}
|
||||
documentSchema := reflector.ReflectFromType(reflect.TypeOf(&syftjsonModel.Document{}))
|
||||
metadataSchema := reflector.ReflectFromType(reflect.TypeOf(&genInt.ArtifactMetadataContainer{}))
|
||||
// TODO: inject source definitions
|
||||
|
||||
// inject the definitions of all metadatas into the schema definitions
|
||||
|
||||
var metadataNames []string
|
||||
for name, definition := range metadataSchema.Definitions {
|
||||
if name == reflect.TypeOf(genInt.ArtifactMetadataContainer{}).Name() {
|
||||
// ignore the definition for the fake container
|
||||
continue
|
||||
}
|
||||
documentSchema.Definitions[name] = definition
|
||||
if strings.HasSuffix(name, "Metadata") {
|
||||
metadataNames = append(metadataNames, name)
|
||||
}
|
||||
}
|
||||
|
||||
// ensure the generated list of names is stable between runs
|
||||
sort.Strings(metadataNames)
|
||||
|
||||
var metadataTypes = []map[string]string{
|
||||
// allow for no metadata to be provided
|
||||
{"type": "null"},
|
||||
}
|
||||
for _, name := range metadataNames {
|
||||
metadataTypes = append(metadataTypes, map[string]string{
|
||||
"$ref": fmt.Sprintf("#/$defs/%s", name),
|
||||
})
|
||||
}
|
||||
|
||||
// set the "anyOf" field for Package.Metadata to be a conjunction of several types
|
||||
documentSchema.Definitions["Package"].Properties.Set("metadata", map[string][]map[string]string{
|
||||
"anyOf": metadataTypes,
|
||||
})
|
||||
|
||||
return documentSchema
|
||||
}
|
||||
|
||||
func encode(schema *jsonschema.Schema) []byte {
|
||||
var newSchemaBuffer = new(bytes.Buffer)
|
||||
enc := json.NewEncoder(newSchemaBuffer)
|
||||
// prevent > and < from being escaped in the payload
|
||||
enc.SetEscapeHTML(false)
|
||||
enc.SetIndent("", " ")
|
||||
err := enc.Encode(&schema)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return newSchemaBuffer.Bytes()
|
||||
}
|
||||
|
||||
func write(schema []byte) {
|
||||
filename := fmt.Sprintf("schema-%s.json", schemaVersion)
|
||||
|
||||
if _, err := os.Stat(filename); !os.IsNotExist(err) {
|
||||
// check if the schema is the same...
|
||||
existingFh, err := os.Open(filename)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
existingSchemaBytes, err := io.ReadAll(existingFh)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if bytes.Equal(existingSchemaBytes, schema) {
|
||||
// the generated schema is the same, bail with no error :)
|
||||
fmt.Println("No change to the existing schema!")
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// the generated schema is different, bail with error :(
|
||||
fmt.Printf("Cowardly refusing to overwrite existing schema (%s)!\nSee the schema/json/README.md for how to increment\n", filename)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fh, err := os.Create(filename)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
_, err = fh.Write(schema)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
defer fh.Close()
|
||||
|
||||
fmt.Printf("Wrote new schema to %q\n", filename)
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/schema/json/internal"
|
||||
)
|
||||
|
||||
func TestAllMetadataRepresented(t *testing.T) {
|
||||
// this test checks that all the metadata types are represented in the currently generated ArtifactMetadataContainer struct
|
||||
// such that PRs will reflect when there is drift from the implemented set of metadata types and the generated struct
|
||||
// which controls the JSON schema content.
|
||||
expected, err := internal.AllSyftMetadataTypeNames()
|
||||
require.NoError(t, err)
|
||||
actual := allTypeNamesFromStruct(internal.ArtifactMetadataContainer{})
|
||||
if !assert.ElementsMatch(t, expected, actual) {
|
||||
t.Errorf("metadata types not fully represented: \n%s", cmp.Diff(expected, actual))
|
||||
t.Log("did you add a new pkg.*Metadata type without updating the JSON schema?")
|
||||
t.Log("if so, you need to update the schema version and regenerate the JSON schema (make generate-json-schema)")
|
||||
}
|
||||
}
|
||||
|
||||
func allTypeNamesFromStruct(instance any) []string {
|
||||
// get all the type names from the struct (not recursively)
|
||||
var typeNames []string
|
||||
tt := reflect.TypeOf(instance)
|
||||
for i := 0; i < tt.NumField(); i++ {
|
||||
field := tt.Field(i)
|
||||
typeNames = append(typeNames, field.Type.Name())
|
||||
}
|
||||
sort.Strings(typeNames)
|
||||
return typeNames
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user