Add enterprise upload capability (#285)

* add support to upload results to enterprise

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add package sbom upload

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add dockerfile support

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add manifest, index, and dockerfile import functions

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* schema version to json output + enhance json schema generation

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* modify package SBOM shape to be entire syft document + add etui updates

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add import image config and manifest support

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add config options for import to enterprise

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* incorporate final stereoscope and client-go deps

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2020-12-09 22:20:53 -05:00
committed by GitHub
parent 2d0c127419
commit 52bac6e2fd
42 changed files with 2284 additions and 319 deletions
+39
View File
@@ -0,0 +1,39 @@
# JSON Schema
This is the JSON schema for output from the JSON presenter (`syft <img> -o json`). The required inputs for defining the JSON schema are as follows:
- the value of `internal.JSONSchemaVersion` that governs the schema filename
- the `Document` struct definition within `syft/presenters/json/document.go` that governs the overall document shape
- the `metadataContainer` struct definition within `schema/json/generate.go` that governs the allowable shapes of `pkg.Package.Metadata`
With regard to testing the JSON schema, integration test cases provided by the developer are used as examples to validate that JSON output from Syft is always valid relative to the `schema/json/schema-$VERSION.json` file.
## Versioning
Versioning the JSON schema must be done manually by changing the `JSONSchemaVersion` constant within `internal/constants.go`.
This schema is being versioned based off of the "SchemaVer" guidelines, which slightly diverges from Semantic Versioning to tailor for the purposes of data models.
Given a version number format `MODEL.REVISION.ADDITION`:
- `MODEL`: increment when you make a breaking schema change which will prevent interaction with any historical data
- `REVISION`: increment when you make a schema change which may prevent interaction with some historical data
- `ADDITION`: increment when you make a schema change that is compatible with all historical data
## Adding a New `pkg.*Metadata` Type
When adding a new `pkg.*Metadata` that is assigned to the `pkg.Package.Metadata` struct field it is important that a few things
are done:
- a new integration test case is added to `test/integration/pkg_cases_test.go` that exercises the new package type with the new metadata
- the new metadata struct is added to the `metadataContainer` struct within `schema/json/generate.go`
## Generating a New Schema
Create the new schema by running `cd schema/json && go run generate.go` (note you must be in the `schema/json` dir while running this):
- If there is **not** an existing schema for the given version, then the new schema file will be written to `schema/json/schema-$VERSION.json`
- If there is an existing schema for the given version and the new schema matches the existing schema, no action is taken
- If there is an existing schema for the given version and the new schema **does not** match the existing schema, an error is shown indicating to increment the version appropriately (see the "Versioning" section)
***Note: never delete a JSON schema and never change an existing JSON schema once it has been published in a release!*** Only add new schemas with a newly incremented version. All previous schema files must be stored in the `schema/json/` directory.
+97 -7
View File
@@ -1,11 +1,17 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"sort"
"strings"
"github.com/alecthomas/jsonschema"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/pkg"
jsonPresenter "github.com/anchore/syft/syft/presenter/json"
)
@@ -16,20 +22,104 @@ are not captured (empty interfaces). This means that pkg.Package.Metadata is not
can be extended to include specific package metadata struct shapes in the future.
*/
// This should represent all possible metadatas represented in the pkg.Package.Metadata field (an interface{}).
// When a new package metadata definition is created it will need to be manually added here. The variable name does
// not matter as long as it is exported.
type metadataContainer struct {
Apk pkg.ApkMetadata
Dpkg pkg.DpkgMetadata
Gem pkg.GemMetadata
Java pkg.JavaMetadata
Npm pkg.NpmPackageJSONMetadata
Python pkg.PythonPackageMetadata
Rpm pkg.RpmdbMetadata
}
// nolint:funlen
func main() {
j := jsonschema.Reflect(&jsonPresenter.Document{})
filename := "schema.json"
fh, err := os.OpenFile("schema.json", os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
panic(err)
metadataSchema := jsonschema.Reflect(&metadataContainer{})
documentSchema := jsonschema.Reflect(&jsonPresenter.Document{})
// TODO: inject source definitions
// inject the definitions of all metadatas into the schema definitions
var metadataNames []string
for name, definition := range metadataSchema.Definitions {
if name == "metadataContainer" {
// ignore the definition for the fake container
continue
}
documentSchema.Definitions[name] = definition
if strings.HasSuffix(name, "Metadata") {
metadataNames = append(metadataNames, name)
}
}
enc := json.NewEncoder(fh)
// ensure the generated list of names is stable between runs
sort.Strings(metadataNames)
var metadataTypes = []map[string]string{
// allow for no metadata to be provided
{"type": "null"},
}
for _, name := range metadataNames {
metadataTypes = append(metadataTypes, map[string]string{
"$ref": fmt.Sprintf("#/definitions/%s", name),
})
}
// set the "anyOf" field for Package.Metadata to be a conjunction of several types
documentSchema.Definitions["Package"].Properties.Set("metadata", map[string][]map[string]string{
"anyOf": metadataTypes,
})
filename := fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion)
var newSchemaBuffer = new(bytes.Buffer)
enc := json.NewEncoder(newSchemaBuffer)
// prevent > and < from being escaped in the payload
enc.SetEscapeHTML(false)
enc.SetIndent("", " ")
err = enc.Encode(&j)
err := enc.Encode(&documentSchema)
if err != nil {
panic(err)
}
if _, err := os.Stat(filename); !os.IsNotExist(err) {
// check if the schema is the same...
existingFh, err := os.Open(filename)
if err != nil {
panic(err)
}
existingSchemaBytes, err := ioutil.ReadAll(existingFh)
if err != nil {
panic(err)
}
if bytes.Equal(existingSchemaBytes, newSchemaBuffer.Bytes()) {
// the generated schema is the same, bail with no error :)
fmt.Println("No change to the existing schema!")
os.Exit(0)
}
// the generated schema is different, bail with error :(
fmt.Printf("Cowardly refusing to overwrite existing schema (%s)!\nSee the scheam/json/README.md for how to increment\n", filename)
os.Exit(1)
}
fh, err := os.Create(filename)
if err != nil {
panic(err)
}
_, err = fh.Write(newSchemaBuffer.Bytes())
if err != nil {
panic(err)
}
defer fh.Close()
fmt.Printf("wrote new schema to %q\n", filename)
}
+678
View File
@@ -0,0 +1,678 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Document",
"definitions": {
"ApkFileRecord": {
"required": [
"path"
],
"properties": {
"path": {
"type": "string"
},
"ownerUid": {
"type": "string"
},
"ownerGid": {
"type": "string"
},
"permissions": {
"type": "string"
},
"checksum": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"ApkMetadata": {
"required": [
"package",
"originPackage",
"maintainer",
"version",
"license",
"architecture",
"url",
"description",
"size",
"installedSize",
"pullDependencies",
"pullChecksum",
"gitCommitOfApkPort",
"files"
],
"properties": {
"package": {
"type": "string"
},
"originPackage": {
"type": "string"
},
"maintainer": {
"type": "string"
},
"version": {
"type": "string"
},
"license": {
"type": "string"
},
"architecture": {
"type": "string"
},
"url": {
"type": "string"
},
"description": {
"type": "string"
},
"size": {
"type": "integer"
},
"installedSize": {
"type": "integer"
},
"pullDependencies": {
"type": "string"
},
"pullChecksum": {
"type": "string"
},
"gitCommitOfApkPort": {
"type": "string"
},
"files": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/ApkFileRecord"
},
"type": "array"
}
},
"additionalProperties": false,
"type": "object"
},
"Descriptor": {
"required": [
"name",
"version"
],
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"Distribution": {
"required": [
"name",
"version",
"idLike"
],
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"idLike": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"Document": {
"required": [
"artifacts",
"source",
"distro",
"descriptor",
"schema"
],
"properties": {
"artifacts": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Package"
},
"type": "array"
},
"source": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Source"
},
"distro": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Distribution"
},
"descriptor": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Descriptor"
},
"schema": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Schema"
}
},
"additionalProperties": false,
"type": "object"
},
"DpkgFileRecord": {
"required": [
"path",
"md5"
],
"properties": {
"path": {
"type": "string"
},
"md5": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"DpkgMetadata": {
"required": [
"package",
"source",
"version",
"architecture",
"maintainer",
"installedSize",
"files"
],
"properties": {
"package": {
"type": "string"
},
"source": {
"type": "string"
},
"version": {
"type": "string"
},
"architecture": {
"type": "string"
},
"maintainer": {
"type": "string"
},
"installedSize": {
"type": "integer"
},
"files": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/DpkgFileRecord"
},
"type": "array"
}
},
"additionalProperties": false,
"type": "object"
},
"GemMetadata": {
"required": [
"name",
"version"
],
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"files": {
"items": {
"type": "string"
},
"type": "array"
},
"authors": {
"items": {
"type": "string"
},
"type": "array"
},
"licenses": {
"items": {
"type": "string"
},
"type": "array"
},
"homepage": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"JavaManifest": {
"properties": {
"main": {
"patternProperties": {
".*": {
"type": "string"
}
},
"type": "object"
},
"namedSections": {
"patternProperties": {
".*": {
"patternProperties": {
".*": {
"type": "string"
}
},
"type": "object"
}
},
"type": "object"
}
},
"additionalProperties": false,
"type": "object"
},
"JavaMetadata": {
"required": [
"virtualPath"
],
"properties": {
"virtualPath": {
"type": "string"
},
"manifest": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/JavaManifest"
},
"pomProperties": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/PomProperties"
}
},
"additionalProperties": false,
"type": "object"
},
"Location": {
"required": [
"path"
],
"properties": {
"path": {
"type": "string"
},
"layerID": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"NpmPackageJSONMetadata": {
"required": [
"author",
"licenses",
"homepage",
"description",
"url"
],
"properties": {
"files": {
"items": {
"type": "string"
},
"type": "array"
},
"author": {
"type": "string"
},
"licenses": {
"items": {
"type": "string"
},
"type": "array"
},
"homepage": {
"type": "string"
},
"description": {
"type": "string"
},
"url": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"Package": {
"required": [
"name",
"version",
"type",
"foundBy",
"locations",
"licenses",
"language",
"cpes",
"purl",
"metadataType",
"metadata"
],
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"type": {
"type": "string"
},
"foundBy": {
"type": "string"
},
"locations": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Location"
},
"type": "array"
},
"licenses": {
"items": {
"type": "string"
},
"type": "array"
},
"language": {
"type": "string"
},
"cpes": {
"items": {
"type": "string"
},
"type": "array"
},
"purl": {
"type": "string"
},
"metadataType": {
"type": "string"
},
"metadata": {
"anyOf": [
{
"type": "null"
},
{
"$ref": "#/definitions/ApkMetadata"
},
{
"$ref": "#/definitions/DpkgMetadata"
},
{
"$ref": "#/definitions/GemMetadata"
},
{
"$ref": "#/definitions/JavaMetadata"
},
{
"$ref": "#/definitions/NpmPackageJSONMetadata"
},
{
"$ref": "#/definitions/PythonPackageMetadata"
},
{
"$ref": "#/definitions/RpmdbMetadata"
}
]
}
},
"additionalProperties": false,
"type": "object"
},
"PomProperties": {
"required": [
"path",
"name",
"groupId",
"artifactId",
"version",
"extraFields"
],
"properties": {
"path": {
"type": "string"
},
"name": {
"type": "string"
},
"groupId": {
"type": "string"
},
"artifactId": {
"type": "string"
},
"version": {
"type": "string"
},
"extraFields": {
"patternProperties": {
".*": {
"type": "string"
}
},
"type": "object"
}
},
"additionalProperties": false,
"type": "object"
},
"PythonFileDigest": {
"required": [
"algorithm",
"value"
],
"properties": {
"algorithm": {
"type": "string"
},
"value": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"PythonFileRecord": {
"required": [
"path"
],
"properties": {
"path": {
"type": "string"
},
"digest": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/PythonFileDigest"
},
"size": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"PythonPackageMetadata": {
"required": [
"name",
"version",
"license",
"author",
"authorEmail",
"platform",
"sitePackagesRootPath"
],
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"license": {
"type": "string"
},
"author": {
"type": "string"
},
"authorEmail": {
"type": "string"
},
"platform": {
"type": "string"
},
"files": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/PythonFileRecord"
},
"type": "array"
},
"sitePackagesRootPath": {
"type": "string"
},
"topLevelPackages": {
"items": {
"type": "string"
},
"type": "array"
}
},
"additionalProperties": false,
"type": "object"
},
"RpmdbFileRecord": {
"required": [
"path",
"mode",
"size",
"sha256"
],
"properties": {
"path": {
"type": "string"
},
"mode": {
"type": "integer"
},
"size": {
"type": "integer"
},
"sha256": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"RpmdbMetadata": {
"required": [
"name",
"version",
"epoch",
"architecture",
"release",
"sourceRpm",
"size",
"license",
"vendor",
"files"
],
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"epoch": {
"type": "integer"
},
"architecture": {
"type": "string"
},
"release": {
"type": "string"
},
"sourceRpm": {
"type": "string"
},
"size": {
"type": "integer"
},
"license": {
"type": "string"
},
"vendor": {
"type": "string"
},
"files": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/RpmdbFileRecord"
},
"type": "array"
}
},
"additionalProperties": false,
"type": "object"
},
"Schema": {
"required": [
"version",
"url"
],
"properties": {
"version": {
"type": "string"
},
"url": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"Source": {
"required": [
"type",
"target"
],
"properties": {
"type": {
"type": "string"
},
"target": {
"additionalProperties": true
}
},
"additionalProperties": false,
"type": "object"
}
}
}
-165
View File
@@ -1,165 +0,0 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Document",
"definitions": {
"Descriptor": {
"required": [
"name",
"version"
],
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"Distribution": {
"required": [
"name",
"version",
"idLike"
],
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"idLike": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"Document": {
"required": [
"artifacts",
"source",
"distro",
"descriptor"
],
"properties": {
"artifacts": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Package"
},
"type": "array"
},
"source": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Source"
},
"distro": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Distribution"
},
"descriptor": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Descriptor"
}
},
"additionalProperties": false,
"type": "object"
},
"Location": {
"required": [
"path"
],
"properties": {
"path": {
"type": "string"
},
"layerID": {
"type": "string"
}
},
"additionalProperties": false,
"type": "object"
},
"Package": {
"required": [
"name",
"version",
"type",
"foundBy",
"locations",
"licenses",
"language",
"cpes",
"purl",
"metadataType"
],
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"type": {
"type": "string"
},
"foundBy": {
"type": "string"
},
"locations": {
"items": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/Location"
},
"type": "array"
},
"licenses": {
"items": {
"type": "string"
},
"type": "array"
},
"language": {
"type": "string"
},
"cpes": {
"items": {
"type": "string"
},
"type": "array"
},
"purl": {
"type": "string"
},
"metadataType": {
"type": "string"
},
"metadata": {
"additionalProperties": true
}
},
"additionalProperties": false,
"type": "object"
},
"Source": {
"required": [
"type",
"target"
],
"properties": {
"type": {
"type": "string"
},
"target": {
"additionalProperties": true
}
},
"additionalProperties": false,
"type": "object"
}
}
}