add bom descriptor schema + test against xml schemas in pipeline (#163)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

.circleci/config.yml

@@ -15,7 +15,6 @@ jobs:
     # 2CPU / 4GB RAM
     resource_class: medium
     steps:
-
       - checkout
 
       - restore_cache:
@@ -48,7 +47,6 @@ jobs:
     # 2CPU / 4GB RAM
     resource_class: medium
     steps:
-
       - checkout
 
       - restore_cache:
@@ -69,15 +67,15 @@ jobs:
       - run:
           name: enable docker client
           command: |
-              # all of this to enable "circleci local execute ..." cli commands for /var/run/docker.sock
+            # all of this to enable "circleci local execute ..." cli commands for /var/run/docker.sock
-              mkdir -p ${HOME}/.local/bin
+            mkdir -p ${HOME}/.local/bin
-              cat \<< EOF > ${HOME}/.local/bin/docker
+            cat \<< EOF > ${HOME}/.local/bin/docker
-              #!/bin/bash
+            #!/bin/bash
-              set -xue
+            set -xue
-              sudo -E ${HOME}/.local/bin/docker.bin \$@
+            sudo -E ${HOME}/.local/bin/docker.bin \$@
-              EOF
+            EOF
-              sudo mv /usr/bin/docker ${HOME}/.local/bin/docker.bin
+            sudo mv /usr/bin/docker ${HOME}/.local/bin/docker.bin
-              chmod 755 ${HOME}/.local/bin/docker
+            chmod 755 ${HOME}/.local/bin/docker
 
       - run:
           name: build cache key for java test-fixture blobs
@@ -96,6 +94,10 @@ jobs:
           paths:
             - "syft/cataloger/java/test-fixtures/java-builds/packages"
 
+      - run:
+          name: validate syft output against the CycloneDX schema
+          command: make validate-cyclonedx-schema
+
       - run:
           name: build hash key for integration test-fixtures blobs
           command: make integration-fingerprint
@@ -124,4 +126,4 @@ workflows:
           version: "1.13"
       - run-tests:
           name: "Unit & Integration Tests (go-latest)"
-          version: "latest"
+          version: "latest"

Makefile

@@ -60,7 +60,7 @@ all: clean static-analysis test ## Run all linux-based checks (linting, license
 	@printf '$(SUCCESS)All checks pass!$(RESET)\n'
 
 .PHONY: test
-test: unit integration acceptance-linux ## Run all tests (currently unit, integration, and linux acceptance tests)
+test: unit validate-cyclonedx-schema integration acceptance-linux ## Run all tests (currently unit, integration, and linux acceptance tests)
 
 .PHONY: help
 help:
@@ -68,7 +68,7 @@ help:
 
 .PHONY: ci-bootstrap
 ci-bootstrap: bootstrap
-	sudo apt update && sudo apt install -y bc jq
+	DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils
 
 .PHONY: bootstrap
 bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir)
@@ -111,6 +111,10 @@ lint-fix: ## Auto-format all source code + run golangci lint fixers
 check-licenses:
 	$(TEMPDIR)/bouncer check
 
+.PHONY: validate-cyclonedx-schema
+validate-cyclonedx-schema:
+	cd schema/cyclonedx && make
+
 .PHONY: unit
 unit: fixtures ## Run unit tests (with coverage)
 	$(call title,Running unit tests)
@@ -143,7 +147,7 @@ generate-json-schema: clean-json-schema-examples integration ## Generate a new j
 	docker run \
 		-i \
 		--rm \
-		-v $(shell pwd)/json-schema:/work \
+		-v $(shell pwd)/schema/json:/work \
 		-w /work \
 		python:3.8 \
 			bash -x -c "\
@@ -269,4 +273,4 @@ clean-dist:
 
 .PHONY: clean-json-schema-examples
 clean-json-schema-examples:
-	rm -f json-schema/examples/*
+	rm -f schema/json/examples/*

schema/cyclonedx/.gitignore

@@ -0,0 +1 @@
+bom.xml

schema/cyclonedx/Makefile

@@ -0,0 +1,5 @@
+.DEFAULT_GOAL := validate-schema
+.PHONY: validate-schema
+validate-schema:
+	go run ../../main.go ubuntu:latest -vv -o cyclonedx > bom.xml
+	xmllint --noout --schema ./cyclonedx.xsd bom.xml

schema/cyclonedx/README.md

@@ -0,0 +1,7 @@
+# CycloneDX Schemas
+
+`syft` generates a CycloneDX BOm output. We want to be able to validate the CycloneDX schemas
+(and dependent schemas) against generated syft output. The best way to do this is with `xmllint`,
+however, this tool does not know how to deal with references from HTTP, only the local filesystem.
+For this reason we've included a copy of all schemas needed to validate `syft` output, modified
+to reference local copies of dependent schemas.

schema/cyclonedx/bd.xsd

@@ -0,0 +1,183 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+CycloneDX BOM Descriptor Extension
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+           xmlns:vc="http://www.w3.org/2007/XMLSchema-versioning"
+           xmlns:bom="http://cyclonedx.org/schema/bom/1.1"
+           xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0"
+           elementFormDefault="qualified"
+           targetNamespace="http://cyclonedx.org/schema/ext/bom-descriptor/1.0"
+           vc:minVersion="1.0"
+           vc:maxVersion="1.1"
+           version="1.0">
+
+  <xs:annotation>
+    <xs:documentation>
+      <name>CycloneDX BOM Descriptor Extension</name>
+      <url>https://cyclonedx.org/ext/bom-descriptor</url>
+      <license uri="http://www.apache.org/licenses/LICENSE-2.0"
+               version="2.0">Apache License, Version 2.0</license>
+      <authors>
+        <author>Steve Springett</author>
+      </authors>
+    </xs:documentation>
+  </xs:annotation>
+
+  <xs:import namespace="http://cyclonedx.org/schema/bom/1.1" schemaLocation="http://cyclonedx.org/schema/bom/1.1"/>
+
+  <xs:complexType name="metadata">
+    <xs:sequence minOccurs="0" maxOccurs="1">
+      <xs:element name="timestamp" type="xs:dateTime" minOccurs="0">
+        <xs:annotation>
+          <xs:documentation>The date and time (timestamp) when the document was created.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="tool" minOccurs="0" type="bd:toolType">
+        <xs:annotation>
+          <xs:documentation>The tool used to create the BOM.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="authors" minOccurs="0" maxOccurs="1">
+        <xs:annotation>
+          <xs:documentation>The person(s) who created the BOM. Authors are common in BOMs created through
+            manual processes. BOMs created through automated means may not have authors.</xs:documentation>
+        </xs:annotation>
+        <xs:complexType>
+          <xs:sequence minOccurs="0" maxOccurs="unbounded">
+            <xs:element name="author" type="bd:organizationalPerson"/>
+          </xs:sequence>
+        </xs:complexType>
+      </xs:element>
+      <xs:element name="component" type="bom:component" minOccurs="0">
+        <xs:annotation>
+          <xs:documentation>The component that the BOM describes.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="manufacture" type="bd:organizationalEntity" minOccurs="0" maxOccurs="unbounded">
+        <xs:annotation>
+          <xs:documentation>The organization that manufactured the component that the BOM describes.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="supplier" type="bd:organizationalEntity" minOccurs="0" maxOccurs="unbounded">
+        <xs:annotation>
+          <xs:documentation>The organization that supplied the component that the BOM describes. The
+            supplier may often be the manufacture, but may also be a distributor or repackager.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+    </xs:sequence>
+    <xs:anyAttribute namespace="##other" processContents="lax">
+      <xs:annotation>
+        <xs:documentation>User-defined attributes may be used on this element as long as they
+          do not have the same name as an existing attribute used by the schema.</xs:documentation>
+      </xs:annotation>
+    </xs:anyAttribute>
+  </xs:complexType>
+
+  <xs:complexType name="organizationalEntity">
+    <xs:sequence minOccurs="0" maxOccurs="1">
+      <xs:element name="name" type="xs:normalizedString" minOccurs="0" maxOccurs="1">
+        <xs:annotation>
+          <xs:documentation>The name of the organization</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="url" type="xs:anyURI" minOccurs="0" maxOccurs="unbounded">
+        <xs:annotation>
+          <xs:documentation>The URL of the organization. Multiple URLs are allowed.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="contact" type="bd:organizationalPerson" minOccurs="0" maxOccurs="unbounded">
+        <xs:annotation>
+          <xs:documentation>A contact person at the organization. Multiple contacts are allowed.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+    </xs:sequence>
+    <xs:anyAttribute namespace="##other" processContents="lax">
+      <xs:annotation>
+        <xs:documentation>User-defined attributes may be used on this element as long as they
+          do not have the same name as an existing attribute used by the schema.</xs:documentation>
+      </xs:annotation>
+    </xs:anyAttribute>
+  </xs:complexType>
+
+  <xs:complexType name="toolType">
+    <xs:annotation>
+      <xs:documentation>Specifies a tool (manual or automated).</xs:documentation>
+    </xs:annotation>
+    <xs:sequence minOccurs="0" maxOccurs="1">
+      <xs:element name="vendor" minOccurs="0" maxOccurs="1" type="xs:normalizedString">
+        <xs:annotation>
+          <xs:documentation>The vendor of the tool used to create the BOM.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="name" minOccurs="0" maxOccurs="1" type="xs:normalizedString">
+        <xs:annotation>
+          <xs:documentation>The name of the tool used to create the BOM.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="version" minOccurs="0" maxOccurs="1" type="xs:normalizedString">
+        <xs:annotation>
+          <xs:documentation>The version of the tool used to create the BOM.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="hashes" minOccurs="0" maxOccurs="1">
+        <xs:complexType>
+          <xs:sequence minOccurs="0" maxOccurs="unbounded">
+            <xs:element name="hash" type="bom:hashType"/>
+          </xs:sequence>
+        </xs:complexType>
+      </xs:element>
+    </xs:sequence>
+    <xs:anyAttribute namespace="##other" processContents="lax">
+      <xs:annotation>
+        <xs:documentation>User-defined attributes may be used on this element as long as they
+          do not have the same name as an existing attribute used by the schema.</xs:documentation>
+      </xs:annotation>
+    </xs:anyAttribute>
+  </xs:complexType>
+
+  <xs:complexType name="organizationalPerson">
+    <xs:sequence minOccurs="0" maxOccurs="1">
+      <xs:element name="name" type="xs:normalizedString" minOccurs="0" maxOccurs="1">
+        <xs:annotation>
+          <xs:documentation>The name of the person</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="email" type="xs:normalizedString" minOccurs="0" maxOccurs="unbounded">
+        <xs:annotation>
+          <xs:documentation>The email address of the person. Multiple email addresses are allowed.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="phone" type="xs:normalizedString" minOccurs="0" maxOccurs="unbounded">
+        <xs:annotation>
+          <xs:documentation>The phone number of the person. Multiple phone numbers are allowed.</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+    </xs:sequence>
+    <xs:anyAttribute namespace="##other" processContents="lax">
+      <xs:annotation>
+        <xs:documentation>User-defined attributes may be used on this element as long as they
+          do not have the same name as an existing attribute used by the schema.</xs:documentation>
+      </xs:annotation>
+    </xs:anyAttribute>
+  </xs:complexType>
+
+  <xs:element name="metadata" type="bd:metadata">
+    <xs:annotation>
+      <xs:documentation>Provides additional information about a BOM.</xs:documentation>
+    </xs:annotation>
+  </xs:element>
+
+</xs:schema>

syft/presenter/cyclonedx/document.go

@@ -13,6 +13,7 @@ import (
 type Document struct {
 	XMLName       xml.Name       `xml:"bom"`
 	XMLNs         string         `xml:"xmlns,attr"`
+	XMLNsBd       string         `xml:"xmlns:bd,attr"`
 	Version       int            `xml:"version,attr"`
 	SerialNumber  string         `xml:"serialNumber,attr"`
 	Components    []Component    `xml:"components>component"` // The BOM contents
@@ -23,6 +24,7 @@ type Document struct {
 func NewDocument() Document {
 	return Document{
 		XMLNs:        "http://cyclonedx.org/schema/bom/1.2",
+		XMLNsBd:      "http://cyclonedx.org/schema/ext/bom-descriptor/1.0",
 		Version:      1,
 		SerialNumber: uuid.New().URN(),
 	}

syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
- <bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:1519f630-0721-40f5-8462-277e6534761d">
+ <bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:0667955f-34d0-49c1-8062-996dbc636974">
    <components>
      <component type="library">
        <name>package-1</name>
@@ -19,7 +19,7 @@
      </component>
    </components>
    <bd:metadata>
-     <bd:timestamp>2020-08-24T17:37:37-04:00</bd:timestamp>
+     <bd:timestamp>2020-08-26T15:31:39-04:00</bd:timestamp>
      <bd:tool>
        <bd:vendor>anchore</bd:vendor>
        <bd:name>syft</bd:name>

syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
- <bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:4448d650-a65e-49e4-b826-30d2010dfcd9">
+ <bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:2e7af7d0-83a4-4730-9175-66858406b285">
    <components>
      <component type="library">
        <name>package-1</name>
@@ -19,7 +19,7 @@
      </component>
    </components>
    <bd:metadata>
-     <bd:timestamp>2020-08-24T17:37:37-04:00</bd:timestamp>
+     <bd:timestamp>2020-08-26T15:31:39-04:00</bd:timestamp>
      <bd:tool>
        <bd:vendor>anchore</bd:vendor>
        <bd:name>syft</bd:name>

test/integration/json_schema_test.go

@@ -5,21 +5,23 @@ package integration
 import (
 	"bytes"
 	"fmt"
-	"github.com/anchore/go-testutils"
-	"github.com/anchore/syft/syft"
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/presenter"
-	"github.com/anchore/syft/syft/scope"
-	"github.com/xeipuuv/gojsonschema"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/anchore/go-testutils"
+	"github.com/anchore/syft/syft"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/presenter"
+	"github.com/anchore/syft/syft/scope"
+	"github.com/xeipuuv/gojsonschema"
 )
 
-const jsonSchemaExamplesPath = "json-schema/examples"
+const jsonSchemaPath = "schema/json"
+const jsonSchemaExamplesPath = jsonSchemaPath + "/examples"
 
 func repoRoot(t *testing.T) string {
 	t.Helper()
@@ -35,7 +37,7 @@ func repoRoot(t *testing.T) string {
 }
 
 func validateAgainstV1Schema(t *testing.T, json string) {
-	fullSchemaPath := path.Join(repoRoot(t), "json-schema", "schema.json")
+	fullSchemaPath := path.Join(repoRoot(t), jsonSchemaPath, "schema.json")
 	schemaLoader := gojsonschema.NewReferenceLoader(fmt.Sprintf("file://%s", fullSchemaPath))
 	documentLoader := gojsonschema.NewStringLoader(json)