Split the sbom.Format interface by encode and decode use cases (#2186)

* split up sbom.Format into encode and decode ops

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update cmd pkg to inject format configs

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* bump cyclonedx schema to 1.5

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* redact image metadata from github encoder tests

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add more testing around format decoder identify

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add test case for format version options

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix cli tests

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix CLI test

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* [wip] - review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* keep encoder creation out of post load function

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* keep decider and identify functions

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add a few more doc comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove format encoder default function helpers

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address PR feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* move back to streaming based decode functions

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* with common convention for encoder constructors

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix tests and allow for encoders to be created from cli options

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix cli tests

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix linting

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* buffer reads from stdin to support seeking

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2023-10-25 13:43:06 +00:00
committed by GitHub
parent 7315f83f9d
commit 7392d607b6
254 changed files with 10450 additions and 2714 deletions
@@ -0,0 +1,33 @@
package cyclonedxutil
import (
"fmt"
"io"
"github.com/CycloneDX/cyclonedx-go"
)
type Decoder struct {
format cyclonedx.BOMFileFormat
}
func NewDecoder(format cyclonedx.BOMFileFormat) Decoder {
return Decoder{
format: format,
}
}
func (d Decoder) Decode(reader io.ReadSeeker) (*cyclonedx.BOM, error) {
doc := &cyclonedx.BOM{
Components: &[]cyclonedx.Component{},
}
if _, err := reader.Seek(0, io.SeekStart); err != nil {
return nil, fmt.Errorf("unable to seek to start of CycloneDX SBOM: %w", err)
}
err := cyclonedx.NewBOMDecoder(reader, d.format).Decode(doc)
if err != nil {
return nil, err
}
return doc, nil
}
@@ -0,0 +1,21 @@
package cyclonedxutil
import (
"bytes"
"encoding/json"
"testing"
"github.com/CycloneDX/cyclonedx-go"
"github.com/stretchr/testify/assert"
)
func Test_missingComponentsDecode(t *testing.T) {
bom := &cyclonedx.BOM{
SpecVersion: cyclonedx.SpecVersion1_4,
}
bomBytes, _ := json.Marshal(&bom)
dec := NewDecoder(cyclonedx.BOMFileFormatJSON)
_, err := dec.Decode(bytes.NewReader(bomBytes))
assert.NoError(t, err)
}
@@ -0,0 +1,37 @@
package cyclonedxutil
import (
"io"
"github.com/CycloneDX/cyclonedx-go"
"github.com/anchore/syft/syft/format/common/cyclonedxhelpers"
"github.com/anchore/syft/syft/sbom"
)
const DefaultVersion = "1.5"
type Encoder struct {
version cyclonedx.SpecVersion
format cyclonedx.BOMFileFormat
}
func NewEncoder(version string, format cyclonedx.BOMFileFormat) (Encoder, error) {
specVersion, err := SpecVersionFromString(version)
if err != nil {
return Encoder{}, err
}
return Encoder{
version: specVersion,
format: format,
}, nil
}
func (e Encoder) Encode(writer io.Writer, s sbom.SBOM) error {
bom := cyclonedxhelpers.ToFormatModel(s)
enc := cyclonedx.NewBOMEncoder(writer, e.format)
enc.SetPretty(true)
enc.SetEscapeHTML(false)
return enc.EncodeVersion(bom, e.version)
}
@@ -0,0 +1,66 @@
package cyclonedxutil
import (
"fmt"
"github.com/CycloneDX/cyclonedx-go"
"github.com/anchore/syft/syft/sbom"
)
const (
XMLFormatID sbom.FormatID = "cyclonedx-xml"
JSONFormatID sbom.FormatID = "cyclonedx-json"
)
func SupportedVersions(id sbom.FormatID) []string {
versions := []string{
"1.2",
"1.3",
"1.4",
"1.5",
}
if id != JSONFormatID {
// JSON format not supported for version < 1.2
versions = append([]string{"1.0", "1.1"}, versions...)
}
return versions
}
func SpecVersionFromString(v string) (cyclonedx.SpecVersion, error) {
switch v {
case "1.0":
return cyclonedx.SpecVersion1_0, nil
case "1.1":
return cyclonedx.SpecVersion1_1, nil
case "1.2":
return cyclonedx.SpecVersion1_2, nil
case "1.3":
return cyclonedx.SpecVersion1_3, nil
case "1.4":
return cyclonedx.SpecVersion1_4, nil
case "1.5":
return cyclonedx.SpecVersion1_5, nil
}
return -1, fmt.Errorf("unsupported CycloneDX version %q", v)
}
func VersionFromSpecVersion(spec cyclonedx.SpecVersion) string {
switch spec {
case cyclonedx.SpecVersion1_0:
return "1.0"
case cyclonedx.SpecVersion1_1:
return "1.1"
case cyclonedx.SpecVersion1_2:
return "1.2"
case cyclonedx.SpecVersion1_3:
return "1.3"
case cyclonedx.SpecVersion1_4:
return "1.4"
case cyclonedx.SpecVersion1_5:
return "1.5"
}
return ""
}
+26
View File
@@ -0,0 +1,26 @@
package spdxutil
import (
"github.com/anchore/syft/syft/sbom"
)
const DefaultVersion = "2.3"
const (
JSONFormatID sbom.FormatID = "spdx-json"
TagValueFormatID sbom.FormatID = "spdx-tag-value"
)
func SupportedVersions(id sbom.FormatID) []string {
versions := []string{
"2.2",
"2.3",
}
if id != JSONFormatID {
// JSON format is not supported in v2.1
return append([]string{"2.1"}, versions...)
}
return versions
}
@@ -0,0 +1,204 @@
package testutil
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
func DirectoryInput(t testing.TB, dir string) sbom.SBOM {
catalog := newDirectoryCatalog()
path := filepath.Join(dir, "some", "path")
require.NoError(t, os.MkdirAll(path, 0755))
src, err := source.NewFromDirectory(
source.DirectoryConfig{
Path: path,
Base: dir,
},
)
require.NoError(t, err)
return sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: catalog,
LinuxDistribution: &linux.Release{
PrettyName: "debian",
Name: "debian",
ID: "debian",
IDLike: []string{"like!"},
Version: "1.2.3",
VersionID: "1.2.3",
},
},
Source: src.Describe(),
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
// the application configuration should be persisted here, however, we do not want to import
// the application configuration in this package (it's reserved only for ingestion by the cmd package)
Configuration: map[string]string{
"config-key": "config-value",
},
},
}
}
func DirectoryInputWithAuthorField(t testing.TB) sbom.SBOM {
catalog := newDirectoryCatalogWithAuthorField()
dir := t.TempDir()
path := filepath.Join(dir, "some", "path")
require.NoError(t, os.MkdirAll(path, 0755))
src, err := source.NewFromDirectory(
source.DirectoryConfig{
Path: path,
Base: dir,
},
)
require.NoError(t, err)
return sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: catalog,
LinuxDistribution: &linux.Release{
PrettyName: "debian",
Name: "debian",
ID: "debian",
IDLike: []string{"like!"},
Version: "1.2.3",
VersionID: "1.2.3",
},
},
Source: src.Describe(),
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
// the application configuration should be persisted here, however, we do not want to import
// the application configuration in this package (it's reserved only for ingestion by the cmd package)
Configuration: map[string]string{
"config-key": "config-value",
},
},
}
}
func newDirectoryCatalog() *pkg.Collection {
catalog := pkg.NewCollection()
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
),
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
Files: []pkg.PythonFileRecord{
{
Path: "/some/path/pkg1/dependencies/foo",
},
},
},
PURL: "a-purl-2", // intentionally a bad pURL for test fixtures
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
PURL: "pkg:deb/debian/package-2@2.0.1",
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
return catalog
}
func newDirectoryCatalogWithAuthorField() *pkg.Collection {
catalog := pkg.NewCollection()
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
),
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
Author: "test-author",
Files: []pkg.PythonFileRecord{
{
Path: "/some/path/pkg1/dependencies/foo",
},
},
},
PURL: "a-purl-2", // intentionally a bad pURL for test fixtures
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
PURL: "pkg:deb/debian/package-2@2.0.1",
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
return catalog
}
@@ -0,0 +1,32 @@
package testutil
import (
"math/rand"
"time"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/sbom"
)
//nolint:gosec
func AddSampleFileRelationships(s *sbom.SBOM) {
catalog := s.Artifacts.Packages.Sorted()
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
files := []string{"/f1", "/f2", "/d1/f3", "/d2/f4", "/z1/f5", "/a1/f6"}
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
rnd.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] })
for _, f := range files {
meta := file.Metadata{}
coords := file.Coordinates{RealPath: f}
s.Artifacts.FileMetadata[coords] = meta
s.Relationships = append(s.Relationships, artifact.Relationship{
From: catalog[0],
To: coords,
Type: artifact.ContainsRelationship,
})
}
}
@@ -0,0 +1,142 @@
package testutil
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
func ImageInput(t testing.TB, testImage string, options ...ImageOption) sbom.SBOM {
t.Helper()
catalog := pkg.NewCollection()
var cfg imageCfg
var img *image.Image
for _, opt := range options {
opt(&cfg)
}
defer changeToDirectoryWithGoldenFixture(t, testImage)()
switch cfg.fromSnapshot {
case true:
img = imagetest.GetGoldenFixtureImage(t, testImage)
default:
img = imagetest.GetFixtureImage(t, "docker-archive", testImage)
}
populateImageCatalog(catalog, img)
// this is a hard coded value that is not given by the fixture helper and must be provided manually
img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
src, err := source.NewFromStereoscopeImageObject(img, "user-image-input", nil)
assert.NoError(t, err)
return sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: catalog,
LinuxDistribution: &linux.Release{
PrettyName: "debian",
Name: "debian",
ID: "debian",
IDLike: []string{"like!"},
Version: "1.2.3",
VersionID: "1.2.3",
},
},
Source: src.Describe(),
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
// the application configuration should be persisted here, however, we do not want to import
// the application configuration in this package (it's reserved only for ingestion by the cmd package)
Configuration: map[string]string{
"config-key": "config-value",
},
},
}
}
func changeToDirectoryWithGoldenFixture(t testing.TB, testImage string) func() {
// check if test fixture exists... if not, check if there is a shared fixture relative to this dir
fn := func() {}
path := filepath.Join("test-fixtures", testImage)
if _, err := os.Stat(path); err != nil {
// change dir, restore as defer
wd, err := os.Getwd()
require.NoError(t, err)
fn = func() {
require.NoError(t, os.Chdir(wd))
}
// change dir to the testutil dir
require.NoError(t, os.Chdir(filepath.Join(wd, "..", "internal", "testutil")))
t.Cleanup(fn)
if _, err := os.Stat(path); err != nil {
t.Fatalf("unable to find test fixture: %s", path)
}
}
return fn
}
func populateImageCatalog(catalog *pkg.Collection, img *image.Image) {
_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Locations: file.NewLocationSet(
file.NewLocationFromImage(string(ref1.RealPath), *ref1.Reference, img),
),
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
),
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
},
PURL: "a-purl-1", // intentionally a bad pURL for test fixtures
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"),
},
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Locations: file.NewLocationSet(
file.NewLocationFromImage(string(ref2.RealPath), *ref2.Reference, img),
),
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
PURL: "pkg:deb/debian/package-2@2.0.1",
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
}
+142
View File
@@ -0,0 +1,142 @@
package testutil
import (
"bytes"
"regexp"
)
var (
_ Redactor = (*RedactorFn)(nil)
_ Redactor = (*PatternReplacement)(nil)
_ Redactor = (*ValueReplacement)(nil)
_ Redactor = (*Redactions)(nil)
)
type Redactor interface {
Redact([]byte) []byte
}
// Replace by function //////////////////////////////
type RedactorFn func([]byte) []byte
func (r RedactorFn) Redact(b []byte) []byte {
return r(b)
}
// Replace by regex //////////////////////////////
type PatternReplacement struct {
Search *regexp.Regexp
Replace string
}
func NewPatternReplacement(r *regexp.Regexp) PatternReplacement {
return PatternReplacement{
Search: r,
Replace: "redacted",
}
}
func (p PatternReplacement) Redact(b []byte) []byte {
return p.Search.ReplaceAll(b, []byte(p.Replace))
}
// Replace by value //////////////////////////////
type ValueReplacement struct {
Search string
Replace string
}
func NewValueReplacement(v string) ValueReplacement {
return ValueReplacement{
Search: v,
Replace: "redacted",
}
}
func (v ValueReplacement) Redact(b []byte) []byte {
return bytes.ReplaceAll(b, []byte(v.Search), []byte(v.Replace))
}
// Handle a collection of redactors //////////////////////////////
type Redactions struct {
redactors []Redactor
}
func NewRedactions(redactors ...Redactor) *Redactions {
r := &Redactions{
redactors: redactors,
}
return r.WithFunctions(carriageRedactor)
}
func (r *Redactions) WithPatternRedactors(values map[string]string) *Redactions {
for k, v := range values {
r.redactors = append(r.redactors,
PatternReplacement{
Search: regexp.MustCompile(k),
Replace: v,
},
)
}
return r
}
func (r *Redactions) WithValueRedactors(values map[string]string) *Redactions {
for k, v := range values {
r.redactors = append(r.redactors,
ValueReplacement{
Search: k,
Replace: v,
},
)
}
return r
}
func (r *Redactions) WithPatternsRedacted(values ...string) *Redactions {
for _, pattern := range values {
r.redactors = append(r.redactors,
NewPatternReplacement(regexp.MustCompile(pattern)),
)
}
return r
}
func (r *Redactions) WithValuesRedacted(values ...string) *Redactions {
for _, v := range values {
r.redactors = append(r.redactors,
NewValueReplacement(v),
)
}
return r
}
func (r *Redactions) WithFunctions(values ...func([]byte) []byte) *Redactions {
for _, fn := range values {
r.redactors = append(r.redactors,
RedactorFn(fn),
)
}
return r
}
func (r *Redactions) WithRedactors(rs ...Redactor) *Redactions {
r.redactors = append(r.redactors, rs...)
return r
}
func (r Redactions) Redact(b []byte) []byte {
for _, redactor := range r.redactors {
b = redactor.Redact(b)
}
return b
}
func carriageRedactor(s []byte) []byte {
return bytes.ReplaceAll(s, []byte("\r\n"), []byte("\n"))
}
+95
View File
@@ -0,0 +1,95 @@
package testutil
import (
"bytes"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/sbom"
)
type imageCfg struct {
fromSnapshot bool
}
type ImageOption func(cfg *imageCfg)
func FromSnapshot() ImageOption {
return func(cfg *imageCfg) {
cfg.fromSnapshot = true
}
}
type EncoderSnapshotTestConfig struct {
Subject sbom.SBOM
Format sbom.FormatEncoder
UpdateSnapshot bool
PersistRedactionsInSnapshot bool
IsJSON bool
Redactor Redactor
}
func AssertEncoderAgainstGoldenSnapshot(t *testing.T, cfg EncoderSnapshotTestConfig) {
t.Helper()
var buffer bytes.Buffer
err := cfg.Format.Encode(&buffer, cfg.Subject)
assert.NoError(t, err)
actual := buffer.Bytes()
if cfg.UpdateSnapshot && !cfg.PersistRedactionsInSnapshot {
// replace the expected snapshot contents with the current (unredacted) encoder contents
testutils.UpdateGoldenFileContents(t, actual)
return
}
var expected []byte
if cfg.Redactor != nil {
actual = cfg.Redactor.Redact(actual)
expected = cfg.Redactor.Redact(testutils.GetGoldenFileContents(t))
} else {
expected = testutils.GetGoldenFileContents(t)
}
if cfg.UpdateSnapshot && cfg.PersistRedactionsInSnapshot {
// replace the expected snapshot contents with the current (redacted) encoder contents
testutils.UpdateGoldenFileContents(t, actual)
return
}
if cfg.IsJSON {
require.JSONEq(t, string(expected), string(actual))
} else {
requireEqual(t, expected, actual)
}
}
func requireEqual(t *testing.T, expected any, actual any) {
if diff := cmp.Diff(expected, actual); diff != "" {
// uncomment to debug
// t.Logf("expected: %s", expected)
// t.Logf("actual: %s", actual)
t.Fatalf("mismatched output: %s", diff)
}
}
type ImageSnapshotTestConfig struct {
Image string
UpdateImageSnapshot bool
}
func AssertEncoderAgainstGoldenImageSnapshot(t *testing.T, imgCfg ImageSnapshotTestConfig, cfg EncoderSnapshotTestConfig) {
if imgCfg.UpdateImageSnapshot {
defer changeToDirectoryWithGoldenFixture(t, imgCfg.Image)()
// grab the latest image contents and persist
imagetest.UpdateGoldenFixtureImage(t, imgCfg.Image)
}
AssertEncoderAgainstGoldenSnapshot(t, cfg)
}
@@ -0,0 +1,4 @@
# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one.
FROM scratch
ADD file-1.txt /somefile-1.txt
ADD file-2.txt /somefile-2.txt
@@ -0,0 +1 @@
this file has contents
@@ -0,0 +1 @@
file-2 contents!