oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-04-01 06:33:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

Refactor source API (#1846)

Browse Source 
* refactor source API and syft json source block

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update source detection and format test utils

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* generate list of all source metadata types

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* extract base and root normalization into helper functions

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* preserve syftjson model package name import ref

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* alias should not be a pointer

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman 2023-06-30 10:19:16 -04:00 committed by GitHub
parent 608dbded06
commit 4da3be864f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
126 changed files with 7384 additions and 3190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
/.bin
CHANGELOG.md
VERSION
/test/results

2
Makefile
View File

@ -302,7 +302,7 @@ compare-test-rpm-package-install: $(TEMP_DIR) $(SNAPSHOT_DIR)
.PHONY: generate-json-schema
generate-json-schema: ## Generate a new json schema
cd schema/json && go generate . && go run .
cd syft/internal && go generate . && cd jsonschema && go run .
.PHONY: generate-license-list
generate-license-list: ## Generate an updated spdx license list

67
cmd/syft/cli/attest/attest.go
View File

@ -12,6 +12,7 @@ import (
"golang.org/x/exp/slices"
"github.com/anchore/stereoscope"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/cmd/syft/cli/eventloop"
"github.com/anchore/syft/cmd/syft/cli/options"
"github.com/anchore/syft/cmd/syft/cli/packages"
@ -34,17 +35,8 @@ func Run(_ context.Context, app *config.Application, args []string) error {
return err
}
// could be an image or a directory, with or without a scheme
// TODO: validate that source is image
// note: must be a container image
userInput := args[0]
si, err := source.ParseInputWithNameVersion(userInput, app.Platform, app.SourceName, app.SourceVersion, app.DefaultImagePullSource)
if err != nil {
return fmt.Errorf("could not generate source input for packages command: %w", err)
}
if si.Scheme != source.ImageScheme {
return fmt.Errorf("attestations are only supported for oci images at this time")
}
eventBus := partybus.NewBus()
stereoscope.SetBus(eventBus)
@ -52,7 +44,7 @@ func Run(_ context.Context, app *config.Application, args []string) error {
subscription := eventBus.Subscribe()
return eventloop.EventLoop(
execWorker(app, *si),
execWorker(app, userInput),
eventloop.SetupSignals(),
subscription,
stereoscope.Cleanup,
@ -60,13 +52,48 @@ func Run(_ context.Context, app *config.Application, args []string) error {
)
}
func buildSBOM(app *config.Application, si source.Input, errs chan error) (*sbom.SBOM, error) {
src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions)
if cleanup != nil {
defer cleanup()
func buildSBOM(app *config.Application, userInput string, errs chan error) (*sbom.SBOM, error) {
cfg := source.DetectConfig{
DefaultImageSource: app.DefaultImagePullSource,
}
detection, err := source.Detect(userInput, cfg)
if err != nil {
return nil, fmt.Errorf("could not deteremine source: %w", err)
}
if detection.IsContainerImage() {
return nil, fmt.Errorf("attestations are only supported for oci images at this time")
}
var platform *image.Platform
if app.Platform != "" {
platform, err = image.NewPlatform(app.Platform)
if err != nil {
return nil, fmt.Errorf("invalid platform: %w", err)
}
}
src, err := detection.NewSource(
source.DetectionSourceConfig{
Alias: source.Alias{
Name: app.SourceName,
Version: app.SourceVersion,
},
RegistryOptions: app.Registry.ToOptions(),
Platform: platform,
Exclude: source.ExcludeConfig{
Paths: app.Exclusions,
},
DigestAlgorithms: nil,
},
)
if src != nil {
defer src.Close()
}
if err != nil {
return nil, fmt.Errorf("failed to construct source from user input %q: %w", si.UserInput, err)
return nil, fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
}
s, err := packages.GenerateSBOM(src, errs, app)
@ -75,20 +102,20 @@ func buildSBOM(app *config.Application, si source.Input, errs chan error) (*sbom
}
if s == nil {
return nil, fmt.Errorf("no SBOM produced for %q", si.UserInput)
return nil, fmt.Errorf("no SBOM produced for %q", userInput)
}
return s, nil
}
//nolint:funlen
func execWorker(app *config.Application, si source.Input) <-chan error {
func execWorker(app *config.Application, userInput string) <-chan error {
errs := make(chan error)
go func() {
defer close(errs)
defer bus.Publish(partybus.Event{Type: event.Exit})
s, err := buildSBOM(app, si, errs)
s, err := buildSBOM(app, userInput, errs)
if err != nil {
errs <- fmt.Errorf("unable to build SBOM: %w", err)
return
@ -136,7 +163,7 @@ func execWorker(app *config.Application, si source.Input) <-chan error {
predicateType = "custom"
}
args := []string{"attest", si.UserInput, "--predicate", f.Name(), "--type", predicateType}
args := []string{"attest", userInput, "--predicate", f.Name(), "--type", predicateType}
if app.Attest.Key != "" {
args = append(args, "--key", app.Attest.Key)
}

14
cmd/syft/cli/eventloop/tasks.go
View File

@ -16,7 +16,7 @@ import (
"github.com/anchore/syft/syft/source"
)
type Task func(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error)
type Task func(*sbom.Artifacts, source.Source) ([]artifact.Relationship, error)
func Tasks(app *config.Application) ([]Task, error) {
var tasks []Task
@ -48,7 +48,7 @@ func generateCatalogPackagesTask(app *config.Application) (Task, error) {
return nil, nil
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, app.ToCatalogerConfig())
results.Packages = packageCatalog
@ -67,7 +67,7 @@ func generateCatalogFileMetadataTask(app *config.Application) (Task, error) {
metadataCataloger := filemetadata.NewCataloger()
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
if err != nil {
return nil, err
@ -110,7 +110,7 @@ func generateCatalogFileDigestsTask(app *config.Application) (Task, error) {
digestsCataloger := filedigest.NewCataloger(hashes)
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
if err != nil {
return nil, err
@ -142,7 +142,7 @@ func generateCatalogSecretsTask(app *config.Application) (Task, error) {
return nil, err
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.Secrets.Cataloger.ScopeOpt)
if err != nil {
return nil, err
@ -169,7 +169,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) {
return nil, err
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.FileContents.Cataloger.ScopeOpt)
if err != nil {
return nil, err
@ -186,7 +186,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) {
return task, nil
}
func RunTask(t Task, a *sbom.Artifacts, src *source.Source, c chan<- artifact.Relationship, errs chan<- error) {
func RunTask(t Task, a *sbom.Artifacts, src source.Source, c chan<- artifact.Relationship, errs chan<- error) {
defer close(c)
relationships, err := t(a, src)

60
cmd/syft/cli/packages/packages.go
View File

@ -7,6 +7,7 @@ import (
"github.com/wagoodman/go-partybus"
"github.com/anchore/stereoscope"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/cmd/syft/cli/eventloop"
"github.com/anchore/syft/cmd/syft/cli/options"
"github.com/anchore/syft/internal"
@ -35,10 +36,6 @@ func Run(_ context.Context, app *config.Application, args []string) error {
// could be an image or a directory, with or without a scheme
userInput := args[0]
si, err := source.ParseInputWithNameVersion(userInput, app.Platform, app.SourceName, app.SourceVersion, app.DefaultImagePullSource)
if err != nil {
return fmt.Errorf("could not generate source input for packages command: %w", err)
}
eventBus := partybus.NewBus()
stereoscope.SetBus(eventBus)
@ -46,7 +43,7 @@ func Run(_ context.Context, app *config.Application, args []string) error {
subscription := eventBus.Subscribe()
return eventloop.EventLoop(
execWorker(app, *si, writer),
execWorker(app, userInput, writer),
eventloop.SetupSignals(),
subscription,
stereoscope.Cleanup,
@ -54,17 +51,52 @@ func Run(_ context.Context, app *config.Application, args []string) error {
)
}
func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-chan error {
func execWorker(app *config.Application, userInput string, writer sbom.Writer) <-chan error {
errs := make(chan error)
go func() {
defer close(errs)
src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions)
if cleanup != nil {
defer cleanup()
detection, err := source.Detect(
userInput,
source.DetectConfig{
DefaultImageSource: app.DefaultImagePullSource,
},
)
if err != nil {
errs <- fmt.Errorf("could not deteremine source: %w", err)
return
}
var platform *image.Platform
if app.Platform != "" {
platform, err = image.NewPlatform(app.Platform)
if err != nil {
errs <- fmt.Errorf("invalid platform: %w", err)
return
}
}
src, err := detection.NewSource(
source.DetectionSourceConfig{
Alias: source.Alias{
Name: app.SourceName,
Version: app.SourceVersion,
},
RegistryOptions: app.Registry.ToOptions(),
Platform: platform,
Exclude: source.ExcludeConfig{
Paths: app.Exclusions,
},
DigestAlgorithms: nil,
},
)
if src != nil {
defer src.Close()
}
if err != nil {
errs <- fmt.Errorf("failed to construct source from user input %q: %w", si.UserInput, err)
errs <- fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
return
}
@ -75,7 +107,7 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-
}
if s == nil {
errs <- fmt.Errorf("no SBOM produced for %q", si.UserInput)
errs <- fmt.Errorf("no SBOM produced for %q", userInput)
}
bus.Publish(partybus.Event{
@ -86,14 +118,14 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-
return errs
}
func GenerateSBOM(src *source.Source, errs chan error, app *config.Application) (*sbom.SBOM, error) {
func GenerateSBOM(src source.Source, errs chan error, app *config.Application) (*sbom.SBOM, error) {
tasks, err := eventloop.Tasks(app)
if err != nil {
return nil, err
}
s := sbom.SBOM{
Source: src.Metadata,
Source: src.Describe(),
Descriptor: sbom.Descriptor{
Name: internal.ApplicationName,
Version: version.FromBuild().Version,
@ -106,7 +138,7 @@ func GenerateSBOM(src *source.Source, errs chan error, app *config.Application)
return &s, nil
}
func buildRelationships(s *sbom.SBOM, src *source.Source, tasks []eventloop.Task, errs chan error) {
func buildRelationships(s *sbom.SBOM, src source.Source, tasks []eventloop.Task, errs chan error) {
var relationships []<-chan artifact.Relationship
for _, task := range tasks {
c := make(chan artifact.Relationship)

55
cmd/syft/cli/poweruser/poweruser.go
View File

@ -9,6 +9,7 @@ import (
"github.com/wagoodman/go-partybus"
"github.com/anchore/stereoscope"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/cmd/syft/cli/eventloop"
"github.com/anchore/syft/cmd/syft/cli/options"
"github.com/anchore/syft/cmd/syft/cli/packages"
@ -38,10 +39,6 @@ func Run(_ context.Context, app *config.Application, args []string) error {
}()
userInput := args[0]
si, err := source.ParseInputWithNameVersion(userInput, app.Platform, app.SourceName, app.SourceVersion, app.DefaultImagePullSource)
if err != nil {
return fmt.Errorf("could not generate source input for packages command: %w", err)
}
eventBus := partybus.NewBus()
stereoscope.SetBus(eventBus)
@ -49,7 +46,7 @@ func Run(_ context.Context, app *config.Application, args []string) error {
subscription := eventBus.Subscribe()
return eventloop.EventLoop(
execWorker(app, *si, writer),
execWorker(app, userInput, writer),
eventloop.SetupSignals(),
subscription,
stereoscope.Cleanup,
@ -57,7 +54,8 @@ func Run(_ context.Context, app *config.Application, args []string) error {
)
}
func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-chan error {
//nolint:funlen
func execWorker(app *config.Application, userInput string, writer sbom.Writer) <-chan error {
errs := make(chan error)
go func() {
defer close(errs)
@ -72,17 +70,52 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-
return
}
src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions)
detection, err := source.Detect(
userInput,
source.DetectConfig{
DefaultImageSource: app.DefaultImagePullSource,
},
)
if err != nil {
errs <- err
errs <- fmt.Errorf("could not deteremine source: %w", err)
return
}
if cleanup != nil {
defer cleanup()
var platform *image.Platform
if app.Platform != "" {
platform, err = image.NewPlatform(app.Platform)
if err != nil {
errs <- fmt.Errorf("invalid platform: %w", err)
return
}
}
src, err := detection.NewSource(
source.DetectionSourceConfig{
Alias: source.Alias{
Name: app.SourceName,
Version: app.SourceVersion,
},
RegistryOptions: app.Registry.ToOptions(),
Platform: platform,
Exclude: source.ExcludeConfig{
Paths: app.Exclusions,
},
DigestAlgorithms: nil,
},
)
if src != nil {
defer src.Close()
}
if err != nil {
errs <- fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
return
}
s := sbom.SBOM{
Source: src.Metadata,
Source: src.Describe(),
Descriptor: sbom.Descriptor{
Name: internal.ApplicationName,
Version: version.FromBuild().Version,

2
internal/constants.go
View File

@ -6,5 +6,5 @@ const (
// JSONSchemaVersion is the current schema version output by the JSON encoder
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
JSONSchemaVersion = "8.0.1"
JSONSchemaVersion = "9.0.0"
)

12
schema/json/README.md
View File

@ -3,8 +3,8 @@
This is the JSON schema for output from the JSON presenters (`syft packages <img> -o json` and `syft power-user <img>`). The required inputs for defining the JSON schema are as follows:
- the value of `internal.JSONSchemaVersion` that governs the schema filename
- the `Document` struct definition within `internal/presenters/poweruser/json_document.go` that governs the overall document shape
- the `artifactMetadataContainer` struct definition within `schema/json/generate.go` that governs the allowable shapes of `pkg.Package.Metadata`
- the `Document` struct definition within `github.com/anchore/syft/syft/formats/syftjson/model/document.go` that governs the overall document shape
- generated `AllTypes()` helper function within the `syft/internal/sourcemetadata` and `syft/internal/packagemetadata` packages
With regard to testing the JSON schema, integration test cases provided by the developer are used as examples to validate that JSON output from Syft is always valid relative to the `schema/json/schema-$VERSION.json` file.
@ -22,15 +22,13 @@ Given a version number format `MODEL.REVISION.ADDITION`:
## Adding a New `pkg.*Metadata` Type
When adding a new `pkg.*Metadata` that is assigned to the `pkg.Package.Metadata` struct field it is important that a few things
are done:
When adding a new `pkg.*Metadata` that is assigned to the `pkg.Package.Metadata` struct field you must add a test case to `test/integration/catalog_packages_cases_test.go` that exercises the new package type with the new metadata.
- a new integration test case is added to `test/integration/catalog_packages_cases_test.go` that exercises the new package type with the new metadata
- the new metadata struct is added to the `artifactMetadataContainer` struct within `schema/json/generate.go`
Additionally it is important to generate a new JSON schema since the `pkg.Package.Metadata` field is covered by the schema.
## Generating a New Schema
Create the new schema by running `cd schema/json && go run generate.go` (note you must be in the `schema/json` dir while running this):
Create the new schema by running `make generate-json-schema` from the root of the repo:
- If there is **not** an existing schema for the given version, then the new schema file will be written to `schema/json/schema-$VERSION.json`
- If there is an existing schema for the given version and the new schema matches the existing schema, no action is taken

50
schema/json/generate/main.go
View File

@ -1,50 +0,0 @@
package main
import (
"fmt"
"os"
"github.com/dave/jennifer/jen"
"github.com/anchore/syft/schema/json/internal"
)
// This program generates internal/generated.go.
const (
pkgImport = "github.com/anchore/syft/syft/pkg"
path = "internal/generated.go"
)
func main() {
typeNames, err := internal.AllSyftMetadataTypeNames()
if err != nil {
panic(fmt.Errorf("unable to get all metadata type names: %w", err))
}
fmt.Printf("updating metadata container object with %+v types\n", len(typeNames))
f := jen.NewFile("internal")
f.HeaderComment("DO NOT EDIT: generated by schema/json/generate/main.go")
f.ImportName(pkgImport, "pkg")
f.Comment("ArtifactMetadataContainer is a struct that contains all the metadata types for a package, as represented in the pkg.Package.Metadata field.")
f.Type().Id("ArtifactMetadataContainer").StructFunc(func(g *jen.Group) {
for _, typeName := range typeNames {
g.Id(typeName).Qual(pkgImport, typeName)
}
})
rendered := fmt.Sprintf("%#v", f)
fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
panic(fmt.Errorf("unable to open file: %w", err))
}
_, err = fh.WriteString(rendered)
if err != nil {
panic(fmt.Errorf("unable to write file: %w", err))
}
if err := fh.Close(); err != nil {
panic(fmt.Errorf("unable to close file: %w", err))
}
}

39
schema/json/internal/generated.go
View File

@ -1,39 +0,0 @@
// DO NOT EDIT: generated by schema/json/generate/main.go
package internal
import "github.com/anchore/syft/syft/pkg"
// ArtifactMetadataContainer is a struct that contains all the metadata types for a package, as represented in the pkg.Package.Metadata field.
type ArtifactMetadataContainer struct {
AlpmMetadata pkg.AlpmMetadata
ApkMetadata pkg.ApkMetadata
BinaryMetadata pkg.BinaryMetadata
CargoPackageMetadata pkg.CargoPackageMetadata
CocoapodsMetadata pkg.CocoapodsMetadata
ConanLockMetadata pkg.ConanLockMetadata
ConanMetadata pkg.ConanMetadata
DartPubMetadata pkg.DartPubMetadata
DotnetDepsMetadata pkg.DotnetDepsMetadata
DpkgMetadata pkg.DpkgMetadata
GemMetadata pkg.GemMetadata
GolangBinMetadata pkg.GolangBinMetadata
GolangModMetadata pkg.GolangModMetadata
HackageMetadata pkg.HackageMetadata
JavaMetadata pkg.JavaMetadata
KbPackageMetadata pkg.KbPackageMetadata
LinuxKernelMetadata pkg.LinuxKernelMetadata
LinuxKernelModuleMetadata pkg.LinuxKernelModuleMetadata
MixLockMetadata pkg.MixLockMetadata
NixStoreMetadata pkg.NixStoreMetadata
NpmPackageJSONMetadata pkg.NpmPackageJSONMetadata
NpmPackageLockJSONMetadata pkg.NpmPackageLockJSONMetadata
PhpComposerJSONMetadata pkg.PhpComposerJSONMetadata
PortageMetadata pkg.PortageMetadata
PythonPackageMetadata pkg.PythonPackageMetadata
PythonPipfileLockMetadata pkg.PythonPipfileLockMetadata
PythonRequirementsMetadata pkg.PythonRequirementsMetadata
RDescriptionFileMetadata pkg.RDescriptionFileMetadata
RebarLockMetadata pkg.RebarLockMetadata
RpmMetadata pkg.RpmMetadata
}

39
schema/json/main_test.go
View File

@ -1,39 +0,0 @@
package main
import (
"reflect"
"sort"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/schema/json/internal"
)
func TestAllMetadataRepresented(t *testing.T) {
// this test checks that all the metadata types are represented in the currently generated ArtifactMetadataContainer struct
// such that PRs will reflect when there is drift from the implemented set of metadata types and the generated struct
// which controls the JSON schema content.
expected, err := internal.AllSyftMetadataTypeNames()
require.NoError(t, err)
actual := allTypeNamesFromStruct(internal.ArtifactMetadataContainer{})
if !assert.ElementsMatch(t, expected, actual) {
t.Errorf("metadata types not fully represented: \n%s", cmp.Diff(expected, actual))
t.Log("did you add a new pkg.*Metadata type without updating the JSON schema?")
t.Log("if so, you need to update the schema version and regenerate the JSON schema (make generate-json-schema)")
}
}
func allTypeNamesFromStruct(instance any) []string {
// get all the type names from the struct (not recursively)
var typeNames []string
tt := reflect.TypeOf(instance)
for i := 0; i < tt.NumField(); i++ {
field := tt.Field(i)
typeNames = append(typeNames, field.Type.Name())
}
sort.Strings(typeNames)
return typeNames
}

1881
schema/json/schema-9.0.0.json Normal file
View File

File diff suppressed because it is too large Load Diff

4
syft/file/cataloger/filedigest/cataloger_test.go
View File

@ -75,7 +75,7 @@ func TestDigestsCataloger(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
c := NewCataloger(test.digests)
src, err := source.NewFromDirectory("test-fixtures/last/")
src, err := source.NewFromDirectoryPath("test-fixtures/last/")
require.NoError(t, err)
resolver, err := src.FileResolver(source.SquashedScope)
@ -94,7 +94,7 @@ func TestDigestsCataloger_MixFileTypes(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
src, err := source.NewFromImage(img, "---")
src, err := source.NewFromStereoscopeImageObject(img, testImage, nil)
if err != nil {
t.Fatalf("could not create source: %+v", err)
}

2
syft/file/cataloger/filemetadata/cataloger_test.go
View File

@ -20,7 +20,7 @@ func TestFileMetadataCataloger(t *testing.T) {
c := NewCataloger()
src, err := source.NewFromImage(img, "---")
src, err := source.NewFromStereoscopeImageObject(img, testImage, nil)
if err != nil {
t.Fatalf("could not create source: %+v", err)
}

4
syft/file/cataloger/internal/all_regular_files_test.go
View File

@ -27,7 +27,7 @@ func Test_allRegularFiles(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
s, err := source.NewFromImage(img, "---")
s, err := source.NewFromStereoscopeImageObject(img, testImage, nil)
require.NoError(t, err)
r, err := s.FileResolver(source.SquashedScope)
@ -41,7 +41,7 @@ func Test_allRegularFiles(t *testing.T) {
{
name: "directory",
setup: func() file.Resolver {
s, err := source.NewFromDirectory("test-fixtures/symlinked-root/nested/link-root")
s, err := source.NewFromDirectoryPath("test-fixtures/symlinked-root/nested/link-root")
require.NoError(t, err)
r, err := s.FileResolver(source.SquashedScope)
require.NoError(t, err)

2
syft/file/test-fixtures/req-resp/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
path/to/abs-inside.txt
path/to/the/abs-outside.txt

1
syft/file/test-fixtures/req-resp/path/to/rel-inside.txt Symbolic link
View File

@ -0,0 +1 @@
./the/file.txt

1
syft/file/test-fixtures/req-resp/path/to/the/file.txt Normal file
View File

@ -0,0 +1 @@
file-1

1
syft/file/test-fixtures/req-resp/path/to/the/rel-outside.txt Symbolic link
View File

@ -0,0 +1 @@
../../../somewhere/outside.txt

1
syft/file/test-fixtures/req-resp/root-link Symbolic link
View File

@ -0,0 +1 @@
./

1
syft/file/test-fixtures/req-resp/somewhere/outside.txt Normal file
View File

@ -0,0 +1 @@
file-2

34
syft/formats/common/cyclonedxhelpers/decoder.go
View File

@ -229,32 +229,34 @@ func collectRelationships(bom *cyclonedx.BOM, s *sbom.SBOM, idMap map[string]int
}
}
func extractComponents(meta *cyclonedx.Metadata) source.Metadata {
func extractComponents(meta *cyclonedx.Metadata) source.Description {
if meta == nil || meta.Component == nil {
return source.Metadata{}
return source.Description{}
}
c := meta.Component
image := source.ImageMetadata{
UserInput: c.Name,
ID: c.BOMRef,
ManifestDigest: c.Version,
}
switch c.Type {
case cyclonedx.ComponentTypeContainer:
return source.Metadata{
Scheme: source.ImageScheme,
ImageMetadata: image,
return source.Description{
ID: "",
// TODO: can we decode alias name-version somehow? (it isn't be encoded in the first place yet)
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: c.Name,
ID: c.BOMRef,
ManifestDigest: c.Version,
},
}
case cyclonedx.ComponentTypeFile:
return source.Metadata{
Scheme: source.FileScheme, // or source.DirectoryScheme
Path: c.Name,
ImageMetadata: image,
// TODO: can we decode alias name-version somehow? (it isn't be encoded in the first place yet)
// TODO: this is lossy... we can't know if this is a file or a directory
return source.Description{
ID: "",
Metadata: source.FileSourceMetadata{Path: c.Name},
}
}
return source.Metadata{}
return source.Description{}
}
// if there is more than one tool in meta.Tools' list the last item will be used

49
syft/formats/common/cyclonedxhelpers/format.go
View File

@ -110,7 +110,7 @@ func formatCPE(cpeString string) string {
}
// NewBomDescriptor returns a new BomDescriptor tailored for the current time and "syft" tool details.
func toBomDescriptor(name, version string, srcMetadata source.Metadata) *cyclonedx.Metadata {
func toBomDescriptor(name, version string, srcMetadata source.Description) *cyclonedx.Metadata {
return &cyclonedx.Metadata{
Timestamp: time.Now().Format(time.RFC3339),
Tools: &[]cyclonedx.Tool{
@ -170,35 +170,56 @@ func toDependencies(relationships []artifact.Relationship) []cyclonedx.Dependenc
return result
}
func toBomDescriptorComponent(srcMetadata source.Metadata) *cyclonedx.Component {
func toBomDescriptorComponent(srcMetadata source.Description) *cyclonedx.Component {
name := srcMetadata.Name
switch srcMetadata.Scheme {
case source.ImageScheme:
version := srcMetadata.Version
switch metadata := srcMetadata.Metadata.(type) {
case source.StereoscopeImageSourceMetadata:
if name == "" {
name = srcMetadata.ImageMetadata.UserInput
name = metadata.UserInput
}
bomRef, err := artifact.IDByHash(srcMetadata.ImageMetadata.ID)
if version == "" {
version = metadata.ManifestDigest
}
bomRef, err := artifact.IDByHash(metadata.ID)
if err != nil {
log.Warnf("unable to get fingerprint of image metadata=%s: %+v", srcMetadata.ImageMetadata.ID, err)
log.Warnf("unable to get fingerprint of source image metadata=%s: %+v", metadata.ID, err)
}
return &cyclonedx.Component{
BOMRef: string(bomRef),
Type: cyclonedx.ComponentTypeContainer,
Name: name,
Version: srcMetadata.ImageMetadata.ManifestDigest,
Version: version,
}
case source.DirectoryScheme, source.FileScheme:
case source.DirectorySourceMetadata:
if name == "" {
name = srcMetadata.Path
name = metadata.Path
}
bomRef, err := artifact.IDByHash(srcMetadata.Path)
bomRef, err := artifact.IDByHash(metadata.Path)
if err != nil {
log.Warnf("unable to get fingerprint of source metadata path=%s: %+v", srcMetadata.Path, err)
log.Warnf("unable to get fingerprint of source directory metadata path=%s: %+v", metadata.Path, err)
}
return &cyclonedx.Component{
BOMRef: string(bomRef),
Type: cyclonedx.ComponentTypeFile,
Name: name,
// TODO: this is lossy... we can't know if this is a file or a directory
Type: cyclonedx.ComponentTypeFile,
Name: name,
Version: version,
}
case source.FileSourceMetadata:
if name == "" {
name = metadata.Path
}
bomRef, err := artifact.IDByHash(metadata.Path)
if err != nil {
log.Warnf("unable to get fingerprint of source file metadata path=%s: %+v", metadata.Path, err)
}
return &cyclonedx.Component{
BOMRef: string(bomRef),
// TODO: this is lossy... we can't know if this is a file or a directory
Type: cyclonedx.ComponentTypeFile,
Name: name,
Version: version,
}
}

14
syft/formats/common/spdxhelpers/document_name.go
View File

@ -4,16 +4,18 @@ import (
"github.com/anchore/syft/syft/source"
)
func DocumentName(srcMetadata source.Metadata) string {
func DocumentName(srcMetadata source.Description) string {
if srcMetadata.Name != "" {
return srcMetadata.Name
}
switch srcMetadata.Scheme {
case source.ImageScheme:
return srcMetadata.ImageMetadata.UserInput
case source.DirectoryScheme, source.FileScheme:
return srcMetadata.Path
switch metadata := srcMetadata.Metadata.(type) {
case source.StereoscopeImageSourceMetadata:
return metadata.UserInput
case source.DirectorySourceMetadata:
return metadata.Path
case source.FileSourceMetadata:
return metadata.Path
default:
return "unknown"
}

31
syft/formats/common/spdxhelpers/document_name_test.go
View File

@ -5,31 +5,27 @@ import (
"strings"
"testing"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/internal/sourcemetadata"
"github.com/anchore/syft/syft/source"
)
func Test_DocumentName(t *testing.T) {
allSchemes := strset.New()
for _, s := range source.AllSchemes {
allSchemes.Add(string(s))
}
testedSchemes := strset.New()
tracker := sourcemetadata.NewCompletionTester(t)
tests := []struct {
name string
inputName string
srcMetadata source.Metadata
srcMetadata source.Description
expected string
}{
{
name: "image",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.ImageScheme,
ImageMetadata: source.ImageMetadata{
srcMetadata: source.Description{
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "image-repo/name:tag",
ID: "id",
ManifestDigest: "digest",
@ -40,18 +36,16 @@ func Test_DocumentName(t *testing.T) {
{
name: "directory",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.DirectoryScheme,
Path: "some/path/to/place",
srcMetadata: source.Description{
Metadata: source.DirectorySourceMetadata{Path: "some/path/to/place"},
},
expected: "some/path/to/place",
},
{
name: "file",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.FileScheme,
Path: "some/path/to/place",
srcMetadata: source.Description{
Metadata: source.FileSourceMetadata{Path: "some/path/to/place"},
},
expected: "some/path/to/place",
},
@ -62,10 +56,7 @@ func Test_DocumentName(t *testing.T) {
assert.True(t, strings.HasPrefix(actual, test.expected), fmt.Sprintf("actual name %q", actual))
// track each scheme tested (passed or not)
testedSchemes.Add(string(test.srcMetadata.Scheme))
tracker.Tested(t, test.srcMetadata.Metadata)
})
}
// assert all possible schemes were under test
assert.ElementsMatch(t, allSchemes.List(), testedSchemes.List(), "not all source.Schemes are under test")
}

16
syft/formats/common/spdxhelpers/document_namespace.go
View File

@ -18,20 +18,20 @@ const (
inputFile = "file"
)
func DocumentNameAndNamespace(srcMetadata source.Metadata) (string, string) {
name := DocumentName(srcMetadata)
return name, DocumentNamespace(name, srcMetadata)
func DocumentNameAndNamespace(src source.Description) (string, string) {
name := DocumentName(src)
return name, DocumentNamespace(name, src)
}
func DocumentNamespace(name string, srcMetadata source.Metadata) string {
func DocumentNamespace(name string, src source.Description) string {
name = cleanName(name)
input := "unknown-source-type"
switch srcMetadata.Scheme {
case source.ImageScheme:
switch src.Metadata.(type) {
case source.StereoscopeImageSourceMetadata:
input = inputImage
case source.DirectoryScheme:
case source.DirectorySourceMetadata:
input = inputDirectory
case source.FileScheme:
case source.FileSourceMetadata:
input = inputFile
}

42
syft/formats/common/spdxhelpers/document_namespace_test.go
View File

@ -5,31 +5,26 @@ import (
"strings"
"testing"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/internal/sourcemetadata"
"github.com/anchore/syft/syft/source"
)
func Test_documentNamespace(t *testing.T) {
allSchemes := strset.New()
for _, s := range source.AllSchemes {
allSchemes.Add(string(s))
}
testedSchemes := strset.New()
tracker := sourcemetadata.NewCompletionTester(t)
tests := []struct {
name string
inputName string
srcMetadata source.Metadata
expected string
name string
inputName string
src source.Description
expected string
}{
{
name: "image",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.ImageScheme,
ImageMetadata: source.ImageMetadata{
src: source.Description{
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "image-repo/name:tag",
ID: "id",
ManifestDigest: "digest",
@ -40,33 +35,32 @@ func Test_documentNamespace(t *testing.T) {
{
name: "directory",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.DirectoryScheme,
Path: "some/path/to/place",
src: source.Description{
Metadata: source.DirectorySourceMetadata{
Path: "some/path/to/place",
},
},
expected: "https://anchore.com/syft/dir/my-name-",
},
{
name: "file",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.FileScheme,
Path: "some/path/to/place",
src: source.Description{
Metadata: source.FileSourceMetadata{
Path: "some/path/to/place",
},
},
expected: "https://anchore.com/syft/file/my-name-",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual := DocumentNamespace(test.inputName, test.srcMetadata)
actual := DocumentNamespace(test.inputName, test.src)
// note: since the namespace ends with a UUID we check the prefix
assert.True(t, strings.HasPrefix(actual, test.expected), fmt.Sprintf("actual namespace %q", actual))
// track each scheme tested (passed or not)
testedSchemes.Add(string(test.srcMetadata.Scheme))
tracker.Tested(t, test.src.Metadata)
})
}
// assert all possible schemes were under test
assert.ElementsMatch(t, allSchemes.List(), testedSchemes.List(), "not all source.Schemes are under test")
}

23
syft/formats/common/spdxhelpers/to_syft_model.go
View File

@ -28,8 +28,7 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
spdxIDMap := make(map[string]interface{})
src := source.Metadata{Scheme: source.UnknownScheme}
src.Scheme = extractSchemeFromNamespace(doc.DocumentNamespace)
src := extractSourceFromNamespace(doc.DocumentNamespace)
s := &sbom.SBOM{
Source: src,
@ -54,24 +53,32 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
// image, directory, for example. This is our best effort to determine
// the scheme. Syft-generated SBOMs have in the namespace
// field a type encoded, which we try to identify here.
func extractSchemeFromNamespace(ns string) source.Scheme {
func extractSourceFromNamespace(ns string) source.Description {
u, err := url.Parse(ns)
if err != nil {
return source.UnknownScheme
return source.Description{
Metadata: nil,
}
}
parts := strings.Split(u.Path, "/")
for _, p := range parts {
switch p {
case inputFile:
return source.FileScheme
return source.Description{
Metadata: source.FileSourceMetadata{},
}
case inputImage:
return source.ImageScheme
return source.Description{
Metadata: source.StereoscopeImageSourceMetadata{},
}
case inputDirectory:
return source.DirectoryScheme
return source.Description{
Metadata: source.DirectorySourceMetadata{},
}
}
}
return source.UnknownScheme
return source.Description{}
}
func findLinuxReleaseByPURL(doc *spdx.Document) *linux.Release {

27
syft/formats/common/spdxhelpers/to_syft_model_test.go
View File

@ -1,6 +1,7 @@
package spdxhelpers
import (
"reflect"
"testing"
"github.com/spdx/tools-golang/spdx"
@ -197,36 +198,46 @@ func Test_extractMetadata(t *testing.T) {
func TestExtractSourceFromNamespaces(t *testing.T) {
tests := []struct {
namespace string
expected source.Scheme
expected any
}{
{
namespace: "https://anchore.com/syft/file/d42b01d0-7325-409b-b03f-74082935c4d3",
expected: source.FileScheme,
expected: source.FileSourceMetadata{},
},
{
namespace: "https://anchore.com/syft/image/d42b01d0-7325-409b-b03f-74082935c4d3",
expected: source.ImageScheme,
expected: source.StereoscopeImageSourceMetadata{},
},
{
namespace: "https://anchore.com/syft/dir/d42b01d0-7325-409b-b03f-74082935c4d3",
expected: source.DirectoryScheme,
expected: source.DirectorySourceMetadata{},
},
{
namespace: "https://another-host/blob/123",
expected: source.UnknownScheme,
expected: nil,
},
{
namespace: "bla bla",
expected: source.UnknownScheme,
expected: nil,
},
{
namespace: "",
expected: source.UnknownScheme,
expected: nil,
},
}
for _, tt := range tests {
require.Equal(t, tt.expected, extractSchemeFromNamespace(tt.namespace))
desc := extractSourceFromNamespace(tt.namespace)
if tt.expected == nil && desc.Metadata == nil {
return
}
if tt.expected != nil && desc.Metadata == nil {
t.Fatal("expected metadata but got nil")
}
if tt.expected == nil && desc.Metadata != nil {
t.Fatal("expected nil metadata but got something")
}
require.Equal(t, reflect.TypeOf(tt.expected), reflect.TypeOf(desc.Metadata))
}
}

69
syft/formats/cyclonedxjson/encoder_test.go
View File

@ -2,49 +2,62 @@ package cyclonedxjson
import (
"flag"
"regexp"
"testing"
"github.com/anchore/syft/syft/formats/internal/testutils"
)
var updateCycloneDx = flag.Bool("update-cyclonedx", false, "update the *.golden files for cyclone-dx encoders")
var updateSnapshot = flag.Bool("update-cyclonedx-json", false, "update the *.golden files for cyclone-dx JSON encoders")
var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing")
func TestCycloneDxDirectoryEncoder(t *testing.T) {
dir := t.TempDir()
testutils.AssertEncoderAgainstGoldenSnapshot(t,
Format(),
testutils.DirectoryInput(t),
*updateCycloneDx,
true,
cycloneDxRedactor,
testutils.EncoderSnapshotTestConfig{
Subject: testutils.DirectoryInput(t, dir),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: true,
Redactor: redactor(dir),
},
)
}
func TestCycloneDxImageEncoder(t *testing.T) {
testImage := "image-simple"
testutils.AssertEncoderAgainstGoldenImageSnapshot(t,
Format(),
testutils.ImageInput(t, testImage),
testImage,
*updateCycloneDx,
true,
cycloneDxRedactor,
testutils.ImageSnapshotTestConfig{
Image: testImage,
UpdateImageSnapshot: *updateImage,
},
testutils.EncoderSnapshotTestConfig{
Subject: testutils.ImageInput(t, testImage),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: true,
Redactor: redactor(),
},
)
}
func cycloneDxRedactor(s []byte) []byte {
replacements := map[string]string{
// UUIDs
`urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`: `urn:uuid:redacted`,
// timestamps
`([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`: `timestamp:redacted`,
// image hashes
`sha256:[A-Fa-f0-9]{64}`: `sha256:redacted`,
// bom-refs
`"bom-ref":\s*"[^"]+"`: `"bom-ref": "redacted"`,
}
for pattern, replacement := range replacements {
s = regexp.MustCompile(pattern).ReplaceAll(s, []byte(replacement))
}
return s
func redactor(values ...string) testutils.Redactor {
return testutils.NewRedactions().
WithValuesRedacted(values...).
WithPatternRedactors(
map[string]string{
// UUIDs
`urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`: `urn:uuid:redacted`,
// timestamps
`([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`: `timestamp:redacted`,
// image hashes
`sha256:[A-Fa-f0-9]{64}`: `sha256:redacted`,
// BOM refs
`"bom-ref":\s*"[^"]+"`: `"bom-ref":"redacted"`,
},
)
}

12
syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden
View File

@ -2,10 +2,10 @@
"$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json",
"bomFormat": "CycloneDX",
"specVersion": "1.4",
"serialNumber": "urn:uuid:1b71a5b4-4bc5-4548-a51a-212e631976cd",
"serialNumber": "urn:uuid:redacted",
"version": 1,
"metadata": {
"timestamp": "2023-05-08T14:40:32-04:00",
"timestamp": "timestamp:redacted",
"tools": [
{
"vendor": "anchore",
@ -14,14 +14,14 @@
}
],
"component": {
"bom-ref": "163686ac6e30c752",
"bom-ref":"redacted",
"type": "file",
"name": "/some/path"
"name": "some/path"
}
},
"components": [
{
"bom-ref": "8c7e1242588c971a",
"bom-ref":"redacted",
"type": "library",
"name": "package-1",
"version": "1.0.1",
@ -58,7 +58,7 @@
]
},
{
"bom-ref": "pkg:deb/debian/package-2@2.0.1?package-id=db4abfe497c180d3",
"bom-ref":"redacted",
"type": "library",
"name": "package-2",
"version": "2.0.1",

16
syft/formats/cyclonedxjson/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden
View File

@ -2,10 +2,10 @@
"$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json",
"bomFormat": "CycloneDX",
"specVersion": "1.4",
"serialNumber": "urn:uuid:1695d6ae-0ddf-4e77-9c9d-74df1bdd8d5b",
"serialNumber": "urn:uuid:redacted",
"version": 1,
"metadata": {
"timestamp": "2023-05-08T14:40:32-04:00",
"timestamp": "timestamp:redacted",
"tools": [
{
"vendor": "anchore",
@ -14,15 +14,15 @@
}
],
"component": {
"bom-ref": "38160ebc2a6876e8",
"bom-ref":"redacted",
"type": "container",
"name": "user-image-input",
"version": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
"version": "sha256:redacted"
}
},
"components": [
{
"bom-ref": "ec2e0c93617507ef",
"bom-ref":"redacted",
"type": "library",
"name": "package-1",
"version": "1.0.1",
@ -54,7 +54,7 @@
},
{
"name": "syft:location:0:layerID",
"value": "sha256:ab62016f9bec7286af65604081564cadeeb364a48faca2346c3f5a5a1f5ef777"
"value": "sha256:redacted"
},
{
"name": "syft:location:0:path",
@ -63,7 +63,7 @@
]
},
{
"bom-ref": "pkg:deb/debian/package-2@2.0.1?package-id=958443e2d9304af4",
"bom-ref":"redacted",
"type": "library",
"name": "package-2",
"version": "2.0.1",
@ -84,7 +84,7 @@
},
{
"name": "syft:location:0:layerID",
"value": "sha256:f1803845b6747d94d6e4ecce2331457e5f1c4fb97de5216f392a76f4582f63b2"
"value": "sha256:redacted"
},
{
"name": "syft:location:0:path",

65
syft/formats/cyclonedxxml/encoder_test.go
View File

@ -2,51 +2,62 @@ package cyclonedxxml
import (
"flag"
"regexp"
"testing"
"github.com/anchore/syft/syft/formats/internal/testutils"
)
var updateCycloneDx = flag.Bool("update-cyclonedx", false, "update the *.golden files for cyclone-dx encoders")
var updateSnapshot = flag.Bool("update-cyclonedx-xml", false, "update the *.golden files for cyclone-dx XML encoders")
var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing")
func TestCycloneDxDirectoryEncoder(t *testing.T) {
dir := t.TempDir()
testutils.AssertEncoderAgainstGoldenSnapshot(t,
Format(),
testutils.DirectoryInput(t),
*updateCycloneDx,
false,
cycloneDxRedactor,
testutils.EncoderSnapshotTestConfig{
Subject: testutils.DirectoryInput(t, dir),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: false,
Redactor: redactor(dir),
},
)
}
func TestCycloneDxImageEncoder(t *testing.T) {
testImage := "image-simple"
testutils.AssertEncoderAgainstGoldenImageSnapshot(t,
Format(),
testutils.ImageInput(t, testImage),
testImage,
*updateCycloneDx,
false,
cycloneDxRedactor,
testutils.ImageSnapshotTestConfig{
Image: testImage,
UpdateImageSnapshot: *updateImage,
},
testutils.EncoderSnapshotTestConfig{
Subject: testutils.ImageInput(t, testImage),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: false,
Redactor: redactor(),
},
)
}
func cycloneDxRedactor(s []byte) []byte {
serialPattern := regexp.MustCompile(`serialNumber="[a-zA-Z0-9\-:]+"`)
rfc3339Pattern := regexp.MustCompile(`([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`)
sha256Pattern := regexp.MustCompile(`sha256:[A-Fa-f0-9]{64}`)
func redactor(values ...string) testutils.Redactor {
return testutils.NewRedactions().
WithValuesRedacted(values...).
WithPatternRedactors(
map[string]string{
// serial numbers
`serialNumber="[a-zA-Z0-9\-:]+`: `serialNumber="redacted`,
for _, pattern := range []*regexp.Regexp{serialPattern, rfc3339Pattern, sha256Pattern} {
s = pattern.ReplaceAll(s, []byte("redacted"))
}
// dates
`([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`: `redacted`,
// the bom-ref will be autogenerated every time, the value here should not be directly tested in snapshot tests
bomRefPattern := regexp.MustCompile(` bom-ref="[a-zA-Z0-9\-:]+"`)
bomRef3339Pattern := regexp.MustCompile(`([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`)
for _, pattern := range []*regexp.Regexp{bomRefPattern, bomRef3339Pattern} {
s = pattern.ReplaceAll(s, []byte(""))
}
// image hashes
`sha256:[A-Za-z0-9]{64}`: `sha256:redacted`,
return s
// BOM refs
`bom-ref="[a-zA-Z0-9\-:]+"`: `bom-ref:redacted`,
},
)
}

10
syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxDirectoryEncoder.golden
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<bom xmlns="http://cyclonedx.org/schema/bom/1.4" serialNumber="urn:uuid:60f4e726-e884-4ae3-9b0e-18a918fbb02e" version="1">
<bom xmlns="http://cyclonedx.org/schema/bom/1.4" serialNumber="redacted" version="1">
<metadata>
<timestamp>2023-05-08T14:40:52-04:00</timestamp>
<timestamp>redacted</timestamp>
<tools>
<tool>
<vendor>anchore</vendor>
@ -9,12 +9,12 @@
<version>v0.42.0-bogus</version>
</tool>
</tools>
<component bom-ref="163686ac6e30c752" type="file">
<name>/some/path</name>
<component bom-ref="redacted" type="file">
<name>some/path</name>
</component>
</metadata>
<components>
<component bom-ref="8c7e1242588c971a" type="library">
<component bom-ref="redacted" type="library">
<name>package-1</name>
<version>1.0.1</version>
<licenses>

14
syft/formats/cyclonedxxml/test-fixtures/snapshot/TestCycloneDxImageEncoder.golden
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<bom xmlns="http://cyclonedx.org/schema/bom/1.4" serialNumber="urn:uuid:c8894728-c156-4fc5-8f5d-3e397eede5a7" version="1">
<bom xmlns="http://cyclonedx.org/schema/bom/1.4" serialNumber="redacted" version="1">
<metadata>
<timestamp>2023-05-08T14:40:52-04:00</timestamp>
<timestamp>redacted</timestamp>
<tools>
<tool>
<vendor>anchore</vendor>
@ -9,13 +9,13 @@
<version>v0.42.0-bogus</version>
</tool>
</tools>
<component bom-ref="38160ebc2a6876e8" type="container">
<component bom-ref="redacted" type="container">
<name>user-image-input</name>
<version>sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368</version>
<version>sha256:redacted</version>
</component>
</metadata>
<components>
<component bom-ref="ec2e0c93617507ef" type="library">
<component bom-ref="redacted" type="library">
<name>package-1</name>
<version>1.0.1</version>
<licenses>
@ -30,7 +30,7 @@
<property name="syft:package:language">python</property>
<property name="syft:package:metadataType">PythonPackageMetadata</property>
<property name="syft:package:type">python</property>
<property name="syft:location:0:layerID">sha256:ab62016f9bec7286af65604081564cadeeb364a48faca2346c3f5a5a1f5ef777</property>
<property name="syft:location:0:layerID">sha256:redacted</property>
<property name="syft:location:0:path">/somefile-1.txt</property>
</properties>
</component>
@ -43,7 +43,7 @@
<property name="syft:package:foundBy">the-cataloger-2</property>
<property name="syft:package:metadataType">DpkgMetadata</property>
<property name="syft:package:type">deb</property>
<property name="syft:location:0:layerID">sha256:f1803845b6747d94d6e4ecce2331457e5f1c4fb97de5216f392a76f4582f63b2</property>
<property name="syft:location:0:layerID">sha256:redacted</property>
<property name="syft:location:0:path">/somefile-2.txt</property>
<property name="syft:metadata:installedSize">0</property>
</properties>

107
syft/formats/github/encoder.go
View File

@ -64,45 +64,6 @@ func filesystem(p pkg.Package) string {
return ""
}
// isArchive returns true if the path appears to be an archive
func isArchive(path string) bool {
_, err := archiver.ByExtension(path)
return err == nil
}
// toPath Generates a string representation of the package location, optionally including the layer hash
func toPath(s source.Metadata, p pkg.Package) string {
inputPath := strings.TrimPrefix(s.Path, "./")
if inputPath == "." {
inputPath = ""
}
locations := p.Locations.ToSlice()
if len(locations) > 0 {
location := locations[0]
packagePath := location.RealPath
if location.VirtualPath != "" {
packagePath = location.VirtualPath
}
packagePath = strings.TrimPrefix(packagePath, "/")
switch s.Scheme {
case source.ImageScheme:
image := strings.ReplaceAll(s.ImageMetadata.UserInput, ":/", "//")
return fmt.Sprintf("%s:/%s", image, packagePath)
case source.FileScheme:
if isArchive(inputPath) {
return fmt.Sprintf("%s:/%s", inputPath, packagePath)
}
return inputPath
case source.DirectoryScheme:
if inputPath != "" {
return fmt.Sprintf("%s/%s", inputPath, packagePath)
}
return packagePath
}
}
return fmt.Sprintf("%s%s", inputPath, s.ImageMetadata.UserInput)
}
// toGithubManifests manifests, each of which represents a specific location that has dependencies
func toGithubManifests(s *sbom.SBOM) Manifests {
manifests := map[string]*Manifest{}
@ -144,6 +105,63 @@ func toGithubManifests(s *sbom.SBOM) Manifests {
return out
}
// toPath Generates a string representation of the package location, optionally including the layer hash
func toPath(s source.Description, p pkg.Package) string {
inputPath := trimRelative(s.Name)
locations := p.Locations.ToSlice()
if len(locations) > 0 {
location := locations[0]
packagePath := location.RealPath
if location.VirtualPath != "" {
packagePath = location.VirtualPath
}
packagePath = strings.TrimPrefix(packagePath, "/")
switch metadata := s.Metadata.(type) {
case source.StereoscopeImageSourceMetadata:
image := strings.ReplaceAll(metadata.UserInput, ":/", "//")
return fmt.Sprintf("%s:/%s", image, packagePath)
case source.FileSourceMetadata:
path := trimRelative(metadata.Path)
if isArchive(metadata.Path) {
return fmt.Sprintf("%s:/%s", path, packagePath)
}
return path
case source.DirectorySourceMetadata:
path := trimRelative(metadata.Path)
if path != "" {
return fmt.Sprintf("%s/%s", path, packagePath)
}
return packagePath
}
}
return inputPath
}
func trimRelative(s string) string {
s = strings.TrimPrefix(s, "./")
if s == "." {
s = ""
}
return s
}
// isArchive returns true if the path appears to be an archive
func isArchive(path string) bool {
_, err := archiver.ByExtension(path)
return err == nil
}
func toDependencies(s *sbom.SBOM, p pkg.Package) (out []string) {
for _, r := range s.Relationships {
if r.From.ID() == p.ID() {
if p, ok := r.To.(pkg.Package); ok {
out = append(out, dependencyName(p))
}
}
}
return
}
// dependencyName to make things a little nicer to read; this might end up being lossy
func dependencyName(p pkg.Package) string {
purl, err := packageurl.FromString(p.PURL)
@ -171,14 +189,3 @@ func toDependencyMetadata(_ pkg.Package) Metadata {
// so we don't need anything here yet
return Metadata{}
}
func toDependencies(s *sbom.SBOM, p pkg.Package) (out []string) {
for _, r := range s.Relationships {
if r.From.ID() == p.ID() {
if p, ok := r.To.(pkg.Package); ok {
out = append(out, dependencyName(p))
}
}
}
return
}

182
syft/formats/github/encoder_test.go
View File

@ -1,24 +1,25 @@
package github
import (
"encoding/json"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/assert"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/internal/sourcemetadata"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
func Test_toGithubModel(t *testing.T) {
func sbomFixture() sbom.SBOM {
s := sbom.SBOM{
Source: source.Metadata{
Scheme: source.ImageScheme,
ImageMetadata: source.ImageMetadata{
Source: source.Description{
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "ubuntu:18.04",
Architecture: "amd64",
},
@ -75,88 +76,121 @@ func Test_toGithubModel(t *testing.T) {
s.Artifacts.Packages.Add(p)
}
actual := toGithubModel(&s)
return s
}
expected := DependencySnapshot{
Version: 0,
Detector: DetectorMetadata{
Name: "syft",
Version: "0.0.0-dev",
URL: "https://github.com/anchore/syft",
},
Metadata: Metadata{
"syft:distro": "pkg:generic/ubuntu@18.04?like=debian",
},
Scanned: actual.Scanned,
Manifests: Manifests{
"ubuntu:18.04:/usr/lib": Manifest{
Name: "ubuntu:18.04:/usr/lib",
File: FileInfo{
SourceLocation: "ubuntu:18.04:/usr/lib",
func Test_toGithubModel(t *testing.T) {
tracker := sourcemetadata.NewCompletionTester(t)
tests := []struct {
name string
metadata any
testPath string
expected *DependencySnapshot
}{
{
name: "image",
expected: &DependencySnapshot{
Version: 0,
Detector: DetectorMetadata{
Name: "syft",
Version: "0.0.0-dev",
URL: "https://github.com/anchore/syft",
},
Metadata: Metadata{
"syft:filesystem": "fsid-1",
"syft:distro": "pkg:generic/ubuntu@18.04?like=debian",
},
Resolved: DependencyGraph{
"pkg:generic/pkg-1@1.0.1": DependencyNode{
PackageURL: "pkg:generic/pkg-1@1.0.1",
Scope: DependencyScopeRuntime,
Relationship: DependencyRelationshipDirect,
//Scanned: actual.Scanned,
Manifests: Manifests{
"ubuntu:18.04:/usr/lib": Manifest{
Name: "ubuntu:18.04:/usr/lib",
File: FileInfo{
SourceLocation: "ubuntu:18.04:/usr/lib",
},
Metadata: Metadata{
"syft:filesystem": "fsid-1",
},
Resolved: DependencyGraph{
"pkg:generic/pkg-1@1.0.1": DependencyNode{
PackageURL: "pkg:generic/pkg-1@1.0.1",
Scope: DependencyScopeRuntime,
Relationship: DependencyRelationshipDirect,
Metadata: Metadata{},
},
"pkg:generic/pkg-2@2.0.2": DependencyNode{
PackageURL: "pkg:generic/pkg-2@2.0.2",
Scope: DependencyScopeRuntime,
Relationship: DependencyRelationshipDirect,
Metadata: Metadata{},
},
},
},
"pkg:generic/pkg-2@2.0.2": DependencyNode{
PackageURL: "pkg:generic/pkg-2@2.0.2",
Scope: DependencyScopeRuntime,
Relationship: DependencyRelationshipDirect,
},
},
},
"ubuntu:18.04:/etc": Manifest{
Name: "ubuntu:18.04:/etc",
File: FileInfo{
SourceLocation: "ubuntu:18.04:/etc",
},
Metadata: Metadata{
"syft:filesystem": "fsid-1",
},
Resolved: DependencyGraph{
"pkg:generic/pkg-3@3.0.3": DependencyNode{
PackageURL: "pkg:generic/pkg-3@3.0.3",
Scope: DependencyScopeRuntime,
Relationship: DependencyRelationshipDirect,
"ubuntu:18.04:/etc": Manifest{
Name: "ubuntu:18.04:/etc",
File: FileInfo{
SourceLocation: "ubuntu:18.04:/etc",
},
Metadata: Metadata{
"syft:filesystem": "fsid-1",
},
Resolved: DependencyGraph{
"pkg:generic/pkg-3@3.0.3": DependencyNode{
PackageURL: "pkg:generic/pkg-3@3.0.3",
Scope: DependencyScopeRuntime,
Relationship: DependencyRelationshipDirect,
Metadata: Metadata{},
},
},
},
},
},
},
{
name: "current directory",
metadata: source.DirectorySourceMetadata{Path: "."},
testPath: "etc",
},
{
name: "relative directory",
metadata: source.DirectorySourceMetadata{Path: "./artifacts"},
testPath: "artifacts/etc",
},
{
name: "absolute directory",
metadata: source.DirectorySourceMetadata{Path: "/artifacts"},
testPath: "/artifacts/etc",
},
{
name: "file",
metadata: source.FileSourceMetadata{Path: "./executable"},
testPath: "executable",
},
{
name: "archive",
metadata: source.FileSourceMetadata{Path: "./archive.tar.gz"},
testPath: "archive.tar.gz:/etc",
},
}
// just using JSONEq because it gives a comprehensible diff
s1, _ := json.Marshal(expected)
s2, _ := json.Marshal(actual)
assert.JSONEq(t, string(s1), string(s2))
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
s := sbomFixture()
// Just test the other schemes:
s.Source.Path = "."
s.Source.Scheme = source.DirectoryScheme
actual = toGithubModel(&s)
assert.Equal(t, "etc", actual.Manifests["etc"].Name)
if test.metadata != nil {
s.Source.Metadata = test.metadata
}
actual := toGithubModel(&s)
s.Source.Path = "./artifacts"
s.Source.Scheme = source.DirectoryScheme
actual = toGithubModel(&s)
assert.Equal(t, "artifacts/etc", actual.Manifests["artifacts/etc"].Name)
if test.expected != nil {
if d := cmp.Diff(*test.expected, actual, cmpopts.IgnoreFields(DependencySnapshot{}, "Scanned")); d != "" {
t.Errorf("unexpected result (-want +got):\n%s", d)
}
}
s.Source.Path = "/artifacts"
s.Source.Scheme = source.DirectoryScheme
actual = toGithubModel(&s)
assert.Equal(t, "/artifacts/etc", actual.Manifests["/artifacts/etc"].Name)
assert.Equal(t, test.testPath, actual.Manifests[test.testPath].Name)
s.Source.Path = "./executable"
s.Source.Scheme = source.FileScheme
actual = toGithubModel(&s)
assert.Equal(t, "executable", actual.Manifests["executable"].Name)
s.Source.Path = "./archive.tar.gz"
s.Source.Scheme = source.FileScheme
actual = toGithubModel(&s)
assert.Equal(t, "archive.tar.gz:/etc", actual.Manifests["archive.tar.gz:/etc"].Name)
// track each scheme tested (passed or not)
tracker.Tested(t, s.Source.Metadata)
})
}
}

204
syft/formats/internal/testutils/directory_input.go Normal file
View File

@ -0,0 +1,204 @@
package testutils
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
func DirectoryInput(t testing.TB, dir string) sbom.SBOM {
catalog := newDirectoryCatalog()
path := filepath.Join(dir, "some", "path")
require.NoError(t, os.MkdirAll(path, 0755))
src, err := source.NewFromDirectory(
source.DirectoryConfig{
Path: path,
Base: dir,
},
)
require.NoError(t, err)
return sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: catalog,
LinuxDistribution: &linux.Release{
PrettyName: "debian",
Name: "debian",
ID: "debian",
IDLike: []string{"like!"},
Version: "1.2.3",
VersionID: "1.2.3",
},
},
Source: src.Describe(),
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
// the application configuration should be persisted here, however, we do not want to import
// the application configuration in this package (it's reserved only for ingestion by the cmd package)
Configuration: map[string]string{
"config-key": "config-value",
},
},
}
}
func DirectoryInputWithAuthorField(t testing.TB) sbom.SBOM {
catalog := newDirectoryCatalogWithAuthorField()
dir := t.TempDir()
path := filepath.Join(dir, "some", "path")
require.NoError(t, os.MkdirAll(path, 0755))
src, err := source.NewFromDirectory(
source.DirectoryConfig{
Path: path,
Base: dir,
},
)
require.NoError(t, err)
return sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: catalog,
LinuxDistribution: &linux.Release{
PrettyName: "debian",
Name: "debian",
ID: "debian",
IDLike: []string{"like!"},
Version: "1.2.3",
VersionID: "1.2.3",
},
},
Source: src.Describe(),
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
// the application configuration should be persisted here, however, we do not want to import
// the application configuration in this package (it's reserved only for ingestion by the cmd package)
Configuration: map[string]string{
"config-key": "config-value",
},
},
}
}
func newDirectoryCatalog() *pkg.Collection {
catalog := pkg.NewCollection()
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
),
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
Files: []pkg.PythonFileRecord{
{
Path: "/some/path/pkg1/dependencies/foo",
},
},
},
PURL: "a-purl-2", // intentionally a bad pURL for test fixtures
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
PURL: "pkg:deb/debian/package-2@2.0.1",
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
return catalog
}
func newDirectoryCatalogWithAuthorField() *pkg.Collection {
catalog := pkg.NewCollection()
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
),
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
Author: "test-author",
Files: []pkg.PythonFileRecord{
{
Path: "/some/path/pkg1/dependencies/foo",
},
},
},
PURL: "a-purl-2", // intentionally a bad pURL for test fixtures
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
PURL: "pkg:deb/debian/package-2@2.0.1",
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
return catalog
}

32
syft/formats/internal/testutils/file_relationships.go Normal file
View File

@ -0,0 +1,32 @@
package testutils
import (
"math/rand"
"time"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/sbom"
)
//nolint:gosec
func AddSampleFileRelationships(s *sbom.SBOM) {
catalog := s.Artifacts.Packages.Sorted()
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
files := []string{"/f1", "/f2", "/d1/f3", "/d2/f4", "/z1/f5", "/a1/f6"}
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
rnd.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] })
for _, f := range files {
meta := file.Metadata{}
coords := file.Coordinates{RealPath: f}
s.Artifacts.FileMetadata[coords] = meta
s.Relationships = append(s.Relationships, artifact.Relationship{
From: catalog[0],
To: coords,
Type: artifact.ContainsRelationship,
})
}
}

113
syft/formats/internal/testutils/image_input.go Normal file
View File

@ -0,0 +1,113 @@
package testutils
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
func ImageInput(t testing.TB, testImage string, options ...ImageOption) sbom.SBOM {
t.Helper()
catalog := pkg.NewCollection()
var cfg imageCfg
var img *image.Image
for _, opt := range options {
opt(&cfg)
}
switch cfg.fromSnapshot {
case true:
img = imagetest.GetGoldenFixtureImage(t, testImage)
default:
img = imagetest.GetFixtureImage(t, "docker-archive", testImage)
}
populateImageCatalog(catalog, img)
// this is a hard coded value that is not given by the fixture helper and must be provided manually
img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
src, err := source.NewFromStereoscopeImageObject(img, "user-image-input", nil)
assert.NoError(t, err)
return sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: catalog,
LinuxDistribution: &linux.Release{
PrettyName: "debian",
Name: "debian",
ID: "debian",
IDLike: []string{"like!"},
Version: "1.2.3",
VersionID: "1.2.3",
},
},
Source: src.Describe(),
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
// the application configuration should be persisted here, however, we do not want to import
// the application configuration in this package (it's reserved only for ingestion by the cmd package)
Configuration: map[string]string{
"config-key": "config-value",
},
},
}
}
func populateImageCatalog(catalog *pkg.Collection, img *image.Image) {
_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Locations: file.NewLocationSet(
file.NewLocationFromImage(string(ref1.RealPath), *ref1.Reference, img),
),
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
),
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
},
PURL: "a-purl-1", // intentionally a bad pURL for test fixtures
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"),
},
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Locations: file.NewLocationSet(
file.NewLocationFromImage(string(ref2.RealPath), *ref2.Reference, img),
),
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
PURL: "pkg:deb/debian/package-2@2.0.1",
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
}

142
syft/formats/internal/testutils/redactor.go Normal file
View File

@ -0,0 +1,142 @@
package testutils
import (
"bytes"
"regexp"
)
var (
_ Redactor = (*RedactorFn)(nil)
_ Redactor = (*PatternReplacement)(nil)
_ Redactor = (*ValueReplacement)(nil)
_ Redactor = (*Redactions)(nil)
)
type Redactor interface {
Redact([]byte) []byte
}
// Replace by function //////////////////////////////
type RedactorFn func([]byte) []byte
func (r RedactorFn) Redact(b []byte) []byte {
return r(b)
}
// Replace by regex //////////////////////////////
type PatternReplacement struct {
Search *regexp.Regexp
Replace string
}
func NewPatternReplacement(r *regexp.Regexp) PatternReplacement {
return PatternReplacement{
Search: r,
Replace: "redacted",
}
}
func (p PatternReplacement) Redact(b []byte) []byte {
return p.Search.ReplaceAll(b, []byte(p.Replace))
}
// Replace by value //////////////////////////////
type ValueReplacement struct {
Search string
Replace string
}
func NewValueReplacement(v string) ValueReplacement {
return ValueReplacement{
Search: v,
Replace: "redacted",
}
}
func (v ValueReplacement) Redact(b []byte) []byte {
return bytes.ReplaceAll(b, []byte(v.Search), []byte(v.Replace))
}
// Handle a collection of redactors //////////////////////////////
type Redactions struct {
redactors []Redactor
}
func NewRedactions(redactors ...Redactor) *Redactions {
r := &Redactions{
redactors: redactors,
}
return r.WithFunctions(carriageRedactor)
}
func (r *Redactions) WithPatternRedactors(values map[string]string) *Redactions {
for k, v := range values {
r.redactors = append(r.redactors,
PatternReplacement{
Search: regexp.MustCompile(k),
Replace: v,
},
)
}
return r
}
func (r *Redactions) WithValueRedactors(values map[string]string) *Redactions {
for k, v := range values {
r.redactors = append(r.redactors,
ValueReplacement{
Search: k,
Replace: v,
},
)
}
return r
}
func (r *Redactions) WithPatternsRedacted(values ...string) *Redactions {
for _, pattern := range values {
r.redactors = append(r.redactors,
NewPatternReplacement(regexp.MustCompile(pattern)),
)
}
return r
}
func (r *Redactions) WithValuesRedacted(values ...string) *Redactions {
for _, v := range values {
r.redactors = append(r.redactors,
NewValueReplacement(v),
)
}
return r
}
func (r *Redactions) WithFunctions(values ...func([]byte) []byte) *Redactions {
for _, fn := range values {
r.redactors = append(r.redactors,
RedactorFn(fn),
)
}
return r
}
func (r *Redactions) WithRedactors(rs ...Redactor) *Redactions {
r.redactors = append(r.redactors, rs...)
return r
}
func (r Redactions) Redact(b []byte) []byte {
for _, redactor := range r.redactors {
b = redactor.Redact(b)
}
return b
}
func carriageRedactor(s []byte) []byte {
return bytes.ReplaceAll(s, []byte("\r\n"), []byte("\n"))
}

88
syft/formats/internal/testutils/snapshot.go Normal file
View File

@ -0,0 +1,88 @@
package testutils
import (
"bytes"
"testing"
"github.com/sergi/go-diff/diffmatchpatch"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/sbom"
)
type imageCfg struct {
fromSnapshot bool
}
type ImageOption func(cfg *imageCfg)
func FromSnapshot() ImageOption {
return func(cfg *imageCfg) {
cfg.fromSnapshot = true
}
}
type EncoderSnapshotTestConfig struct {
Subject sbom.SBOM
Format sbom.Format
UpdateSnapshot bool
PersistRedactionsInSnapshot bool
IsJSON bool
Redactor Redactor
}
func AssertEncoderAgainstGoldenSnapshot(t *testing.T, cfg EncoderSnapshotTestConfig) {
t.Helper()
var buffer bytes.Buffer
err := cfg.Format.Encode(&buffer, cfg.Subject)
assert.NoError(t, err)
actual := buffer.Bytes()
if cfg.UpdateSnapshot && !cfg.PersistRedactionsInSnapshot {
// replace the expected snapshot contents with the current (unredacted) encoder contents
testutils.UpdateGoldenFileContents(t, actual)
return
}
var expected []byte
if cfg.Redactor != nil {
actual = cfg.Redactor.Redact(actual)
expected = cfg.Redactor.Redact(testutils.GetGoldenFileContents(t))
} else {
expected = testutils.GetGoldenFileContents(t)
}
if cfg.UpdateSnapshot && cfg.PersistRedactionsInSnapshot {
// replace the expected snapshot contents with the current (redacted) encoder contents
testutils.UpdateGoldenFileContents(t, actual)
return
}
if cfg.IsJSON {
require.JSONEq(t, string(expected), string(actual))
} else if !bytes.Equal(expected, actual) {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(string(expected), string(actual), true)
t.Logf("len: %d\nexpected: %s", len(expected), expected)
t.Logf("len: %d\nactual: %s", len(actual), actual)
t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
}
}
type ImageSnapshotTestConfig struct {
Image string
UpdateImageSnapshot bool
}
func AssertEncoderAgainstGoldenImageSnapshot(t *testing.T, imgCfg ImageSnapshotTestConfig, cfg EncoderSnapshotTestConfig) {
if imgCfg.UpdateImageSnapshot {
// grab the latest image contents and persist
imagetest.UpdateGoldenFixtureImage(t, imgCfg.Image)
}
AssertEncoderAgainstGoldenSnapshot(t, cfg)
}

396
syft/formats/internal/testutils/utils.go
View File

@ -1,396 +0,0 @@
package testutils
import (
"bytes"
"math/rand"
"strings"
"testing"
"time"
"github.com/sergi/go-diff/diffmatchpatch"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
type redactor func(s []byte) []byte
type imageCfg struct {
fromSnapshot bool
}
type ImageOption func(cfg *imageCfg)
func FromSnapshot() ImageOption {
return func(cfg *imageCfg) {
cfg.fromSnapshot = true
}
}
func AssertEncoderAgainstGoldenImageSnapshot(t *testing.T, format sbom.Format, sbom sbom.SBOM, testImage string, updateSnapshot bool, json bool, redactors ...redactor) {
var buffer bytes.Buffer
// grab the latest image contents and persist
if updateSnapshot {
imagetest.UpdateGoldenFixtureImage(t, testImage)
}
err := format.Encode(&buffer, sbom)
assert.NoError(t, err)
actual := buffer.Bytes()
// replace the expected snapshot contents with the current encoder contents
if updateSnapshot {
testutils.UpdateGoldenFileContents(t, actual)
}
actual = redact(actual, redactors...)
expected := redact(testutils.GetGoldenFileContents(t), redactors...)
if json {
require.JSONEq(t, string(expected), string(actual))
} else if !bytes.Equal(expected, actual) {
// assert that the golden file snapshot matches the actual contents
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(string(expected), string(actual), true)
t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
}
}
func AssertEncoderAgainstGoldenSnapshot(t *testing.T, format sbom.Format, sbom sbom.SBOM, updateSnapshot bool, json bool, redactors ...redactor) {
var buffer bytes.Buffer
err := format.Encode(&buffer, sbom)
assert.NoError(t, err)
actual := buffer.Bytes()
// replace the expected snapshot contents with the current encoder contents
if updateSnapshot {
testutils.UpdateGoldenFileContents(t, actual)
}
actual = redact(actual, redactors...)
expected := redact(testutils.GetGoldenFileContents(t), redactors...)
if json {
require.JSONEq(t, string(expected), string(actual))
} else if !bytes.Equal(expected, actual) {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(string(expected), string(actual), true)
t.Logf("len: %d\nexpected: %s", len(expected), expected)
t.Logf("len: %d\nactual: %s", len(actual), actual)
t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
}
}
func ImageInput(t testing.TB, testImage string, options ...ImageOption) sbom.SBOM {
t.Helper()
catalog := pkg.NewCollection()
var cfg imageCfg
var img *image.Image
for _, opt := range options {
opt(&cfg)
}
switch cfg.fromSnapshot {
case true:
img = imagetest.GetGoldenFixtureImage(t, testImage)
default:
img = imagetest.GetFixtureImage(t, "docker-archive", testImage)
}
populateImageCatalog(catalog, img)
// this is a hard coded value that is not given by the fixture helper and must be provided manually
img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
src, err := source.NewFromImage(img, "user-image-input")
assert.NoError(t, err)
return sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: catalog,
LinuxDistribution: &linux.Release{
PrettyName: "debian",
Name: "debian",
ID: "debian",
IDLike: []string{"like!"},
Version: "1.2.3",
VersionID: "1.2.3",
},
},
Source: src.Metadata,
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
// the application configuration should be persisted here, however, we do not want to import
// the application configuration in this package (it's reserved only for ingestion by the cmd package)
Configuration: map[string]string{
"config-key": "config-value",
},
},
}
}
func carriageRedactor(s []byte) []byte {
msg := strings.ReplaceAll(string(s), "\r\n", "\n")
return []byte(msg)
}
func populateImageCatalog(catalog *pkg.Collection, img *image.Image) {
_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Locations: file.NewLocationSet(
file.NewLocationFromImage(string(ref1.RealPath), *ref1.Reference, img),
),
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
),
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
},
PURL: "a-purl-1", // intentionally a bad pURL for test fixtures
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"),
},
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Locations: file.NewLocationSet(
file.NewLocationFromImage(string(ref2.RealPath), *ref2.Reference, img),
),
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
PURL: "pkg:deb/debian/package-2@2.0.1",
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
}
func DirectoryInput(t testing.TB) sbom.SBOM {
catalog := newDirectoryCatalog()
src, err := source.NewFromDirectory("/some/path")
assert.NoError(t, err)
return sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: catalog,
LinuxDistribution: &linux.Release{
PrettyName: "debian",
Name: "debian",
ID: "debian",
IDLike: []string{"like!"},
Version: "1.2.3",
VersionID: "1.2.3",
},
},
Source: src.Metadata,
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
// the application configuration should be persisted here, however, we do not want to import
// the application configuration in this package (it's reserved only for ingestion by the cmd package)
Configuration: map[string]string{
"config-key": "config-value",
},
},
}
}
func DirectoryInputWithAuthorField(t testing.TB) sbom.SBOM {
catalog := newDirectoryCatalogWithAuthorField()
src, err := source.NewFromDirectory("/some/path")
assert.NoError(t, err)
return sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: catalog,
LinuxDistribution: &linux.Release{
PrettyName: "debian",
Name: "debian",
ID: "debian",
IDLike: []string{"like!"},
Version: "1.2.3",
VersionID: "1.2.3",
},
},
Source: src.Metadata,
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
// the application configuration should be persisted here, however, we do not want to import
// the application configuration in this package (it's reserved only for ingestion by the cmd package)
Configuration: map[string]string{
"config-key": "config-value",
},
},
}
}
func newDirectoryCatalog() *pkg.Collection {
catalog := pkg.NewCollection()
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
),
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
Files: []pkg.PythonFileRecord{
{
Path: "/some/path/pkg1/dependencies/foo",
},
},
},
PURL: "a-purl-2", // intentionally a bad pURL for test fixtures
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
PURL: "pkg:deb/debian/package-2@2.0.1",
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
return catalog
}
func newDirectoryCatalogWithAuthorField() *pkg.Collection {
catalog := pkg.NewCollection()
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
),
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
Author: "test-author",
Files: []pkg.PythonFileRecord{
{
Path: "/some/path/pkg1/dependencies/foo",
},
},
},
PURL: "a-purl-2", // intentionally a bad pURL for test fixtures
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
PURL: "pkg:deb/debian/package-2@2.0.1",
CPEs: []cpe.CPE{
cpe.Must("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
},
})
return catalog
}
//nolint:gosec
func AddSampleFileRelationships(s *sbom.SBOM) {
catalog := s.Artifacts.Packages.Sorted()
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
files := []string{"/f1", "/f2", "/d1/f3", "/d2/f4", "/z1/f5", "/a1/f6"}
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
rnd.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] })
for _, f := range files {
meta := file.Metadata{}
coords := file.Coordinates{RealPath: f}
s.Artifacts.FileMetadata[coords] = meta
s.Relationships = append(s.Relationships, artifact.Relationship{
From: catalog[0],
To: coords,
Type: artifact.ContainsRelationship,
})
}
}
// remove dynamic values, which should be tested independently
func redact(b []byte, redactors ...redactor) []byte {
redactors = append(redactors, carriageRedactor)
for _, r := range redactors {
b = r(b)
}
return b
}

76
syft/formats/spdxjson/encoder_test.go
View File

@ -2,57 +2,81 @@ package spdxjson
import (
"flag"
"regexp"
"testing"
"github.com/anchore/syft/syft/formats/internal/testutils"
)
var updateSpdxJson = flag.Bool("update-spdx-json", false, "update the *.golden files for spdx-json encoders")
var updateSnapshot = flag.Bool("update-spdx-json", false, "update the *.golden files for spdx-json encoders")
var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing")
func TestSPDXJSONDirectoryEncoder(t *testing.T) {
dir := t.TempDir()
testutils.AssertEncoderAgainstGoldenSnapshot(t,
Format(),
testutils.DirectoryInput(t),
*updateSpdxJson,
true,
spdxJsonRedactor,
testutils.EncoderSnapshotTestConfig{
Subject: testutils.DirectoryInput(t, dir),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: true,
Redactor: redactor(dir),
},
)
}
func TestSPDXJSONImageEncoder(t *testing.T) {
testImage := "image-simple"
testutils.AssertEncoderAgainstGoldenImageSnapshot(t,
Format(),
testutils.ImageInput(t, testImage, testutils.FromSnapshot()),
testImage,
*updateSpdxJson,
true,
spdxJsonRedactor,
testutils.ImageSnapshotTestConfig{
Image: testImage,
UpdateImageSnapshot: *updateImage,
},
testutils.EncoderSnapshotTestConfig{
Subject: testutils.ImageInput(t, testImage, testutils.FromSnapshot()),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: true,
Redactor: redactor(),
},
)
}
func TestSPDXRelationshipOrder(t *testing.T) {
testImage := "image-simple"
s := testutils.ImageInput(t, testImage, testutils.FromSnapshot())
testutils.AddSampleFileRelationships(&s)
testutils.AssertEncoderAgainstGoldenImageSnapshot(t,
Format(),
s,
testImage,
*updateSpdxJson,
true,
spdxJsonRedactor,
testutils.ImageSnapshotTestConfig{
Image: testImage,
UpdateImageSnapshot: *updateImage,
},
testutils.EncoderSnapshotTestConfig{
Subject: s,
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: true,
Redactor: redactor(),
},
)
}
func spdxJsonRedactor(s []byte) []byte {
// each SBOM reports the time it was generated, which is not useful during snapshot testing
s = regexp.MustCompile(`"created":\s+"[^"]*"`).ReplaceAll(s, []byte(`"created":""`))
func redactor(values ...string) testutils.Redactor {
return testutils.NewRedactions().
WithValuesRedacted(values...).
WithPatternRedactors(
map[string]string{
// each SBOM reports the time it was generated, which is not useful during snapshot testing
`"created":\s+"[^"]*"`: `"created":"redacted"`,
// each SBOM reports a unique documentNamespace when generated, this is not useful for snapshot testing
s = regexp.MustCompile(`"documentNamespace":\s+"[^"]*"`).ReplaceAll(s, []byte(`"documentNamespace":""`))
// each SBOM reports a unique documentNamespace when generated, this is not useful for snapshot testing
`"documentNamespace":\s+"[^"]*"`: `"documentNamespace":"redacted"`,
// the license list will be updated periodically, the value here should not be directly tested in snapshot tests
return regexp.MustCompile(`"licenseListVersion":\s+"[^"]*"`).ReplaceAll(s, []byte(`"licenseListVersion":""`))
// the license list will be updated periodically, the value here should not be directly tested in snapshot tests
`"licenseListVersion":\s+"[^"]*"`: `"licenseListVersion":"redacted"`,
},
)
}

8
syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXJSONDirectoryEncoder.golden
View File

@ -2,15 +2,15 @@
"spdxVersion": "SPDX-2.3",
"dataLicense": "CC0-1.0",
"SPDXID": "SPDXRef-DOCUMENT",
"name": "/some/path",
"documentNamespace": "https://anchore.com/syft/dir/some/path-303fccb4-22d1-4039-9061-553bc875f086",
"name": "some/path",
"documentNamespace":"redacted",
"creationInfo": {
"licenseListVersion": "3.20",
"licenseListVersion":"redacted",
"creators": [
"Organization: Anchore, Inc",
"Tool: syft-v0.42.0-bogus"
],
"created": "2023-06-05T18:49:13Z"
"created":"redacted"
},
"packages": [
{

6
syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXJSONImageEncoder.golden
View File

@ -3,14 +3,14 @@
"dataLicense": "CC0-1.0",
"SPDXID": "SPDXRef-DOCUMENT",
"name": "user-image-input",
"documentNamespace": "https://anchore.com/syft/image/user-image-input-5b9aac79-334c-4d6a-b2e6-95a819c1d45a",
"documentNamespace":"redacted",
"creationInfo": {
"licenseListVersion": "3.20",
"licenseListVersion":"redacted",
"creators": [
"Organization: Anchore, Inc",
"Tool: syft-v0.42.0-bogus"
],
"created": "2023-06-05T18:49:14Z"
"created":"redacted"
},
"packages": [
{

6
syft/formats/spdxjson/test-fixtures/snapshot/TestSPDXRelationshipOrder.golden
View File

@ -3,14 +3,14 @@
"dataLicense": "CC0-1.0",
"SPDXID": "SPDXRef-DOCUMENT",
"name": "user-image-input",
"documentNamespace": "https://anchore.com/syft/image/user-image-input-2a1392ab-7eb5-4f2a-86f6-777aef3232e1",
"documentNamespace":"redacted",
"creationInfo": {
"licenseListVersion": "3.20",
"licenseListVersion":"redacted",
"creators": [
"Organization: Anchore, Inc",
"Tool: syft-v0.42.0-bogus"
],
"created": "2023-06-05T18:49:14Z"
"created":"redacted"
},
"packages": [
{

121
syft/formats/spdxtagvalue/encoder_test.go
View File

@ -2,7 +2,6 @@ package spdxtagvalue
import (
"flag"
"regexp"
"testing"
"github.com/anchore/syft/syft/formats/internal/testutils"
@ -11,28 +10,38 @@ import (
"github.com/anchore/syft/syft/source"
)
var updateSpdxTagValue = flag.Bool("update-spdx-tv", false, "update the *.golden files for spdx-tv encoders")
var updateSnapshot = flag.Bool("update-spdx-tv", false, "update the *.golden files for spdx-tv encoders")
var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing")
func TestSPDXTagValueDirectoryEncoder(t *testing.T) {
dir := t.TempDir()
testutils.AssertEncoderAgainstGoldenSnapshot(t,
Format(),
testutils.DirectoryInput(t),
*updateSpdxTagValue,
false,
spdxTagValueRedactor,
testutils.EncoderSnapshotTestConfig{
Subject: testutils.DirectoryInput(t, dir),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: false,
Redactor: redactor(dir),
},
)
}
func TestSPDXTagValueImageEncoder(t *testing.T) {
testImage := "image-simple"
testutils.AssertEncoderAgainstGoldenImageSnapshot(t,
Format(),
testutils.ImageInput(t, testImage, testutils.FromSnapshot()),
testImage,
*updateSpdxTagValue,
false,
spdxTagValueRedactor,
testutils.ImageSnapshotTestConfig{
Image: testImage,
UpdateImageSnapshot: *updateImage,
},
testutils.EncoderSnapshotTestConfig{
Subject: testutils.ImageInput(t, testImage, testutils.FromSnapshot()),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: false,
Redactor: redactor(),
},
)
}
@ -45,28 +54,33 @@ func TestSPDXJSONSPDXIDs(t *testing.T) {
p.SetID()
pkgs = append(pkgs, p)
}
testutils.AssertEncoderAgainstGoldenSnapshot(t,
Format(),
sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: pkg.NewCollection(pkgs...),
},
Relationships: nil,
Source: source.Metadata{
Scheme: source.DirectoryScheme,
Path: "foobar/baz", // in this case, foobar is used as the spdx docment name
},
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
Configuration: map[string]string{
"config-key": "config-value",
},
s := sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: pkg.NewCollection(pkgs...),
},
Relationships: nil,
Source: source.Description{
Metadata: source.DirectorySourceMetadata{Path: "foobar/baz"}, // in this case, foobar is used as the spdx docment name
},
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
Configuration: map[string]string{
"config-key": "config-value",
},
},
*updateSpdxTagValue,
false,
spdxTagValueRedactor,
}
testutils.AssertEncoderAgainstGoldenSnapshot(t,
testutils.EncoderSnapshotTestConfig{
Subject: s,
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: false,
Redactor: redactor(),
},
)
}
@ -74,23 +88,36 @@ func TestSPDXRelationshipOrder(t *testing.T) {
testImage := "image-simple"
s := testutils.ImageInput(t, testImage, testutils.FromSnapshot())
testutils.AddSampleFileRelationships(&s)
testutils.AssertEncoderAgainstGoldenImageSnapshot(t,
Format(),
s,
testImage,
*updateSpdxTagValue,
false,
spdxTagValueRedactor,
testutils.ImageSnapshotTestConfig{
Image: testImage,
UpdateImageSnapshot: *updateImage,
},
testutils.EncoderSnapshotTestConfig{
Subject: s,
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: false,
Redactor: redactor(),
},
)
}
func spdxTagValueRedactor(s []byte) []byte {
// each SBOM reports the time it was generated, which is not useful during snapshot testing
s = regexp.MustCompile(`Created: .*`).ReplaceAll(s, []byte("redacted"))
func redactor(values ...string) testutils.Redactor {
return testutils.NewRedactions().
WithValuesRedacted(values...).
WithPatternRedactors(
map[string]string{
// each SBOM reports the time it was generated, which is not useful during snapshot testing
`Created: .*`: "Created: redacted",
// each SBOM reports a unique documentNamespace when generated, this is not useful for snapshot testing
s = regexp.MustCompile(`DocumentNamespace: https://anchore.com/syft/.*`).ReplaceAll(s, []byte("redacted"))
// each SBOM reports a unique documentNamespace when generated, this is not useful for snapshot testing
`DocumentNamespace: https://anchore.com/syft/.*`: "DocumentNamespace: redacted",
// the license list will be updated periodically, the value here should not be directly tested in snapshot tests
return regexp.MustCompile(`LicenseListVersion: .*`).ReplaceAll(s, []byte("redacted"))
// the license list will be updated periodically, the value here should not be directly tested in snapshot tests
`LicenseListVersion: .*`: "LicenseListVersion: redacted",
},
)
}

6
syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXJSONSPDXIDs.golden
View File

@ -2,11 +2,11 @@ SPDXVersion: SPDX-2.3
DataLicense: CC0-1.0
SPDXID: SPDXRef-DOCUMENT
DocumentName: foobar/baz
DocumentNamespace: https://anchore.com/syft/dir/foobar/baz-1813dede-1ac5-4c44-a640-4c56e213d575
LicenseListVersion: 3.20
DocumentNamespace: redacted
LicenseListVersion: redacted
Creator: Organization: Anchore, Inc
Creator: Tool: syft-v0.42.0-bogus
Created: 2023-05-09T17:11:49Z
Created: redacted
##### Package: @at-sign

6
syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXRelationshipOrder.golden
View File

@ -2,11 +2,11 @@ SPDXVersion: SPDX-2.3
DataLicense: CC0-1.0
SPDXID: SPDXRef-DOCUMENT
DocumentName: user-image-input
DocumentNamespace: https://anchore.com/syft/image/user-image-input-96ea886a-3297-4847-b211-6da405ff1f8f
LicenseListVersion: 3.20
DocumentNamespace: redacted
LicenseListVersion: redacted
Creator: Organization: Anchore, Inc
Creator: Tool: syft-v0.42.0-bogus
Created: 2023-05-09T17:11:49Z
Created: redacted
##### Unpackaged files

8
syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXTagValueDirectoryEncoder.golden
View File

@ -1,12 +1,12 @@
SPDXVersion: SPDX-2.3
DataLicense: CC0-1.0
SPDXID: SPDXRef-DOCUMENT
DocumentName: /some/path
DocumentNamespace: https://anchore.com/syft/dir/some/path-f7bdb1ee-7fef-48e7-a386-6ee3836d4a28
LicenseListVersion: 3.20
DocumentName: some/path
DocumentNamespace: redacted
LicenseListVersion: redacted
Creator: Organization: Anchore, Inc
Creator: Tool: syft-v0.42.0-bogus
Created: 2023-05-09T17:11:49Z
Created: redacted
##### Package: package-2

6
syft/formats/spdxtagvalue/test-fixtures/snapshot/TestSPDXTagValueImageEncoder.golden
View File

@ -2,11 +2,11 @@ SPDXVersion: SPDX-2.3
DataLicense: CC0-1.0
SPDXID: SPDXRef-DOCUMENT
DocumentName: user-image-input
DocumentNamespace: https://anchore.com/syft/image/user-image-input-44d44a85-2207-4b51-bd73-d0c7b080f6d3
LicenseListVersion: 3.20
DocumentNamespace: redacted
LicenseListVersion: redacted
Creator: Organization: Anchore, Inc
Creator: Tool: syft-v0.42.0-bogus
Created: 2023-05-09T17:11:49Z
Created: redacted
##### Package: package-2

75
syft/formats/syftjson/encoder_test.go
View File

@ -2,7 +2,6 @@ package syftjson
import (
"flag"
"regexp"
"testing"
stereoFile "github.com/anchore/stereoscope/pkg/file"
@ -16,36 +15,41 @@ import (
"github.com/anchore/syft/syft/source"
)
var updateJson = flag.Bool("update-json", false, "update the *.golden files for json encoders")
var updateSnapshot = flag.Bool("update-json", false, "update the *.golden files for json encoders")
var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing")
func TestDirectoryEncoder(t *testing.T) {
dir := t.TempDir()
testutils.AssertEncoderAgainstGoldenSnapshot(t,
Format(),
testutils.DirectoryInput(t),
*updateJson,
true,
schemaVersionRedactor,
testutils.EncoderSnapshotTestConfig{
Subject: testutils.DirectoryInput(t, dir),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: true,
Redactor: redactor(dir),
},
)
}
func TestImageEncoder(t *testing.T) {
testImage := "image-simple"
testutils.AssertEncoderAgainstGoldenImageSnapshot(t,
Format(),
testutils.ImageInput(t, testImage, testutils.FromSnapshot()),
testImage,
*updateJson,
true,
schemaVersionRedactor,
testutils.ImageSnapshotTestConfig{
Image: testImage,
UpdateImageSnapshot: *updateImage,
},
testutils.EncoderSnapshotTestConfig{
Subject: testutils.ImageInput(t, testImage, testutils.FromSnapshot()),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: true,
Redactor: redactor(),
},
)
}
func schemaVersionRedactor(s []byte) []byte {
pattern := regexp.MustCompile(`,?\s*"schema":\s*\{[^}]*}`)
out := pattern.ReplaceAll(s, []byte(""))
return out
}
func TestEncodeFullJSONDocument(t *testing.T) {
catalog := pkg.NewCollection()
@ -176,10 +180,9 @@ func TestEncodeFullJSONDocument(t *testing.T) {
},
},
},
Source: source.Metadata{
ID: "c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
Scheme: source.ImageScheme,
ImageMetadata: source.ImageMetadata{
Source: source.Description{
ID: "c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "user-image-input",
ID: "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
ManifestDigest: "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
@ -188,7 +191,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
"stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b",
},
Size: 38,
Layers: []source.LayerMetadata{
Layers: []source.StereoscopeLayerMetadata{
{
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
Digest: "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b",
@ -217,10 +220,24 @@ func TestEncodeFullJSONDocument(t *testing.T) {
}
testutils.AssertEncoderAgainstGoldenSnapshot(t,
Format(),
s,
*updateJson,
true,
schemaVersionRedactor,
testutils.EncoderSnapshotTestConfig{
Subject: s,
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: true,
Redactor: redactor(),
},
)
}
func redactor(values ...string) testutils.Redactor {
return testutils.NewRedactions().
WithValuesRedacted(values...).
WithPatternRedactors(
map[string]string{
// remove schema version (don't even show the key or value)
`,?\s*"schema":\s*\{[^}]*}`: "",
},
)
}

4
syft/formats/syftjson/model/package.go
View File

@ -102,7 +102,7 @@ func (p *Package) UnmarshalJSON(b []byte) error {
return err
}
err := unpackMetadata(p, unpacker)
err := unpackPkgMetadata(p, unpacker)
if errors.Is(err, errUnknownMetadataType) {
log.Warnf("unknown package metadata type=%q for packageID=%q", p.MetadataType, p.ID)
return nil
@ -111,7 +111,7 @@ func (p *Package) UnmarshalJSON(b []byte) error {
return err
}
func unpackMetadata(p *Package, unpacker packageMetadataUnpacker) error {
func unpackPkgMetadata(p *Package, unpacker packageMetadataUnpacker) error {
p.MetadataType = pkg.CleanMetadataType(unpacker.MetadataType)
typ, ok := pkg.MetadataTypeByName[p.MetadataType]

2
syft/formats/syftjson/model/package_test.go
View File

@ -362,7 +362,7 @@ func Test_unpackMetadata(t *testing.T) {
var unpacker packageMetadataUnpacker
require.NoError(t, json.Unmarshal(test.packageData, &unpacker))
err := unpackMetadata(p, unpacker)
err := unpackPkgMetadata(p, unpacker)
assert.Equal(t, test.metadataType, p.MetadataType)
test.wantErr(t, err)

101
syft/formats/syftjson/model/source.go
View File

@ -3,53 +3,114 @@ package model
import (
"encoding/json"
"fmt"
"reflect"
"strconv"
"strings"
"github.com/anchore/syft/syft/internal/sourcemetadata"
"github.com/anchore/syft/syft/source"
)
// Source object represents the thing that was cataloged
type Source struct {
ID string `json:"id"`
Type string `json:"type"`
Target interface{} `json:"target"`
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
Type string `json:"type"`
Metadata interface{} `json:"metadata"`
}
// sourceUnpacker is used to unmarshal Source objects
type sourceUnpacker struct {
ID string `json:"id,omitempty"`
Type string `json:"type"`
Target json.RawMessage `json:"target"`
ID string `json:"id,omitempty"`
Name string `json:"name"`
Version string `json:"version"`
Type string `json:"type"`
Metadata json.RawMessage `json:"metadata"`
Target json.RawMessage `json:"target"` // pre-v9 schema support
}
// UnmarshalJSON populates a source object from JSON bytes.
func (s *Source) UnmarshalJSON(b []byte) error {
var unpacker sourceUnpacker
if err := json.Unmarshal(b, &unpacker); err != nil {
err := json.Unmarshal(b, &unpacker)
if err != nil {
return err
}
s.Name = unpacker.Name
s.Version = unpacker.Version
s.Type = unpacker.Type
s.ID = unpacker.ID
switch s.Type {
case "directory", "file":
if target, err := strconv.Unquote(string(unpacker.Target)); err == nil {
s.Target = target
} else {
s.Target = string(unpacker.Target[:])
if len(unpacker.Target) > 0 {
s.Type = cleanPreSchemaV9MetadataType(s.Type)
s.Metadata, err = extractPreSchemaV9Metadata(s.Type, unpacker.Target)
if err != nil {
return fmt.Errorf("unable to extract pre-schema-v9 source metadata: %w", err)
}
return nil
}
case "image":
var payload source.ImageMetadata
if err := json.Unmarshal(unpacker.Target, &payload); err != nil {
return unpackSrcMetadata(s, unpacker)
}
func unpackSrcMetadata(s *Source, unpacker sourceUnpacker) error {
rt := sourcemetadata.ReflectTypeFromJSONName(s.Type)
if rt == nil {
return fmt.Errorf("unable to find source metadata type=%q", s.Type)
}
val := reflect.New(rt).Interface()
if len(unpacker.Metadata) > 0 {
if err := json.Unmarshal(unpacker.Metadata, val); err != nil {
return err
}
s.Target = payload
default:
return fmt.Errorf("unsupported package metadata type: %+v", s.Type)
}
s.Metadata = reflect.ValueOf(val).Elem().Interface()
return nil
}
func cleanPreSchemaV9MetadataType(t string) string {
t = strings.ToLower(t)
if t == "dir" {
return "directory"
}
return t
}
func extractPreSchemaV9Metadata(t string, target []byte) (interface{}, error) {
switch t {
case "directory", "dir":
cleanTarget, err := strconv.Unquote(string(target))
if err != nil {
cleanTarget = string(target)
}
return source.DirectorySourceMetadata{
Path: cleanTarget,
}, nil
case "file":
cleanTarget, err := strconv.Unquote(string(target))
if err != nil {
cleanTarget = string(target)
}
return source.FileSourceMetadata{
Path: cleanTarget,
}, nil
case "image":
var payload source.StereoscopeImageSourceMetadata
if err := json.Unmarshal(target, &payload); err != nil {
return nil, err
}
return payload, nil
default:
return nil, fmt.Errorf("unsupported package metadata type: %+v", t)
}
}

203
syft/formats/syftjson/model/source_test.go
View File

@ -6,17 +6,194 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/internal/sourcemetadata"
"github.com/anchore/syft/syft/source"
)
func TestSource_UnmarshalJSON(t *testing.T) {
tracker := sourcemetadata.NewCompletionTester(t)
cases := []struct {
name string
input []byte
expected *Source
wantErr require.ErrorAssertionFunc
}{
{
name: "directory",
input: []byte(`{
"id": "foobar",
"type": "directory",
"metadata": {"path": "/var/lib/foo", "base":"/nope"}
}`),
expected: &Source{
ID: "foobar",
Type: "directory",
Metadata: source.DirectorySourceMetadata{
Path: "/var/lib/foo",
//Base: "/nope", // note: should be ignored entirely
},
},
},
{
name: "image",
input: []byte(`{
"id": "foobar",
"type": "image",
"metadata": {
"userInput": "alpine:3.10",
"imageID": "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a",
"manifestDigest": "sha256:e515aad2ed234a5072c4d2ef86a1cb77d5bfe4b11aa865d9214875734c4eeb3c",
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"tags": [],
"imageSize": 5576169,
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:9fb3aa2f8b8023a4bebbf92aa567caf88e38e969ada9f0ac12643b2847391635",
"size": 5576169
}
],
"manifest": "ewogICAic2NoZW1hVmVyc2lvbiI6IDIsCiAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5kaXN0cmlidXRpb24ubWFuaWZlc3QudjIranNvbiIsCiAgICJjb25maWciOiB7CiAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5jb250YWluZXIuaW1hZ2UudjEranNvbiIsCiAgICAgICJzaXplIjogMTQ3MiwKICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6ZTdiMzAwYWVlOWY5YmYzNDMzZDMyYmM5MzA1YmZkZDIyMTgzYmViNTlkOTMzYjQ4ZDc3YWI1NmJhNTNhMTk3YSIKICAgfSwKICAgImxheWVycyI6IFsKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDI3OTgzMzgsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2OjM5NmMzMTgzNzExNmFjMjkwNDU4YWZjYjkyOGY2OGI2Y2MxYzdiZGQ2OTYzZmM3MmY1MmYzNjVhMmE4OWMxYjUiCiAgICAgIH0KICAgXQp9",
"config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJIb3N0bmFtZSI6IiIsIkRvbWFpbm5hbWUiOiIiLCJVc2VyIjoiIiwiQXR0YWNoU3RkaW4iOmZhbHNlLCJBdHRhY2hTdGRvdXQiOmZhbHNlLCJBdHRhY2hTdGRlcnIiOmZhbHNlLCJUdHkiOmZhbHNlLCJPcGVuU3RkaW4iOmZhbHNlLCJTdGRpbk9uY2UiOmZhbHNlLCJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpbIi9iaW4vc2giXSwiSW1hZ2UiOiJzaGEyNTY6ZWIyMDgwYzQ1NWU5NGMyMmFlMzViM2FlZjllMDc4YzQ5MmEwMDc5NTQxMmUwMjZlNGQ2YjQxZWY2NGJjN2RkOCIsIlZvbHVtZXMiOm51bGwsIldvcmtpbmdEaXIiOiIiLCJFbnRyeXBvaW50IjpudWxsLCJPbkJ1aWxkIjpudWxsLCJMYWJlbHMiOm51bGx9LCJjb250YWluZXIiOiJmZGI3ZTgwZTMzMzllOGQwNTk5MjgyZTYwNmM5MDdhYTU4ODFlZTRjNjY4YTY4MTM2MTE5ZTZkZmFjNmNlM2E0IiwiY29udGFpbmVyX2NvbmZpZyI6eyJIb3N0bmFtZSI6ImZkYjdlODBlMzMzOSIsIkRvbWFpbm5hbWUiOiIiLCJVc2VyIjoiIiwiQXR0YWNoU3RkaW4iOmZhbHNlLCJBdHRhY2hTdGRvdXQiOmZhbHNlLCJBdHRhY2hTdGRlcnIiOmZhbHNlLCJUdHkiOmZhbHNlLCJPcGVuU3RkaW4iOmZhbHNlLCJTdGRpbk9uY2UiOmZhbHNlLCJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpbIi9iaW4vc2giLCItYyIsIiMobm9wKSAiLCJDTUQgW1wiL2Jpbi9zaFwiXSJdLCJJbWFnZSI6InNoYTI1NjplYjIwODBjNDU1ZTk0YzIyYWUzNWIzYWVmOWUwNzhjNDkyYTAwNzk1NDEyZTAyNmU0ZDZiNDFlZjY0YmM3ZGQ4IiwiVm9sdW1lcyI6bnVsbCwiV29ya2luZ0RpciI6IiIsIkVudHJ5cG9pbnQiOm51bGwsIk9uQnVpbGQiOm51bGwsIkxhYmVscyI6e319LCJjcmVhdGVkIjoiMjAyMS0wNC0xNFQxOToyMDowNS4zMzgzOTc3NjFaIiwiZG9ja2VyX3ZlcnNpb24iOiIxOS4wMy4xMiIsImhpc3RvcnkiOlt7ImNyZWF0ZWQiOiIyMDIxLTA0LTE0VDE5OjIwOjA0Ljk4NzIxOTEyNFoiLCJjcmVhdGVkX2J5IjoiL2Jpbi9zaCAtYyAjKG5vcCkgQUREIGZpbGU6YzUzNzdlYWE5MjZiZjQxMmRkOGQ0YTA4YjBhMWYyMzk5Y2ZkNzA4NzQzNTMzYjBhYTAzYjUzZDE0Y2I0YmI0ZSBpbiAvICJ9LHsiY3JlYXRlZCI6IjIwMjEtMDQtMTRUMTk6MjA6MDUuMzM4Mzk3NzYxWiIsImNyZWF0ZWRfYnkiOiIvYmluL3NoIC1jICMobm9wKSAgQ01EIFtcIi9iaW4vc2hcIl0iLCJlbXB0eV9sYXllciI6dHJ1ZX1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6OWZiM2FhMmY4YjgwMjNhNGJlYmJmOTJhYTU2N2NhZjg4ZTM4ZTk2OWFkYTlmMGFjMTI2NDNiMjg0NzM5MTYzNSJdfX0=",
"repoDigests": [
"index.docker.io/library/alpine@sha256:451eee8bedcb2f029756dc3e9d73bab0e7943c1ac55cff3a4861c52a0fdd3e98"
]
}
}`),
expected: &Source{
ID: "foobar",
Type: "image",
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "alpine:3.10",
ID: "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a",
ManifestDigest: "sha256:e515aad2ed234a5072c4d2ef86a1cb77d5bfe4b11aa865d9214875734c4eeb3c",
MediaType: "application/vnd.docker.distribution.manifest.v2+json",
Tags: []string{},
Size: 5576169,
Layers: []source.StereoscopeLayerMetadata{
{
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
Digest: "sha256:9fb3aa2f8b8023a4bebbf92aa567caf88e38e969ada9f0ac12643b2847391635",
Size: 5576169,
},
},
RawManifest: []byte(`{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": {
"mediaType": "application/vnd.docker.container.image.v1+json",
"size": 1472,
"digest": "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a"
},
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": 2798338,
"digest": "sha256:396c31837116ac290458afcb928f68b6cc1c7bdd6963fc72f52f365a2a89c1b5"
}
]
}`),
RawConfig: []byte(`{"architecture":"amd64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh"],"Image":"sha256:eb2080c455e94c22ae35b3aef9e078c492a00795412e026e4d6b41ef64bc7dd8","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"fdb7e80e3339e8d0599282e606c907aa5881ee4c668a68136119e6dfac6ce3a4","container_config":{"Hostname":"fdb7e80e3339","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","#(nop) ","CMD [\"/bin/sh\"]"],"Image":"sha256:eb2080c455e94c22ae35b3aef9e078c492a00795412e026e4d6b41ef64bc7dd8","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":{}},"created":"2021-04-14T19:20:05.338397761Z","docker_version":"19.03.12","history":[{"created":"2021-04-14T19:20:04.987219124Z","created_by":"/bin/sh -c #(nop) ADD file:c5377eaa926bf412dd8d4a08b0a1f2399cfd708743533b0aa03b53d14cb4bb4e in / "},{"created":"2021-04-14T19:20:05.338397761Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\"]","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:9fb3aa2f8b8023a4bebbf92aa567caf88e38e969ada9f0ac12643b2847391635"]}}`),
RepoDigests: []string{
"index.docker." +
"io/library/alpine@sha256:451eee8bedcb2f029756dc3e9d73bab0e7943c1ac55cff3a4861c52a0fdd3e98",
},
},
},
},
{
name: "file",
input: []byte(`{
"id": "foobar",
"type": "file",
"metadata": {
"path": "/var/lib/foo/go.mod",
"mimeType": "text/plain",
"digests": [
{
"algorithm": "sha256",
"value": "e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a"
}
]
}
}`),
expected: &Source{
ID: "foobar",
Type: "file",
Metadata: source.FileSourceMetadata{
Path: "/var/lib/foo/go.mod",
Digests: []file.Digest{
{
Algorithm: "sha256",
Value: "e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a",
},
},
MIMEType: "text/plain",
},
},
},
{
name: "unknown source type",
input: []byte(`{
"id": "foobar",
"type": "unknown-thing",
"target":"/var/lib/foo"
}`),
expected: &Source{
ID: "foobar",
Type: "unknown-thing",
},
wantErr: require.Error,
},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = require.NoError
}
var src Source
err := json.Unmarshal(tt.input, &src)
tt.wantErr(t, err)
if diff := cmp.Diff(tt.expected, &src); diff != "" {
t.Errorf("unexpected result from Source unmarshaling (-want +got)\n%s", diff)
}
tracker.Tested(t, tt.expected.Metadata)
})
}
}
func TestSource_UnmarshalJSON_PreSchemaV9(t *testing.T) {
cases := []struct {
name string
input []byte
expectedSource *Source
errAssertion assert.ErrorAssertionFunc
}{
{
name: "abbreviated directory",
input: []byte(`{
"id": "foobar",
"type": "dir",
"target":"/var/lib/foo"
}`),
expectedSource: &Source{
ID: "foobar",
Type: "directory",
Metadata: source.DirectorySourceMetadata{
Path: "/var/lib/foo",
},
},
errAssertion: assert.NoError,
},
{
name: "directory",
input: []byte(`{
@ -25,9 +202,11 @@ func TestSource_UnmarshalJSON(t *testing.T) {
"target":"/var/lib/foo"
}`),
expectedSource: &Source{
ID: "foobar",
Type: "directory",
Target: "/var/lib/foo",
ID: "foobar",
Type: "directory",
Metadata: source.DirectorySourceMetadata{
Path: "/var/lib/foo",
},
},
errAssertion: assert.NoError,
},
@ -60,14 +239,14 @@ func TestSource_UnmarshalJSON(t *testing.T) {
expectedSource: &Source{
ID: "foobar",
Type: "image",
Target: source.ImageMetadata{
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "alpine:3.10",
ID: "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a",
ManifestDigest: "sha256:e515aad2ed234a5072c4d2ef86a1cb77d5bfe4b11aa865d9214875734c4eeb3c",
MediaType: "application/vnd.docker.distribution.manifest.v2+json",
Tags: []string{},
Size: 5576169,
Layers: []source.LayerMetadata{
Layers: []source.StereoscopeLayerMetadata{
{
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
Digest: "sha256:9fb3aa2f8b8023a4bebbf92aa567caf88e38e969ada9f0ac12643b2847391635",
@ -107,9 +286,11 @@ func TestSource_UnmarshalJSON(t *testing.T) {
"target":"/var/lib/foo/go.mod"
}`),
expectedSource: &Source{
ID: "foobar",
Type: "file",
Target: "/var/lib/foo/go.mod",
ID: "foobar",
Type: "file",
Metadata: source.FileSourceMetadata{
Path: "/var/lib/foo/go.mod",
},
},
errAssertion: assert.NoError,
},
@ -130,12 +311,12 @@ func TestSource_UnmarshalJSON(t *testing.T) {
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
source := new(Source)
src := new(Source)
err := json.Unmarshal(testCase.input, source)
err := json.Unmarshal(testCase.input, src)
testCase.errAssertion(t, err)
if diff := cmp.Diff(testCase.expectedSource, source); diff != "" {
if diff := cmp.Diff(testCase.expectedSource, src); diff != "" {
t.Errorf("unexpected result from Source unmarshaling (-want +got)\n%s", diff)
}
})

12
syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden
View File

@ -72,9 +72,13 @@
],
"artifactRelationships": [],
"source": {
"id": "eda6cf0b63f1a1d2eaf7792a2a98c832c21a18e6992bcebffe6381781cc85cbc",
"id": "d1563248892cd59af469f406eee907c76fa4f9041f5410d45b93aef903bc4216",
"name": "some/path",
"version": "",
"type": "directory",
"target": "/some/path"
"metadata": {
"path": "redacted/some/path"
}
},
"distro": {
"prettyName": "debian",
@ -92,9 +96,5 @@
"configuration": {
"config-key": "config-value"
}
},
"schema": {
"version": "8.0.0",
"url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-8.0.0.json"
}
}

8
syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden
View File

@ -149,8 +149,10 @@
],
"source": {
"id": "c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
"name": "",
"version": "",
"type": "image",
"target": {
"metadata": {
"userInput": "user-image-input",
"imageID": "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
"manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
@ -192,9 +194,5 @@
"configuration": {
"config-key": "config-value"
}
},
"schema": {
"version": "8.0.0",
"url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-8.0.0.json"
}
}

8
syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden
View File

@ -70,8 +70,10 @@
"artifactRelationships": [],
"source": {
"id": "c8ac88bbaf3d1c036f6a1d601c3d52bafbf05571c97d68322e7cb3a7ecaa304f",
"name": "user-image-input",
"version": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
"type": "image",
"target": {
"metadata": {
"userInput": "user-image-input",
"imageID": "sha256:a3c61dc134d2f31b415c50324e75842d7f91622f39a89468e51938330b3fd3af",
"manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
@ -115,9 +117,5 @@
"configuration": {
"config-key": "config-value"
}
},
"schema": {
"version": "8.0.0",
"url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-8.0.0.json"
}
}

44
syft/formats/syftjson/to_format_model.go
View File

@ -12,6 +12,7 @@ import (
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/formats/syftjson/model"
"github.com/anchore/syft/syft/internal/sourcemetadata"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
@ -20,17 +21,12 @@ import (
// ToFormatModel transforms the sbom import a format-specific model.
func ToFormatModel(s sbom.SBOM) model.Document {
src, err := toSourceModel(s.Source)
if err != nil {
log.Warnf("unable to create syft-json source object: %+v", err)
}
return model.Document{
Artifacts: toPackageModels(s.Artifacts.Packages),
ArtifactRelationships: toRelationshipModel(s.Relationships),
Files: toFile(s),
Secrets: toSecrets(s.Artifacts.Secrets),
Source: src,
Source: toSourceModel(s.Source),
Distro: toLinuxReleaser(s.Artifacts.LinuxDistribution),
Descriptor: toDescriptor(s.Descriptor),
Schema: model.Schema{
@ -267,10 +263,16 @@ func toRelationshipModel(relationships []artifact.Relationship) []model.Relation
}
// toSourceModel creates a new source object to be represented into JSON.
func toSourceModel(src source.Metadata) (model.Source, error) {
switch src.Scheme {
case source.ImageScheme:
metadata := src.ImageMetadata
func toSourceModel(src source.Description) model.Source {
m := model.Source{
ID: src.ID,
Name: src.Name,
Version: src.Version,
Type: sourcemetadata.JSONName(src.Metadata),
Metadata: src.Metadata,
}
if metadata, ok := src.Metadata.(source.StereoscopeImageSourceMetadata); ok {
// ensure that empty collections are not shown as null
if metadata.RepoDigests == nil {
metadata.RepoDigests = []string{}
@ -278,24 +280,8 @@ func toSourceModel(src source.Metadata) (model.Source, error) {
if metadata.Tags == nil {
metadata.Tags = []string{}
}
return model.Source{
ID: src.ID,
Type: "image",
Target: metadata,
}, nil
case source.DirectoryScheme:
return model.Source{
ID: src.ID,
Type: "directory",
Target: src.Path,
}, nil
case source.FileScheme:
return model.Source{
ID: src.ID,
Type: "file",
Target: src.Path,
}, nil
default:
return model.Source{}, fmt.Errorf("unsupported source: %q", src.Scheme)
m.Metadata = metadata
}
return m
}

176
syft/formats/syftjson/to_format_model_test.go
View File

@ -1,62 +1,174 @@
package syftjson
import (
"encoding/json"
"testing"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/formats/syftjson/model"
"github.com/anchore/syft/syft/internal/sourcemetadata"
"github.com/anchore/syft/syft/source"
)
func Test_toSourceModel(t *testing.T) {
allSchemes := strset.New()
for _, s := range source.AllSchemes {
allSchemes.Add(string(s))
func Test_toSourceModel_IgnoreBase(t *testing.T) {
tests := []struct {
name string
src source.Description
}{
{
name: "directory",
src: source.Description{
ID: "test-id",
Metadata: source.DirectorySourceMetadata{
Path: "some/path",
Base: "some/base",
},
},
},
}
testedSchemes := strset.New()
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
// assert the model transformation is correct
actual := toSourceModel(test.src)
by, err := json.Marshal(actual)
require.NoError(t, err)
assert.NotContains(t, string(by), "some/base")
})
}
}
func Test_toSourceModel(t *testing.T) {
tracker := sourcemetadata.NewCompletionTester(t)
tests := []struct {
name string
src source.Metadata
src source.Description
expected model.Source
}{
{
name: "directory",
src: source.Metadata{
ID: "test-id",
Scheme: source.DirectoryScheme,
Path: "some/path",
src: source.Description{
ID: "test-id",
Name: "some-name",
Version: "some-version",
Metadata: source.DirectorySourceMetadata{
Path: "some/path",
Base: "some/base",
},
},
expected: model.Source{
ID: "test-id",
Type: "directory",
Target: "some/path",
ID: "test-id",
Name: "some-name",
Version: "some-version",
Type: "directory",
Metadata: source.DirectorySourceMetadata{
Path: "some/path",
Base: "some/base",
},
},
},
{
name: "file",
src: source.Metadata{
ID: "test-id",
Scheme: source.FileScheme,
Path: "some/path",
src: source.Description{
ID: "test-id",
Name: "some-name",
Version: "some-version",
Metadata: source.FileSourceMetadata{
Path: "some/path",
Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}},
MIMEType: "text/plain",
},
},
expected: model.Source{
ID: "test-id",
Type: "file",
Target: "some/path",
ID: "test-id",
Name: "some-name",
Version: "some-version",
Type: "file",
Metadata: source.FileSourceMetadata{
Path: "some/path",
Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}},
MIMEType: "text/plain",
},
},
},
{
name: "image",
src: source.Metadata{
ID: "test-id",
Scheme: source.ImageScheme,
ImageMetadata: source.ImageMetadata{
src: source.Description{
ID: "test-id",
Name: "some-name",
Version: "some-version",
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "user-input",
ID: "id...",
ManifestDigest: "digest...",
MediaType: "type...",
},
},
expected: model.Source{
ID: "test-id",
Name: "some-name",
Version: "some-version",
Type: "image",
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "user-input",
ID: "id...",
ManifestDigest: "digest...",
MediaType: "type...",
RepoDigests: []string{},
Tags: []string{},
},
},
},
// below are regression tests for when the name/version are not provided
// historically we've hoisted up the name/version from the metadata, now it is a simple pass-through
{
name: "directory - no name/version",
src: source.Description{
ID: "test-id",
Metadata: source.DirectorySourceMetadata{
Path: "some/path",
Base: "some/base",
},
},
expected: model.Source{
ID: "test-id",
Type: "directory",
Metadata: source.DirectorySourceMetadata{
Path: "some/path",
Base: "some/base",
},
},
},
{
name: "file - no name/version",
src: source.Description{
ID: "test-id",
Metadata: source.FileSourceMetadata{
Path: "some/path",
Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}},
MIMEType: "text/plain",
},
},
expected: model.Source{
ID: "test-id",
Type: "file",
Metadata: source.FileSourceMetadata{
Path: "some/path",
Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}},
MIMEType: "text/plain",
},
},
},
{
name: "image - no name/version",
src: source.Description{
ID: "test-id",
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "user-input",
ID: "id...",
ManifestDigest: "digest...",
@ -66,7 +178,7 @@ func Test_toSourceModel(t *testing.T) {
expected: model.Source{
ID: "test-id",
Type: "image",
Target: source.ImageMetadata{
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "user-input",
ID: "id...",
ManifestDigest: "digest...",
@ -79,18 +191,14 @@ func Test_toSourceModel(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
// track each scheme tested (passed or not)
testedSchemes.Add(string(test.src.Scheme))
// assert the model transformation is correct
actual, err := toSourceModel(test.src)
require.NoError(t, err)
actual := toSourceModel(test.src)
assert.Equal(t, test.expected, actual)
// track each scheme tested (passed or not)
tracker.Tested(t, test.expected.Metadata)
})
}
// assert all possible schemes were under test
assert.ElementsMatch(t, allSchemes.List(), testedSchemes.List(), "not all source.Schemes are under test")
}
func Test_toFileType(t *testing.T) {

52
syft/formats/syftjson/to_syft_model.go
View File

@ -202,12 +202,12 @@ func toSyftRelationships(doc *model.Document, catalog *pkg.Collection, relations
return out, conversionErrors
}
func toSyftSource(s model.Source) *source.Source {
newSrc := &source.Source{
Metadata: *toSyftSourceData(s),
func toSyftSource(s model.Source) source.Source {
description := toSyftSourceData(s)
if description == nil {
return nil
}
newSrc.SetID()
return newSrc
return source.FromDescription(*description)
}
func toSyftRelationship(idMap map[string]interface{}, relationship model.Relationship, idAliases map[string]string) (*artifact.Relationship, error) {
@ -257,43 +257,13 @@ func toSyftDescriptor(d model.Descriptor) sbom.Descriptor {
}
}
func toSyftSourceData(s model.Source) *source.Metadata {
switch s.Type {
case "directory":
path, ok := s.Target.(string)
if !ok {
log.Warnf("unable to parse source target as string: %+v", s.Target)
return nil
}
return &source.Metadata{
ID: s.ID,
Scheme: source.DirectoryScheme,
Path: path,
}
case "file":
path, ok := s.Target.(string)
if !ok {
log.Warnf("unable to parse source target as string: %+v", s.Target)
return nil
}
return &source.Metadata{
ID: s.ID,
Scheme: source.FileScheme,
Path: path,
}
case "image":
metadata, ok := s.Target.(source.ImageMetadata)
if !ok {
log.Warnf("unable to parse source target as image metadata: %+v", s.Target)
return nil
}
return &source.Metadata{
ID: s.ID,
Scheme: source.ImageScheme,
ImageMetadata: metadata,
}
func toSyftSourceData(s model.Source) *source.Description {
return &source.Description{
ID: s.ID,
Name: s.Name,
Version: s.Version,
Metadata: s.Metadata,
}
return nil
}
func toSyftCatalog(pkgs []model.Package, idAliases map[string]string) *pkg.Collection {

154
syft/formats/syftjson/to_syft_model_test.go
View File

@ -4,66 +4,154 @@ import (
"errors"
"testing"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
stereoFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/formats/syftjson/model"
"github.com/anchore/syft/syft/internal/sourcemetadata"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
func Test_toSyftSourceData(t *testing.T) {
allSchemes := strset.New()
for _, s := range source.AllSchemes {
allSchemes.Add(string(s))
}
testedSchemes := strset.New()
tracker := sourcemetadata.NewCompletionTester(t)
tests := []struct {
name string
src model.Source
expected source.Metadata
expected *source.Description
}{
{
name: "directory",
expected: source.Metadata{
Scheme: source.DirectoryScheme,
Path: "some/path",
},
src: model.Source{
Type: "directory",
Target: "some/path",
ID: "the-id",
Name: "some-name",
Version: "some-version",
Type: "directory",
Metadata: source.DirectorySourceMetadata{
Path: "some/path",
Base: "some/base",
},
},
expected: &source.Description{
ID: "the-id",
Name: "some-name",
Version: "some-version",
Metadata: source.DirectorySourceMetadata{
Path: "some/path",
Base: "some/base",
},
},
},
{
name: "file",
expected: source.Metadata{
Scheme: source.FileScheme,
Path: "some/path",
},
src: model.Source{
Type: "file",
Target: "some/path",
ID: "the-id",
Name: "some-name",
Version: "some-version",
Type: "file",
Metadata: source.FileSourceMetadata{
Path: "some/path",
Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}},
MIMEType: "text/plain",
},
},
expected: &source.Description{
ID: "the-id",
Name: "some-name",
Version: "some-version",
Metadata: source.FileSourceMetadata{
Path: "some/path",
Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}},
MIMEType: "text/plain",
},
},
},
{
name: "image",
expected: source.Metadata{
Scheme: source.ImageScheme,
ImageMetadata: source.ImageMetadata{
src: model.Source{
ID: "the-id",
Name: "some-name",
Version: "some-version",
Type: "image",
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "user-input",
ID: "id...",
ManifestDigest: "digest...",
MediaType: "type...",
},
},
expected: &source.Description{
ID: "the-id",
Name: "some-name",
Version: "some-version",
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "user-input",
ID: "id...",
ManifestDigest: "digest...",
MediaType: "type...",
},
},
},
// below are regression tests for when the name/version are not provided
// historically we've hoisted up the name/version from the metadata, now it is a simple pass-through
{
name: "directory - no name/version",
src: model.Source{
ID: "the-id",
Type: "directory",
Metadata: source.DirectorySourceMetadata{
Path: "some/path",
Base: "some/base",
},
},
expected: &source.Description{
ID: "the-id",
Metadata: source.DirectorySourceMetadata{
Path: "some/path",
Base: "some/base",
},
},
},
{
name: "file - no name/version",
src: model.Source{
ID: "the-id",
Type: "file",
Metadata: source.FileSourceMetadata{
Path: "some/path",
Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}},
MIMEType: "text/plain",
},
},
expected: &source.Description{
ID: "the-id",
Metadata: source.FileSourceMetadata{
Path: "some/path",
Digests: []file.Digest{{Algorithm: "sha256", Value: "some-digest"}},
MIMEType: "text/plain",
},
},
},
{
name: "image - no name/version",
src: model.Source{
ID: "the-id",
Type: "image",
Target: source.ImageMetadata{
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "user-input",
ID: "id...",
ManifestDigest: "digest...",
MediaType: "type...",
},
},
expected: &source.Description{
ID: "the-id",
Metadata: source.StereoscopeImageSourceMetadata{
UserInput: "user-input",
ID: "id...",
ManifestDigest: "digest...",
@ -76,22 +164,18 @@ func Test_toSyftSourceData(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
// assert the model transformation is correct
actual := toSyftSourceData(test.src)
assert.Equal(t, test.expected, *actual)
assert.Equal(t, test.expected, actual)
// track each scheme tested (passed or not)
testedSchemes.Add(string(test.expected.Scheme))
tracker.Tested(t, test.expected.Metadata)
})
}
// assert all possible schemes were under test
assert.ElementsMatch(t, allSchemes.List(), testedSchemes.List(), "not all source.Schemes are under test")
}
func Test_idsHaveChanged(t *testing.T) {
s, err := toSyftModel(model.Document{
Source: model.Source{
Type: "file",
Target: "some/path",
Type: "file",
Metadata: source.FileSourceMetadata{Path: "some/path"},
},
Artifacts: []model.Package{
{
@ -116,17 +200,17 @@ func Test_idsHaveChanged(t *testing.T) {
},
})
assert.NoError(t, err)
assert.Len(t, s.Relationships, 1)
require.NoError(t, err)
require.Len(t, s.Relationships, 1)
r := s.Relationships[0]
from := s.Artifacts.Packages.Package(r.From.ID())
assert.NotNil(t, from)
require.NotNil(t, from)
assert.Equal(t, "pkg-1", from.Name)
to := s.Artifacts.Packages.Package(r.To.ID())
assert.NotNil(t, to)
require.NotNil(t, to)
assert.Equal(t, "pkg-2", to.Name)
}

13
syft/formats/table/encoder_test.go
View File

@ -9,14 +9,17 @@ import (
"github.com/anchore/syft/syft/formats/internal/testutils"
)
var updateTableGoldenFiles = flag.Bool("update-table", false, "update the *.golden files for table format")
var updateSnapshot = flag.Bool("update-table", false, "update the *.golden files for table format")
func TestTableEncoder(t *testing.T) {
testutils.AssertEncoderAgainstGoldenSnapshot(t,
Format(),
testutils.DirectoryInput(t),
*updateTableGoldenFiles,
false,
testutils.EncoderSnapshotTestConfig{
Subject: testutils.DirectoryInput(t, t.TempDir()),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: false,
},
)
}

27
syft/formats/template/encoder_test.go
View File

@ -9,19 +9,21 @@ import (
"github.com/anchore/syft/syft/formats/internal/testutils"
)
var updateTmpl = flag.Bool("update-tmpl", false, "update the *.golden files for json encoders")
var updateSnapshot = flag.Bool("update-template", false, "update the *.golden files for json encoders")
func TestFormatWithOption(t *testing.T) {
f := OutputFormat{}
f.SetTemplatePath("test-fixtures/csv.template")
testutils.AssertEncoderAgainstGoldenSnapshot(t,
f,
testutils.DirectoryInput(t),
*updateTmpl,
false,
testutils.EncoderSnapshotTestConfig{
Subject: testutils.DirectoryInput(t, t.TempDir()),
Format: f,
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: false,
},
)
}
func TestFormatWithOptionAndHasField(t *testing.T) {
@ -29,16 +31,19 @@ func TestFormatWithOptionAndHasField(t *testing.T) {
f.SetTemplatePath("test-fixtures/csv-hasField.template")
testutils.AssertEncoderAgainstGoldenSnapshot(t,
f,
testutils.DirectoryInputWithAuthorField(t),
*updateTmpl,
false,
testutils.EncoderSnapshotTestConfig{
Subject: testutils.DirectoryInputWithAuthorField(t),
Format: f,
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: false,
},
)
}
func TestFormatWithoutOptions(t *testing.T) {
f := Format()
err := f.Encode(nil, testutils.DirectoryInput(t))
err := f.Encode(nil, testutils.DirectoryInput(t, t.TempDir()))
assert.ErrorContains(t, err, "no template file: please provide a template path")
}

14
syft/formats/text/encoder.go
View File

@ -14,13 +14,15 @@ func encoder(output io.Writer, s sbom.SBOM) error {
w := new(tabwriter.Writer)
w.Init(output, 0, 8, 0, '\t', tabwriter.AlignRight)
switch s.Source.Scheme {
case source.DirectoryScheme, source.FileScheme:
fmt.Fprintf(w, "[Path: %s]\n", s.Source.Path)
case source.ImageScheme:
switch metadata := s.Source.Metadata.(type) {
case source.DirectorySourceMetadata:
fmt.Fprintf(w, "[Path: %s]\n", metadata.Path)
case source.FileSourceMetadata:
fmt.Fprintf(w, "[Path: %s]\n", metadata.Path)
case source.StereoscopeImageSourceMetadata:
fmt.Fprintln(w, "[Image]")
for idx, l := range s.Source.ImageMetadata.Layers {
for idx, l := range metadata.Layers {
fmt.Fprintln(w, " Layer:\t", idx)
fmt.Fprintln(w, " Digest:\t", l.Digest)
fmt.Fprintln(w, " Size:\t", l.Size)
@ -29,7 +31,7 @@ func encoder(output io.Writer, s sbom.SBOM) error {
w.Flush()
}
default:
return fmt.Errorf("unsupported source: %T", s.Source.Scheme)
return fmt.Errorf("unsupported source: %T", s.Source.Metadata)
}
// populate artifacts...

38
syft/formats/text/encoder_test.go
View File

@ -7,24 +7,42 @@ import (
"github.com/anchore/syft/syft/formats/internal/testutils"
)
var updateTextEncoderGoldenFiles = flag.Bool("update-text", false, "update the *.golden files for text encoder")
var updateSnapshot = flag.Bool("update-text", false, "update the *.golden files for text encoder")
var updateImage = flag.Bool("update-image", false, "update the golden image used for image encoder testing")
func TestTextDirectoryEncoder(t *testing.T) {
dir := t.TempDir()
testutils.AssertEncoderAgainstGoldenSnapshot(t,
Format(),
testutils.DirectoryInput(t),
*updateTextEncoderGoldenFiles,
false,
testutils.EncoderSnapshotTestConfig{
Subject: testutils.DirectoryInput(t, dir),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: false,
Redactor: redactor(dir),
},
)
}
func TestTextImageEncoder(t *testing.T) {
testImage := "image-simple"
testutils.AssertEncoderAgainstGoldenImageSnapshot(t,
Format(),
testutils.ImageInput(t, testImage, testutils.FromSnapshot()),
testImage,
*updateTextEncoderGoldenFiles,
false,
testutils.ImageSnapshotTestConfig{
Image: testImage,
UpdateImageSnapshot: *updateImage,
},
testutils.EncoderSnapshotTestConfig{
Subject: testutils.ImageInput(t, testImage, testutils.FromSnapshot()),
Format: Format(),
UpdateSnapshot: *updateSnapshot,
PersistRedactionsInSnapshot: true,
IsJSON: false,
Redactor: redactor(),
},
)
}
func redactor(values ...string) testutils.Redactor {
return testutils.NewRedactions().
WithValuesRedacted(values...)
}

2
syft/formats/text/test-fixtures/snapshot/TestTextDirectoryEncoder.golden
View File

@ -1,4 +1,4 @@
[Path: /some/path]
[Path: redacted/some/path]
[package-1]
Version: 1.0.1
Type: python

165
syft/internal/fileresolver/chroot_context.go Normal file
View File

@ -0,0 +1,165 @@
package fileresolver
import (
"fmt"
"os"
"path"
"path/filepath"
"strings"
"github.com/anchore/syft/syft/internal/windows"
)
// ChrootContext helps to modify path from a real filesystem to a chroot-like filesystem, taking into account
// the user given root, the base path (if any) to consider as the root, and the current working directory.
// Note: this only works on a real filesystem, not on a virtual filesystem (such as a stereoscope filetree).
type ChrootContext struct {
root string
base string
cwd string
cwdRelativeToRoot string
}
func NewChrootContextFromCWD(root, base string) (*ChrootContext, error) {
currentWD, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("could not get current working directory: %w", err)
}
return NewChrootContext(root, base, currentWD)
}
func NewChrootContext(root, base, cwd string) (*ChrootContext, error) {
cleanRoot, err := NormalizeRootDirectory(root)
if err != nil {
return nil, err
}
cleanBase, err := NormalizeBaseDirectory(base)
if err != nil {
return nil, err
}
chroot := &ChrootContext{
root: cleanRoot,
base: cleanBase,
cwd: cwd,
}
return chroot, chroot.ChangeDirectory(cwd)
}
func NormalizeRootDirectory(root string) (string, error) {
cleanRoot, err := filepath.EvalSymlinks(root)
if err != nil {
return "", fmt.Errorf("could not evaluate root=%q symlinks: %w", root, err)
}
return cleanRoot, nil
}
func NormalizeBaseDirectory(base string) (string, error) {
if base == "" {
return "", nil
}
cleanBase, err := filepath.EvalSymlinks(base)
if err != nil {
return "", fmt.Errorf("could not evaluate base=%q symlinks: %w", base, err)
}
return filepath.Abs(cleanBase)
}
// Root returns the root path with all symlinks evaluated.
func (r ChrootContext) Root() string {
return r.root
}
// Base returns the absolute base path with all symlinks evaluated.
func (r ChrootContext) Base() string {
return r.base
}
// ChangeRoot swaps the path for the chroot.
func (r *ChrootContext) ChangeRoot(dir string) error {
newR, err := NewChrootContext(dir, r.base, r.cwd)
if err != nil {
return fmt.Errorf("could not change root: %w", err)
}
*r = *newR
return nil
}
// ChangeDirectory changes the current working directory so that any relative paths passed
// into ToNativePath() and ToChrootPath() honor the new CWD. If the process changes the CWD in-flight, this should be
// called again to ensure correct functionality of ToNativePath() and ToChrootPath().
func (r *ChrootContext) ChangeDirectory(dir string) error {
var (
cwdRelativeToRoot string
err error
)
dir, err = filepath.Abs(dir)
if err != nil {
return fmt.Errorf("could not determine absolute path to CWD: %w", err)
}
if path.IsAbs(r.root) {
cwdRelativeToRoot, err = filepath.Rel(dir, r.root)
if err != nil {
return fmt.Errorf("could not determine given root path to CWD: %w", err)
}
} else {
cwdRelativeToRoot = filepath.Clean(r.root)
}
r.cwd = dir
r.cwdRelativeToRoot = cwdRelativeToRoot
return nil
}
// ToNativePath takes a path in the context of the chroot-like filesystem and converts it to a path in the underlying fs domain.
func (r ChrootContext) ToNativePath(chrootPath string) (string, error) {
responsePath := chrootPath
if filepath.IsAbs(responsePath) {
// don't allow input to potentially hop above root path
responsePath = path.Join(r.root, responsePath)
} else {
// ensure we take into account any relative difference between the root path and the CWD for relative requests
responsePath = path.Join(r.cwdRelativeToRoot, responsePath)
}
var err error
responsePath, err = filepath.Abs(responsePath)
if err != nil {
return "", err
}
return responsePath, nil
}
// ToChrootPath takes a path from the underlying fs domain and converts it to a path that is relative to the current root context.
func (r ChrootContext) ToChrootPath(nativePath string) string {
responsePath := nativePath
// check to see if we need to encode back to Windows from posix
if windows.HostRunningOnWindows() {
responsePath = windows.FromPosix(responsePath)
}
// clean references to the request path (either the root, or the base if set)
if filepath.IsAbs(responsePath) {
var prefix string
if r.base != "" {
prefix = r.base
} else {
// we need to account for the cwd relative to the running process and the given root for the directory resolver
prefix = filepath.Clean(filepath.Join(r.cwd, r.cwdRelativeToRoot))
prefix += string(filepath.Separator)
}
responsePath = strings.TrimPrefix(responsePath, prefix)
}
return responsePath
}

481
syft/internal/fileresolver/chroot_context_test.go Normal file
View File

@ -0,0 +1,481 @@
package fileresolver
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_ChrootContext_RequestResponse(t *testing.T) {
// /
// somewhere/
// outside.txt
// root-link -> ./
// path/
// to/
// abs-inside.txt -> /path/to/the/file.txt # absolute link to somewhere inside of the root
// rel-inside.txt -> ./the/file.txt # relative link to somewhere inside of the root
// the/
// file.txt
// abs-outside.txt -> /somewhere/outside.txt # absolute link to outside of the root
// rel-outside -> ../../../somewhere/outside.txt # relative link to outside of the root
//
testDir, err := os.Getwd()
require.NoError(t, err)
relative := filepath.Join("test-fixtures", "req-resp")
absolute := filepath.Join(testDir, relative)
absPathToTheFile := filepath.Join(absolute, "path", "to", "the", "file.txt")
absAbsInsidePath := filepath.Join(absolute, "path", "to", "abs-inside.txt")
absAbsOutsidePath := filepath.Join(absolute, "path", "to", "the", "abs-outside.txt")
absRelOutsidePath := filepath.Join(absolute, "path", "to", "the", "rel-outside.txt")
relViaLink := filepath.Join(relative, "root-link")
absViaLink := filepath.Join(absolute, "root-link")
absViaLinkPathToTheFile := filepath.Join(absViaLink, "path", "to", "the", "file.txt")
absViaLinkAbsOutsidePath := filepath.Join(absViaLink, "path", "to", "the", "abs-outside.txt")
absViaLinkRelOutsidePath := filepath.Join(absViaLink, "path", "to", "the", "rel-outside.txt")
relViaDoubleLink := filepath.Join(relative, "root-link", "root-link")
absViaDoubleLink := filepath.Join(absolute, "root-link", "root-link")
absViaDoubleLinkPathToTheFile := filepath.Join(absViaDoubleLink, "path", "to", "the", "file.txt")
absViaDoubleLinkRelOutsidePath := filepath.Join(absViaDoubleLink, "path", "to", "the", "rel-outside.txt")
cleanup := func() {
_ = os.Remove(absAbsInsidePath)
_ = os.Remove(absAbsOutsidePath)
}
// ensure the absolute symlinks are cleaned up from any previous runs
cleanup()
require.NoError(t, os.Symlink(filepath.Join(absolute, "path", "to", "the", "file.txt"), absAbsInsidePath))
require.NoError(t, os.Symlink(filepath.Join(absolute, "somewhere", "outside.txt"), absAbsOutsidePath))
t.Cleanup(cleanup)
cases := []struct {
name string
cwd string
root string
base string
input string
expectedNativePath string
expectedChrootPath string
}{
{
name: "relative root, relative request, direct",
root: relative,
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, relative request, direct",
root: absolute,
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "relative root, abs request, direct",
root: relative,
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, abs request, direct",
root: absolute,
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
// cwd within root...
{
name: "relative root, relative request, direct, cwd within root",
cwd: filepath.Join(relative, "path/to"),
root: "../../",
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, relative request, direct, cwd within root",
cwd: filepath.Join(relative, "path/to"),
root: absolute,
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "relative root, abs request, direct, cwd within root",
cwd: filepath.Join(relative, "path/to"),
root: "../../",
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, abs request, direct, cwd within root",
cwd: filepath.Join(relative, "path/to"),
root: absolute,
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
// cwd within symlink root...
{
name: "relative root, relative request, direct, cwd within symlink root",
cwd: relViaLink,
root: "./",
input: "path/to/the/file.txt",
expectedNativePath: absViaLinkPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, relative request, direct, cwd within symlink root",
cwd: relViaLink,
root: absViaLink,
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "relative root, abs request, direct, cwd within symlink root",
cwd: relViaLink,
root: "./",
input: "/path/to/the/file.txt",
expectedNativePath: absViaLinkPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, abs request, direct, cwd within symlink root",
cwd: relViaLink,
root: absViaLink,
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
// cwd within symlink root, request nested within...
{
name: "relative root, relative nested request, direct, cwd within symlink root",
cwd: relViaLink,
root: "./path",
input: "to/the/file.txt",
expectedNativePath: absViaLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, relative nested request, direct, cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absViaLink, "path"),
input: "to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "relative root, abs nested request, direct, cwd within symlink root",
cwd: relViaLink,
root: "./path",
input: "/to/the/file.txt",
expectedNativePath: absViaLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, abs nested request, direct, cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absViaLink, "path"),
input: "/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
// cwd within DOUBLE symlink root...
{
name: "relative root, relative request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: "./",
input: "path/to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, relative request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: absViaDoubleLink,
input: "path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "relative root, abs request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: "./",
input: "/path/to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
{
name: "abs root, abs request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: absViaDoubleLink,
input: "/path/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "path/to/the/file.txt",
},
// cwd within DOUBLE symlink root, request nested within...
{
name: "relative root, relative nested request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: "./path",
input: "to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, relative nested request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: filepath.Join(absViaDoubleLink, "path"),
input: "to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "relative root, abs nested request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: "./path",
input: "/to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, abs nested request, direct, cwd within (double) symlink root",
cwd: relViaDoubleLink,
root: filepath.Join(absViaDoubleLink, "path"),
input: "/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
// cwd within DOUBLE symlink root, request nested DEEP within...
{
name: "relative root, relative nested request, direct, cwd deep within (double) symlink root",
cwd: filepath.Join(relViaDoubleLink, "path", "to"),
root: "../",
input: "to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, relative nested request, direct, cwd deep within (double) symlink root",
cwd: filepath.Join(relViaDoubleLink, "path", "to"),
root: filepath.Join(absViaDoubleLink, "path"),
input: "to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "relative root, abs nested request, direct, cwd deep within (double) symlink root",
cwd: filepath.Join(relViaDoubleLink, "path", "to"),
root: "../",
input: "/to/the/file.txt",
expectedNativePath: absViaDoubleLinkPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
{
name: "abs root, abs nested request, direct, cwd deep within (double) symlink root",
cwd: filepath.Join(relViaDoubleLink, "path", "to"),
root: filepath.Join(absViaDoubleLink, "path"),
input: "/to/the/file.txt",
expectedNativePath: absPathToTheFile,
expectedChrootPath: "to/the/file.txt",
},
// link to outside of root cases...
{
name: "relative root, relative request, abs indirect (outside of root)",
root: filepath.Join(relative, "path"),
input: "to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "abs root, relative request, abs indirect (outside of root)",
root: filepath.Join(absolute, "path"),
input: "to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "relative root, abs request, abs indirect (outside of root)",
root: filepath.Join(relative, "path"),
input: "/to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "abs root, abs request, abs indirect (outside of root)",
root: filepath.Join(absolute, "path"),
input: "/to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "relative root, relative request, relative indirect (outside of root)",
root: filepath.Join(relative, "path"),
input: "to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, relative request, relative indirect (outside of root)",
root: filepath.Join(absolute, "path"),
input: "to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "relative root, abs request, relative indirect (outside of root)",
root: filepath.Join(relative, "path"),
input: "/to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, abs request, relative indirect (outside of root)",
root: filepath.Join(absolute, "path"),
input: "/to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
// link to outside of root cases... cwd within symlink root
{
name: "relative root, relative request, abs indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: "path",
input: "to/the/abs-outside.txt",
expectedNativePath: absViaLinkAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "abs root, relative request, abs indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absolute, "path"),
input: "to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "relative root, abs request, abs indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: "path",
input: "/to/the/abs-outside.txt",
expectedNativePath: absViaLinkAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "abs root, abs request, abs indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absolute, "path"),
input: "/to/the/abs-outside.txt",
expectedNativePath: absAbsOutsidePath,
expectedChrootPath: "to/the/abs-outside.txt",
},
{
name: "relative root, relative request, relative indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: "path",
input: "to/the/rel-outside.txt",
expectedNativePath: absViaLinkRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, relative request, relative indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absolute, "path"),
input: "to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "relative root, abs request, relative indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: "path",
input: "/to/the/rel-outside.txt",
expectedNativePath: absViaLinkRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, abs request, relative indirect (outside of root), cwd within symlink root",
cwd: relViaLink,
root: filepath.Join(absolute, "path"),
input: "/to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "relative root, relative request, relative indirect (outside of root), cwd within DOUBLE symlink root",
cwd: relViaDoubleLink,
root: "path",
input: "to/the/rel-outside.txt",
expectedNativePath: absViaDoubleLinkRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, relative request, relative indirect (outside of root), cwd within DOUBLE symlink root",
cwd: relViaDoubleLink,
root: filepath.Join(absolute, "path"),
input: "to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "relative root, abs request, relative indirect (outside of root), cwd within DOUBLE symlink root",
cwd: relViaDoubleLink,
root: "path",
input: "/to/the/rel-outside.txt",
expectedNativePath: absViaDoubleLinkRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
{
name: "abs root, abs request, relative indirect (outside of root), cwd within DOUBLE symlink root",
cwd: relViaDoubleLink,
root: filepath.Join(absolute, "path"),
input: "/to/the/rel-outside.txt",
expectedNativePath: absRelOutsidePath,
expectedChrootPath: "to/the/rel-outside.txt",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
// we need to mimic a shell, otherwise we won't get a path within a symlink
targetPath := filepath.Join(testDir, c.cwd)
t.Setenv("PWD", filepath.Clean(targetPath))
require.NoError(t, err)
require.NoError(t, os.Chdir(targetPath))
t.Cleanup(func() {
require.NoError(t, os.Chdir(testDir))
})
chroot, err := NewChrootContextFromCWD(c.root, c.base)
require.NoError(t, err)
require.NotNil(t, chroot)
req, err := chroot.ToNativePath(c.input)
require.NoError(t, err)
assert.Equal(t, c.expectedNativePath, req, "native path different")
resp := chroot.ToChrootPath(req)
assert.Equal(t, c.expectedChrootPath, resp, "chroot path different")
})
}
}

21
syft/internal/fileresolver/container_image_squash_test.go
View File

@ -6,6 +6,7 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -513,6 +514,26 @@ func Test_imageSquashResolver_resolvesLinks(t *testing.T) {
}
func compareLocations(t *testing.T, expected, actual []file.Location) {
t.Helper()
ignoreUnexported := cmpopts.IgnoreUnexported(file.LocationData{})
ignoreMetadata := cmpopts.IgnoreFields(file.LocationMetadata{}, "Annotations")
ignoreFS := cmpopts.IgnoreFields(file.Coordinates{}, "FileSystemID")
sort.Sort(file.Locations(expected))
sort.Sort(file.Locations(actual))
if d := cmp.Diff(expected, actual,
ignoreUnexported,
ignoreFS,
ignoreMetadata,
); d != "" {
t.Errorf("unexpected locations (-want +got):\n%s", d)
}
}
func TestSquashResolver_AllLocations(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-files-deleted")

135
syft/internal/fileresolver/directory.go
View File

@ -5,19 +5,14 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"runtime"
"strings"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/internal/windows"
)
const WindowsOS = "windows"
var unixSystemRuntimePrefixes = []string{
"/proc",
"/dev",
@ -30,14 +25,12 @@ var _ file.Resolver = (*Directory)(nil)
// Directory implements path and content access for the directory data source.
type Directory struct {
path string
base string
currentWdRelativeToRoot string
currentWd string
tree filetree.Reader
index filetree.IndexReader
searchContext filetree.Searcher
indexer *directoryIndexer
path string
chroot ChrootContext
tree filetree.Reader
index filetree.IndexReader
searchContext filetree.Searcher
indexer *directoryIndexer
}
func NewFromDirectory(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
@ -50,46 +43,20 @@ func NewFromDirectory(root string, base string, pathFilters ...PathIndexVisitor)
}
func newFromDirectoryWithoutIndex(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
currentWD, err := os.Getwd()
chroot, err := NewChrootContextFromCWD(root, base)
if err != nil {
return nil, fmt.Errorf("could not get CWD: %w", err)
return nil, fmt.Errorf("unable to interpret chroot context: %w", err)
}
cleanRoot, err := filepath.EvalSymlinks(root)
if err != nil {
return nil, fmt.Errorf("could not evaluate root=%q symlinks: %w", root, err)
}
cleanBase := ""
if base != "" {
cleanBase, err = filepath.EvalSymlinks(base)
if err != nil {
return nil, fmt.Errorf("could not evaluate base=%q symlinks: %w", base, err)
}
cleanBase, err = filepath.Abs(cleanBase)
if err != nil {
return nil, err
}
}
var currentWdRelRoot string
if path.IsAbs(cleanRoot) {
currentWdRelRoot, err = filepath.Rel(currentWD, cleanRoot)
if err != nil {
return nil, fmt.Errorf("could not determine given root path to CWD: %w", err)
}
} else {
currentWdRelRoot = filepath.Clean(cleanRoot)
}
cleanRoot := chroot.Root()
cleanBase := chroot.Base()
return &Directory{
path: cleanRoot,
base: cleanBase,
currentWd: currentWD,
currentWdRelativeToRoot: currentWdRelRoot,
tree: filetree.New(),
index: filetree.NewIndex(),
indexer: newDirectoryIndexer(cleanRoot, cleanBase, pathFilters...),
path: cleanRoot,
chroot: *chroot,
tree: filetree.New(),
index: filetree.NewIndex(),
indexer: newDirectoryIndexer(cleanRoot, cleanBase, pathFilters...),
}, nil
}
@ -110,43 +77,12 @@ func (r *Directory) buildIndex() error {
}
func (r Directory) requestPath(userPath string) (string, error) {
if filepath.IsAbs(userPath) {
// don't allow input to potentially hop above root path
userPath = path.Join(r.path, userPath)
} else {
// ensure we take into account any relative difference between the root path and the CWD for relative requests
userPath = path.Join(r.currentWdRelativeToRoot, userPath)
}
var err error
userPath, err = filepath.Abs(userPath)
if err != nil {
return "", err
}
return userPath, nil
return r.chroot.ToNativePath(userPath)
}
// responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the directory resolver.
func (r Directory) responsePath(path string) string {
// check to see if we need to encode back to Windows from posix
if runtime.GOOS == WindowsOS {
path = posixToWindows(path)
}
// clean references to the request path (either the root, or the base if set)
if filepath.IsAbs(path) {
var prefix string
if r.base != "" {
prefix = r.base
} else {
// we need to account for the cwd relative to the running process and the given root for the directory resolver
prefix = filepath.Clean(filepath.Join(r.currentWd, r.currentWdRelativeToRoot))
prefix += string(filepath.Separator)
}
path = strings.TrimPrefix(path, prefix)
}
return path
return r.chroot.ToChrootPath(path)
}
// HasPath indicates if the given path exists in the underlying source.
@ -196,8 +132,8 @@ func (r Directory) FilesByPath(userPaths ...string) ([]file.Location, error) {
continue
}
if runtime.GOOS == WindowsOS {
userStrPath = windowsToPosix(userStrPath)
if windows.HostRunningOnWindows() {
userStrPath = windows.ToPosix(userStrPath)
}
if ref.HasReference() {
@ -286,8 +222,8 @@ func (r Directory) FileContentsByLocation(location file.Location) (io.ReadCloser
// RealPath is posix so for windows directory resolver we need to translate
// to its true on disk path.
filePath := string(location.Reference().RealPath)
if runtime.GOOS == WindowsOS {
filePath = posixToWindows(filePath)
if windows.HostRunningOnWindows() {
filePath = windows.FromPosix(filePath)
}
return stereoscopeFile.NewLazyReadCloser(filePath), nil
@ -338,30 +274,3 @@ func (r *Directory) FilesByMIMEType(types ...string) ([]file.Location, error) {
return uniqueLocations, nil
}
func windowsToPosix(windowsPath string) (posixPath string) {
// volume should be encoded at the start (e.g /c/<path>) where c is the volume
volumeName := filepath.VolumeName(windowsPath)
pathWithoutVolume := strings.TrimPrefix(windowsPath, volumeName)
volumeLetter := strings.ToLower(strings.TrimSuffix(volumeName, ":"))
// translate non-escaped backslash to forwardslash
translatedPath := strings.ReplaceAll(pathWithoutVolume, "\\", "/")
// always have `/` as the root... join all components, e.g.:
// convert: C:\\some\windows\Place
// into: /c/some/windows/Place
return path.Clean("/" + strings.Join([]string{volumeLetter, translatedPath}, "/"))
}
func posixToWindows(posixPath string) (windowsPath string) {
// decode the volume (e.g. /c/<path> --> C:\\) - There should always be a volume name.
pathFields := strings.Split(posixPath, "/")
volumeName := strings.ToUpper(pathFields[1]) + `:\\`
// translate non-escaped forward slashes into backslashes
remainingTranslatedPath := strings.Join(pathFields[2:], "\\")
// combine volume name and backslash components
return filepath.Clean(volumeName + remainingTranslatedPath)
}

6
syft/internal/fileresolver/directory_indexer.go
View File

@ -7,7 +7,6 @@ import (
"os"
"path"
"path/filepath"
"runtime"
"strings"
"github.com/wagoodman/go-partybus"
@ -19,6 +18,7 @@ import (
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/internal/windows"
)
type PathIndexVisitor func(string, os.FileInfo, error) error
@ -263,8 +263,8 @@ func (r *directoryIndexer) indexPath(path string, info os.FileInfo, err error) (
}
// here we check to see if we need to normalize paths to posix on the way in coming from windows
if runtime.GOOS == WindowsOS {
path = windowsToPosix(path)
if windows.HostRunningOnWindows() {
path = windows.ToPosix(path)
}
newRoot, err := r.addPathToIndex(path, info)

4
syft/internal/fileresolver/excluding_file.go
View File

@ -16,9 +16,9 @@ type excluding struct {
excludeFn excludeFn
}
// NewExcluding create a new resolver which wraps the provided delegate and excludes
// NewExcludingDecorator create a new resolver which wraps the provided delegate and excludes
// entries based on a provided path exclusion function
func NewExcluding(delegate file.Resolver, excludeFn excludeFn) file.Resolver {
func NewExcludingDecorator(delegate file.Resolver, excludeFn excludeFn) file.Resolver {
return &excluding{
delegate,
excludeFn,

2
syft/internal/fileresolver/excluding_file_test.go
View File

@ -56,7 +56,7 @@ func TestExcludingResolver(t *testing.T) {
resolver := &mockResolver{
locations: test.locations,
}
er := NewExcluding(resolver, test.excludeFn)
er := NewExcludingDecorator(resolver, test.excludeFn)
locations, _ := er.FilesByPath()
assert.ElementsMatch(t, locationPaths(locations), test.expected)

2
syft/internal/fileresolver/test-fixtures/req-resp/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
path/to/abs-inside.txt
path/to/the/abs-outside.txt

21
syft/internal/fileresolver/unindexed_directory_test.go
View File

@ -14,7 +14,6 @@ import (
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -1263,23 +1262,3 @@ func testWithTimeout(t *testing.T, timeout time.Duration, test func(*testing.T))
case <-done:
}
}
func compareLocations(t *testing.T, expected, actual []file.Location) {
t.Helper()
ignoreUnexported := cmpopts.IgnoreFields(file.LocationData{}, "ref")
ignoreMetadata := cmpopts.IgnoreFields(file.LocationMetadata{}, "Annotations")
ignoreFS := cmpopts.IgnoreFields(file.Coordinates{}, "FileSystemID")
sort.Sort(file.Locations(expected))
sort.Sort(file.Locations(actual))
if d := cmp.Diff(expected, actual,
ignoreUnexported,
ignoreFS,
ignoreMetadata,
); d != "" {
t.Errorf("unexpected locations (-want +got):\n%s", d)
}
}

4
syft/internal/generate.go Normal file
View File

@ -0,0 +1,4 @@
package internal
//go:generate go run ./sourcemetadata/generate/main.go
//go:generate go run ./packagemetadata/generate/main.go

1
syft/internal/jsonschema/README.md Normal file
View File

@ -0,0 +1 @@
Please see [schema/json/README.md](../../../schema/json/README.md) for more information on the JSON schema files in this directory.

71
schema/json/main.go → syft/internal/jsonschema/main.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
@ -13,8 +14,8 @@ import (
"github.com/invopop/jsonschema"
"github.com/anchore/syft/internal"
genInt "github.com/anchore/syft/schema/json/internal"
syftjsonModel "github.com/anchore/syft/syft/formats/syftjson/model"
syftJsonModel "github.com/anchore/syft/syft/formats/syftjson/model"
"github.com/anchore/syft/syft/internal/packagemetadata"
)
/*
@ -24,30 +25,59 @@ are not captured (empty interfaces). This means that pkg.Package.Metadata is not
can be extended to include specific package metadata struct shapes in the future.
*/
//go:generate go run ./generate/main.go
const schemaVersion = internal.JSONSchemaVersion
func main() {
write(encode(build()))
}
func schemaID() jsonschema.ID {
// Today we do not host the schemas at this address, but per the JSON schema spec we should be referencing
// the schema by a URL in a domain we control. This is a placeholder for now.
return jsonschema.ID(fmt.Sprintf("anchore.io/schema/syft/json/%s", internal.JSONSchemaVersion))
}
func assembleTypeContainer(items []any) any {
structFields := make([]reflect.StructField, len(items))
for i, item := range items {
itemType := reflect.TypeOf(item)
fieldName := itemType.Name()
structFields[i] = reflect.StructField{
Name: fieldName,
Type: itemType,
}
}
structType := reflect.StructOf(structFields)
return reflect.New(structType).Elem().Interface()
}
func build() *jsonschema.Schema {
reflector := &jsonschema.Reflector{
BaseSchemaID: schemaID(),
AllowAdditionalProperties: true,
Namer: func(r reflect.Type) string {
return strings.TrimPrefix(r.Name(), "JSON")
},
}
documentSchema := reflector.ReflectFromType(reflect.TypeOf(&syftjsonModel.Document{}))
metadataSchema := reflector.ReflectFromType(reflect.TypeOf(&genInt.ArtifactMetadataContainer{}))
// TODO: inject source definitions
// inject the definitions of all metadatas into the schema definitions
pkgMetadataContainer := assembleTypeContainer(packagemetadata.AllTypes())
pkgMetadataContainerType := reflect.TypeOf(pkgMetadataContainer)
// srcMetadataContainer := assembleTypeContainer(sourcemetadata.AllTypes())
// srcMetadataContainerType := reflect.TypeOf(srcMetadataContainer)
documentSchema := reflector.ReflectFromType(reflect.TypeOf(&syftJsonModel.Document{}))
pkgMetadataSchema := reflector.ReflectFromType(reflect.TypeOf(pkgMetadataContainer))
// srcMetadataSchema := reflector.ReflectFromType(reflect.TypeOf(srcMetadataContainer))
// TODO: add source metadata types
// inject the definitions of all packages metadatas into the schema definitions
var metadataNames []string
for name, definition := range metadataSchema.Definitions {
if name == reflect.TypeOf(genInt.ArtifactMetadataContainer{}).Name() {
for name, definition := range pkgMetadataSchema.Definitions {
if name == pkgMetadataContainerType.Name() {
// ignore the definition for the fake container
continue
}
@ -93,11 +123,16 @@ func encode(schema *jsonschema.Schema) []byte {
}
func write(schema []byte) {
filename := fmt.Sprintf("schema-%s.json", schemaVersion)
repoRoot, err := packagemetadata.RepoRoot()
if err != nil {
fmt.Println("unable to determine repo root")
os.Exit(1)
}
schemaPath := filepath.Join(repoRoot, "schema", "json", fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion))
if _, err := os.Stat(filename); !os.IsNotExist(err) {
if _, err := os.Stat(schemaPath); !os.IsNotExist(err) {
// check if the schema is the same...
existingFh, err := os.Open(filename)
existingFh, err := os.Open(schemaPath)
if err != nil {
panic(err)
}
@ -114,11 +149,11 @@ func write(schema []byte) {
}
// the generated schema is different, bail with error :(
fmt.Printf("Cowardly refusing to overwrite existing schema (%s)!\nSee the schema/json/README.md for how to increment\n", filename)
fmt.Printf("Cowardly refusing to overwrite existing schema (%s)!\nSee the schema/json/README.md for how to increment\n", schemaPath)
os.Exit(1)
}
fh, err := os.Create(filename)
fh, err := os.Create(schemaPath)
if err != nil {
panic(err)
}
@ -130,5 +165,5 @@ func write(schema []byte) {
defer fh.Close()
fmt.Printf("Wrote new schema to %q\n", filename)
fmt.Printf("Wrote new schema to %q\n", schemaPath)
}

8
schema/json/internal/metadata_types.go → syft/internal/packagemetadata/discover_type_names.go
View File

@ -1,4 +1,4 @@
package internal
package packagemetadata
import (
"fmt"
@ -18,8 +18,8 @@ var metadataExceptions = strset.New(
"FileMetadata",
)
func AllSyftMetadataTypeNames() ([]string, error) {
root, err := repoRoot()
func DiscoverTypeNames() ([]string, error) {
root, err := RepoRoot()
if err != nil {
return nil, err
}
@ -30,7 +30,7 @@ func AllSyftMetadataTypeNames() ([]string, error) {
return findMetadataDefinitionNames(files...)
}
func repoRoot() (string, error) {
func RepoRoot() (string, error) {
root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
if err != nil {
return "", fmt.Errorf("unable to find repo root dir: %+v", err)

55
syft/internal/packagemetadata/generate/main.go Normal file
View File

@ -0,0 +1,55 @@
package main
import (
"fmt"
"os"
"github.com/dave/jennifer/jen"
"github.com/anchore/syft/syft/internal/packagemetadata"
)
// This program is invoked from syft/internal and generates packagemetadata/generated.go
const (
pkgImport = "github.com/anchore/syft/syft/pkg"
path = "packagemetadata/generated.go"
)
func main() {
typeNames, err := packagemetadata.DiscoverTypeNames()
if err != nil {
panic(fmt.Errorf("unable to get all metadata type names: %w", err))
}
fmt.Printf("updating package metadata type list with %+v types\n", len(typeNames))
f := jen.NewFile("packagemetadata")
f.HeaderComment("DO NOT EDIT: generated by syft/internal/packagemetadata/generate/main.go")
f.ImportName(pkgImport, "pkg")
f.Comment("AllTypes returns a list of all pkg metadata types that syft supports (that are represented in the pkg.Package.Metadata field).")
f.Func().Id("AllTypes").Params().Index().Any().BlockFunc(func(g *jen.Group) {
g.ReturnFunc(func(g *jen.Group) {
g.Index().Any().ValuesFunc(func(g *jen.Group) {
for _, typeName := range typeNames {
g.Qual(pkgImport, typeName).Values()
}
})
})
})
rendered := fmt.Sprintf("%#v", f)
fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
panic(fmt.Errorf("unable to open file: %w", err))
}
_, err = fh.WriteString(rendered)
if err != nil {
panic(fmt.Errorf("unable to write file: %w", err))
}
if err := fh.Close(); err != nil {
panic(fmt.Errorf("unable to close file: %w", err))
}
}

10
syft/internal/packagemetadata/generated.go Normal file
View File

@ -0,0 +1,10 @@
// DO NOT EDIT: generated by syft/internal/packagemetadata/generate/main.go
package packagemetadata
import "github.com/anchore/syft/syft/pkg"
// AllTypes returns a list of all pkg metadata types that syft supports (that are represented in the pkg.Package.Metadata field).
func AllTypes() []any {
return []any{pkg.AlpmMetadata{}, pkg.ApkMetadata{}, pkg.BinaryMetadata{}, pkg.CargoPackageMetadata{}, pkg.CocoapodsMetadata{}, pkg.ConanLockMetadata{}, pkg.ConanMetadata{}, pkg.DartPubMetadata{}, pkg.DotnetDepsMetadata{}, pkg.DpkgMetadata{}, pkg.GemMetadata{}, pkg.GolangBinMetadata{}, pkg.GolangModMetadata{}, pkg.HackageMetadata{}, pkg.JavaMetadata{}, pkg.KbPackageMetadata{}, pkg.LinuxKernelMetadata{}, pkg.LinuxKernelModuleMetadata{}, pkg.MixLockMetadata{}, pkg.NixStoreMetadata{}, pkg.NpmPackageJSONMetadata{}, pkg.NpmPackageLockJSONMetadata{}, pkg.PhpComposerJSONMetadata{}, pkg.PortageMetadata{}, pkg.PythonPackageMetadata{}, pkg.PythonPipfileLockMetadata{}, pkg.PythonRequirementsMetadata{}, pkg.RDescriptionFileMetadata{}, pkg.RebarLockMetadata{}, pkg.RpmMetadata{}}
}

13
syft/internal/packagemetadata/names.go Normal file
View File

@ -0,0 +1,13 @@
package packagemetadata
import (
"reflect"
)
func AllNames() []string {
names := make([]string, 0)
for _, t := range AllTypes() {
names = append(names, reflect.TypeOf(t).Name())
}
return names
}

25
syft/internal/packagemetadata/names_test.go Normal file
View File

@ -0,0 +1,25 @@
package packagemetadata
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAllNames(t *testing.T) {
// note: this is a form of completion testing relative to the current code base.
expected, err := DiscoverTypeNames()
require.NoError(t, err)
actual := AllNames()
// ensure that the codebase (from ast analysis) reflects the latest code generated state
if !assert.ElementsMatch(t, expected, actual) {
t.Errorf("metadata types not fully represented: \n%s", cmp.Diff(expected, actual))
t.Log("did you add a new pkg.*Metadata type without updating the JSON schema?")
t.Log("if so, you need to update the schema version and regenerate the JSON schema (make generate-json-schema)")
}
}

69
syft/internal/sourcemetadata/completion_tester.go Normal file
View File

@ -0,0 +1,69 @@
package sourcemetadata
import (
"reflect"
"testing"
)
type CompletionTester struct {
saw []any
valid []any
ignore []any
}
func NewCompletionTester(t testing.TB, ignore ...any) *CompletionTester {
tester := &CompletionTester{
valid: AllTypes(),
ignore: ignore,
}
t.Cleanup(func() {
t.Helper()
tester.validate(t)
})
return tester
}
func (tr *CompletionTester) Tested(t testing.TB, m any) {
t.Helper()
if m == nil {
return
}
if len(tr.valid) == 0 {
t.Fatal("no valid metadata types to test against")
}
ty := reflect.TypeOf(m)
for _, v := range tr.valid {
if reflect.TypeOf(v) == ty {
tr.saw = append(tr.saw, m)
return
}
}
t.Fatalf("tested metadata type is not valid: %s", ty.Name())
}
func (tr *CompletionTester) validate(t testing.TB) {
t.Helper()
count := make(map[reflect.Type]int)
for _, m := range tr.saw {
count[reflect.TypeOf(m)]++
}
validations:
for _, v := range tr.valid {
ty := reflect.TypeOf(v)
for _, ignore := range tr.ignore {
if ty == reflect.TypeOf(ignore) {
// skip ignored types
continue validations
}
}
if c, exists := count[ty]; c == 0 || !exists {
t.Errorf("metadata type %s is not covered by a test", ty.Name())
}
}
}

148
syft/internal/sourcemetadata/discover_type_names.go Normal file
View File

@ -0,0 +1,148 @@
package sourcemetadata
import (
"fmt"
"go/ast"
"go/parser"
"go/token"
"os/exec"
"path/filepath"
"sort"
"strings"
"unicode"
"github.com/scylladb/go-set/strset"
)
var metadataExceptions = strset.New()
func DiscoverTypeNames() ([]string, error) {
root, err := repoRoot()
if err != nil {
return nil, err
}
files, err := filepath.Glob(filepath.Join(root, "syft/source/*.go"))
if err != nil {
return nil, err
}
return findMetadataDefinitionNames(files...)
}
func repoRoot() (string, error) {
root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
if err != nil {
return "", fmt.Errorf("unable to find repo root dir: %+v", err)
}
absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root)))
if err != nil {
return "", fmt.Errorf("unable to get abs path to repo root: %w", err)
}
return absRepoRoot, nil
}
func findMetadataDefinitionNames(paths ...string) ([]string, error) {
names := strset.New()
usedNames := strset.New()
for _, path := range paths {
metadataDefinitions, usedTypeNames, err := findMetadataDefinitionNamesInFile(path)
if err != nil {
return nil, err
}
// useful for debugging...
// fmt.Println(path)
// fmt.Println("Defs:", metadataDefinitions)
// fmt.Println("Used Types:", usedTypeNames)
// fmt.Println()
names.Add(metadataDefinitions...)
usedNames.Add(usedTypeNames...)
}
// any definition that is used within another struct should not be considered a top-level metadata definition
names.Remove(usedNames.List()...)
strNames := names.List()
sort.Strings(strNames)
// note: 3 is a point-in-time gut check. This number could be updated if new metadata definitions are added, but is not required.
// it is really intended to catch any major issues with the generation process that would generate, say, 0 definitions.
if len(strNames) < 3 {
return nil, fmt.Errorf("not enough metadata definitions found (discovered: " + fmt.Sprintf("%d", len(strNames)) + ")")
}
return strNames, nil
}
func findMetadataDefinitionNamesInFile(path string) ([]string, []string, error) {
// set up the parser
fs := token.NewFileSet()
f, err := parser.ParseFile(fs, path, nil, parser.ParseComments)
if err != nil {
return nil, nil, err
}
var metadataDefinitions []string
var usedTypeNames []string
for _, decl := range f.Decls {
// check if the declaration is a type declaration
spec, ok := decl.(*ast.GenDecl)
if !ok || spec.Tok != token.TYPE {
continue
}
// loop over all types declared in the type declaration
for _, typ := range spec.Specs {
// check if the type is a struct type
spec, ok := typ.(*ast.TypeSpec)
if !ok || spec.Type == nil {
continue
}
structType, ok := spec.Type.(*ast.StructType)
if !ok {
continue
}
// check if the struct type ends with "Metadata"
name := spec.Name.String()
// only look for exported types that end with "Metadata"
if isMetadataTypeCandidate(name) {
// print the full declaration of the struct type
metadataDefinitions = append(metadataDefinitions, name)
usedTypeNames = append(usedTypeNames, typeNamesUsedInStruct(structType)...)
}
}
}
return metadataDefinitions, usedTypeNames, nil
}
func typeNamesUsedInStruct(structType *ast.StructType) []string {
// recursively find all type names used in the struct type
var names []string
for i := range structType.Fields.List {
// capture names of all of the types (not field names)
ast.Inspect(structType.Fields.List[i].Type, func(n ast.Node) bool {
ident, ok := n.(*ast.Ident)
if !ok {
return true
}
// add the type name to the list
names = append(names, ident.Name)
// continue inspecting
return true
})
}
return names
}
func isMetadataTypeCandidate(name string) bool {
return len(name) > 0 &&
strings.HasSuffix(name, "Metadata") &&
unicode.IsUpper(rune(name[0])) && // must be exported
!metadataExceptions.Has(name)
}

55
syft/internal/sourcemetadata/generate/main.go Normal file
View File

@ -0,0 +1,55 @@
package main
import (
"fmt"
"os"
"github.com/dave/jennifer/jen"
"github.com/anchore/syft/syft/internal/sourcemetadata"
)
// This program is invoked from syft/internal and generates sourcemetadata/generated.go
const (
srcImport = "github.com/anchore/syft/syft/source"
path = "sourcemetadata/generated.go"
)
func main() {
typeNames, err := sourcemetadata.DiscoverTypeNames()
if err != nil {
panic(fmt.Errorf("unable to get all metadata type names: %w", err))
}
fmt.Printf("updating source metadata type list with %+v types\n", len(typeNames))
f := jen.NewFile("sourcemetadata")
f.HeaderComment("DO NOT EDIT: generated by syft/internal/sourcemetadata/generate/main.go")
f.ImportName(srcImport, "source")
f.Comment("AllTypes returns a list of all source metadata types that syft supports (that are represented in the source.Description.Metadata field).")
f.Func().Id("AllTypes").Params().Index().Any().BlockFunc(func(g *jen.Group) {
g.ReturnFunc(func(g *jen.Group) {
g.Index().Any().ValuesFunc(func(g *jen.Group) {
for _, typeName := range typeNames {
g.Qual(srcImport, typeName).Values()
}
})
})
})
rendered := fmt.Sprintf("%#v", f)
fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
panic(fmt.Errorf("unable to open file: %w", err))
}
_, err = fh.WriteString(rendered)
if err != nil {
panic(fmt.Errorf("unable to write file: %w", err))
}
if err := fh.Close(); err != nil {
panic(fmt.Errorf("unable to close file: %w", err))
}
}

10
syft/internal/sourcemetadata/generated.go Normal file
View File

@ -0,0 +1,10 @@
// DO NOT EDIT: generated by syft/internal/sourcemetadata/generate/main.go
package sourcemetadata
import "github.com/anchore/syft/syft/source"
// AllTypes returns a list of all source metadata types that syft supports (that are represented in the source.Description.Metadata field).
func AllTypes() []any {
return []any{source.DirectorySourceMetadata{}, source.FileSourceMetadata{}, source.StereoscopeImageSourceMetadata{}}
}

41
syft/internal/sourcemetadata/names.go Normal file
View File

@ -0,0 +1,41 @@
package sourcemetadata
import (
"reflect"
"strings"
"github.com/anchore/syft/syft/source"
)
var jsonNameFromType = map[reflect.Type][]string{
reflect.TypeOf(source.DirectorySourceMetadata{}): {"directory", "dir"},
reflect.TypeOf(source.FileSourceMetadata{}): {"file"},
reflect.TypeOf(source.StereoscopeImageSourceMetadata{}): {"image"},
}
func AllNames() []string {
names := make([]string, 0)
for _, t := range AllTypes() {
names = append(names, reflect.TypeOf(t).Name())
}
return names
}
func JSONName(metadata any) string {
if vs, exists := jsonNameFromType[reflect.TypeOf(metadata)]; exists {
return vs[0]
}
return ""
}
func ReflectTypeFromJSONName(name string) reflect.Type {
name = strings.ToLower(name)
for t, vs := range jsonNameFromType {
for _, v := range vs {
if v == name {
return t
}
}
}
return nil
}

29
syft/internal/sourcemetadata/names_test.go Normal file
View File

@ -0,0 +1,29 @@
package sourcemetadata
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAllNames(t *testing.T) {
// note: this is a form of completion testing relative to the current code base.
expected, err := DiscoverTypeNames()
require.NoError(t, err)
actual := AllNames()
// ensure that the codebase (from ast analysis) reflects the latest code generated state
if !assert.ElementsMatch(t, expected, actual) {
t.Errorf("metadata types not fully represented: \n%s", cmp.Diff(expected, actual))
t.Log("did you add a new source.*Metadata type without updating the JSON schema?")
t.Log("if so, you need to update the schema version and regenerate the JSON schema (make generate-json-schema)")
}
for _, ty := range AllTypes() {
assert.NotEmpty(t, JSONName(ty), "metadata type %q does not have a JSON name", ty)
}
}

41
syft/internal/windows/path.go Normal file
View File

@ -0,0 +1,41 @@
package windows
import (
"path"
"path/filepath"
"runtime"
"strings"
)
const windowsGoOS = "windows"
func HostRunningOnWindows() bool {
return runtime.GOOS == windowsGoOS
}
func ToPosix(windowsPath string) (posixPath string) {
// volume should be encoded at the start (e.g /c/<path>) where c is the volume
volumeName := filepath.VolumeName(windowsPath)
pathWithoutVolume := strings.TrimPrefix(windowsPath, volumeName)
volumeLetter := strings.ToLower(strings.TrimSuffix(volumeName, ":"))
// translate non-escaped backslash to forwardslash
translatedPath := strings.ReplaceAll(pathWithoutVolume, "\\", "/")
// always have `/` as the root... join all components, e.g.:
// convert: C:\\some\windows\Place
// into: /c/some/windows/Place
return path.Clean("/" + strings.Join([]string{volumeLetter, translatedPath}, "/"))
}
func FromPosix(posixPath string) (windowsPath string) {
// decode the volume (e.g. /c/<path> --> C:\\) - There should always be a volume name.
pathFields := strings.Split(posixPath, "/")
volumeName := strings.ToUpper(pathFields[1]) + `:\\`
// translate non-escaped forward slashes into backslashes
remainingTranslatedPath := strings.Join(pathFields[2:], "\\")
// combine volume name and backslash components
return filepath.Clean(volumeName + remainingTranslatedPath)
}

17
syft/lib.go
View File

@ -34,7 +34,7 @@ import (
// CatalogPackages takes an inventory of packages from the given image from a particular perspective
// (e.g. squashed source, all-layers source). Returns the discovered set of packages, the identified Linux
// distribution, and the source object used to wrap the data source.
func CatalogPackages(src *source.Source, cfg cataloger.Config) (*pkg.Collection, []artifact.Relationship, *linux.Release, error) {
func CatalogPackages(src source.Source, cfg cataloger.Config) (*pkg.Collection, []artifact.Relationship, *linux.Release, error) {
resolver, err := src.FileResolver(cfg.Search.Scope)
if err != nil {
return nil, nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err)
@ -54,18 +54,21 @@ func CatalogPackages(src *source.Source, cfg cataloger.Config) (*pkg.Collection,
catalogers = cataloger.AllCatalogers(cfg)
} else {
// otherwise conditionally use the correct set of loggers based on the input type (container image or directory)
switch src.Metadata.Scheme {
case source.ImageScheme:
// TODO: this is bad, we should not be using the concrete type to determine the cataloger set
// instead this should be a caller concern (pass the catalogers you want to use). The SBOM build PR will do this.
switch src.(type) {
case *source.StereoscopeImageSource:
log.Info("cataloging an image")
catalogers = cataloger.ImageCatalogers(cfg)
case source.FileScheme:
case *source.FileSource:
log.Info("cataloging a file")
catalogers = cataloger.AllCatalogers(cfg)
case source.DirectoryScheme:
case *source.DirectorySource:
log.Info("cataloging a directory")
catalogers = cataloger.DirectoryCatalogers(cfg)
default:
return nil, nil, nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", src.Metadata.Scheme)
return nil, nil, nil, fmt.Errorf("unsupported source type: %T", src)
}
}
@ -76,7 +79,7 @@ func CatalogPackages(src *source.Source, cfg cataloger.Config) (*pkg.Collection,
return catalog, relationships, release, err
}
func newSourceRelationshipsFromCatalog(src *source.Source, c *pkg.Collection) []artifact.Relationship {
func newSourceRelationshipsFromCatalog(src source.Source, c *pkg.Collection) []artifact.Relationship {
relationships := make([]artifact.Relationship, 0) // Should we pre-allocate this by giving catalog a Len() method?
for p := range c.Enumerate() {
relationships = append(relationships, artifact.Relationship{

2
syft/linux/identify_release_test.go
View File

@ -336,7 +336,7 @@ func TestIdentifyRelease(t *testing.T) {
for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
s, err := source.NewFromDirectory(test.fixture)
s, err := source.NewFromDirectoryPath(test.fixture)
require.NoError(t, err)
resolver, err := s.FileResolver(source.SquashedScope)

6
syft/pkg/cataloger/binary/cataloger_test.go
View File

@ -649,7 +649,7 @@ func Test_Cataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
c := NewCataloger()
src, err := source.NewFromDirectory(test.fixtureDir)
src, err := source.NewFromDirectoryPath(test.fixtureDir)
require.NoError(t, err)
resolver, err := src.FileResolver(source.SquashedScope)
@ -688,7 +688,7 @@ func Test_Cataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) {
c := NewCataloger()
img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureImage)
src, err := source.NewFromImage(img, "test-img")
src, err := source.NewFromStereoscopeImageObject(img, test.fixtureImage, nil)
require.NoError(t, err)
resolver, err := src.FileResolver(source.SquashedScope)
@ -718,7 +718,7 @@ func Test_Cataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) {
func TestClassifierCataloger_DefaultClassifiers_NegativeCases(t *testing.T) {
c := NewCataloger()
src, err := source.NewFromDirectory("test-fixtures/classifiers/negative")
src, err := source.NewFromDirectoryPath("test-fixtures/classifiers/negative")
assert.NoError(t, err)
resolver, err := src.FileResolver(source.SquashedScope)

4
syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go
View File

@ -85,7 +85,7 @@ func DefaultLicenseComparer(x, y pkg.License) bool {
func (p *CatalogTester) FromDirectory(t *testing.T, path string) *CatalogTester {
t.Helper()
s, err := source.NewFromDirectory(path)
s, err := source.NewFromDirectoryPath(path)
require.NoError(t, err)
resolver, err := s.FileResolver(source.AllLayersScope)
@ -149,7 +149,7 @@ func (p *CatalogTester) WithImageResolver(t *testing.T, fixtureName string) *Cat
t.Helper()
img := imagetest.GetFixtureImage(t, "docker-archive", fixtureName)
s, err := source.NewFromImage(img, fixtureName)
s, err := source.NewFromStereoscopeImageObject(img, fixtureName, nil)
require.NoError(t, err)
r, err := s.FileResolver(source.SquashedScope)

4
syft/pkg/cataloger/search_config.go
View File

@ -1,6 +1,8 @@
package cataloger
import "github.com/anchore/syft/syft/source"
import (
"github.com/anchore/syft/syft/source"
)
type SearchConfig struct {
IncludeIndexedArchives bool

2
syft/sbom/sbom.go
View File

@ -15,7 +15,7 @@ import (
type SBOM struct {
Artifacts Artifacts
Relationships []artifact.Relationship
Source source.Metadata
Source source.Description
Descriptor Descriptor
}

13
syft/source/alias.go Normal file
View File

@ -0,0 +1,13 @@
package source
type Alias struct {
Name string `json:"name" yaml:"name" mapstructure:"name"`
Version string `json:"version" yaml:"version" mapstructure:"version"`
}
func (a *Alias) IsEmpty() bool {
if a == nil {
return true
}
return a.Name == "" && a.Version == ""
}

Some files were not shown because too many files have changed in this diff Show More