Replace core SBOM-creation API with builder pattern (#1383)

* remove existing cataloging API

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add file cataloging config

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add package cataloging config

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add configs for cross-cutting concerns

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename CLI option configs to not require import aliases later

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update all nested structs for the Catalog struct

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update Catalog cli options

- add new cataloger selection options (selection and default)
- remove the excludeBinaryOverlapByOwnership
- deprecate "catalogers" flag
- add new javascript configuration

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* migrate relationship capabilities to separate internal package

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor golang cataloger to use configuration options when creating packages

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* create internal object to facilitate reading from and writing to an SBOM

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* create a command-like object (task) to facilitate partial SBOM creation

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add cataloger selection capability

- be able to parse string expressions into a set of resolved actions against sets
- be able to use expressions to select/add/remove tasks to/from the final set of tasks to run

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add package, file, and environment related tasks

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update existing file catalogers to use nested UI elements

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add CreateSBOMConfig that drives the SBOM creation process

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* capture SBOM creation info as a struct

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add CreateSBOM() function

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix tests

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update docs with SBOM selection help + breaking changes

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix multiple override default inputs

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix deprecation flag printing to stdout

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor cataloger selection description to separate object

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* keep expression errors and show specific suggestions only

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address additional review feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address more review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* addressed additional PR review feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix file selection references

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove guess language data generation option

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tests for coordinatesForSelection

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename relationship attributes

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add descriptions to relationships config fields

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* improve documentation around configuration options

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add explicit errors around legacy config entries

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2024-01-12 17:39:13 -05:00
committed by GitHub
parent 1bffc6ef0c
commit b0ab75fd89
126 changed files with 5740 additions and 1841 deletions
@@ -11,3 +11,13 @@ func DefaultArchiveSearchConfig() ArchiveSearchConfig {
IncludeUnindexedArchives: false,
}
}
func (c ArchiveSearchConfig) WithIncludeIndexedArchives(include bool) ArchiveSearchConfig {
c.IncludeIndexedArchives = include
return c
}
func (c ArchiveSearchConfig) WithIncludeUnindexedArchives(include bool) ArchiveSearchConfig {
c.IncludeUnindexedArchives = include
return c
}
+16
View File
@@ -0,0 +1,16 @@
package cataloging
type DataGenerationConfig struct {
GenerateCPEs bool `yaml:"generate-cpes" json:"generate-cpes" mapstructure:"generate-cpes"`
}
func DefaultDataGenerationConfig() DataGenerationConfig {
return DataGenerationConfig{
GenerateCPEs: true,
}
}
func (c DataGenerationConfig) WithGenerateCPEs(generate bool) DataGenerationConfig {
c.GenerateCPEs = generate
return c
}
+78
View File
@@ -0,0 +1,78 @@
package filecataloging
import (
"crypto"
"encoding/json"
"fmt"
"strings"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/file/cataloger/filecontent"
)
type Config struct {
Selection file.Selection `yaml:"selection" json:"selection" mapstructure:"selection"`
Hashers []crypto.Hash `yaml:"hashers" json:"hashers" mapstructure:"hashers"`
Content filecontent.Config `yaml:"content" json:"content" mapstructure:"content"`
}
type configMarshaledForm struct {
Selection file.Selection `yaml:"selection" json:"selection" mapstructure:"selection"`
Hashers []string `yaml:"hashers" json:"hashers" mapstructure:"hashers"`
Content filecontent.Config `yaml:"content" json:"content" mapstructure:"content"`
}
func DefaultConfig() Config {
hashers, err := intFile.Hashers("sha256")
if err != nil {
log.WithFields("error", err).Warn("unable to create file hashers")
}
return Config{
Selection: file.FilesOwnedByPackageSelection,
Hashers: hashers,
Content: filecontent.DefaultConfig(),
}
}
func (cfg Config) MarshalJSON() ([]byte, error) {
marshaled := configMarshaledForm{
Selection: cfg.Selection,
Hashers: hashersToString(cfg.Hashers),
}
return json.Marshal(marshaled)
}
func hashersToString(hashers []crypto.Hash) []string {
var result []string
for _, h := range hashers {
result = append(result, strings.ToLower(h.String()))
}
return result
}
func (cfg *Config) UnmarshalJSON(data []byte) error {
var marshaled configMarshaledForm
if err := json.Unmarshal(data, &marshaled); err != nil {
return err
}
hashers, err := intFile.Hashers(marshaled.Hashers...)
if err != nil {
return fmt.Errorf("unable to parse configured hashers: %w", err)
}
cfg.Selection = marshaled.Selection
cfg.Hashers = hashers
return nil
}
func (cfg Config) WithSelection(selection file.Selection) Config {
cfg.Selection = selection
return cfg
}
func (cfg Config) WithHashers(hashers []crypto.Hash) Config {
cfg.Hashers = hashers
return cfg
}
@@ -0,0 +1,74 @@
package filecataloging
import (
"crypto"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/file"
)
func TestConfig_MarshalJSON(t *testing.T) {
tests := []struct {
name string
cfg Config
want []byte
wantErr require.ErrorAssertionFunc
}{
{
name: "converts hashers to strings",
cfg: Config{
Selection: file.FilesOwnedByPackageSelection,
Hashers: []crypto.Hash{crypto.SHA256},
},
want: []byte(`{"selection":"owned-by-package","hashers":["sha-256"],"content":{"globs":null,"skip-files-above-size":0}}`),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = require.NoError
}
got, err := tt.cfg.MarshalJSON()
tt.wantErr(t, err)
if err != nil {
return
}
if d := cmp.Diff(got, tt.want); d != "" {
t.Errorf("MarshalJSON() mismatch (-want +got):\n%s", d)
}
})
}
}
func TestConfig_UnmarshalJSON(t *testing.T) {
tests := []struct {
name string
data []byte
want Config
wantErr bool
}{
{
name: "converts strings to hashers",
data: []byte(`{"selection":"owned-by-package","hashers":["sha-256"]}`),
want: Config{
Selection: file.FilesOwnedByPackageSelection,
Hashers: []crypto.Hash{crypto.SHA256},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Config{}
if err := cfg.UnmarshalJSON(tt.data); (err != nil) != tt.wantErr {
t.Errorf("UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr)
}
assert.Equal(t, tt.want, cfg)
})
}
}
@@ -0,0 +1,23 @@
package pkgcataloging
import "github.com/anchore/syft/syft/pkg"
type CatalogerReference struct {
Cataloger pkg.Cataloger
AlwaysEnabled bool
Tags []string
}
func NewCatalogerReference(cataloger pkg.Cataloger, tags []string) CatalogerReference {
return CatalogerReference{
Cataloger: cataloger,
Tags: tags,
}
}
func NewAlwaysEnabledCatalogerReference(cataloger pkg.Cataloger) CatalogerReference {
return CatalogerReference{
Cataloger: cataloger,
AlwaysEnabled: true,
}
}
+58
View File
@@ -0,0 +1,58 @@
package pkgcataloging
import (
"github.com/anchore/syft/syft/pkg/cataloger/binary"
"github.com/anchore/syft/syft/pkg/cataloger/golang"
"github.com/anchore/syft/syft/pkg/cataloger/java"
"github.com/anchore/syft/syft/pkg/cataloger/javascript"
"github.com/anchore/syft/syft/pkg/cataloger/kernel"
"github.com/anchore/syft/syft/pkg/cataloger/python"
)
type Config struct {
Binary binary.CatalogerConfig `yaml:"binary" json:"binary" mapstructure:"binary"`
Golang golang.CatalogerConfig `yaml:"golang" json:"golang" mapstructure:"golang"`
JavaArchive java.ArchiveCatalogerConfig `yaml:"java-archive" json:"java-archive" mapstructure:"java-archive"`
JavaScript javascript.CatalogerConfig `yaml:"javascript" json:"javascript" mapstructure:"javascript"`
LinuxKernel kernel.LinuxKernelCatalogerConfig `yaml:"linux-kernel" json:"linux-kernel" mapstructure:"linux-kernel"`
Python python.CatalogerConfig `yaml:"python" json:"python" mapstructure:"python"`
}
func DefaultConfig() Config {
return Config{
Golang: golang.DefaultCatalogerConfig(),
LinuxKernel: kernel.DefaultLinuxCatalogerConfig(),
Python: python.DefaultCatalogerConfig(),
JavaArchive: java.DefaultArchiveCatalogerConfig(),
}
}
func (c Config) WithBinaryConfig(cfg binary.CatalogerConfig) Config {
c.Binary = cfg
return c
}
func (c Config) WithGolangConfig(cfg golang.CatalogerConfig) Config {
c.Golang = cfg
return c
}
func (c Config) WithJavascriptConfig(cfg javascript.CatalogerConfig) Config {
c.JavaScript = cfg
return c
}
func (c Config) WithLinuxKernelConfig(cfg kernel.LinuxKernelCatalogerConfig) Config {
c.LinuxKernel = cfg
return c
}
func (c Config) WithPythonConfig(cfg python.CatalogerConfig) Config {
c.Python = cfg
return c
}
func (c Config) WithJavaArchiveConfig(cfg java.ArchiveCatalogerConfig) Config {
c.JavaArchive = cfg
return c
}
@@ -0,0 +1,65 @@
package pkgcataloging
import (
"strings"
)
type SelectionRequest struct {
DefaultNamesOrTags []string `json:"default,omitempty"`
SubSelectTags []string `json:"selection,omitempty"`
AddNames []string `json:"addition,omitempty"`
RemoveNamesOrTags []string `json:"removal,omitempty"`
}
func NewSelectionRequest() SelectionRequest {
return SelectionRequest{}
}
func (s SelectionRequest) WithExpression(expressions ...string) SelectionRequest {
expressions = cleanSelection(expressions)
for _, expr := range expressions {
switch {
case strings.HasPrefix(expr, "+"):
s = s.WithAdditions(strings.TrimPrefix(expr, "+"))
case strings.HasPrefix(expr, "-"):
s = s.WithRemovals(strings.TrimPrefix(expr, "-"))
default:
s = s.WithSubSelections(expr)
}
}
return s
}
func (s SelectionRequest) WithDefaults(nameOrTags ...string) SelectionRequest {
s.DefaultNamesOrTags = append(s.DefaultNamesOrTags, nameOrTags...)
return s
}
func (s SelectionRequest) WithSubSelections(tags ...string) SelectionRequest {
s.SubSelectTags = append(s.SubSelectTags, tags...)
return s
}
func (s SelectionRequest) WithAdditions(names ...string) SelectionRequest {
s.AddNames = append(s.AddNames, names...)
return s
}
func (s SelectionRequest) WithRemovals(nameOrTags ...string) SelectionRequest {
s.RemoveNamesOrTags = append(s.RemoveNamesOrTags, nameOrTags...)
return s
}
func cleanSelection(tags []string) []string {
var cleaned []string
for _, tag := range tags {
for _, t := range strings.Split(tag, ",") {
t = strings.TrimSpace(t)
if t == "" {
continue
}
cleaned = append(cleaned, t)
}
}
return cleaned
}
+24
View File
@@ -0,0 +1,24 @@
package pkgcataloging
const (
// InstalledTag is to identify packages found to be positively installed.
InstalledTag = "installed"
// DeclaredTag is to identify packages described but not necessarily installed.
DeclaredTag = "declared"
// ImageTag indicates the cataloger should be used when cataloging images.
ImageTag = "image"
// DirectoryTag indicates the cataloger should be used when cataloging directories.
DirectoryTag = "directory"
// PackageTag should be used to identify catalogers that are package-based.
PackageTag = "package"
// OSTag should be used to identify catalogers that cataloging OS packages.
OSTag = "os"
// LanguageTag should be used to identify catalogers that cataloging language-specific packages.
LanguageTag = "language"
)
+39
View File
@@ -0,0 +1,39 @@
package cataloging
type RelationshipsConfig struct {
// PackageFileOwnership will include package-to-file relationships that indicate which files are owned by which packages.
PackageFileOwnership bool `yaml:"package-file-ownership" json:"package-file-ownership" mapstructure:"package-file-ownership"`
// PackageFileOwnershipOverlap will include package-to-package relationships that indicate one package is owned by another due to files claimed to be owned by one package are also evidence of another package's existence.
// For example, if an RPM package is installed and claims to own /etc/app/package.lock and a separate NPM package was discovered by cataloging /etc/app/package.lock, then the two packages will
// have ownership overlap relationship.
PackageFileOwnershipOverlap bool `yaml:"package-file-ownership-overlap" json:"package-file-ownership-overlap" mapstructure:"package-file-ownership-overlap"`
// ExcludeBinaryPackagesWithFileOwnershipOverlap will exclude binary packages from the package catalog that are evident by files also owned by another package.
// For example, if a binary package representing the /bin/python binary is discovered and there is a python RPM package installed which claims to
// orn /bin/python, then the binary package will be excluded from the catalog altogether if this configuration is set to true.
ExcludeBinaryPackagesWithFileOwnershipOverlap bool `yaml:"exclude-binary-packages-with-file-ownership-overlap" json:"exclude-binary-packages-with-file-ownership-overlap" mapstructure:"exclude-binary-packages-with-file-ownership-overlap"`
}
func DefaultRelationshipsConfig() RelationshipsConfig {
return RelationshipsConfig{
PackageFileOwnership: true,
PackageFileOwnershipOverlap: true,
ExcludeBinaryPackagesWithFileOwnershipOverlap: true,
}
}
func (c RelationshipsConfig) WithPackageFileOwnership(ownership bool) RelationshipsConfig {
c.PackageFileOwnership = ownership
return c
}
func (c RelationshipsConfig) WithPackageFileOwnershipOverlap(overlap bool) RelationshipsConfig {
c.PackageFileOwnershipOverlap = overlap
return c
}
func (c RelationshipsConfig) WithExcludeBinaryPackagesWithFileOwnershipOverlap(exclude bool) RelationshipsConfig {
c.ExcludeBinaryPackagesWithFileOwnershipOverlap = exclude
return c
}
+18
View File
@@ -0,0 +1,18 @@
package cataloging
import "github.com/anchore/syft/syft/source"
type SearchConfig struct {
Scope source.Scope `yaml:"scope" json:"scope" mapstructure:"scope"`
}
func DefaultSearchConfig() SearchConfig {
return SearchConfig{
Scope: source.SquashedScope,
}
}
func (c SearchConfig) WithScope(scope source.Scope) SearchConfig {
c.Scope = scope
return c
}
+80
View File
@@ -0,0 +1,80 @@
package syft
import (
"encoding/json"
"reflect"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/filecataloging"
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
)
// configurationAuditTrail is all input configuration was used to generate the SBOM
type configurationAuditTrail struct {
Search cataloging.SearchConfig `json:"search" yaml:"search" mapstructure:"search"`
Relationships cataloging.RelationshipsConfig `json:"relationships" yaml:"relationships" mapstructure:"relationships"`
DataGeneration cataloging.DataGenerationConfig `json:"data-generation" yaml:"data-generation" mapstructure:"data-generation"`
Packages pkgcataloging.Config `json:"packages" yaml:"packages" mapstructure:"packages"`
Files filecataloging.Config `json:"files" yaml:"files" mapstructure:"files"`
Catalogers catalogerManifest `json:"catalogers" yaml:"catalogers" mapstructure:"catalogers"`
ExtraConfigs any `json:"extra,omitempty" yaml:"extra" mapstructure:"extra"`
}
type catalogerManifest struct {
Requested pkgcataloging.SelectionRequest `json:"requested" yaml:"requested" mapstructure:"requested"`
Used []string `json:"used" yaml:"used" mapstructure:"used"`
}
type marshalAPIConfiguration configurationAuditTrail
func (cfg configurationAuditTrail) MarshalJSON() ([]byte, error) {
// since the api configuration is placed into the SBOM in an empty interface, and we want a stable ordering of
// keys (not guided by the struct ordering) we need to convert the struct to a map. This is best done with
// simply marshalling and unmarshalling. Mapstructure is used to ensure we are honoring all json struct
// tags. Once we have a map, we can lean on the stable ordering of json map keys in the stdlib. This is an
// implementation detail that can be at least relied on until Go 2 (at which point it can change).
// This dance allows us to guarantee ordering of keys in the configuration section of the SBOM.
initialJSON, err := json.Marshal(marshalAPIConfiguration(cfg))
if err != nil {
return nil, err
}
var dataMap map[string]interface{}
if err := json.Unmarshal(initialJSON, &dataMap); err != nil {
return nil, err
}
if v, exists := dataMap["extra"]; exists && v == nil {
// remove the extra key if it renders as nil
delete(dataMap, "extra")
}
return marshalSorted(dataMap)
}
// marshalSorted recursively marshals a map with sorted keys
func marshalSorted(m interface{}) ([]byte, error) {
if reflect.TypeOf(m).Kind() != reflect.Map {
return json.Marshal(m)
}
val := reflect.ValueOf(m)
sortedMap := make(map[string]interface{})
for _, key := range val.MapKeys() {
value := val.MapIndex(key).Interface()
if value != nil && reflect.TypeOf(value).Kind() == reflect.Map {
sortedValue, err := marshalSorted(value)
if err != nil {
return nil, err
}
sortedMap[key.String()] = json.RawMessage(sortedValue)
} else {
sortedMap[key.String()] = value
}
}
return json.Marshal(sortedMap)
}
+346
View File
@@ -0,0 +1,346 @@
package syft
import (
"bytes"
"crypto"
"encoding/json"
"fmt"
"io"
"reflect"
"sort"
"testing"
"github.com/hashicorp/go-multierror"
"github.com/iancoleman/strcase"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/cataloging/filecataloging"
"github.com/anchore/syft/syft/file"
)
func Test_configurationAuditTrail_StructTags(t *testing.T) {
// we need to ensure that the output for any configuration is well-formed and follows conventions.
// We ensure that:
// 1. all fields have a JSON tag
// 2. the tag value follows lowercase kebab-case style
jsonTags := getJSONTags(t, configurationAuditTrail{})
for _, tag := range jsonTags {
assertLowercaseKebab(t, tag)
}
}
func getJSONTags(t *testing.T, v interface{}) []string {
var tags []string
err := collectJSONTags(t, reflect.ValueOf(v), &tags, "", "")
require.NoError(t, err)
return tags
}
func collectJSONTags(t *testing.T, v reflect.Value, tags *[]string, parentTag string, path string) error {
var errs error
if v.Kind() == reflect.Ptr {
v = v.Elem()
}
if v.Kind() != reflect.Struct {
return errs
}
tType := v.Type()
for i := 0; i < v.NumField(); i++ {
field := v.Field(i)
fieldType := tType.Field(i)
curPath := path + "." + fieldType.Name
// account for embeddings
if fieldType.Anonymous {
embeddedField := field
if embeddedField.Kind() == reflect.Ptr {
// this can be enhanced in the future if the need arises...
errs = multierror.Append(errs, fmt.Errorf("field '%s' is a pointer to an embedded struct, this is not supported in the test helper", curPath))
}
if embeddedField.Kind() == reflect.Struct {
err := collectJSONTags(t, field, tags, parentTag, curPath)
if err != nil {
errs = multierror.Append(errs, err)
}
}
continue
}
var tag string
var ok bool
if fieldType.PkgPath == "" {
tag, ok = fieldType.Tag.Lookup("json")
if !ok || (tag == "" && parentTag == "") {
errs = multierror.Append(errs, fmt.Errorf("field '%s' does not have a json tag", curPath))
return errs
}
if tag != "" && tag != "-" {
*tags = append(*tags, tag)
}
}
if field.Kind() == reflect.Struct || (field.Kind() == reflect.Ptr && field.Elem().Kind() == reflect.Struct) {
err := collectJSONTags(t, field, tags, tag, curPath)
if err != nil {
errs = multierror.Append(errs, err)
}
}
}
return errs
}
func assertLowercaseKebab(t *testing.T, tag string) {
t.Helper()
require.NotEmpty(t, tag)
assert.Equal(t, strcase.ToKebab(tag), tag)
}
func Test_collectJSONTags(t *testing.T) {
// though this is not used in production, this is a sensitive and complex enough of a check to warrant testing the test helper.
type good struct {
A string `json:"a"`
}
type missing struct {
A string `json:"a"`
B string
}
type exclude struct {
A string `json:"a"`
B string `json:"-"`
}
type goodEmbedded struct {
good `json:""`
}
type badEmbedded struct {
missing `json:""`
}
// simply not covered and require further development to support
type goodPtrEmbedded struct {
*good `json:""`
}
// simply not covered and require further development to support
type badPtrEmbedded struct {
*missing `json:""`
}
tests := []struct {
name string
v interface{}
want []string
wantErr require.ErrorAssertionFunc
}{
{
name: "good",
v: good{},
want: []string{
"a",
},
},
{
name: "missing",
v: missing{},
wantErr: require.Error,
},
{
name: "exclude",
v: exclude{},
want: []string{
"a",
},
},
{
name: "bad embedded",
v: badEmbedded{},
wantErr: require.Error,
},
{
name: "good embedded",
v: goodEmbedded{},
want: []string{
"a",
},
},
// these cases are simply not covered and require further development to support
{
name: "bad ptr embedded",
v: badPtrEmbedded{},
wantErr: require.Error,
},
{
name: "good ptr embedded",
v: goodPtrEmbedded{},
want: []string{
"a",
},
wantErr: require.Error,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = require.NoError
}
var tags []string
err := collectJSONTags(t, reflect.ValueOf(tt.v), &tags, "", "")
tt.wantErr(t, err)
if err != nil {
return
}
assert.Equal(t, tt.want, tags)
})
}
}
func Test_configurationAuditTrail_MarshalJSON(t *testing.T) {
tests := []struct {
name string
cfg configurationAuditTrail
assert func(t *testing.T, got []byte)
}{
{
name: "ensure other marshallers are called",
cfg: configurationAuditTrail{
Files: filecataloging.Config{
Selection: file.FilesOwnedByPackageSelection,
Hashers: []crypto.Hash{
crypto.SHA256,
},
},
},
// the custom file marshaller swaps ints for strings for hashers
assert: func(t *testing.T, got []byte) {
assert.Contains(t, string(got), `"hashers":["sha-256"]`)
},
},
{
name: "ensure maps are sorted",
cfg: configurationAuditTrail{},
assert: func(t *testing.T, got []byte) {
assert.NoError(t, assertJSONKeysSorted(got))
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.cfg.MarshalJSON()
require.NoError(t, err)
if tt.assert == nil {
t.Fatal("assert function must be provided")
}
tt.assert(t, got)
})
}
}
// assertJSONKeysSorted asserts that all keys in JSON maps are sorted.
func assertJSONKeysSorted(jsonBytes []byte) error {
var errs error
decoder := json.NewDecoder(bytes.NewReader(jsonBytes))
var keys []string
var inObject bool
for {
token, err := decoder.Token()
if err != nil {
if err == io.EOF {
break
}
errs = multierror.Append(errs, fmt.Errorf("error decoding JSON: %w", err))
}
switch v := token.(type) {
case json.Delim:
switch v {
case '{':
inObject = true
keys = nil // Reset keys for a new object
case '}':
inObject = false
if !sort.StringsAreSorted(keys) {
errs = multierror.Append(errs, fmt.Errorf("Keys are not sorted: %v", keys))
}
}
case string:
if inObject && v != "" {
keys = append(keys, v)
}
}
}
return errs
}
func Test_assertJSONKeysSorted(t *testing.T) {
// this test function is sufficiently complicated enough to warrant its own test...
sorted := []byte(`{"a":1,"b":2}`)
unsorted := []byte(`{"b":2,"a":1}`)
nestedSorted := []byte(`{"a":1,"b":{"a":1,"b":2}}`)
nestedUnsorted := []byte(`{"a":1,"b":{"b":2,"a":1}}`)
tests := []struct {
name string
json []byte
wantErr require.ErrorAssertionFunc
}{
{
name: "sorted",
json: sorted,
wantErr: require.NoError,
},
{
name: "unsorted",
json: unsorted,
wantErr: require.Error,
},
{
name: "nested sorted",
json: nestedSorted,
wantErr: require.NoError,
},
{
name: "nested unsorted",
json: nestedUnsorted,
wantErr: require.Error,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = require.NoError
}
err := assertJSONKeysSorted(tt.json)
tt.wantErr(t, err)
})
}
}
+145
View File
@@ -0,0 +1,145 @@
package syft
import (
"context"
"fmt"
"sort"
"time"
"github.com/dustin/go-humanize"
"github.com/scylladb/go-set/strset"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/relationship"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/internal/task"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
// CreateSBOM creates a software bill-of-materials from the given source. If the CreateSBOMConfig is nil, then
// default options will be used.
func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (*sbom.SBOM, error) {
if cfg == nil {
cfg = DefaultCreateSBOMConfig()
}
if err := cfg.validate(); err != nil {
return nil, fmt.Errorf("invalid configuration: %w", err)
}
srcMetadata := src.Describe()
taskGroups, audit, err := cfg.makeTaskGroups(srcMetadata)
if err != nil {
return nil, err
}
resolver, err := src.FileResolver(cfg.Search.Scope)
if err != nil {
return nil, fmt.Errorf("unable to get file resolver: %w", err)
}
s := sbom.SBOM{
Source: srcMetadata,
Descriptor: sbom.Descriptor{
Name: cfg.ToolName,
Version: cfg.ToolVersion,
Configuration: configurationAuditTrail{
Search: cfg.Search,
Relationships: cfg.Relationships,
DataGeneration: cfg.DataGeneration,
Packages: cfg.Packages,
Files: cfg.Files,
Catalogers: *audit,
ExtraConfigs: cfg.ToolConfiguration,
},
},
Artifacts: sbom.Artifacts{
Packages: pkg.NewCollection(),
},
}
catalogingProgress := monitorCatalogingTask(src.ID(), taskGroups)
packageCatalogingProgress := monitorPackageCatalogingTask(s.Artifacts.Packages)
builder := sbomsync.NewBuilder(&s)
for i := range taskGroups {
err := task.NewTaskExecutor(taskGroups[i], cfg.Parallelism).Execute(ctx, resolver, builder, catalogingProgress)
if err != nil {
// TODO: tie this to the open progress monitors...
return nil, fmt.Errorf("failed to run tasks: %w", err)
}
}
packageCatalogingProgress.SetCompleted()
catalogingProgress.SetCompleted()
relationship.Finalize(builder, cfg.Relationships, src)
return &s, nil
}
func monitorPackageCatalogingTask(pkgs *pkg.Collection) *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Packages",
},
ID: monitor.PackageCatalogingTaskID,
HideOnSuccess: false,
ParentID: monitor.TopLevelCatalogingTaskID,
}
prog := bus.StartCatalogerTask(info, -1, "")
go func() {
ticker := time.NewTicker(200 * time.Millisecond)
defer ticker.Stop()
for {
<-ticker.C
count := humanize.Comma(int64(pkgs.PackageCount()))
prog.AtomicStage.Set(fmt.Sprintf("%s packages", count))
if progress.IsCompleted(prog) {
break
}
}
}()
return prog
}
func monitorCatalogingTask(srcID artifact.ID, tasks [][]task.Task) *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog contents",
WhileRunning: "Cataloging contents",
OnSuccess: "Cataloged contents",
},
ID: monitor.TopLevelCatalogingTaskID,
Context: string(srcID),
HideOnSuccess: false,
}
var length int64
for _, tg := range tasks {
length += int64(len(tg))
}
return bus.StartCatalogerTask(info, length, "")
}
func formatTaskNames(tasks []task.Task) []string {
set := strset.New()
for _, td := range tasks {
set.Add(td.Name())
}
list := set.List()
sort.Strings(list)
return list
}
+335
View File
@@ -0,0 +1,335 @@
package syft
import (
"context"
"errors"
"fmt"
"strings"
"github.com/anchore/syft/internal/task"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/filecataloging"
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
// CreateSBOMConfig specifies all parameters needed for creating an SBOM.
type CreateSBOMConfig struct {
// required configuration input to specify how cataloging should be performed
Search cataloging.SearchConfig
Relationships cataloging.RelationshipsConfig
DataGeneration cataloging.DataGenerationConfig
Packages pkgcataloging.Config
Files filecataloging.Config
Parallelism int
CatalogerSelection pkgcataloging.SelectionRequest
// audit what tool is being used to generate the SBOM
ToolName string
ToolVersion string
ToolConfiguration interface{}
packageTaskFactories task.PackageTaskFactories
packageCatalogerReferences []pkgcataloging.CatalogerReference
}
func DefaultCreateSBOMConfig() *CreateSBOMConfig {
return &CreateSBOMConfig{
Search: cataloging.DefaultSearchConfig(),
Relationships: cataloging.DefaultRelationshipsConfig(),
DataGeneration: cataloging.DefaultDataGenerationConfig(),
Packages: pkgcataloging.DefaultConfig(),
Files: filecataloging.DefaultConfig(),
Parallelism: 1,
packageTaskFactories: task.DefaultPackageTaskFactories(),
}
}
// WithTool allows for setting the specific name, version, and any additional configuration that is not captured
// in the syft default API configuration. This could cover inputs for catalogers that were user-provided, thus,
// is not visible to the syft API, but would be useful to see in the SBOM output.
func (c *CreateSBOMConfig) WithTool(name, version string, cfg ...any) *CreateSBOMConfig {
c.ToolName = name
c.ToolVersion = version
c.ToolConfiguration = cfg
return c
}
// WithParallelism allows for setting the number of concurrent cataloging tasks that can be performed at once
func (c *CreateSBOMConfig) WithParallelism(p int) *CreateSBOMConfig {
if p < 1 {
// TODO: warn?
p = 1
}
c.Parallelism = p
return c
}
// WithSearchConfig allows for setting the specific search configuration for cataloging.
func (c *CreateSBOMConfig) WithSearchConfig(cfg cataloging.SearchConfig) *CreateSBOMConfig {
c.Search = cfg
return c
}
// WithRelationshipsConfig allows for defining the specific relationships that should be captured during cataloging.
func (c *CreateSBOMConfig) WithRelationshipsConfig(cfg cataloging.RelationshipsConfig) *CreateSBOMConfig {
c.Relationships = cfg
return c
}
// WithDataGenerationConfig allows for defining what data elements that cannot be discovered from the underlying
// target being scanned that should be generated after package creation.
func (c *CreateSBOMConfig) WithDataGenerationConfig(cfg cataloging.DataGenerationConfig) *CreateSBOMConfig {
c.DataGeneration = cfg
return c
}
// WithPackagesConfig allows for defining any specific behavior for syft-implemented catalogers.
func (c *CreateSBOMConfig) WithPackagesConfig(cfg pkgcataloging.Config) *CreateSBOMConfig {
c.Packages = cfg
return c
}
// WithFilesConfig allows for defining file-based cataloging parameters.
func (c *CreateSBOMConfig) WithFilesConfig(cfg filecataloging.Config) *CreateSBOMConfig {
c.Files = cfg
return c
}
// WithoutFiles allows for disabling file cataloging altogether.
func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig {
c.Files = filecataloging.Config{
Selection: file.NoFilesSelection,
Hashers: nil,
}
return c
}
// WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag.
func (c *CreateSBOMConfig) WithCatalogerSelection(selection pkgcataloging.SelectionRequest) *CreateSBOMConfig {
c.CatalogerSelection = selection
return c
}
// WithoutCatalogers removes all catalogers from the final set of catalogers. This is useful if you want to only use
// user-provided catalogers (without the default syft-provided catalogers).
func (c *CreateSBOMConfig) WithoutCatalogers() *CreateSBOMConfig {
c.packageTaskFactories = nil
c.packageCatalogerReferences = nil
return c
}
// WithCatalogers allows for adding user-provided catalogers to the final set of catalogers that will always be run
// regardless of the source type or any cataloger selections provided.
func (c *CreateSBOMConfig) WithCatalogers(catalogerRefs ...pkgcataloging.CatalogerReference) *CreateSBOMConfig {
c.packageCatalogerReferences = append(c.packageCatalogerReferences, catalogerRefs...)
return c
}
// makeTaskGroups considers the entire configuration and finalizes the set of tasks to be run. Tasks are run in
// groups, where each task in a group can be run concurrently, while tasks in different groups must be run serially.
// The final set of task groups is returned along with a cataloger manifest that describes the catalogers that were
// selected and the tokens that were sensitive to this selection (both for adding and removing from the final set).
func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task, *catalogerManifest, error) {
var taskGroups [][]task.Task
// generate package and file tasks based on the configuration
environmentTasks := c.environmentTasks()
relationshipsTasks := c.relationshipTasks(src)
fileTasks := c.fileTasks()
pkgTasks, selectionEvidence, err := c.packageTasks(src)
if err != nil {
return nil, nil, err
}
// combine the user-provided and configured tasks
if c.Files.Selection == file.FilesOwnedByPackageSelection {
// special case: we need the package info when we are cataloging files owned by packages
taskGroups = append(taskGroups, pkgTasks, fileTasks)
} else {
taskGroups = append(taskGroups, append(pkgTasks, fileTasks...))
}
// all relationship work must be done after all nodes (files and packages) have been cataloged
if len(relationshipsTasks) > 0 {
taskGroups = append(taskGroups, relationshipsTasks)
}
// identifying the environment (i.e. the linux release) must be done first as this is required for package cataloging
taskGroups = append(
[][]task.Task{
environmentTasks,
},
taskGroups...,
)
return taskGroups, &catalogerManifest{
Requested: selectionEvidence.Request,
Used: formatTaskNames(pkgTasks),
}, nil
}
// fileTasks returns the set of tasks that should be run to catalog files.
func (c *CreateSBOMConfig) fileTasks() []task.Task {
var tsks []task.Task
if t := task.NewFileDigestCatalogerTask(c.Files.Selection, c.Files.Hashers...); t != nil {
tsks = append(tsks, t)
}
if t := task.NewFileMetadataCatalogerTask(c.Files.Selection); t != nil {
tsks = append(tsks, t)
}
if t := task.NewFileContentCatalogerTask(c.Files.Content); t != nil {
tsks = append(tsks, t)
}
return tsks
}
// packageTasks returns the set of tasks that should be run to catalog packages.
func (c *CreateSBOMConfig) packageTasks(src source.Description) ([]task.Task, *task.Selection, error) {
cfg := task.CatalogingFactoryConfig{
SearchConfig: c.Search,
RelationshipsConfig: c.Relationships,
DataGenerationConfig: c.DataGeneration,
PackagesConfig: c.Packages,
}
persistentTasks, selectableTasks, err := c.allPackageTasks(cfg)
if err != nil {
return nil, nil, fmt.Errorf("unable to create package cataloger tasks: %w", err)
}
req, err := finalSelectionRequest(c.CatalogerSelection, src)
if err != nil {
return nil, nil, err
}
finalTasks, selection, err := task.Select(selectableTasks, *req)
if err != nil {
return nil, nil, err
}
finalTasks = append(finalTasks, persistentTasks...)
if len(finalTasks) == 0 {
return nil, nil, fmt.Errorf("no catalogers selected")
}
return finalTasks, &selection, nil
}
func finalSelectionRequest(req pkgcataloging.SelectionRequest, src source.Description) (*pkgcataloging.SelectionRequest, error) {
if len(req.DefaultNamesOrTags) == 0 {
defaultTag, err := findDefaultTag(src)
if err != nil {
return nil, fmt.Errorf("unable to determine default cataloger tag: %w", err)
}
if defaultTag != "" {
req.DefaultNamesOrTags = append(req.DefaultNamesOrTags, defaultTag)
}
req.RemoveNamesOrTags = replaceDefaultTagReferences(defaultTag, req.RemoveNamesOrTags)
req.SubSelectTags = replaceDefaultTagReferences(defaultTag, req.SubSelectTags)
}
return &req, nil
}
func (c *CreateSBOMConfig) allPackageTasks(cfg task.CatalogingFactoryConfig) ([]task.Task, []task.Task, error) {
persistentPackageTasks, selectablePackageTasks, err := c.userPackageTasks(cfg)
if err != nil {
return nil, nil, err
}
tsks, err := c.packageTaskFactories.Tasks(cfg)
if err != nil {
return nil, nil, fmt.Errorf("unable to create package cataloger tasks: %w", err)
}
return persistentPackageTasks, append(tsks, selectablePackageTasks...), nil
}
func (c *CreateSBOMConfig) userPackageTasks(cfg task.CatalogingFactoryConfig) ([]task.Task, []task.Task, error) {
var (
persistentPackageTasks []task.Task
selectablePackageTasks []task.Task
)
for _, catalogerRef := range c.packageCatalogerReferences {
if catalogerRef.Cataloger == nil {
return nil, nil, errors.New("provided cataloger reference without a cataloger")
}
if catalogerRef.AlwaysEnabled {
persistentPackageTasks = append(persistentPackageTasks, task.NewPackageTask(cfg, catalogerRef.Cataloger, catalogerRef.Tags...))
continue
}
if len(catalogerRef.Tags) == 0 {
return nil, nil, errors.New("provided cataloger reference without tags")
}
selectablePackageTasks = append(selectablePackageTasks, task.NewPackageTask(cfg, catalogerRef.Cataloger, catalogerRef.Tags...))
}
return persistentPackageTasks, selectablePackageTasks, nil
}
// relationshipTasks returns the set of tasks that should be run to generate additional relationships as well as
// prune existing relationships.
func (c *CreateSBOMConfig) relationshipTasks(src source.Description) []task.Task {
var tsks []task.Task
if t := task.NewRelationshipsTask(c.Relationships, src); t != nil {
tsks = append(tsks, t)
}
return tsks
}
// environmentTasks returns the set of tasks that should be run to identify what is being scanned or the context
// of where it is being scanned. Today this is used to identify the linux distribution release for container images
// being scanned.
func (c *CreateSBOMConfig) environmentTasks() []task.Task {
var tsks []task.Task
if t := task.NewEnvironmentTask(); t != nil {
tsks = append(tsks, t)
}
return tsks
}
func (c *CreateSBOMConfig) validate() error {
if c.Relationships.ExcludeBinaryPackagesWithFileOwnershipOverlap {
if !c.Relationships.PackageFileOwnershipOverlap {
return fmt.Errorf("invalid configuration: to exclude binary packages based on file ownership overlap relationships, cataloging file ownership overlap relationships must be enabled")
}
}
return nil
}
// Create creates an SBOM from the given source with the current SBOM configuration.
func (c *CreateSBOMConfig) Create(ctx context.Context, src source.Source) (*sbom.SBOM, error) {
return CreateSBOM(ctx, src, c)
}
func findDefaultTag(src source.Description) (string, error) {
switch m := src.Metadata.(type) {
case source.StereoscopeImageSourceMetadata:
return pkgcataloging.ImageTag, nil
case source.FileSourceMetadata, source.DirectorySourceMetadata:
return pkgcataloging.DirectoryTag, nil
default:
return "", fmt.Errorf("unable to determine default cataloger tag for source type=%T", m)
}
}
func replaceDefaultTagReferences(defaultTag string, lst []string) []string {
for i, tag := range lst {
if strings.ToLower(tag) == "default" {
lst[i] = defaultTag
}
}
return lst
}
+502
View File
@@ -0,0 +1,502 @@
package syft
import (
"sort"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/task"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/cataloging/filecataloging"
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
var _ pkg.Cataloger = (*dummyCataloger)(nil)
type dummyCataloger struct {
name string
}
func newDummyCataloger(name string) pkg.Cataloger {
return dummyCataloger{name: name}
}
func (d dummyCataloger) Name() string {
return d.name
}
func (d dummyCataloger) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
return nil, nil, nil
}
func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
pkgIntersect := func(intersect ...string) []string {
var sets []*strset.Set
for _, s := range intersect {
sets = append(sets, strset.New(pkgCatalogerNamesWithTagOrName(t, s)...))
}
intersectSet := strset.Intersection(sets...)
slice := intersectSet.List()
sort.Strings(slice)
return slice
}
addTo := func(slice []string, add ...string) []string {
slice = append(slice, add...)
sort.Strings(slice)
return slice
}
imgSrc := source.Description{
Metadata: source.StereoscopeImageSourceMetadata{},
}
dirSrc := source.Description{
Metadata: source.DirectorySourceMetadata{},
}
fileSrc := source.Description{
Metadata: source.FileSourceMetadata{},
}
tests := []struct {
name string
src source.Description
cfg *CreateSBOMConfig
wantTaskNames [][]string
wantManifest *catalogerManifest
wantErr require.ErrorAssertionFunc
}{
{
name: "default catalogers for image source",
src: imgSrc,
cfg: DefaultCreateSBOMConfig(),
wantTaskNames: [][]string{
environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(true, true),
relationshipCatalogerNames(),
},
wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"},
},
Used: pkgCatalogerNamesWithTagOrName(t, "image"),
},
wantErr: require.NoError,
},
{
name: "default catalogers for directory source",
src: dirSrc,
cfg: DefaultCreateSBOMConfig(),
wantTaskNames: [][]string{
environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "directory"),
fileCatalogerNames(true, true),
relationshipCatalogerNames(),
},
wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"directory"},
},
Used: pkgCatalogerNamesWithTagOrName(t, "directory"),
},
wantErr: require.NoError,
},
{
// note, the file source acts like a directory scan
name: "default catalogers for file source",
src: fileSrc,
cfg: DefaultCreateSBOMConfig(),
wantTaskNames: [][]string{
environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "directory"),
fileCatalogerNames(true, true),
relationshipCatalogerNames(),
},
wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"directory"},
},
Used: pkgCatalogerNamesWithTagOrName(t, "directory"),
},
wantErr: require.NoError,
},
{
name: "no file digest cataloger",
src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithHashers(nil)),
wantTaskNames: [][]string{
environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(false, true), // note: the digest cataloger is not included
relationshipCatalogerNames(),
},
wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"},
},
Used: pkgCatalogerNamesWithTagOrName(t, "image"),
},
wantErr: require.NoError,
},
{
name: "select no file catalogers",
src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.NoFilesSelection)),
wantTaskNames: [][]string{
environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"),
// note: there are no file catalogers in their own group
relationshipCatalogerNames(),
},
wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"},
},
Used: pkgCatalogerNamesWithTagOrName(t, "image"),
},
wantErr: require.NoError,
},
{
name: "select all file catalogers",
src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)),
wantTaskNames: [][]string{
environmentCatalogerNames(),
// note: there is a single group of catalogers for pkgs and files
append(
pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(true, true)...,
),
relationshipCatalogerNames(),
},
wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"},
},
Used: pkgCatalogerNamesWithTagOrName(t, "image"),
},
wantErr: require.NoError,
},
{
name: "user-provided persistent cataloger is always run (image)",
src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithCatalogers(
pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")),
),
wantTaskNames: [][]string{
environmentCatalogerNames(),
addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"),
fileCatalogerNames(true, true),
relationshipCatalogerNames(),
},
wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"},
},
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"),
},
wantErr: require.NoError,
},
{
name: "user-provided persistent cataloger is always run (directory)",
src: dirSrc,
cfg: DefaultCreateSBOMConfig().WithCatalogers(
pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")),
),
wantTaskNames: [][]string{
environmentCatalogerNames(),
addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"),
fileCatalogerNames(true, true),
relationshipCatalogerNames(),
},
wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"directory"},
},
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"),
},
wantErr: require.NoError,
},
{
name: "user-provided persistent cataloger is always run (user selection does not affect this)",
src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithCatalogers(
pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")),
).WithCatalogerSelection(pkgcataloging.NewSelectionRequest().WithSubSelections("javascript")),
wantTaskNames: [][]string{
environmentCatalogerNames(),
addTo(pkgIntersect("image", "javascript"), "persistent"),
fileCatalogerNames(true, true),
relationshipCatalogerNames(),
},
wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"},
SubSelectTags: []string{"javascript"},
},
Used: addTo(pkgIntersect("image", "javascript"), "persistent"),
},
wantErr: require.NoError,
},
{
name: "user-provided cataloger runs when selected",
src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithCatalogers(
pkgcataloging.NewCatalogerReference(newDummyCataloger("user-provided"), []string{"image"}),
),
wantTaskNames: [][]string{
environmentCatalogerNames(),
addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"),
fileCatalogerNames(true, true),
relationshipCatalogerNames(),
},
wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"},
},
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"),
},
wantErr: require.NoError,
},
{
name: "user-provided cataloger NOT run when NOT selected",
src: imgSrc,
cfg: DefaultCreateSBOMConfig().WithCatalogers(
pkgcataloging.NewCatalogerReference(newDummyCataloger("user-provided"), []string{"bogus-selector-will-never-be-used"}),
),
wantTaskNames: [][]string{
environmentCatalogerNames(),
pkgCatalogerNamesWithTagOrName(t, "image"),
fileCatalogerNames(true, true),
relationshipCatalogerNames(),
},
wantManifest: &catalogerManifest{
Requested: pkgcataloging.SelectionRequest{
DefaultNamesOrTags: []string{"image"},
},
Used: pkgCatalogerNamesWithTagOrName(t, "image"),
},
wantErr: require.NoError,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = require.NoError
}
// sanity check
require.NotEmpty(t, tt.wantTaskNames)
for _, group := range tt.wantTaskNames {
require.NotEmpty(t, group)
}
// test the subject
gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src)
tt.wantErr(t, err)
if err != nil {
return
}
gotNames := taskGroupNames(gotTasks)
if d := cmp.Diff(
tt.wantTaskNames,
gotNames,
// order within a group does not matter
cmpopts.SortSlices(func(a, b string) bool {
return a < b
}),
); d != "" {
t.Errorf("mismatched task group names (-want +got):\n%s", d)
}
if d := cmp.Diff(tt.wantManifest, gotManifest); d != "" {
t.Errorf("mismatched cataloger manifest (-want +got):\n%s", d)
}
})
}
}
func pkgCatalogerNamesWithTagOrName(t *testing.T, token string) []string {
var names []string
cfg := task.DefaultCatalogingFactoryConfig()
for _, factory := range task.DefaultPackageTaskFactories() {
cat := factory(cfg)
name := cat.Name()
if selector, ok := cat.(task.Selector); ok {
if selector.HasAllSelectors(token) {
names = append(names, name)
continue
}
}
if name == token {
names = append(names, name)
}
}
// these thresholds are arbitrary but should be large enough to catch any major changes
switch token {
case "image":
require.Greater(t, len(names), 18, "minimum cataloger sanity check failed token")
case "directory":
require.Greater(t, len(names), 25, "minimum cataloger sanity check failed token")
default:
require.Greater(t, len(names), 0, "minimum cataloger sanity check failed token")
}
sort.Strings(names)
return names
}
func fileCatalogerNames(digest, metadata bool) []string {
var names []string
if digest {
names = append(names, "file-digest-cataloger")
}
if metadata {
names = append(names, "file-metadata-cataloger")
}
return names
}
func relationshipCatalogerNames() []string {
return []string{"relationships-cataloger"}
}
func environmentCatalogerNames() []string {
return []string{"environment-cataloger"}
}
func taskGroupNames(groups [][]task.Task) [][]string {
var names [][]string
for _, group := range groups {
var groupNames []string
for _, tsk := range group {
groupNames = append(groupNames, tsk.Name())
}
names = append(names, groupNames)
}
return names
}
func Test_replaceDefaultTagReferences(t *testing.T) {
tests := []struct {
name string
lst []string
want []string
}{
{
name: "no default tag",
lst: []string{"foo", "bar"},
want: []string{"foo", "bar"},
},
{
name: "replace default tag",
lst: []string{"foo", "default", "bar"},
want: []string{"foo", "replacement", "bar"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, replaceDefaultTagReferences("replacement", tt.lst))
})
}
}
func Test_findDefaultTag(t *testing.T) {
tests := []struct {
name string
src source.Description
want string
wantErr require.ErrorAssertionFunc
}{
{
name: "image",
src: source.Description{
Metadata: source.StereoscopeImageSourceMetadata{},
},
want: pkgcataloging.ImageTag,
},
{
name: "directory",
src: source.Description{
Metadata: source.DirectorySourceMetadata{},
},
want: pkgcataloging.DirectoryTag,
},
{
name: "file",
src: source.Description{
Metadata: source.FileSourceMetadata{},
},
want: pkgcataloging.DirectoryTag, // not a mistake...
},
{
name: "unknown",
src: source.Description{
Metadata: struct{}{},
},
wantErr: require.Error,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = require.NoError
}
got, err := findDefaultTag(tt.src)
tt.wantErr(t, err)
if err != nil {
return
}
assert.Equal(t, tt.want, got)
})
}
}
func TestCreateSBOMConfig_validate(t *testing.T) {
tests := []struct {
name string
cfg *CreateSBOMConfig
wantErr assert.ErrorAssertionFunc
}{
{
name: "incompatible ExcludeBinaryPackagesWithFileOwnershipOverlap selection",
cfg: DefaultCreateSBOMConfig().
WithRelationshipsConfig(
cataloging.DefaultRelationshipsConfig().
WithExcludeBinaryPackagesWithFileOwnershipOverlap(true).
WithPackageFileOwnershipOverlap(false),
),
wantErr: assert.Error,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = assert.NoError
}
tt.wantErr(t, tt.cfg.validate())
})
}
}
@@ -4,6 +4,11 @@ import (
"github.com/wagoodman/go-progress"
)
const (
TopLevelCatalogingTaskID = "cataloging"
PackageCatalogingTaskID = "package-cataloging"
)
type CatalogerTaskProgress struct {
*progress.AtomicStage
*progress.Manual
+49 -5
View File
@@ -6,21 +6,40 @@ import (
"fmt"
"io"
"github.com/dustin/go-humanize"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
)
type Config struct {
// Globs are the file patterns that must be matched for a file to be considered for cataloging.
Globs []string `yaml:"globs" json:"globs" mapstructure:"globs"`
// SkipFilesAboveSize is the maximum file size (in bytes) to allow to be considered while cataloging. If the file is larger than this size it will be skipped.
SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"`
}
type Cataloger struct {
globs []string
skipFilesAboveSizeInBytes int64
}
func NewCataloger(globs []string, skipFilesAboveSize int64) (*Cataloger, error) {
func DefaultConfig() Config {
return Config{
SkipFilesAboveSize: 250 * intFile.KB,
}
}
func NewCataloger(cfg Config) *Cataloger {
return &Cataloger{
globs: globs,
skipFilesAboveSizeInBytes: skipFilesAboveSize,
}, nil
globs: cfg.Globs,
skipFilesAboveSizeInBytes: cfg.SkipFilesAboveSize,
}
}
func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string, error) {
@@ -31,9 +50,15 @@ func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string
if err != nil {
return nil, err
}
prog := catalogingProgress(int64(len(locations)))
for _, location := range locations {
prog.AtomicStage.Set(location.Path())
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
prog.SetError(err)
return nil, err
}
@@ -47,12 +72,20 @@ func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string
continue
}
if err != nil {
prog.SetError(err)
return nil, err
}
prog.Increment()
results[location.Coordinates] = result
}
log.Debugf("file contents cataloger processed %d files", len(results))
prog.AtomicStage.Set(fmt.Sprintf("%s files", humanize.Comma(prog.Current())))
prog.SetCompleted()
return results, nil
}
@@ -66,7 +99,7 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
buf := &bytes.Buffer{}
encoder := base64.NewEncoder(base64.StdEncoding, buf)
if _, err = io.Copy(encoder, contentReader); err != nil {
return "", internal.ErrPath{Context: "contents-cataloger", Path: location.RealPath, Err: err}
return "", internal.ErrPath{Context: "content-cataloger", Path: location.RealPath, Err: err}
}
// note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes
if err := encoder.Close(); err != nil {
@@ -75,3 +108,14 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
return buf.String(), nil
}
func catalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "File contents",
},
ParentID: monitor.TopLevelCatalogingTaskID,
}
return bus.StartCatalogerTask(info, locations, "")
}
@@ -67,8 +67,10 @@ func TestContentsCataloger(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c, err := NewCataloger(test.globs, test.maxSize)
assert.NoError(t, err)
c := NewCataloger(Config{
Globs: test.globs,
SkipFilesAboveSize: test.maxSize,
})
resolver := file.NewMockResolverForPaths(test.files...)
actual, err := c.Catalog(resolver)
+11 -10
View File
@@ -45,32 +45,34 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
}
}
prog := digestsCatalogingProgress(int64(len(locations)))
prog := catalogingProgress(int64(len(locations)))
for _, location := range locations {
prog.Increment()
prog.AtomicStage.Set(location.Path())
result, err := i.catalogLocation(resolver, location)
if errors.Is(err, ErrUndigestableFile) {
continue
}
prog.AtomicStage.Set(location.Path())
if internal.IsErrPathPermission(err) {
log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
continue
}
if err != nil {
return nil, err
prog.SetError(err)
return nil, fmt.Errorf("failed to process file %q: %w", location.RealPath, err)
}
prog.Increment()
results[location.Coordinates] = result
}
log.Debugf("file digests cataloger processed %d files", prog.Current())
prog.AtomicStage.Set(fmt.Sprintf("%s digests", humanize.Comma(prog.Current())))
prog.AtomicStage.Set(fmt.Sprintf("%s files", humanize.Comma(prog.Current())))
prog.SetCompleted()
return results, nil
@@ -101,13 +103,12 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
return digests, nil
}
func digestsCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
func catalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog file digests",
WhileRunning: "Cataloging file digests",
OnSuccess: "Cataloged file digests",
Default: "File digests",
},
ParentID: monitor.TopLevelCatalogingTaskID,
}
return bus.StartCatalogerTask(info, locations, "")
@@ -43,16 +43,18 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
}()
}
prog := metadataCatalogingProgress(int64(len(locations)))
prog := catalogingProgress(-1)
for location := range locations {
prog.Increment()
prog.AtomicStage.Set(location.Path())
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
prog.SetError(err)
return nil, err
}
prog.Increment()
results[location.Coordinates] = metadata
}
@@ -64,13 +66,12 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
return results, nil
}
func metadataCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
func catalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog file metadata",
WhileRunning: "Cataloging file metadata",
OnSuccess: "Cataloged file metadata",
Default: "File metadata",
},
ParentID: monitor.TopLevelCatalogingTaskID,
}
return bus.StartCatalogerTask(info, locations, "")
+9
View File
@@ -0,0 +1,9 @@
package file
const (
NoFilesSelection Selection = "none"
FilesOwnedByPackageSelection Selection = "owned-by-package"
AllFilesSelection Selection = "all"
)
type Selection string
-90
View File
@@ -17,103 +17,13 @@ Similar to the cataloging process, Linux distribution identification is also per
package syft
import (
"fmt"
"github.com/wagoodman/go-partybus"
"github.com/anchore/go-logger"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger"
"github.com/anchore/syft/syft/source"
)
// CatalogPackages takes an inventory of packages from the given image from a particular perspective
// (e.g. squashed source, all-layers source). Returns the discovered set of packages, the identified Linux
// distribution, and the source object used to wrap the data source.
func CatalogPackages(src source.Source, cfg cataloger.Config) (*pkg.Collection, []artifact.Relationship, *linux.Release, error) {
resolver, err := src.FileResolver(cfg.Search.Scope)
if err != nil {
return nil, nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err)
}
// find the distro
release := linux.IdentifyRelease(resolver)
if release != nil {
log.Infof("identified distro: %s", release.String())
} else {
log.Info("could not identify distro")
}
// if the catalogers have been configured, use them regardless of input type
var catalogers []pkg.Cataloger
if len(cfg.Catalogers) > 0 {
catalogers = cataloger.AllCatalogers(cfg)
} else {
// otherwise conditionally use the correct set of loggers based on the input type (container image or directory)
// TODO: this is bad, we should not be using the concrete type to determine the cataloger set
// instead this should be a caller concern (pass the catalogers you want to use). The SBOM build PR will do this.
switch src.(type) {
case *source.StereoscopeImageSource:
log.Info("cataloging an image")
catalogers = cataloger.ImageCatalogers(cfg)
case *source.FileSource:
log.Info("cataloging a file")
catalogers = cataloger.AllCatalogers(cfg)
case *source.DirectorySource:
log.Info("cataloging a directory")
catalogers = cataloger.DirectoryCatalogers(cfg)
default:
return nil, nil, nil, fmt.Errorf("unsupported source type: %T", src)
}
}
catalog, relationships, err := cataloger.Catalog(resolver, release, cfg.Parallelism, catalogers...)
// apply exclusions to the package catalog
// default config value for this is true
// https://github.com/anchore/syft/issues/931
if cfg.ExcludeBinaryOverlapByOwnership {
for _, r := range relationships {
if cataloger.ExcludeBinaryByFileOwnershipOverlap(r, catalog) {
catalog.Delete(r.To.ID())
relationships = removeRelationshipsByID(relationships, r.To.ID())
}
}
}
// no need to consider source relationships for os -> binary exclusions
relationships = append(relationships, newSourceRelationshipsFromCatalog(src, catalog)...)
return catalog, relationships, release, err
}
func removeRelationshipsByID(relationships []artifact.Relationship, id artifact.ID) []artifact.Relationship {
var filtered []artifact.Relationship
for _, r := range relationships {
if r.To.ID() != id && r.From.ID() != id {
filtered = append(filtered, r)
}
}
return filtered
}
func newSourceRelationshipsFromCatalog(src source.Source, c *pkg.Collection) []artifact.Relationship {
relationships := make([]artifact.Relationship, 0) // Should we pre-allocate this by giving catalog a Len() method?
for p := range c.Enumerate() {
relationships = append(relationships, artifact.Relationship{
From: src,
To: p,
Type: artifact.ContainsRelationship,
})
}
return relationships
}
// SetLogger sets the logger object used for all syft logging calls.
func SetLogger(logger logger.Logger) {
log.Set(logger)
+2 -1
View File
@@ -5,6 +5,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/relationship"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
)
@@ -34,7 +35,7 @@ func Test_removeRelationshipsByID(t *testing.T) {
for _, r := range relationships {
if r.From.ID() == "1" || r.From.ID() == "2" {
relationships = removeRelationshipsByID(relationships, r.From.ID())
relationships = relationship.RemoveRelationshipsByID(relationships, r.From.ID())
}
}
+1 -1
View File
@@ -13,7 +13,7 @@ import (
const catalogerName = "binary-cataloger"
type CatalogerConfig struct {
Classifiers []Classifier
Classifiers []Classifier `yaml:"classifiers" json:"classifiers" mapstructure:"classifiers"`
}
func DefaultCatalogerConfig() CatalogerConfig {
+6 -6
View File
@@ -23,25 +23,25 @@ import (
// Classifier is a generic package classifier that can be used to match a package definition
// to a file that meets the given content criteria of the EvidenceMatcher.
type Classifier struct {
Class string
Class string `json:"class"`
// FileGlob is a selector to narrow down file inspection using the **/glob* syntax
FileGlob string
FileGlob string `json:"fileGlob"`
// EvidenceMatcher is what will be used to match against the file in the source
// location. If the matcher returns a package, the file will be considered a candidate.
EvidenceMatcher EvidenceMatcher
EvidenceMatcher EvidenceMatcher `json:"-"`
// Information below is used to specify the Package information when returned
// Package is the name to use for the package
Package string
Package string `json:"package"`
// PURL is the Package URL to use when generating a package
PURL packageurl.PackageURL
PURL packageurl.PackageURL `json:"purl"`
// CPEs are the specific CPEs we want to include for this binary with updated version information
CPEs []cpe.CPE
CPEs []cpe.CPE `json:"cpes"`
}
// EvidenceMatcher is a function called to catalog Packages that match some sort of evidence
-229
View File
@@ -1,229 +0,0 @@
package cataloger
import (
"fmt"
"math"
"runtime/debug"
"sync"
"github.com/dustin/go-humanize"
"github.com/hashicorp/go-multierror"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common/cpe"
)
// Monitor provides progress-related data for observing the progress of a Catalog() call (published on the event bus).
type Monitor struct {
FilesProcessed progress.Monitorable // the number of files selected and contents analyzed from all registered catalogers
PackagesDiscovered progress.Monitorable // the number of packages discovered from all registered catalogers
}
// catalogResult provides the result of running a single cataloger against source
type catalogResult struct {
Packages []pkg.Package
Relationships []artifact.Relationship
// Discovered may sometimes be more than len(packages)
Discovered int64
Error error
}
func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerResult *catalogResult, err error) {
// handle individual cataloger panics
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("%v at:\n%s", e, string(debug.Stack()))
}
}()
catalogerResult = new(catalogResult)
// find packages from the underlying raw data
log.WithFields("cataloger", cataloger.Name()).Trace("cataloging started")
packages, relationships, err := cataloger.Catalog(resolver)
if err != nil {
log.WithFields("cataloger", cataloger.Name()).Warn("error while cataloging")
return catalogerResult, err
}
catalogedPackages := len(packages)
log.WithFields("cataloger", cataloger.Name()).Debugf("discovered %d packages", catalogedPackages)
catalogerResult.Discovered = int64(catalogedPackages)
for _, p := range packages {
// generate CPEs (note: this is excluded from package ID, so is safe to mutate)
// we might have binary classified CPE already with the package so we want to append here
dictionaryCPE, ok := cpe.DictionaryFind(p)
if ok {
log.Debugf("used CPE dictionary to find CPE for %s package %q: %s", p.Type, p.Name, dictionaryCPE.BindToFmtString())
p.CPEs = append(p.CPEs, dictionaryCPE)
} else {
p.CPEs = append(p.CPEs, cpe.Generate(p)...)
}
// if we were not able to identify the language we have an opportunity
// to try and get this value from the PURL. Worst case we assert that
// we could not identify the language at either stage and set UnknownLanguage
if p.Language == "" {
p.Language = pkg.LanguageFromPURL(p.PURL)
}
// create file-to-package relationships for files owned by the package
owningRelationships, err := packageFileOwnershipRelationships(p, resolver)
if err != nil {
log.WithFields("cataloger", cataloger.Name(), "package", p.Name, "error", err).Warnf("unable to create any package-file relationships")
} else {
catalogerResult.Relationships = append(catalogerResult.Relationships, owningRelationships...)
}
catalogerResult.Packages = append(catalogerResult.Packages, p)
}
catalogerResult.Relationships = append(catalogerResult.Relationships, relationships...)
log.WithFields("cataloger", cataloger.Name()).Trace("cataloging complete")
return catalogerResult, err
}
// Catalog a given source (container image or filesystem) with the given catalogers, returning all discovered packages.
// In order to efficiently retrieve contents from a underlying container image the content fetch requests are
// done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single
// request.
//
//nolint:funlen
func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, catalogers ...pkg.Cataloger) (*pkg.Collection, []artifact.Relationship, error) {
catalog := pkg.NewCollection()
var allRelationships []artifact.Relationship
prog := monitorPackageCatalogingTask()
// perform analysis, accumulating errors for each failed analysis
var errs error
nCatalogers := len(catalogers)
// we do not need more parallelism than there are `catalogers`.
parallelism = int(math.Min(float64(nCatalogers), math.Max(1.0, float64(parallelism))))
log.WithFields("parallelism", parallelism, "catalogers", nCatalogers).Debug("cataloging packages")
jobs := make(chan pkg.Cataloger, nCatalogers)
results := make(chan *catalogResult, nCatalogers)
waitGroup := sync.WaitGroup{}
var totalPackagesDiscovered int64
for i := 0; i < parallelism; i++ {
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
// wait for / get the next cataloger job available.
for cataloger := range jobs {
result, err := runCataloger(cataloger, resolver)
// ensure we set the error to be aggregated
result.Error = err
prog.Add(result.Discovered)
totalPackagesDiscovered += result.Discovered
count := humanize.Comma(totalPackagesDiscovered)
prog.AtomicStage.Set(fmt.Sprintf("%s packages", count))
results <- result
}
}()
}
// Enqueue the jobs
for _, cataloger := range catalogers {
jobs <- cataloger
}
close(jobs)
// Wait for the jobs to finish
waitGroup.Wait()
close(results)
// collect the results
for result := range results {
if result.Error != nil {
errs = multierror.Append(errs, result.Error)
}
for _, p := range result.Packages {
catalog.Add(p)
}
allRelationships = append(allRelationships, result.Relationships...)
}
allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...)
if errs != nil {
prog.SetError(errs)
} else {
prog.SetCompleted()
}
return catalog, allRelationships, errs
}
func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver) ([]artifact.Relationship, error) {
fileOwner, ok := p.Metadata.(pkg.FileOwner)
if !ok {
return nil, nil
}
locations := map[artifact.ID]file.Location{}
for _, path := range fileOwner.OwnedFiles() {
pathRefs, err := resolver.FilesByPath(path)
if err != nil {
return nil, fmt.Errorf("unable to find path for path=%q: %w", path, err)
}
if len(pathRefs) == 0 {
// ideally we want to warn users about missing files from a package, however, it is very common for
// container image authors to delete files that are not needed in order to keep image sizes small. Adding
// a warning here would be needlessly noisy (even for popular base images).
continue
}
for _, ref := range pathRefs {
if oldRef, ok := locations[ref.Coordinates.ID()]; ok {
log.Debugf("found path duplicate of %s", oldRef.RealPath)
}
locations[ref.Coordinates.ID()] = ref
}
}
var relationships []artifact.Relationship
for _, location := range locations {
relationships = append(relationships, artifact.Relationship{
From: p,
To: location.Coordinates,
Type: artifact.ContainsRelationship,
})
}
return relationships, nil
}
func monitorPackageCatalogingTask() *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog packages",
WhileRunning: "Cataloging packages",
OnSuccess: "Cataloged packages",
},
HideOnSuccess: false,
}
return bus.StartCatalogerTask(info, -1, "")
}
-67
View File
@@ -1,67 +0,0 @@
package cataloger
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
)
func Test_CatalogPanicHandling(t *testing.T) {
catalog, relationships, err := Catalog(
file.NewMockResolverForPaths(),
&linux.Release{},
1,
panickingCataloger{},
returningCataloger{},
)
require.Error(t, err)
require.Contains(t, err.Error(), "catalog_test.go")
require.Len(t, catalog.Sorted(), 2)
require.Len(t, relationships, 1)
}
type panickingCataloger struct{}
func (p panickingCataloger) Name() string {
return "panicking-cataloger"
}
func (p panickingCataloger) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
panic("something bad happened")
}
var _ pkg.Cataloger = (*panickingCataloger)(nil)
type returningCataloger struct{}
func (p returningCataloger) Name() string {
return "returning-cataloger"
}
func (p returningCataloger) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
pkg1 := pkg.Package{
Name: "package-1",
Version: "1.0",
}
pkg1.SetID()
pkg2 := pkg.Package{
Name: "package-2",
Version: "2.0",
}
pkg2.SetID()
return []pkg.Package{pkg1, pkg2}, []artifact.Relationship{
{
From: pkg1,
To: pkg2,
Type: artifact.DependencyOfRelationship,
},
}, nil
}
var _ pkg.Cataloger = (*returningCataloger)(nil)
-212
View File
@@ -1,212 +0,0 @@
/*
Package cataloger provides the ability to process files from a container image or file system and discover packages
(gems, wheels, jars, rpms, debs, etc). Specifically, this package contains both a catalog function to utilize all
catalogers defined in child packages as well as the interface definition to implement a cataloger.
*/
package cataloger
import (
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/alpine"
"github.com/anchore/syft/syft/pkg/cataloger/arch"
"github.com/anchore/syft/syft/pkg/cataloger/binary"
"github.com/anchore/syft/syft/pkg/cataloger/cpp"
"github.com/anchore/syft/syft/pkg/cataloger/dart"
"github.com/anchore/syft/syft/pkg/cataloger/debian"
"github.com/anchore/syft/syft/pkg/cataloger/dotnet"
"github.com/anchore/syft/syft/pkg/cataloger/elixir"
"github.com/anchore/syft/syft/pkg/cataloger/erlang"
"github.com/anchore/syft/syft/pkg/cataloger/gentoo"
"github.com/anchore/syft/syft/pkg/cataloger/githubactions"
"github.com/anchore/syft/syft/pkg/cataloger/golang"
"github.com/anchore/syft/syft/pkg/cataloger/haskell"
"github.com/anchore/syft/syft/pkg/cataloger/java"
"github.com/anchore/syft/syft/pkg/cataloger/javascript"
"github.com/anchore/syft/syft/pkg/cataloger/kernel"
"github.com/anchore/syft/syft/pkg/cataloger/nix"
"github.com/anchore/syft/syft/pkg/cataloger/php"
"github.com/anchore/syft/syft/pkg/cataloger/python"
"github.com/anchore/syft/syft/pkg/cataloger/r"
"github.com/anchore/syft/syft/pkg/cataloger/redhat"
"github.com/anchore/syft/syft/pkg/cataloger/ruby"
"github.com/anchore/syft/syft/pkg/cataloger/rust"
"github.com/anchore/syft/syft/pkg/cataloger/sbom"
"github.com/anchore/syft/syft/pkg/cataloger/swift"
)
const AllCatalogersPattern = "all"
// ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
func ImageCatalogers(cfg Config) []pkg.Cataloger {
return filterCatalogers([]pkg.Cataloger{
arch.NewDBCataloger(),
alpine.NewDBCataloger(),
binary.NewCataloger(cfg.Binary),
cpp.NewConanInfoCataloger(),
debian.NewDBCataloger(),
dotnet.NewDotnetPortableExecutableCataloger(),
golang.NewGoModuleBinaryCataloger(cfg.Golang),
java.NewArchiveCataloger(cfg.JavaConfig()),
java.NewNativeImageCataloger(),
javascript.NewPackageCataloger(),
nix.NewStoreCataloger(),
php.NewComposerInstalledCataloger(),
gentoo.NewPortageCataloger(),
python.NewInstalledPackageCataloger(),
r.NewPackageCataloger(),
redhat.NewDBCataloger(),
ruby.NewInstalledGemSpecCataloger(),
sbom.NewCataloger(),
}, cfg.Catalogers)
}
// DirectoryCatalogers returns a slice of locally implemented catalogers that are fit for detecting packages from index files (and select installations)
func DirectoryCatalogers(cfg Config) []pkg.Cataloger {
return filterCatalogers([]pkg.Cataloger{
arch.NewDBCataloger(),
alpine.NewDBCataloger(),
binary.NewCataloger(cfg.Binary),
cpp.NewConanCataloger(),
dart.NewPubspecLockCataloger(),
debian.NewDBCataloger(),
dotnet.NewDotnetDepsCataloger(),
dotnet.NewDotnetPortableExecutableCataloger(),
elixir.NewMixLockCataloger(),
erlang.NewRebarLockCataloger(),
githubactions.NewActionUsageCataloger(),
githubactions.NewWorkflowUsageCataloger(),
golang.NewGoModuleFileCataloger(cfg.Golang),
golang.NewGoModuleBinaryCataloger(cfg.Golang),
haskell.NewHackageCataloger(),
java.NewArchiveCataloger(cfg.JavaConfig()),
java.NewGradleLockfileCataloger(),
java.NewPomCataloger(cfg.JavaConfig()),
java.NewNativeImageCataloger(),
javascript.NewLockCataloger(cfg.Javascript),
nix.NewStoreCataloger(),
php.NewComposerLockCataloger(),
gentoo.NewPortageCataloger(),
python.NewPackageCataloger(cfg.Python),
python.NewInstalledPackageCataloger(),
redhat.NewArchiveCataloger(),
redhat.NewDBCataloger(),
ruby.NewGemFileLockCataloger(),
ruby.NewGemSpecCataloger(),
rust.NewCargoLockCataloger(),
sbom.NewCataloger(),
swift.NewCocoapodsCataloger(),
swift.NewSwiftPackageManagerCataloger(),
}, cfg.Catalogers)
}
// AllCatalogers returns all implemented catalogers
func AllCatalogers(cfg Config) []pkg.Cataloger {
return filterCatalogers([]pkg.Cataloger{
arch.NewDBCataloger(),
alpine.NewDBCataloger(),
binary.NewCataloger(cfg.Binary),
cpp.NewConanCataloger(),
dart.NewPubspecLockCataloger(),
debian.NewDBCataloger(),
dotnet.NewDotnetDepsCataloger(),
dotnet.NewDotnetPortableExecutableCataloger(),
elixir.NewMixLockCataloger(),
erlang.NewRebarLockCataloger(),
githubactions.NewActionUsageCataloger(),
githubactions.NewWorkflowUsageCataloger(),
golang.NewGoModuleFileCataloger(cfg.Golang),
golang.NewGoModuleBinaryCataloger(cfg.Golang),
haskell.NewHackageCataloger(),
java.NewArchiveCataloger(cfg.JavaConfig()),
java.NewGradleLockfileCataloger(),
java.NewPomCataloger(cfg.JavaConfig()),
java.NewNativeImageCataloger(),
javascript.NewLockCataloger(cfg.Javascript),
javascript.NewPackageCataloger(),
kernel.NewLinuxKernelCataloger(cfg.LinuxKernel),
nix.NewStoreCataloger(),
php.NewComposerInstalledCataloger(),
php.NewComposerLockCataloger(),
gentoo.NewPortageCataloger(),
python.NewPackageCataloger(cfg.Python),
python.NewInstalledPackageCataloger(),
r.NewPackageCataloger(),
redhat.NewArchiveCataloger(),
redhat.NewDBCataloger(),
ruby.NewGemFileLockCataloger(),
ruby.NewGemSpecCataloger(),
ruby.NewInstalledGemSpecCataloger(),
rust.NewAuditBinaryCataloger(),
rust.NewCargoLockCataloger(),
sbom.NewCataloger(),
swift.NewCocoapodsCataloger(),
swift.NewSwiftPackageManagerCataloger(),
}, cfg.Catalogers)
}
func RequestedAllCatalogers(cfg Config) bool {
for _, enableCatalogerPattern := range cfg.Catalogers {
if enableCatalogerPattern == AllCatalogersPattern {
return true
}
}
return false
}
func filterCatalogers(catalogers []pkg.Cataloger, enabledCatalogerPatterns []string) []pkg.Cataloger {
// if cataloger is not set, all applicable catalogers are enabled by default
if len(enabledCatalogerPatterns) == 0 {
return catalogers
}
for _, enableCatalogerPattern := range enabledCatalogerPatterns {
if enableCatalogerPattern == AllCatalogersPattern {
return catalogers
}
}
var keepCatalogers []pkg.Cataloger
for _, cataloger := range catalogers {
if contains(enabledCatalogerPatterns, cataloger.Name()) {
keepCatalogers = append(keepCatalogers, cataloger)
continue
}
log.Infof("skipping cataloger %q", cataloger.Name())
}
return keepCatalogers
}
func contains(enabledPartial []string, catalogerName string) bool {
catalogerName = strings.TrimSuffix(catalogerName, "-cataloger")
for _, partial := range enabledPartial {
partial = strings.TrimSuffix(partial, "-cataloger")
if partial == "" {
continue
}
if hasFullWord(partial, catalogerName) {
return true
}
}
return false
}
func hasFullWord(targetPhrase, candidate string) bool {
if targetPhrase == "cataloger" || targetPhrase == "" {
return false
}
start := strings.Index(candidate, targetPhrase)
if start == -1 {
return false
}
if start > 0 && candidate[start-1] != '-' {
return false
}
end := start + len(targetPhrase)
if end < len(candidate) && candidate[end] != '-' {
return false
}
return true
}
-342
View File
@@ -1,342 +0,0 @@
package cataloger
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
)
var _ pkg.Cataloger = (*dummy)(nil)
type dummy struct {
name string
}
func (d dummy) Name() string {
return d.name
}
func (d dummy) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
panic("not implemented")
}
func Test_filterCatalogers(t *testing.T) {
largeCatalogerList := []string{
"alpm-db-cataloger",
"apkdb-cataloger",
"binary-cataloger",
"conan-cataloger",
"dartlang-lock-cataloger",
"dpkg-db-cataloger",
"dotnet-deps-cataloger",
"elixir-mix-lock-cataloger",
"erlang-rebar-lock-cataloger",
"go-module-file-cataloger",
"go-module-binary-cataloger",
"haskell-cataloger",
"graalvm-native-image-cataloger",
"java-cataloger",
"java-pom-cataloger",
"javascript-package-cataloger",
"javascript-lock-cataloger",
"php-composer-installed-cataloger",
"php-composer-lock-cataloger",
"portage-cataloger",
"python-package-cataloger",
"python-installed-package-cataloger",
"rpm-db-cataloger",
"rpm-archive-cataloger",
"ruby-gemfile-cataloger",
"ruby-installed-gemspec-cataloger",
"rust-cargo-lock-cataloger",
"cargo-auditable-binary-cataloger",
"sbom-cataloger",
"cocoapods-cataloger",
}
tests := []struct {
name string
patterns []string
catalogers []string
want []string
}{
{
name: "no filtering",
patterns: nil,
catalogers: []string{
"ruby-installed-gemspec-cataloger",
"python-installed-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkg-db-cataloger",
"rpm-db-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
want: []string{
"ruby-installed-gemspec-cataloger",
"python-installed-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkg-db-cataloger",
"rpm-db-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
},
{
name: "exact name match",
patterns: []string{
"rpm-db-cataloger",
"javascript-package-cataloger",
},
catalogers: []string{
"ruby-installed-gemspec-cataloger",
"python-installed-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkg-db-cataloger",
"rpm-db-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
want: []string{
"javascript-package-cataloger",
"rpm-db-cataloger",
},
},
{
name: "partial name match",
patterns: []string{
"ruby",
"installed",
},
catalogers: []string{
"ruby-installed-gemspec-cataloger",
"ruby-gemfile-cataloger",
"python-installed-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkg-db-cataloger",
"rpm-db-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
want: []string{
"php-composer-installed-cataloger",
"python-installed-package-cataloger",
"ruby-installed-gemspec-cataloger",
"ruby-gemfile-cataloger",
},
},
{
name: "ignore 'cataloger' keyword",
patterns: []string{
"cataloger",
},
catalogers: []string{
"ruby-installed-gemspec-cataloger",
"ruby-gemfile-cataloger",
"python-installed-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkg-db-cataloger",
"rpm-db-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
want: []string{},
},
{
name: "only some patterns match",
patterns: []string{
"cataloger",
"go-module",
},
catalogers: []string{
"ruby-installed-gemspec-cataloger",
"ruby-gemfile-cataloger",
"python-installed-package-cataloger",
"php-composer-installed-cataloger",
"javascript-package-cataloger",
"dpkg-db-cataloger",
"rpm-db-cataloger",
"java-cataloger",
"apkdb-cataloger",
"go-module-binary-cataloger",
},
want: []string{
"go-module-binary-cataloger",
},
},
{
name: "don't cross match ecosystems with matching prefix",
patterns: []string{
"java-cataloger",
},
catalogers: []string{
"javascript-package-cataloger",
"java-cataloger",
},
want: []string{
"java-cataloger",
},
},
{
name: "don't cross match ecosystems with short, common name",
patterns: []string{
"go",
},
catalogers: largeCatalogerList,
want: []string{
"go-module-file-cataloger",
"go-module-binary-cataloger",
//"rust-cargo-lock-cataloger", // with naive "contains" matching
//"cargo-auditable-binary-cataloger", // with naive "contains" matching
},
},
{
name: "ignore partial matches",
patterns: []string{
"mod",
},
catalogers: largeCatalogerList,
want: []string{
// important! these are NOT found
//"go-module-file-cataloger",
//"go-module-binary-cataloger",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var catalogers []pkg.Cataloger
for _, n := range tt.catalogers {
catalogers = append(catalogers, dummy{name: n})
}
got := filterCatalogers(catalogers, tt.patterns)
var gotNames []string
for _, g := range got {
gotNames = append(gotNames, g.Name())
}
assert.ElementsMatch(t, tt.want, gotNames)
})
}
}
func Test_contains(t *testing.T) {
tests := []struct {
name string
enabledCatalogers []string
catalogerName string
want bool
}{
{
name: "keep exact match",
enabledCatalogers: []string{
"php-composer-installed-cataloger",
},
catalogerName: "php-composer-installed-cataloger",
want: true,
},
{
name: "match substring",
enabledCatalogers: []string{
"python",
},
catalogerName: "python-installed-package-cataloger",
want: true,
},
{
name: "dont match on 'cataloger'",
enabledCatalogers: []string{
"cataloger",
},
catalogerName: "python-installed-package-cataloger",
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, contains(tt.enabledCatalogers, tt.catalogerName))
})
}
}
func Test_hasFullWord(t *testing.T) {
tests := []struct {
name string
targetPhrase string
candidate string
want bool
}{
{
name: "exact match",
targetPhrase: "php-composer-installed-cataloger",
candidate: "php-composer-installed-cataloger",
want: true,
},
{
name: "partial, full word match",
targetPhrase: "composer",
candidate: "php-composer-installed-cataloger",
want: true,
},
{
name: "partial, full, multi-word match",
targetPhrase: "php-composer",
candidate: "php-composer-installed-cataloger",
want: true,
},
{
name: "prefix match",
targetPhrase: "php",
candidate: "php-composer-installed-cataloger",
want: true,
},
{
name: "postfix match with -cataloger suffix",
targetPhrase: "installed",
candidate: "php-composer-installed-cataloger",
want: true,
},
{
name: "postfix match",
targetPhrase: "installed",
candidate: "php-composer-installed",
want: true,
},
{
name: "ignore cataloger keyword",
targetPhrase: "cataloger",
candidate: "php-composer-installed-cataloger",
want: false,
},
{
name: "ignore partial match",
targetPhrase: "hp",
candidate: "php-composer-installed-cataloger",
want: false,
},
{
name: "ignore empty string",
targetPhrase: "",
candidate: "php-composer-installed-cataloger",
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, hasFullWord(tt.targetPhrase, tt.candidate), "hasFullWord(%v, %v)", tt.targetPhrase, tt.candidate)
})
}
}
-53
View File
@@ -1,53 +0,0 @@
package cataloger
import (
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/pkg/cataloger/binary"
"github.com/anchore/syft/syft/pkg/cataloger/golang"
"github.com/anchore/syft/syft/pkg/cataloger/java"
"github.com/anchore/syft/syft/pkg/cataloger/javascript"
"github.com/anchore/syft/syft/pkg/cataloger/kernel"
"github.com/anchore/syft/syft/pkg/cataloger/python"
)
// TODO: these field naming vs helper function naming schemes are inconsistent.
type Config struct {
Search SearchConfig
Golang golang.CatalogerConfig
LinuxKernel kernel.LinuxKernelCatalogerConfig
Python python.CatalogerConfig
Java java.ArchiveCatalogerConfig
Javascript javascript.CatalogerConfig
Binary binary.CatalogerConfig
Catalogers []string
Parallelism int
ExcludeBinaryOverlapByOwnership bool
}
func DefaultConfig() Config {
return Config{
Search: DefaultSearchConfig(),
Parallelism: 1,
LinuxKernel: kernel.DefaultLinuxCatalogerConfig(),
Python: python.DefaultCatalogerConfig(),
Java: java.DefaultArchiveCatalogerConfig(),
Javascript: javascript.DefaultCatalogerConfig(),
Binary: binary.DefaultCatalogerConfig(),
ExcludeBinaryOverlapByOwnership: true,
}
}
// JavaConfig merges relevant config values from Config to return a java.Config struct.
// Values like IncludeUnindexedArchives and IncludeIndexedArchives are used across catalogers
// and are not specific to Java requiring this merge.
func (c Config) JavaConfig() java.ArchiveCatalogerConfig {
return java.ArchiveCatalogerConfig{
ArchiveSearchConfig: cataloging.ArchiveSearchConfig{
IncludeUnindexedArchives: c.Search.IncludeUnindexedArchives,
IncludeIndexedArchives: c.Search.IncludeIndexedArchives,
},
UseNetwork: c.Java.UseNetwork,
MavenBaseURL: c.Java.MavenBaseURL,
MaxParentRecursiveDepth: c.Java.MaxParentRecursiveDepth,
}
}
@@ -6,6 +6,7 @@ import (
"sort"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/relationship"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -122,7 +123,7 @@ func parseDotnetDeps(_ file.Resolver, _ *generic.Environment, reader file.Locati
// sort the relationships for deterministic output
// TODO: ideally this would be replaced with artifact.SortRelationships when one exists and is type agnostic.
// this will only consider package-to-package relationships.
pkg.SortRelationships(relationships)
relationship.Sort(relationships)
return pkgs, relationships, nil
}
+9 -4
View File
@@ -18,13 +18,18 @@ import (
var versionCandidateGroups = regexp.MustCompile(`(?P<version>\d+(\.\d+)?(\.\d+)?)(?P<candidate>\w*)`)
const (
modFileCatalogerName = "go-module-file-cataloger"
binaryCatalogerName = "go-module-binary-cataloger"
)
// NewGoModuleFileCataloger returns a new cataloger object that searches within go.mod files.
func NewGoModuleFileCataloger(opts CatalogerConfig) pkg.Cataloger {
c := goModCataloger{
licenses: newGoLicenses(opts),
licenses: newGoLicenses(modFileCatalogerName, opts),
}
return &progressingCataloger{
cataloger: generic.NewCataloger("go-module-file-cataloger").
cataloger: generic.NewCataloger(modFileCatalogerName).
WithParserByGlobs(c.parseGoModFile, "**/go.mod"),
}
}
@@ -32,10 +37,10 @@ func NewGoModuleFileCataloger(opts CatalogerConfig) pkg.Cataloger {
// NewGoModuleBinaryCataloger returns a new cataloger object that searches within binaries built by the go compiler.
func NewGoModuleBinaryCataloger(opts CatalogerConfig) pkg.Cataloger {
c := goBinaryCataloger{
licenses: newGoLicenses(opts),
licenses: newGoLicenses(binaryCatalogerName, opts),
}
return &progressingCataloger{
cataloger: generic.NewCataloger("go-module-binary-cataloger").
cataloger: generic.NewCataloger(binaryCatalogerName).
WithParserByMimeTypes(c.parseGoBinary, internal.ExecutableMIMETypeSet.List()...),
}
}
+4 -1
View File
@@ -30,13 +30,15 @@ import (
)
type goLicenses struct {
catalogerName string
opts CatalogerConfig
localModCacheResolver file.WritableResolver
lowerLicenseFileNames *strset.Set
}
func newGoLicenses(opts CatalogerConfig) goLicenses {
func newGoLicenses(catalogerName string, opts CatalogerConfig) goLicenses {
return goLicenses{
catalogerName: catalogerName,
opts: opts,
localModCacheResolver: modCacheResolver(opts.LocalModCacheDir),
lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...),
@@ -125,6 +127,7 @@ func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([]
OnSuccess: "Downloaded go mod",
},
HideOnSuccess: true,
ParentID: c.catalogerName,
}, -1, "")
fsys, err := getModule(prog, proxies, moduleName, moduleVersion)
+9 -5
View File
@@ -67,6 +67,7 @@ func Test_LocalLicenseSearch(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
l := newGoLicenses(
"",
CatalogerConfig{
SearchLocalModCacheLicenses: true,
LocalModCacheDir: path.Join(wd, "test-fixtures", "licenses", "pkg", "mod"),
@@ -154,11 +155,14 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
l := newGoLicenses(CatalogerConfig{
SearchRemoteLicenses: true,
Proxies: []string{server.URL},
LocalModCacheDir: modDir,
})
l := newGoLicenses(
"",
CatalogerConfig{
SearchRemoteLicenses: true,
Proxies: []string{server.URL},
LocalModCacheDir: modDir,
},
)
licenses, err := l.getLicenses(fileresolver.Empty{}, test.name, test.version)
require.NoError(t, err)
@@ -14,6 +14,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/internal/relationship"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
@@ -346,8 +347,8 @@ func (p *CatalogTester) assertPkgs(t *testing.T, pkgs []pkg.Package, relationshi
opts = append(opts, cmp.Reporter(&r))
// order should not matter
pkg.SortRelationships(p.expectedRelationships)
pkg.SortRelationships(relationships)
relationship.Sort(p.expectedRelationships)
relationship.Sort(relationships)
if diff := cmp.Diff(p.expectedRelationships, relationships, opts...); diff != "" {
t.Log("Specific Differences:\n" + r.String())
-51
View File
@@ -1,51 +0,0 @@
package cataloger
import (
"slices"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
)
var (
osCatalogerTypes = []pkg.Type{
pkg.AlpmPkg,
pkg.ApkPkg,
pkg.DebPkg,
pkg.NixPkg,
pkg.PortagePkg,
pkg.RpmPkg,
}
binaryCatalogerTypes = []pkg.Type{
pkg.BinaryPkg,
}
)
// ExcludeBinaryByFileOwnershipOverlap will remove packages from a collection given the following properties are true
// 1) the relationship between packages is OwnershipByFileOverlap
// 2) the parent is an "os" package
// 3) the child is a synthetic package generated by the binary cataloger
// 4) the package names are identical
// This was implemented as a way to help resolve: https://github.com/anchore/syft/issues/931
func ExcludeBinaryByFileOwnershipOverlap(r artifact.Relationship, c *pkg.Collection) bool {
if artifact.OwnershipByFileOverlapRelationship != r.Type {
return false
}
parent := c.Package(r.From.ID())
if parent == nil {
return false
}
parentInExclusion := slices.Contains(osCatalogerTypes, parent.Type)
if !parentInExclusion {
return false
}
child := c.Package(r.To.ID())
if child == nil {
return false
}
return slices.Contains(binaryCatalogerTypes, child.Type)
}
@@ -1,76 +0,0 @@
package cataloger
import (
"testing"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
)
func TestExclude(t *testing.T) {
packageA := pkg.Package{Name: "package-a", Type: pkg.ApkPkg}
packageB := pkg.Package{Name: "package-a", Type: pkg.PythonPkg}
packageC := pkg.Package{Name: "package-a", Type: pkg.BinaryPkg}
packageD := pkg.Package{Name: "package-d", Type: pkg.BinaryPkg}
for _, p := range []*pkg.Package{&packageA, &packageB, &packageC, &packageD} {
p := p
p.SetID()
}
tests := []struct {
name string
relationship artifact.Relationship
packages *pkg.Collection
shouldExclude bool
}{
{
name: "no exclusions from os -> python",
relationship: artifact.Relationship{
Type: artifact.OwnershipByFileOverlapRelationship,
From: packageA,
To: packageB,
},
packages: pkg.NewCollection(packageA, packageB),
shouldExclude: false,
},
{
name: "exclusions from os -> binary",
relationship: artifact.Relationship{
Type: artifact.OwnershipByFileOverlapRelationship,
From: packageA,
To: packageC,
},
packages: pkg.NewCollection(packageA, packageC),
shouldExclude: true,
},
{
name: "no exclusions from python -> binary",
relationship: artifact.Relationship{
Type: artifact.OwnershipByFileOverlapRelationship,
From: packageB,
To: packageC,
},
packages: pkg.NewCollection(packageB, packageC),
shouldExclude: false,
},
{
name: "no exclusions for different package names",
relationship: artifact.Relationship{
Type: artifact.OwnershipByFileOverlapRelationship,
From: packageA,
To: packageD,
},
packages: pkg.NewCollection(packageA, packageD),
shouldExclude: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if !ExcludeBinaryByFileOwnershipOverlap(test.relationship, test.packages) && test.shouldExclude {
t.Errorf("expected to exclude relationship %+v", test.relationship)
}
})
}
}
-19
View File
@@ -1,19 +0,0 @@
package cataloger
import (
"github.com/anchore/syft/syft/source"
)
type SearchConfig struct {
IncludeIndexedArchives bool
IncludeUnindexedArchives bool
Scope source.Scope
}
func DefaultSearchConfig() SearchConfig {
return SearchConfig{
IncludeIndexedArchives: true,
IncludeUnindexedArchives: false,
Scope: source.SquashedScope,
}
}
-47
View File
@@ -1,47 +0,0 @@
package pkg
import (
"sort"
"github.com/anchore/syft/syft/artifact"
)
func NewRelationships(catalog *Collection) []artifact.Relationship {
rels := RelationshipsByFileOwnership(catalog)
rels = append(rels, RelationshipsEvidentBy(catalog)...)
return rels
}
// SortRelationships takes a set of package-to-package relationships and sorts them in a stable order by name and version.
// Note: this does not consider package-to-other, other-to-package, or other-to-other relationships.
// TODO: ideally this should be replaced with a more type-agnostic sort function that resides in the artifact package.
func SortRelationships(rels []artifact.Relationship) {
sort.SliceStable(rels, func(i, j int) bool {
return relationshipLess(rels[i], rels[j])
})
}
func relationshipLess(i, j artifact.Relationship) bool {
iFrom, ok1 := i.From.(Package)
iTo, ok2 := i.To.(Package)
jFrom, ok3 := j.From.(Package)
jTo, ok4 := j.To.(Package)
if !(ok1 && ok2 && ok3 && ok4) {
return false
}
if iFrom.Name != jFrom.Name {
return iFrom.Name < jFrom.Name
}
if iFrom.Version != jFrom.Version {
return iFrom.Version < jFrom.Version
}
if iTo.Name != jTo.Name {
return iTo.Name < jTo.Name
}
if iTo.Version != jTo.Version {
return iTo.Version < jTo.Version
}
return i.Type < j.Type
}
-115
View File
@@ -1,115 +0,0 @@
package pkg
import (
"sort"
"github.com/bmatcuk/doublestar/v4"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
)
// AltRpmDBGlob allows db matches against new locations introduced in fedora:{36,37}
// See https://github.com/anchore/syft/issues/1077 for larger context
const AltRpmDBGlob = "**/rpm/{Packages,Packages.db,rpmdb.sqlite}"
var globsForbiddenFromBeingOwned = []string{
// any OS DBs should automatically be ignored to prevent cyclic issues (e.g. the "rpm" RPM owns the path to the
// RPM DB, so if not ignored that package would own all other packages on the system).
ApkDBGlob,
DpkgDBGlob,
RpmDBGlob,
AltRpmDBGlob,
// DEB packages share common copyright info between, this does not mean that sharing these paths implies ownership.
"/usr/share/doc/**/copyright",
}
type ownershipByFilesMetadata struct {
Files []string `json:"files"`
}
// RelationshipsByFileOwnership creates a package-to-package relationship based on discovering which packages have
// evidence locations that overlap with ownership claim from another package's package manager metadata.
func RelationshipsByFileOwnership(catalog *Collection) []artifact.Relationship {
var relationships = findOwnershipByFilesRelationships(catalog)
var edges []artifact.Relationship
for parentID, children := range relationships {
for childID, files := range children {
fs := files.List()
sort.Strings(fs)
edges = append(edges, artifact.Relationship{
From: catalog.byID[parentID],
To: catalog.byID[childID],
Type: artifact.OwnershipByFileOverlapRelationship,
Data: ownershipByFilesMetadata{
Files: fs,
},
})
}
}
return edges
}
// findOwnershipByFilesRelationships find overlaps in file ownership with a file that defines another package. Specifically, a .Location.Path of
// a package is found to be owned by another (from the owner's .Metadata.Files[]).
func findOwnershipByFilesRelationships(catalog *Collection) map[artifact.ID]map[artifact.ID]*strset.Set {
var relationships = make(map[artifact.ID]map[artifact.ID]*strset.Set)
if catalog == nil {
return relationships
}
for _, candidateOwnerPkg := range catalog.Sorted() {
id := candidateOwnerPkg.ID()
if candidateOwnerPkg.Metadata == nil {
continue
}
// check to see if this is a file owner
pkgFileOwner, ok := candidateOwnerPkg.Metadata.(FileOwner)
if !ok {
continue
}
for _, ownedFilePath := range pkgFileOwner.OwnedFiles() {
if matchesAny(ownedFilePath, globsForbiddenFromBeingOwned) {
// we skip over known exceptions to file ownership, such as the RPM package owning
// the RPM DB path, otherwise the RPM package would "own" all RPMs, which is not intended
continue
}
// look for package(s) in the catalog that may be owned by this package and mark the relationship
for _, subPackage := range catalog.PackagesByPath(ownedFilePath) {
subID := subPackage.ID()
if subID == id {
continue
}
if _, exists := relationships[id]; !exists {
relationships[id] = make(map[artifact.ID]*strset.Set)
}
if _, exists := relationships[id][subID]; !exists {
relationships[id][subID] = strset.New()
}
relationships[id][subID].Add(ownedFilePath)
}
}
}
return relationships
}
func matchesAny(s string, globs []string) bool {
for _, g := range globs {
matches, err := doublestar.Match(g, s)
if err != nil {
log.Errorf("failed to match glob=%q : %+v", g, err)
}
if matches {
return true
}
}
return false
}
@@ -1,152 +0,0 @@
package pkg
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
)
func TestOwnershipByFilesRelationship(t *testing.T) {
tests := []struct {
name string
setup func(t testing.TB) ([]Package, []artifact.Relationship)
}{
{
name: "owns-by-real-path",
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
parent := Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/a/path", "/another/path"),
file.NewVirtualLocation("/b/path", "/bee/path"),
),
Type: RpmPkg,
Metadata: RpmDBEntry{
Files: []RpmFileRecord{
{Path: "/owning/path/1"},
{Path: "/owning/path/2"},
{Path: "/d/path"},
},
},
}
parent.SetID()
child := Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/c/path", "/another/path"),
file.NewVirtualLocation("/d/path", "/another/path"),
),
Type: NpmPkg,
}
child.SetID()
relationship := artifact.Relationship{
From: parent,
To: child,
Type: artifact.OwnershipByFileOverlapRelationship,
Data: ownershipByFilesMetadata{
Files: []string{
"/d/path",
},
},
}
return []Package{parent, child}, []artifact.Relationship{relationship}
},
},
{
name: "owns-by-virtual-path",
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
parent := Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/a/path", "/some/other/path"),
file.NewVirtualLocation("/b/path", "/bee/path"),
),
Type: RpmPkg,
Metadata: RpmDBEntry{
Files: []RpmFileRecord{
{Path: "/owning/path/1"},
{Path: "/owning/path/2"},
{Path: "/another/path"},
},
},
}
parent.SetID()
child := Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/c/path", "/another/path"),
file.NewLocation("/d/path"),
),
Type: NpmPkg,
}
child.SetID()
relationship := artifact.Relationship{
From: parent,
To: child,
Type: artifact.OwnershipByFileOverlapRelationship,
Data: ownershipByFilesMetadata{
Files: []string{
"/another/path",
},
},
}
return []Package{parent, child}, []artifact.Relationship{relationship}
},
},
{
name: "ignore-empty-path",
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
parent := Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/a/path", "/some/other/path"),
file.NewVirtualLocation("/b/path", "/bee/path"),
),
Type: RpmPkg,
Metadata: RpmDBEntry{
Files: []RpmFileRecord{
{Path: "/owning/path/1"},
{Path: "/owning/path/2"},
{Path: ""},
},
},
}
parent.SetID()
child := Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/c/path", "/another/path"),
file.NewLocation("/d/path"),
),
Type: NpmPkg,
}
child.SetID()
return []Package{parent, child}, nil
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pkgs, expectedRelations := test.setup(t)
c := NewCollection(pkgs...)
relationships := RelationshipsByFileOwnership(c)
assert.Len(t, relationships, len(expectedRelations))
for idx, expectedRelationship := range expectedRelations {
actualRelationship := relationships[idx]
assert.Equal(t, expectedRelationship.From.ID(), actualRelationship.From.ID())
assert.Equal(t, expectedRelationship.To.ID(), actualRelationship.To.ID())
assert.Equal(t, expectedRelationship.Type, actualRelationship.Type)
assert.Equal(t, expectedRelationship.Data, actualRelationship.Data)
}
})
}
}
-25
View File
@@ -1,25 +0,0 @@
package pkg
import (
"github.com/anchore/syft/syft/artifact"
)
func RelationshipsEvidentBy(catalog *Collection) []artifact.Relationship {
var edges []artifact.Relationship
for _, p := range catalog.Sorted() {
for _, l := range p.Locations.ToSlice() {
if v, exists := l.Annotations[EvidenceAnnotationKey]; !exists || v != PrimaryEvidenceAnnotation {
// skip non-primary evidence from being expressed as a relationship.
// note: this may be configurable in the future.
continue
}
edges = append(edges, artifact.Relationship{
From: p,
To: l.Coordinates,
Type: artifact.EvidentByRelationship,
})
}
}
return edges
}
-87
View File
@@ -1,87 +0,0 @@
package pkg
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
)
func TestRelationshipsEvidentBy(t *testing.T) {
c := NewCollection()
coordA := file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
}
coordC := file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
}
coordD := file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
}
pkgA := Package{
Locations: file.NewLocationSet(
// added!
file.NewLocationFromCoordinates(coordA).WithAnnotation(EvidenceAnnotationKey, PrimaryEvidenceAnnotation),
// ignored...
file.NewLocationFromCoordinates(coordC).WithAnnotation(EvidenceAnnotationKey, SupportingEvidenceAnnotation),
file.NewLocationFromCoordinates(coordD),
),
}
pkgA.SetID()
c.Add(pkgA)
coordB := file.Coordinates{
RealPath: "/somewhere-else/real",
FileSystemID: "def",
}
pkgB := Package{
Locations: file.NewLocationSet(
// added!
file.NewLocationFromCoordinates(coordB).WithAnnotation(EvidenceAnnotationKey, PrimaryEvidenceAnnotation),
),
}
pkgB.SetID()
c.Add(pkgB)
tests := []struct {
name string
catalog *Collection
want []artifact.Relationship
}{
{
name: "go case",
catalog: c,
want: []artifact.Relationship{
{
From: pkgB,
To: coordB,
Type: artifact.EvidentByRelationship,
},
{
From: pkgA,
To: coordA,
Type: artifact.EvidentByRelationship,
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual := RelationshipsEvidentBy(tt.catalog)
require.Len(t, actual, len(tt.want))
for i := range actual {
assert.Equal(t, tt.want[i].From.ID(), actual[i].From.ID(), "from mismatch at index %d", i)
assert.Equal(t, tt.want[i].To.ID(), actual[i].To.ID(), "to mismatch at index %d", i)
assert.Equal(t, tt.want[i].Type, actual[i].Type, "type mismatch at index %d", i)
}
})
}
}