mirror of
https://github.com/anchore/syft.git
synced 2026-11-03 16:06:36 +01:00
Replace core SBOM-creation API with builder pattern (#1383)
* remove existing cataloging API Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add file cataloging config Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add package cataloging config Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add configs for cross-cutting concerns Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename CLI option configs to not require import aliases later Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update all nested structs for the Catalog struct Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update Catalog cli options - add new cataloger selection options (selection and default) - remove the excludeBinaryOverlapByOwnership - deprecate "catalogers" flag - add new javascript configuration Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * migrate relationship capabilities to separate internal package Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * refactor golang cataloger to use configuration options when creating packages Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * create internal object to facilitate reading from and writing to an SBOM Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * create a command-like object (task) to facilitate partial SBOM creation Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add cataloger selection capability - be able to parse string expressions into a set of resolved actions against sets - be able to use expressions to select/add/remove tasks to/from the final set of tasks to run Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add package, file, and environment related tasks Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update existing file catalogers to use nested UI elements Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add CreateSBOMConfig that drives the SBOM creation process Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * capture SBOM creation info as a struct Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add CreateSBOM() function Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix tests Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update docs with SBOM selection help + breaking changes Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix multiple override default inputs Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix deprecation flag printing to stdout Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * refactor cataloger selection description to separate object Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address review comments Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * keep expression errors and show specific suggestions only Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address additional review feedback Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address more review comments Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * addressed additional PR review feedback Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix file selection references Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * remove guess language data generation option Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add tests for coordinatesForSelection Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename relationship attributes Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add descriptions to relationships config fields Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * improve documentation around configuration options Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add explicit errors around legacy config entries Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
@@ -11,3 +11,13 @@ func DefaultArchiveSearchConfig() ArchiveSearchConfig {
|
||||
IncludeUnindexedArchives: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (c ArchiveSearchConfig) WithIncludeIndexedArchives(include bool) ArchiveSearchConfig {
|
||||
c.IncludeIndexedArchives = include
|
||||
return c
|
||||
}
|
||||
|
||||
func (c ArchiveSearchConfig) WithIncludeUnindexedArchives(include bool) ArchiveSearchConfig {
|
||||
c.IncludeUnindexedArchives = include
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package cataloging
|
||||
|
||||
type DataGenerationConfig struct {
|
||||
GenerateCPEs bool `yaml:"generate-cpes" json:"generate-cpes" mapstructure:"generate-cpes"`
|
||||
}
|
||||
|
||||
func DefaultDataGenerationConfig() DataGenerationConfig {
|
||||
return DataGenerationConfig{
|
||||
GenerateCPEs: true,
|
||||
}
|
||||
}
|
||||
|
||||
func (c DataGenerationConfig) WithGenerateCPEs(generate bool) DataGenerationConfig {
|
||||
c.GenerateCPEs = generate
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package filecataloging
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/file/cataloger/filecontent"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Selection file.Selection `yaml:"selection" json:"selection" mapstructure:"selection"`
|
||||
Hashers []crypto.Hash `yaml:"hashers" json:"hashers" mapstructure:"hashers"`
|
||||
Content filecontent.Config `yaml:"content" json:"content" mapstructure:"content"`
|
||||
}
|
||||
|
||||
type configMarshaledForm struct {
|
||||
Selection file.Selection `yaml:"selection" json:"selection" mapstructure:"selection"`
|
||||
Hashers []string `yaml:"hashers" json:"hashers" mapstructure:"hashers"`
|
||||
Content filecontent.Config `yaml:"content" json:"content" mapstructure:"content"`
|
||||
}
|
||||
|
||||
func DefaultConfig() Config {
|
||||
hashers, err := intFile.Hashers("sha256")
|
||||
if err != nil {
|
||||
log.WithFields("error", err).Warn("unable to create file hashers")
|
||||
}
|
||||
return Config{
|
||||
Selection: file.FilesOwnedByPackageSelection,
|
||||
Hashers: hashers,
|
||||
Content: filecontent.DefaultConfig(),
|
||||
}
|
||||
}
|
||||
|
||||
func (cfg Config) MarshalJSON() ([]byte, error) {
|
||||
marshaled := configMarshaledForm{
|
||||
Selection: cfg.Selection,
|
||||
Hashers: hashersToString(cfg.Hashers),
|
||||
}
|
||||
return json.Marshal(marshaled)
|
||||
}
|
||||
|
||||
func hashersToString(hashers []crypto.Hash) []string {
|
||||
var result []string
|
||||
for _, h := range hashers {
|
||||
result = append(result, strings.ToLower(h.String()))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (cfg *Config) UnmarshalJSON(data []byte) error {
|
||||
var marshaled configMarshaledForm
|
||||
if err := json.Unmarshal(data, &marshaled); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hashers, err := intFile.Hashers(marshaled.Hashers...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to parse configured hashers: %w", err)
|
||||
}
|
||||
cfg.Selection = marshaled.Selection
|
||||
cfg.Hashers = hashers
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cfg Config) WithSelection(selection file.Selection) Config {
|
||||
cfg.Selection = selection
|
||||
return cfg
|
||||
}
|
||||
|
||||
func (cfg Config) WithHashers(hashers []crypto.Hash) Config {
|
||||
cfg.Hashers = hashers
|
||||
return cfg
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package filecataloging
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
func TestConfig_MarshalJSON(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg Config
|
||||
want []byte
|
||||
wantErr require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "converts hashers to strings",
|
||||
cfg: Config{
|
||||
Selection: file.FilesOwnedByPackageSelection,
|
||||
Hashers: []crypto.Hash{crypto.SHA256},
|
||||
},
|
||||
want: []byte(`{"selection":"owned-by-package","hashers":["sha-256"],"content":{"globs":null,"skip-files-above-size":0}}`),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.wantErr == nil {
|
||||
tt.wantErr = require.NoError
|
||||
}
|
||||
got, err := tt.cfg.MarshalJSON()
|
||||
tt.wantErr(t, err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if d := cmp.Diff(got, tt.want); d != "" {
|
||||
t.Errorf("MarshalJSON() mismatch (-want +got):\n%s", d)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_UnmarshalJSON(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
want Config
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "converts strings to hashers",
|
||||
data: []byte(`{"selection":"owned-by-package","hashers":["sha-256"]}`),
|
||||
want: Config{
|
||||
Selection: file.FilesOwnedByPackageSelection,
|
||||
Hashers: []crypto.Hash{crypto.SHA256},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := Config{}
|
||||
if err := cfg.UnmarshalJSON(tt.data); (err != nil) != tt.wantErr {
|
||||
t.Errorf("UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.want, cfg)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package pkgcataloging
|
||||
|
||||
import "github.com/anchore/syft/syft/pkg"
|
||||
|
||||
type CatalogerReference struct {
|
||||
Cataloger pkg.Cataloger
|
||||
AlwaysEnabled bool
|
||||
Tags []string
|
||||
}
|
||||
|
||||
func NewCatalogerReference(cataloger pkg.Cataloger, tags []string) CatalogerReference {
|
||||
return CatalogerReference{
|
||||
Cataloger: cataloger,
|
||||
Tags: tags,
|
||||
}
|
||||
}
|
||||
|
||||
func NewAlwaysEnabledCatalogerReference(cataloger pkg.Cataloger) CatalogerReference {
|
||||
return CatalogerReference{
|
||||
Cataloger: cataloger,
|
||||
AlwaysEnabled: true,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package pkgcataloging
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/binary"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/golang"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/javascript"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/kernel"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/python"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Binary binary.CatalogerConfig `yaml:"binary" json:"binary" mapstructure:"binary"`
|
||||
Golang golang.CatalogerConfig `yaml:"golang" json:"golang" mapstructure:"golang"`
|
||||
JavaArchive java.ArchiveCatalogerConfig `yaml:"java-archive" json:"java-archive" mapstructure:"java-archive"`
|
||||
JavaScript javascript.CatalogerConfig `yaml:"javascript" json:"javascript" mapstructure:"javascript"`
|
||||
LinuxKernel kernel.LinuxKernelCatalogerConfig `yaml:"linux-kernel" json:"linux-kernel" mapstructure:"linux-kernel"`
|
||||
Python python.CatalogerConfig `yaml:"python" json:"python" mapstructure:"python"`
|
||||
}
|
||||
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
Golang: golang.DefaultCatalogerConfig(),
|
||||
LinuxKernel: kernel.DefaultLinuxCatalogerConfig(),
|
||||
Python: python.DefaultCatalogerConfig(),
|
||||
JavaArchive: java.DefaultArchiveCatalogerConfig(),
|
||||
}
|
||||
}
|
||||
|
||||
func (c Config) WithBinaryConfig(cfg binary.CatalogerConfig) Config {
|
||||
c.Binary = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
func (c Config) WithGolangConfig(cfg golang.CatalogerConfig) Config {
|
||||
c.Golang = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
func (c Config) WithJavascriptConfig(cfg javascript.CatalogerConfig) Config {
|
||||
c.JavaScript = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
func (c Config) WithLinuxKernelConfig(cfg kernel.LinuxKernelCatalogerConfig) Config {
|
||||
c.LinuxKernel = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
func (c Config) WithPythonConfig(cfg python.CatalogerConfig) Config {
|
||||
c.Python = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
func (c Config) WithJavaArchiveConfig(cfg java.ArchiveCatalogerConfig) Config {
|
||||
c.JavaArchive = cfg
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package pkgcataloging
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
type SelectionRequest struct {
|
||||
DefaultNamesOrTags []string `json:"default,omitempty"`
|
||||
SubSelectTags []string `json:"selection,omitempty"`
|
||||
AddNames []string `json:"addition,omitempty"`
|
||||
RemoveNamesOrTags []string `json:"removal,omitempty"`
|
||||
}
|
||||
|
||||
func NewSelectionRequest() SelectionRequest {
|
||||
return SelectionRequest{}
|
||||
}
|
||||
|
||||
func (s SelectionRequest) WithExpression(expressions ...string) SelectionRequest {
|
||||
expressions = cleanSelection(expressions)
|
||||
for _, expr := range expressions {
|
||||
switch {
|
||||
case strings.HasPrefix(expr, "+"):
|
||||
s = s.WithAdditions(strings.TrimPrefix(expr, "+"))
|
||||
case strings.HasPrefix(expr, "-"):
|
||||
s = s.WithRemovals(strings.TrimPrefix(expr, "-"))
|
||||
default:
|
||||
s = s.WithSubSelections(expr)
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s SelectionRequest) WithDefaults(nameOrTags ...string) SelectionRequest {
|
||||
s.DefaultNamesOrTags = append(s.DefaultNamesOrTags, nameOrTags...)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s SelectionRequest) WithSubSelections(tags ...string) SelectionRequest {
|
||||
s.SubSelectTags = append(s.SubSelectTags, tags...)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s SelectionRequest) WithAdditions(names ...string) SelectionRequest {
|
||||
s.AddNames = append(s.AddNames, names...)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s SelectionRequest) WithRemovals(nameOrTags ...string) SelectionRequest {
|
||||
s.RemoveNamesOrTags = append(s.RemoveNamesOrTags, nameOrTags...)
|
||||
return s
|
||||
}
|
||||
|
||||
func cleanSelection(tags []string) []string {
|
||||
var cleaned []string
|
||||
for _, tag := range tags {
|
||||
for _, t := range strings.Split(tag, ",") {
|
||||
t = strings.TrimSpace(t)
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
cleaned = append(cleaned, t)
|
||||
}
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package pkgcataloging
|
||||
|
||||
const (
|
||||
// InstalledTag is to identify packages found to be positively installed.
|
||||
InstalledTag = "installed"
|
||||
|
||||
// DeclaredTag is to identify packages described but not necessarily installed.
|
||||
DeclaredTag = "declared"
|
||||
|
||||
// ImageTag indicates the cataloger should be used when cataloging images.
|
||||
ImageTag = "image"
|
||||
|
||||
// DirectoryTag indicates the cataloger should be used when cataloging directories.
|
||||
DirectoryTag = "directory"
|
||||
|
||||
// PackageTag should be used to identify catalogers that are package-based.
|
||||
PackageTag = "package"
|
||||
|
||||
// OSTag should be used to identify catalogers that cataloging OS packages.
|
||||
OSTag = "os"
|
||||
|
||||
// LanguageTag should be used to identify catalogers that cataloging language-specific packages.
|
||||
LanguageTag = "language"
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
package cataloging
|
||||
|
||||
type RelationshipsConfig struct {
|
||||
// PackageFileOwnership will include package-to-file relationships that indicate which files are owned by which packages.
|
||||
PackageFileOwnership bool `yaml:"package-file-ownership" json:"package-file-ownership" mapstructure:"package-file-ownership"`
|
||||
|
||||
// PackageFileOwnershipOverlap will include package-to-package relationships that indicate one package is owned by another due to files claimed to be owned by one package are also evidence of another package's existence.
|
||||
// For example, if an RPM package is installed and claims to own /etc/app/package.lock and a separate NPM package was discovered by cataloging /etc/app/package.lock, then the two packages will
|
||||
// have ownership overlap relationship.
|
||||
PackageFileOwnershipOverlap bool `yaml:"package-file-ownership-overlap" json:"package-file-ownership-overlap" mapstructure:"package-file-ownership-overlap"`
|
||||
|
||||
// ExcludeBinaryPackagesWithFileOwnershipOverlap will exclude binary packages from the package catalog that are evident by files also owned by another package.
|
||||
// For example, if a binary package representing the /bin/python binary is discovered and there is a python RPM package installed which claims to
|
||||
// orn /bin/python, then the binary package will be excluded from the catalog altogether if this configuration is set to true.
|
||||
ExcludeBinaryPackagesWithFileOwnershipOverlap bool `yaml:"exclude-binary-packages-with-file-ownership-overlap" json:"exclude-binary-packages-with-file-ownership-overlap" mapstructure:"exclude-binary-packages-with-file-ownership-overlap"`
|
||||
}
|
||||
|
||||
func DefaultRelationshipsConfig() RelationshipsConfig {
|
||||
return RelationshipsConfig{
|
||||
PackageFileOwnership: true,
|
||||
PackageFileOwnershipOverlap: true,
|
||||
ExcludeBinaryPackagesWithFileOwnershipOverlap: true,
|
||||
}
|
||||
}
|
||||
|
||||
func (c RelationshipsConfig) WithPackageFileOwnership(ownership bool) RelationshipsConfig {
|
||||
c.PackageFileOwnership = ownership
|
||||
return c
|
||||
}
|
||||
|
||||
func (c RelationshipsConfig) WithPackageFileOwnershipOverlap(overlap bool) RelationshipsConfig {
|
||||
c.PackageFileOwnershipOverlap = overlap
|
||||
return c
|
||||
}
|
||||
|
||||
func (c RelationshipsConfig) WithExcludeBinaryPackagesWithFileOwnershipOverlap(exclude bool) RelationshipsConfig {
|
||||
c.ExcludeBinaryPackagesWithFileOwnershipOverlap = exclude
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package cataloging
|
||||
|
||||
import "github.com/anchore/syft/syft/source"
|
||||
|
||||
type SearchConfig struct {
|
||||
Scope source.Scope `yaml:"scope" json:"scope" mapstructure:"scope"`
|
||||
}
|
||||
|
||||
func DefaultSearchConfig() SearchConfig {
|
||||
return SearchConfig{
|
||||
Scope: source.SquashedScope,
|
||||
}
|
||||
}
|
||||
|
||||
func (c SearchConfig) WithScope(scope source.Scope) SearchConfig {
|
||||
c.Scope = scope
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package syft
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/cataloging/filecataloging"
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
)
|
||||
|
||||
// configurationAuditTrail is all input configuration was used to generate the SBOM
|
||||
type configurationAuditTrail struct {
|
||||
Search cataloging.SearchConfig `json:"search" yaml:"search" mapstructure:"search"`
|
||||
Relationships cataloging.RelationshipsConfig `json:"relationships" yaml:"relationships" mapstructure:"relationships"`
|
||||
DataGeneration cataloging.DataGenerationConfig `json:"data-generation" yaml:"data-generation" mapstructure:"data-generation"`
|
||||
Packages pkgcataloging.Config `json:"packages" yaml:"packages" mapstructure:"packages"`
|
||||
Files filecataloging.Config `json:"files" yaml:"files" mapstructure:"files"`
|
||||
Catalogers catalogerManifest `json:"catalogers" yaml:"catalogers" mapstructure:"catalogers"`
|
||||
ExtraConfigs any `json:"extra,omitempty" yaml:"extra" mapstructure:"extra"`
|
||||
}
|
||||
|
||||
type catalogerManifest struct {
|
||||
Requested pkgcataloging.SelectionRequest `json:"requested" yaml:"requested" mapstructure:"requested"`
|
||||
Used []string `json:"used" yaml:"used" mapstructure:"used"`
|
||||
}
|
||||
|
||||
type marshalAPIConfiguration configurationAuditTrail
|
||||
|
||||
func (cfg configurationAuditTrail) MarshalJSON() ([]byte, error) {
|
||||
// since the api configuration is placed into the SBOM in an empty interface, and we want a stable ordering of
|
||||
// keys (not guided by the struct ordering) we need to convert the struct to a map. This is best done with
|
||||
// simply marshalling and unmarshalling. Mapstructure is used to ensure we are honoring all json struct
|
||||
// tags. Once we have a map, we can lean on the stable ordering of json map keys in the stdlib. This is an
|
||||
// implementation detail that can be at least relied on until Go 2 (at which point it can change).
|
||||
// This dance allows us to guarantee ordering of keys in the configuration section of the SBOM.
|
||||
|
||||
initialJSON, err := json.Marshal(marshalAPIConfiguration(cfg))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var dataMap map[string]interface{}
|
||||
if err := json.Unmarshal(initialJSON, &dataMap); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if v, exists := dataMap["extra"]; exists && v == nil {
|
||||
// remove the extra key if it renders as nil
|
||||
delete(dataMap, "extra")
|
||||
}
|
||||
|
||||
return marshalSorted(dataMap)
|
||||
}
|
||||
|
||||
// marshalSorted recursively marshals a map with sorted keys
|
||||
func marshalSorted(m interface{}) ([]byte, error) {
|
||||
if reflect.TypeOf(m).Kind() != reflect.Map {
|
||||
return json.Marshal(m)
|
||||
}
|
||||
|
||||
val := reflect.ValueOf(m)
|
||||
sortedMap := make(map[string]interface{})
|
||||
|
||||
for _, key := range val.MapKeys() {
|
||||
value := val.MapIndex(key).Interface()
|
||||
|
||||
if value != nil && reflect.TypeOf(value).Kind() == reflect.Map {
|
||||
sortedValue, err := marshalSorted(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sortedMap[key.String()] = json.RawMessage(sortedValue)
|
||||
} else {
|
||||
sortedMap[key.String()] = value
|
||||
}
|
||||
}
|
||||
|
||||
return json.Marshal(sortedMap)
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
package syft
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/iancoleman/strcase"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/cataloging/filecataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
func Test_configurationAuditTrail_StructTags(t *testing.T) {
|
||||
// we need to ensure that the output for any configuration is well-formed and follows conventions.
|
||||
// We ensure that:
|
||||
// 1. all fields have a JSON tag
|
||||
// 2. the tag value follows lowercase kebab-case style
|
||||
|
||||
jsonTags := getJSONTags(t, configurationAuditTrail{})
|
||||
|
||||
for _, tag := range jsonTags {
|
||||
assertLowercaseKebab(t, tag)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func getJSONTags(t *testing.T, v interface{}) []string {
|
||||
var tags []string
|
||||
err := collectJSONTags(t, reflect.ValueOf(v), &tags, "", "")
|
||||
require.NoError(t, err)
|
||||
return tags
|
||||
}
|
||||
|
||||
func collectJSONTags(t *testing.T, v reflect.Value, tags *[]string, parentTag string, path string) error {
|
||||
var errs error
|
||||
|
||||
if v.Kind() == reflect.Ptr {
|
||||
v = v.Elem()
|
||||
}
|
||||
|
||||
if v.Kind() != reflect.Struct {
|
||||
return errs
|
||||
}
|
||||
|
||||
tType := v.Type()
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
field := v.Field(i)
|
||||
fieldType := tType.Field(i)
|
||||
|
||||
curPath := path + "." + fieldType.Name
|
||||
|
||||
// account for embeddings
|
||||
if fieldType.Anonymous {
|
||||
embeddedField := field
|
||||
|
||||
if embeddedField.Kind() == reflect.Ptr {
|
||||
// this can be enhanced in the future if the need arises...
|
||||
errs = multierror.Append(errs, fmt.Errorf("field '%s' is a pointer to an embedded struct, this is not supported in the test helper", curPath))
|
||||
}
|
||||
|
||||
if embeddedField.Kind() == reflect.Struct {
|
||||
err := collectJSONTags(t, field, tags, parentTag, curPath)
|
||||
if err != nil {
|
||||
errs = multierror.Append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
var tag string
|
||||
var ok bool
|
||||
if fieldType.PkgPath == "" {
|
||||
tag, ok = fieldType.Tag.Lookup("json")
|
||||
if !ok || (tag == "" && parentTag == "") {
|
||||
errs = multierror.Append(errs, fmt.Errorf("field '%s' does not have a json tag", curPath))
|
||||
return errs
|
||||
}
|
||||
if tag != "" && tag != "-" {
|
||||
*tags = append(*tags, tag)
|
||||
}
|
||||
}
|
||||
|
||||
if field.Kind() == reflect.Struct || (field.Kind() == reflect.Ptr && field.Elem().Kind() == reflect.Struct) {
|
||||
err := collectJSONTags(t, field, tags, tag, curPath)
|
||||
if err != nil {
|
||||
errs = multierror.Append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func assertLowercaseKebab(t *testing.T, tag string) {
|
||||
t.Helper()
|
||||
require.NotEmpty(t, tag)
|
||||
assert.Equal(t, strcase.ToKebab(tag), tag)
|
||||
}
|
||||
|
||||
func Test_collectJSONTags(t *testing.T) {
|
||||
// though this is not used in production, this is a sensitive and complex enough of a check to warrant testing the test helper.
|
||||
type good struct {
|
||||
A string `json:"a"`
|
||||
}
|
||||
|
||||
type missing struct {
|
||||
A string `json:"a"`
|
||||
B string
|
||||
}
|
||||
|
||||
type exclude struct {
|
||||
A string `json:"a"`
|
||||
B string `json:"-"`
|
||||
}
|
||||
|
||||
type goodEmbedded struct {
|
||||
good `json:""`
|
||||
}
|
||||
|
||||
type badEmbedded struct {
|
||||
missing `json:""`
|
||||
}
|
||||
|
||||
// simply not covered and require further development to support
|
||||
type goodPtrEmbedded struct {
|
||||
*good `json:""`
|
||||
}
|
||||
|
||||
// simply not covered and require further development to support
|
||||
type badPtrEmbedded struct {
|
||||
*missing `json:""`
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
v interface{}
|
||||
want []string
|
||||
wantErr require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "good",
|
||||
v: good{},
|
||||
want: []string{
|
||||
"a",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "missing",
|
||||
v: missing{},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "exclude",
|
||||
v: exclude{},
|
||||
want: []string{
|
||||
"a",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "bad embedded",
|
||||
v: badEmbedded{},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "good embedded",
|
||||
v: goodEmbedded{},
|
||||
want: []string{
|
||||
"a",
|
||||
},
|
||||
},
|
||||
// these cases are simply not covered and require further development to support
|
||||
{
|
||||
name: "bad ptr embedded",
|
||||
v: badPtrEmbedded{},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "good ptr embedded",
|
||||
v: goodPtrEmbedded{},
|
||||
want: []string{
|
||||
"a",
|
||||
},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.wantErr == nil {
|
||||
tt.wantErr = require.NoError
|
||||
}
|
||||
|
||||
var tags []string
|
||||
|
||||
err := collectJSONTags(t, reflect.ValueOf(tt.v), &tags, "", "")
|
||||
|
||||
tt.wantErr(t, err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.want, tags)
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func Test_configurationAuditTrail_MarshalJSON(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg configurationAuditTrail
|
||||
assert func(t *testing.T, got []byte)
|
||||
}{
|
||||
{
|
||||
name: "ensure other marshallers are called",
|
||||
cfg: configurationAuditTrail{
|
||||
|
||||
Files: filecataloging.Config{
|
||||
Selection: file.FilesOwnedByPackageSelection,
|
||||
Hashers: []crypto.Hash{
|
||||
crypto.SHA256,
|
||||
},
|
||||
},
|
||||
},
|
||||
// the custom file marshaller swaps ints for strings for hashers
|
||||
assert: func(t *testing.T, got []byte) {
|
||||
assert.Contains(t, string(got), `"hashers":["sha-256"]`)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ensure maps are sorted",
|
||||
cfg: configurationAuditTrail{},
|
||||
assert: func(t *testing.T, got []byte) {
|
||||
assert.NoError(t, assertJSONKeysSorted(got))
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
got, err := tt.cfg.MarshalJSON()
|
||||
require.NoError(t, err)
|
||||
if tt.assert == nil {
|
||||
t.Fatal("assert function must be provided")
|
||||
}
|
||||
tt.assert(t, got)
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// assertJSONKeysSorted asserts that all keys in JSON maps are sorted.
|
||||
func assertJSONKeysSorted(jsonBytes []byte) error {
|
||||
var errs error
|
||||
decoder := json.NewDecoder(bytes.NewReader(jsonBytes))
|
||||
var keys []string
|
||||
var inObject bool
|
||||
|
||||
for {
|
||||
token, err := decoder.Token()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
errs = multierror.Append(errs, fmt.Errorf("error decoding JSON: %w", err))
|
||||
}
|
||||
|
||||
switch v := token.(type) {
|
||||
case json.Delim:
|
||||
switch v {
|
||||
case '{':
|
||||
inObject = true
|
||||
keys = nil // Reset keys for a new object
|
||||
case '}':
|
||||
inObject = false
|
||||
if !sort.StringsAreSorted(keys) {
|
||||
errs = multierror.Append(errs, fmt.Errorf("Keys are not sorted: %v", keys))
|
||||
}
|
||||
}
|
||||
case string:
|
||||
if inObject && v != "" {
|
||||
keys = append(keys, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func Test_assertJSONKeysSorted(t *testing.T) {
|
||||
// this test function is sufficiently complicated enough to warrant its own test...
|
||||
|
||||
sorted := []byte(`{"a":1,"b":2}`)
|
||||
unsorted := []byte(`{"b":2,"a":1}`)
|
||||
|
||||
nestedSorted := []byte(`{"a":1,"b":{"a":1,"b":2}}`)
|
||||
nestedUnsorted := []byte(`{"a":1,"b":{"b":2,"a":1}}`)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
json []byte
|
||||
wantErr require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "sorted",
|
||||
json: sorted,
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "unsorted",
|
||||
json: unsorted,
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "nested sorted",
|
||||
json: nestedSorted,
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "nested unsorted",
|
||||
json: nestedUnsorted,
|
||||
wantErr: require.Error,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.wantErr == nil {
|
||||
tt.wantErr = require.NoError
|
||||
}
|
||||
|
||||
err := assertJSONKeysSorted(tt.json)
|
||||
tt.wantErr(t, err)
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package syft
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
"github.com/wagoodman/go-progress"
|
||||
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/relationship"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/internal/task"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
// CreateSBOM creates a software bill-of-materials from the given source. If the CreateSBOMConfig is nil, then
|
||||
// default options will be used.
|
||||
func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (*sbom.SBOM, error) {
|
||||
if cfg == nil {
|
||||
cfg = DefaultCreateSBOMConfig()
|
||||
}
|
||||
if err := cfg.validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid configuration: %w", err)
|
||||
}
|
||||
|
||||
srcMetadata := src.Describe()
|
||||
|
||||
taskGroups, audit, err := cfg.makeTaskGroups(srcMetadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resolver, err := src.FileResolver(cfg.Search.Scope)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get file resolver: %w", err)
|
||||
}
|
||||
|
||||
s := sbom.SBOM{
|
||||
Source: srcMetadata,
|
||||
Descriptor: sbom.Descriptor{
|
||||
Name: cfg.ToolName,
|
||||
Version: cfg.ToolVersion,
|
||||
Configuration: configurationAuditTrail{
|
||||
Search: cfg.Search,
|
||||
Relationships: cfg.Relationships,
|
||||
DataGeneration: cfg.DataGeneration,
|
||||
Packages: cfg.Packages,
|
||||
Files: cfg.Files,
|
||||
Catalogers: *audit,
|
||||
ExtraConfigs: cfg.ToolConfiguration,
|
||||
},
|
||||
},
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(),
|
||||
},
|
||||
}
|
||||
|
||||
catalogingProgress := monitorCatalogingTask(src.ID(), taskGroups)
|
||||
packageCatalogingProgress := monitorPackageCatalogingTask(s.Artifacts.Packages)
|
||||
|
||||
builder := sbomsync.NewBuilder(&s)
|
||||
for i := range taskGroups {
|
||||
err := task.NewTaskExecutor(taskGroups[i], cfg.Parallelism).Execute(ctx, resolver, builder, catalogingProgress)
|
||||
if err != nil {
|
||||
// TODO: tie this to the open progress monitors...
|
||||
return nil, fmt.Errorf("failed to run tasks: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
packageCatalogingProgress.SetCompleted()
|
||||
catalogingProgress.SetCompleted()
|
||||
|
||||
relationship.Finalize(builder, cfg.Relationships, src)
|
||||
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
func monitorPackageCatalogingTask(pkgs *pkg.Collection) *monitor.CatalogerTaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "Packages",
|
||||
},
|
||||
ID: monitor.PackageCatalogingTaskID,
|
||||
HideOnSuccess: false,
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
|
||||
prog := bus.StartCatalogerTask(info, -1, "")
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(200 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
<-ticker.C
|
||||
|
||||
count := humanize.Comma(int64(pkgs.PackageCount()))
|
||||
prog.AtomicStage.Set(fmt.Sprintf("%s packages", count))
|
||||
|
||||
if progress.IsCompleted(prog) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return prog
|
||||
}
|
||||
|
||||
func monitorCatalogingTask(srcID artifact.ID, tasks [][]task.Task) *monitor.CatalogerTaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "Catalog contents",
|
||||
WhileRunning: "Cataloging contents",
|
||||
OnSuccess: "Cataloged contents",
|
||||
},
|
||||
ID: monitor.TopLevelCatalogingTaskID,
|
||||
Context: string(srcID),
|
||||
HideOnSuccess: false,
|
||||
}
|
||||
|
||||
var length int64
|
||||
for _, tg := range tasks {
|
||||
length += int64(len(tg))
|
||||
}
|
||||
|
||||
return bus.StartCatalogerTask(info, length, "")
|
||||
}
|
||||
|
||||
func formatTaskNames(tasks []task.Task) []string {
|
||||
set := strset.New()
|
||||
for _, td := range tasks {
|
||||
set.Add(td.Name())
|
||||
}
|
||||
list := set.List()
|
||||
sort.Strings(list)
|
||||
return list
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
package syft
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal/task"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/cataloging/filecataloging"
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
// CreateSBOMConfig specifies all parameters needed for creating an SBOM.
|
||||
type CreateSBOMConfig struct {
|
||||
// required configuration input to specify how cataloging should be performed
|
||||
Search cataloging.SearchConfig
|
||||
Relationships cataloging.RelationshipsConfig
|
||||
DataGeneration cataloging.DataGenerationConfig
|
||||
Packages pkgcataloging.Config
|
||||
Files filecataloging.Config
|
||||
Parallelism int
|
||||
CatalogerSelection pkgcataloging.SelectionRequest
|
||||
|
||||
// audit what tool is being used to generate the SBOM
|
||||
ToolName string
|
||||
ToolVersion string
|
||||
ToolConfiguration interface{}
|
||||
|
||||
packageTaskFactories task.PackageTaskFactories
|
||||
packageCatalogerReferences []pkgcataloging.CatalogerReference
|
||||
}
|
||||
|
||||
func DefaultCreateSBOMConfig() *CreateSBOMConfig {
|
||||
return &CreateSBOMConfig{
|
||||
Search: cataloging.DefaultSearchConfig(),
|
||||
Relationships: cataloging.DefaultRelationshipsConfig(),
|
||||
DataGeneration: cataloging.DefaultDataGenerationConfig(),
|
||||
Packages: pkgcataloging.DefaultConfig(),
|
||||
Files: filecataloging.DefaultConfig(),
|
||||
Parallelism: 1,
|
||||
packageTaskFactories: task.DefaultPackageTaskFactories(),
|
||||
}
|
||||
}
|
||||
|
||||
// WithTool allows for setting the specific name, version, and any additional configuration that is not captured
|
||||
// in the syft default API configuration. This could cover inputs for catalogers that were user-provided, thus,
|
||||
// is not visible to the syft API, but would be useful to see in the SBOM output.
|
||||
func (c *CreateSBOMConfig) WithTool(name, version string, cfg ...any) *CreateSBOMConfig {
|
||||
c.ToolName = name
|
||||
c.ToolVersion = version
|
||||
c.ToolConfiguration = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
// WithParallelism allows for setting the number of concurrent cataloging tasks that can be performed at once
|
||||
func (c *CreateSBOMConfig) WithParallelism(p int) *CreateSBOMConfig {
|
||||
if p < 1 {
|
||||
// TODO: warn?
|
||||
p = 1
|
||||
}
|
||||
c.Parallelism = p
|
||||
return c
|
||||
}
|
||||
|
||||
// WithSearchConfig allows for setting the specific search configuration for cataloging.
|
||||
func (c *CreateSBOMConfig) WithSearchConfig(cfg cataloging.SearchConfig) *CreateSBOMConfig {
|
||||
c.Search = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
// WithRelationshipsConfig allows for defining the specific relationships that should be captured during cataloging.
|
||||
func (c *CreateSBOMConfig) WithRelationshipsConfig(cfg cataloging.RelationshipsConfig) *CreateSBOMConfig {
|
||||
c.Relationships = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
// WithDataGenerationConfig allows for defining what data elements that cannot be discovered from the underlying
|
||||
// target being scanned that should be generated after package creation.
|
||||
func (c *CreateSBOMConfig) WithDataGenerationConfig(cfg cataloging.DataGenerationConfig) *CreateSBOMConfig {
|
||||
c.DataGeneration = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
// WithPackagesConfig allows for defining any specific behavior for syft-implemented catalogers.
|
||||
func (c *CreateSBOMConfig) WithPackagesConfig(cfg pkgcataloging.Config) *CreateSBOMConfig {
|
||||
c.Packages = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
// WithFilesConfig allows for defining file-based cataloging parameters.
|
||||
func (c *CreateSBOMConfig) WithFilesConfig(cfg filecataloging.Config) *CreateSBOMConfig {
|
||||
c.Files = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
// WithoutFiles allows for disabling file cataloging altogether.
|
||||
func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig {
|
||||
c.Files = filecataloging.Config{
|
||||
Selection: file.NoFilesSelection,
|
||||
Hashers: nil,
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag.
|
||||
func (c *CreateSBOMConfig) WithCatalogerSelection(selection pkgcataloging.SelectionRequest) *CreateSBOMConfig {
|
||||
c.CatalogerSelection = selection
|
||||
return c
|
||||
}
|
||||
|
||||
// WithoutCatalogers removes all catalogers from the final set of catalogers. This is useful if you want to only use
|
||||
// user-provided catalogers (without the default syft-provided catalogers).
|
||||
func (c *CreateSBOMConfig) WithoutCatalogers() *CreateSBOMConfig {
|
||||
c.packageTaskFactories = nil
|
||||
c.packageCatalogerReferences = nil
|
||||
return c
|
||||
}
|
||||
|
||||
// WithCatalogers allows for adding user-provided catalogers to the final set of catalogers that will always be run
|
||||
// regardless of the source type or any cataloger selections provided.
|
||||
func (c *CreateSBOMConfig) WithCatalogers(catalogerRefs ...pkgcataloging.CatalogerReference) *CreateSBOMConfig {
|
||||
c.packageCatalogerReferences = append(c.packageCatalogerReferences, catalogerRefs...)
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// makeTaskGroups considers the entire configuration and finalizes the set of tasks to be run. Tasks are run in
|
||||
// groups, where each task in a group can be run concurrently, while tasks in different groups must be run serially.
|
||||
// The final set of task groups is returned along with a cataloger manifest that describes the catalogers that were
|
||||
// selected and the tokens that were sensitive to this selection (both for adding and removing from the final set).
|
||||
func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task, *catalogerManifest, error) {
|
||||
var taskGroups [][]task.Task
|
||||
|
||||
// generate package and file tasks based on the configuration
|
||||
environmentTasks := c.environmentTasks()
|
||||
relationshipsTasks := c.relationshipTasks(src)
|
||||
fileTasks := c.fileTasks()
|
||||
pkgTasks, selectionEvidence, err := c.packageTasks(src)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// combine the user-provided and configured tasks
|
||||
if c.Files.Selection == file.FilesOwnedByPackageSelection {
|
||||
// special case: we need the package info when we are cataloging files owned by packages
|
||||
taskGroups = append(taskGroups, pkgTasks, fileTasks)
|
||||
} else {
|
||||
taskGroups = append(taskGroups, append(pkgTasks, fileTasks...))
|
||||
}
|
||||
|
||||
// all relationship work must be done after all nodes (files and packages) have been cataloged
|
||||
if len(relationshipsTasks) > 0 {
|
||||
taskGroups = append(taskGroups, relationshipsTasks)
|
||||
}
|
||||
|
||||
// identifying the environment (i.e. the linux release) must be done first as this is required for package cataloging
|
||||
taskGroups = append(
|
||||
[][]task.Task{
|
||||
environmentTasks,
|
||||
},
|
||||
taskGroups...,
|
||||
)
|
||||
|
||||
return taskGroups, &catalogerManifest{
|
||||
Requested: selectionEvidence.Request,
|
||||
Used: formatTaskNames(pkgTasks),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// fileTasks returns the set of tasks that should be run to catalog files.
|
||||
func (c *CreateSBOMConfig) fileTasks() []task.Task {
|
||||
var tsks []task.Task
|
||||
|
||||
if t := task.NewFileDigestCatalogerTask(c.Files.Selection, c.Files.Hashers...); t != nil {
|
||||
tsks = append(tsks, t)
|
||||
}
|
||||
if t := task.NewFileMetadataCatalogerTask(c.Files.Selection); t != nil {
|
||||
tsks = append(tsks, t)
|
||||
}
|
||||
if t := task.NewFileContentCatalogerTask(c.Files.Content); t != nil {
|
||||
tsks = append(tsks, t)
|
||||
}
|
||||
|
||||
return tsks
|
||||
}
|
||||
|
||||
// packageTasks returns the set of tasks that should be run to catalog packages.
|
||||
func (c *CreateSBOMConfig) packageTasks(src source.Description) ([]task.Task, *task.Selection, error) {
|
||||
cfg := task.CatalogingFactoryConfig{
|
||||
SearchConfig: c.Search,
|
||||
RelationshipsConfig: c.Relationships,
|
||||
DataGenerationConfig: c.DataGeneration,
|
||||
PackagesConfig: c.Packages,
|
||||
}
|
||||
|
||||
persistentTasks, selectableTasks, err := c.allPackageTasks(cfg)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to create package cataloger tasks: %w", err)
|
||||
}
|
||||
|
||||
req, err := finalSelectionRequest(c.CatalogerSelection, src)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
finalTasks, selection, err := task.Select(selectableTasks, *req)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
finalTasks = append(finalTasks, persistentTasks...)
|
||||
|
||||
if len(finalTasks) == 0 {
|
||||
return nil, nil, fmt.Errorf("no catalogers selected")
|
||||
}
|
||||
|
||||
return finalTasks, &selection, nil
|
||||
}
|
||||
|
||||
func finalSelectionRequest(req pkgcataloging.SelectionRequest, src source.Description) (*pkgcataloging.SelectionRequest, error) {
|
||||
if len(req.DefaultNamesOrTags) == 0 {
|
||||
defaultTag, err := findDefaultTag(src)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to determine default cataloger tag: %w", err)
|
||||
}
|
||||
|
||||
if defaultTag != "" {
|
||||
req.DefaultNamesOrTags = append(req.DefaultNamesOrTags, defaultTag)
|
||||
}
|
||||
|
||||
req.RemoveNamesOrTags = replaceDefaultTagReferences(defaultTag, req.RemoveNamesOrTags)
|
||||
req.SubSelectTags = replaceDefaultTagReferences(defaultTag, req.SubSelectTags)
|
||||
}
|
||||
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
func (c *CreateSBOMConfig) allPackageTasks(cfg task.CatalogingFactoryConfig) ([]task.Task, []task.Task, error) {
|
||||
persistentPackageTasks, selectablePackageTasks, err := c.userPackageTasks(cfg)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
tsks, err := c.packageTaskFactories.Tasks(cfg)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to create package cataloger tasks: %w", err)
|
||||
}
|
||||
|
||||
return persistentPackageTasks, append(tsks, selectablePackageTasks...), nil
|
||||
}
|
||||
|
||||
func (c *CreateSBOMConfig) userPackageTasks(cfg task.CatalogingFactoryConfig) ([]task.Task, []task.Task, error) {
|
||||
var (
|
||||
persistentPackageTasks []task.Task
|
||||
selectablePackageTasks []task.Task
|
||||
)
|
||||
|
||||
for _, catalogerRef := range c.packageCatalogerReferences {
|
||||
if catalogerRef.Cataloger == nil {
|
||||
return nil, nil, errors.New("provided cataloger reference without a cataloger")
|
||||
}
|
||||
if catalogerRef.AlwaysEnabled {
|
||||
persistentPackageTasks = append(persistentPackageTasks, task.NewPackageTask(cfg, catalogerRef.Cataloger, catalogerRef.Tags...))
|
||||
continue
|
||||
}
|
||||
if len(catalogerRef.Tags) == 0 {
|
||||
return nil, nil, errors.New("provided cataloger reference without tags")
|
||||
}
|
||||
selectablePackageTasks = append(selectablePackageTasks, task.NewPackageTask(cfg, catalogerRef.Cataloger, catalogerRef.Tags...))
|
||||
}
|
||||
|
||||
return persistentPackageTasks, selectablePackageTasks, nil
|
||||
}
|
||||
|
||||
// relationshipTasks returns the set of tasks that should be run to generate additional relationships as well as
|
||||
// prune existing relationships.
|
||||
func (c *CreateSBOMConfig) relationshipTasks(src source.Description) []task.Task {
|
||||
var tsks []task.Task
|
||||
|
||||
if t := task.NewRelationshipsTask(c.Relationships, src); t != nil {
|
||||
tsks = append(tsks, t)
|
||||
}
|
||||
return tsks
|
||||
}
|
||||
|
||||
// environmentTasks returns the set of tasks that should be run to identify what is being scanned or the context
|
||||
// of where it is being scanned. Today this is used to identify the linux distribution release for container images
|
||||
// being scanned.
|
||||
func (c *CreateSBOMConfig) environmentTasks() []task.Task {
|
||||
var tsks []task.Task
|
||||
|
||||
if t := task.NewEnvironmentTask(); t != nil {
|
||||
tsks = append(tsks, t)
|
||||
}
|
||||
return tsks
|
||||
}
|
||||
|
||||
func (c *CreateSBOMConfig) validate() error {
|
||||
if c.Relationships.ExcludeBinaryPackagesWithFileOwnershipOverlap {
|
||||
if !c.Relationships.PackageFileOwnershipOverlap {
|
||||
return fmt.Errorf("invalid configuration: to exclude binary packages based on file ownership overlap relationships, cataloging file ownership overlap relationships must be enabled")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create creates an SBOM from the given source with the current SBOM configuration.
|
||||
func (c *CreateSBOMConfig) Create(ctx context.Context, src source.Source) (*sbom.SBOM, error) {
|
||||
return CreateSBOM(ctx, src, c)
|
||||
}
|
||||
|
||||
func findDefaultTag(src source.Description) (string, error) {
|
||||
switch m := src.Metadata.(type) {
|
||||
case source.StereoscopeImageSourceMetadata:
|
||||
return pkgcataloging.ImageTag, nil
|
||||
case source.FileSourceMetadata, source.DirectorySourceMetadata:
|
||||
return pkgcataloging.DirectoryTag, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unable to determine default cataloger tag for source type=%T", m)
|
||||
}
|
||||
}
|
||||
|
||||
func replaceDefaultTagReferences(defaultTag string, lst []string) []string {
|
||||
for i, tag := range lst {
|
||||
if strings.ToLower(tag) == "default" {
|
||||
lst[i] = defaultTag
|
||||
}
|
||||
}
|
||||
return lst
|
||||
}
|
||||
@@ -0,0 +1,502 @@
|
||||
package syft
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/task"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/cataloging/filecataloging"
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
var _ pkg.Cataloger = (*dummyCataloger)(nil)
|
||||
|
||||
type dummyCataloger struct {
|
||||
name string
|
||||
}
|
||||
|
||||
func newDummyCataloger(name string) pkg.Cataloger {
|
||||
return dummyCataloger{name: name}
|
||||
}
|
||||
|
||||
func (d dummyCataloger) Name() string {
|
||||
return d.name
|
||||
}
|
||||
|
||||
func (d dummyCataloger) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
|
||||
pkgIntersect := func(intersect ...string) []string {
|
||||
var sets []*strset.Set
|
||||
for _, s := range intersect {
|
||||
sets = append(sets, strset.New(pkgCatalogerNamesWithTagOrName(t, s)...))
|
||||
}
|
||||
|
||||
intersectSet := strset.Intersection(sets...)
|
||||
|
||||
slice := intersectSet.List()
|
||||
|
||||
sort.Strings(slice)
|
||||
|
||||
return slice
|
||||
}
|
||||
|
||||
addTo := func(slice []string, add ...string) []string {
|
||||
slice = append(slice, add...)
|
||||
sort.Strings(slice)
|
||||
return slice
|
||||
}
|
||||
|
||||
imgSrc := source.Description{
|
||||
Metadata: source.StereoscopeImageSourceMetadata{},
|
||||
}
|
||||
|
||||
dirSrc := source.Description{
|
||||
Metadata: source.DirectorySourceMetadata{},
|
||||
}
|
||||
|
||||
fileSrc := source.Description{
|
||||
Metadata: source.FileSourceMetadata{},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
src source.Description
|
||||
cfg *CreateSBOMConfig
|
||||
wantTaskNames [][]string
|
||||
wantManifest *catalogerManifest
|
||||
wantErr require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "default catalogers for image source",
|
||||
src: imgSrc,
|
||||
cfg: DefaultCreateSBOMConfig(),
|
||||
wantTaskNames: [][]string{
|
||||
environmentCatalogerNames(),
|
||||
pkgCatalogerNamesWithTagOrName(t, "image"),
|
||||
fileCatalogerNames(true, true),
|
||||
relationshipCatalogerNames(),
|
||||
},
|
||||
wantManifest: &catalogerManifest{
|
||||
Requested: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"image"},
|
||||
},
|
||||
Used: pkgCatalogerNamesWithTagOrName(t, "image"),
|
||||
},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "default catalogers for directory source",
|
||||
src: dirSrc,
|
||||
cfg: DefaultCreateSBOMConfig(),
|
||||
wantTaskNames: [][]string{
|
||||
environmentCatalogerNames(),
|
||||
pkgCatalogerNamesWithTagOrName(t, "directory"),
|
||||
fileCatalogerNames(true, true),
|
||||
relationshipCatalogerNames(),
|
||||
},
|
||||
wantManifest: &catalogerManifest{
|
||||
Requested: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"directory"},
|
||||
},
|
||||
Used: pkgCatalogerNamesWithTagOrName(t, "directory"),
|
||||
},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
// note, the file source acts like a directory scan
|
||||
name: "default catalogers for file source",
|
||||
src: fileSrc,
|
||||
cfg: DefaultCreateSBOMConfig(),
|
||||
wantTaskNames: [][]string{
|
||||
environmentCatalogerNames(),
|
||||
pkgCatalogerNamesWithTagOrName(t, "directory"),
|
||||
fileCatalogerNames(true, true),
|
||||
relationshipCatalogerNames(),
|
||||
},
|
||||
wantManifest: &catalogerManifest{
|
||||
Requested: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"directory"},
|
||||
},
|
||||
Used: pkgCatalogerNamesWithTagOrName(t, "directory"),
|
||||
},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "no file digest cataloger",
|
||||
src: imgSrc,
|
||||
cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithHashers(nil)),
|
||||
wantTaskNames: [][]string{
|
||||
environmentCatalogerNames(),
|
||||
pkgCatalogerNamesWithTagOrName(t, "image"),
|
||||
fileCatalogerNames(false, true), // note: the digest cataloger is not included
|
||||
relationshipCatalogerNames(),
|
||||
},
|
||||
wantManifest: &catalogerManifest{
|
||||
Requested: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"image"},
|
||||
},
|
||||
Used: pkgCatalogerNamesWithTagOrName(t, "image"),
|
||||
},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "select no file catalogers",
|
||||
src: imgSrc,
|
||||
cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.NoFilesSelection)),
|
||||
wantTaskNames: [][]string{
|
||||
environmentCatalogerNames(),
|
||||
pkgCatalogerNamesWithTagOrName(t, "image"),
|
||||
// note: there are no file catalogers in their own group
|
||||
relationshipCatalogerNames(),
|
||||
},
|
||||
wantManifest: &catalogerManifest{
|
||||
Requested: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"image"},
|
||||
},
|
||||
Used: pkgCatalogerNamesWithTagOrName(t, "image"),
|
||||
},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "select all file catalogers",
|
||||
src: imgSrc,
|
||||
cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)),
|
||||
wantTaskNames: [][]string{
|
||||
environmentCatalogerNames(),
|
||||
// note: there is a single group of catalogers for pkgs and files
|
||||
append(
|
||||
pkgCatalogerNamesWithTagOrName(t, "image"),
|
||||
fileCatalogerNames(true, true)...,
|
||||
),
|
||||
relationshipCatalogerNames(),
|
||||
},
|
||||
wantManifest: &catalogerManifest{
|
||||
Requested: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"image"},
|
||||
},
|
||||
Used: pkgCatalogerNamesWithTagOrName(t, "image"),
|
||||
},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "user-provided persistent cataloger is always run (image)",
|
||||
src: imgSrc,
|
||||
cfg: DefaultCreateSBOMConfig().WithCatalogers(
|
||||
pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")),
|
||||
),
|
||||
wantTaskNames: [][]string{
|
||||
environmentCatalogerNames(),
|
||||
addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"),
|
||||
fileCatalogerNames(true, true),
|
||||
relationshipCatalogerNames(),
|
||||
},
|
||||
wantManifest: &catalogerManifest{
|
||||
Requested: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"image"},
|
||||
},
|
||||
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"),
|
||||
},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "user-provided persistent cataloger is always run (directory)",
|
||||
src: dirSrc,
|
||||
cfg: DefaultCreateSBOMConfig().WithCatalogers(
|
||||
pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")),
|
||||
),
|
||||
wantTaskNames: [][]string{
|
||||
environmentCatalogerNames(),
|
||||
addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"),
|
||||
fileCatalogerNames(true, true),
|
||||
relationshipCatalogerNames(),
|
||||
},
|
||||
wantManifest: &catalogerManifest{
|
||||
Requested: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"directory"},
|
||||
},
|
||||
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"),
|
||||
},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "user-provided persistent cataloger is always run (user selection does not affect this)",
|
||||
src: imgSrc,
|
||||
cfg: DefaultCreateSBOMConfig().WithCatalogers(
|
||||
pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")),
|
||||
).WithCatalogerSelection(pkgcataloging.NewSelectionRequest().WithSubSelections("javascript")),
|
||||
wantTaskNames: [][]string{
|
||||
environmentCatalogerNames(),
|
||||
addTo(pkgIntersect("image", "javascript"), "persistent"),
|
||||
fileCatalogerNames(true, true),
|
||||
relationshipCatalogerNames(),
|
||||
},
|
||||
wantManifest: &catalogerManifest{
|
||||
Requested: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"image"},
|
||||
SubSelectTags: []string{"javascript"},
|
||||
},
|
||||
Used: addTo(pkgIntersect("image", "javascript"), "persistent"),
|
||||
},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "user-provided cataloger runs when selected",
|
||||
src: imgSrc,
|
||||
cfg: DefaultCreateSBOMConfig().WithCatalogers(
|
||||
pkgcataloging.NewCatalogerReference(newDummyCataloger("user-provided"), []string{"image"}),
|
||||
),
|
||||
wantTaskNames: [][]string{
|
||||
environmentCatalogerNames(),
|
||||
addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"),
|
||||
fileCatalogerNames(true, true),
|
||||
relationshipCatalogerNames(),
|
||||
},
|
||||
wantManifest: &catalogerManifest{
|
||||
Requested: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"image"},
|
||||
},
|
||||
Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"),
|
||||
},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
{
|
||||
name: "user-provided cataloger NOT run when NOT selected",
|
||||
src: imgSrc,
|
||||
cfg: DefaultCreateSBOMConfig().WithCatalogers(
|
||||
pkgcataloging.NewCatalogerReference(newDummyCataloger("user-provided"), []string{"bogus-selector-will-never-be-used"}),
|
||||
),
|
||||
wantTaskNames: [][]string{
|
||||
environmentCatalogerNames(),
|
||||
pkgCatalogerNamesWithTagOrName(t, "image"),
|
||||
fileCatalogerNames(true, true),
|
||||
relationshipCatalogerNames(),
|
||||
},
|
||||
wantManifest: &catalogerManifest{
|
||||
Requested: pkgcataloging.SelectionRequest{
|
||||
DefaultNamesOrTags: []string{"image"},
|
||||
},
|
||||
Used: pkgCatalogerNamesWithTagOrName(t, "image"),
|
||||
},
|
||||
wantErr: require.NoError,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.wantErr == nil {
|
||||
tt.wantErr = require.NoError
|
||||
}
|
||||
|
||||
// sanity check
|
||||
require.NotEmpty(t, tt.wantTaskNames)
|
||||
for _, group := range tt.wantTaskNames {
|
||||
require.NotEmpty(t, group)
|
||||
}
|
||||
|
||||
// test the subject
|
||||
gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src)
|
||||
tt.wantErr(t, err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
gotNames := taskGroupNames(gotTasks)
|
||||
|
||||
if d := cmp.Diff(
|
||||
tt.wantTaskNames,
|
||||
gotNames,
|
||||
// order within a group does not matter
|
||||
cmpopts.SortSlices(func(a, b string) bool {
|
||||
return a < b
|
||||
}),
|
||||
); d != "" {
|
||||
t.Errorf("mismatched task group names (-want +got):\n%s", d)
|
||||
}
|
||||
|
||||
if d := cmp.Diff(tt.wantManifest, gotManifest); d != "" {
|
||||
t.Errorf("mismatched cataloger manifest (-want +got):\n%s", d)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func pkgCatalogerNamesWithTagOrName(t *testing.T, token string) []string {
|
||||
var names []string
|
||||
cfg := task.DefaultCatalogingFactoryConfig()
|
||||
for _, factory := range task.DefaultPackageTaskFactories() {
|
||||
cat := factory(cfg)
|
||||
|
||||
name := cat.Name()
|
||||
|
||||
if selector, ok := cat.(task.Selector); ok {
|
||||
if selector.HasAllSelectors(token) {
|
||||
names = append(names, name)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if name == token {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
|
||||
// these thresholds are arbitrary but should be large enough to catch any major changes
|
||||
switch token {
|
||||
case "image":
|
||||
require.Greater(t, len(names), 18, "minimum cataloger sanity check failed token")
|
||||
case "directory":
|
||||
require.Greater(t, len(names), 25, "minimum cataloger sanity check failed token")
|
||||
default:
|
||||
require.Greater(t, len(names), 0, "minimum cataloger sanity check failed token")
|
||||
}
|
||||
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
func fileCatalogerNames(digest, metadata bool) []string {
|
||||
var names []string
|
||||
if digest {
|
||||
names = append(names, "file-digest-cataloger")
|
||||
}
|
||||
if metadata {
|
||||
names = append(names, "file-metadata-cataloger")
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func relationshipCatalogerNames() []string {
|
||||
return []string{"relationships-cataloger"}
|
||||
}
|
||||
|
||||
func environmentCatalogerNames() []string {
|
||||
return []string{"environment-cataloger"}
|
||||
}
|
||||
|
||||
func taskGroupNames(groups [][]task.Task) [][]string {
|
||||
var names [][]string
|
||||
for _, group := range groups {
|
||||
var groupNames []string
|
||||
for _, tsk := range group {
|
||||
groupNames = append(groupNames, tsk.Name())
|
||||
}
|
||||
names = append(names, groupNames)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func Test_replaceDefaultTagReferences(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
lst []string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "no default tag",
|
||||
lst: []string{"foo", "bar"},
|
||||
want: []string{"foo", "bar"},
|
||||
},
|
||||
{
|
||||
name: "replace default tag",
|
||||
lst: []string{"foo", "default", "bar"},
|
||||
want: []string{"foo", "replacement", "bar"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, replaceDefaultTagReferences("replacement", tt.lst))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_findDefaultTag(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
src source.Description
|
||||
want string
|
||||
wantErr require.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "image",
|
||||
src: source.Description{
|
||||
Metadata: source.StereoscopeImageSourceMetadata{},
|
||||
},
|
||||
want: pkgcataloging.ImageTag,
|
||||
},
|
||||
{
|
||||
name: "directory",
|
||||
src: source.Description{
|
||||
Metadata: source.DirectorySourceMetadata{},
|
||||
},
|
||||
want: pkgcataloging.DirectoryTag,
|
||||
},
|
||||
{
|
||||
name: "file",
|
||||
src: source.Description{
|
||||
Metadata: source.FileSourceMetadata{},
|
||||
},
|
||||
want: pkgcataloging.DirectoryTag, // not a mistake...
|
||||
},
|
||||
{
|
||||
name: "unknown",
|
||||
src: source.Description{
|
||||
Metadata: struct{}{},
|
||||
},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.wantErr == nil {
|
||||
tt.wantErr = require.NoError
|
||||
}
|
||||
got, err := findDefaultTag(tt.src)
|
||||
tt.wantErr(t, err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateSBOMConfig_validate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg *CreateSBOMConfig
|
||||
wantErr assert.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "incompatible ExcludeBinaryPackagesWithFileOwnershipOverlap selection",
|
||||
cfg: DefaultCreateSBOMConfig().
|
||||
WithRelationshipsConfig(
|
||||
cataloging.DefaultRelationshipsConfig().
|
||||
WithExcludeBinaryPackagesWithFileOwnershipOverlap(true).
|
||||
WithPackageFileOwnershipOverlap(false),
|
||||
),
|
||||
wantErr: assert.Error,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.wantErr == nil {
|
||||
tt.wantErr = assert.NoError
|
||||
}
|
||||
tt.wantErr(t, tt.cfg.validate())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,11 @@ import (
|
||||
"github.com/wagoodman/go-progress"
|
||||
)
|
||||
|
||||
const (
|
||||
TopLevelCatalogingTaskID = "cataloging"
|
||||
PackageCatalogingTaskID = "package-cataloging"
|
||||
)
|
||||
|
||||
type CatalogerTaskProgress struct {
|
||||
*progress.AtomicStage
|
||||
*progress.Manual
|
||||
|
||||
@@ -6,21 +6,40 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
// Globs are the file patterns that must be matched for a file to be considered for cataloging.
|
||||
Globs []string `yaml:"globs" json:"globs" mapstructure:"globs"`
|
||||
|
||||
// SkipFilesAboveSize is the maximum file size (in bytes) to allow to be considered while cataloging. If the file is larger than this size it will be skipped.
|
||||
SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"`
|
||||
}
|
||||
|
||||
type Cataloger struct {
|
||||
globs []string
|
||||
skipFilesAboveSizeInBytes int64
|
||||
}
|
||||
|
||||
func NewCataloger(globs []string, skipFilesAboveSize int64) (*Cataloger, error) {
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
SkipFilesAboveSize: 250 * intFile.KB,
|
||||
}
|
||||
}
|
||||
|
||||
func NewCataloger(cfg Config) *Cataloger {
|
||||
return &Cataloger{
|
||||
globs: globs,
|
||||
skipFilesAboveSizeInBytes: skipFilesAboveSize,
|
||||
}, nil
|
||||
globs: cfg.Globs,
|
||||
skipFilesAboveSizeInBytes: cfg.SkipFilesAboveSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string, error) {
|
||||
@@ -31,9 +50,15 @@ func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prog := catalogingProgress(int64(len(locations)))
|
||||
|
||||
for _, location := range locations {
|
||||
prog.AtomicStage.Set(location.Path())
|
||||
|
||||
metadata, err := resolver.FileMetadataByLocation(location)
|
||||
if err != nil {
|
||||
prog.SetError(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -47,12 +72,20 @@ func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
prog.SetError(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prog.Increment()
|
||||
|
||||
results[location.Coordinates] = result
|
||||
}
|
||||
|
||||
log.Debugf("file contents cataloger processed %d files", len(results))
|
||||
|
||||
prog.AtomicStage.Set(fmt.Sprintf("%s files", humanize.Comma(prog.Current())))
|
||||
prog.SetCompleted()
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
@@ -66,7 +99,7 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
|
||||
buf := &bytes.Buffer{}
|
||||
encoder := base64.NewEncoder(base64.StdEncoding, buf)
|
||||
if _, err = io.Copy(encoder, contentReader); err != nil {
|
||||
return "", internal.ErrPath{Context: "contents-cataloger", Path: location.RealPath, Err: err}
|
||||
return "", internal.ErrPath{Context: "content-cataloger", Path: location.RealPath, Err: err}
|
||||
}
|
||||
// note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes
|
||||
if err := encoder.Close(); err != nil {
|
||||
@@ -75,3 +108,14 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
|
||||
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
func catalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "File contents",
|
||||
},
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
|
||||
return bus.StartCatalogerTask(info, locations, "")
|
||||
}
|
||||
|
||||
@@ -67,8 +67,10 @@ func TestContentsCataloger(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
c, err := NewCataloger(test.globs, test.maxSize)
|
||||
assert.NoError(t, err)
|
||||
c := NewCataloger(Config{
|
||||
Globs: test.globs,
|
||||
SkipFilesAboveSize: test.maxSize,
|
||||
})
|
||||
|
||||
resolver := file.NewMockResolverForPaths(test.files...)
|
||||
actual, err := c.Catalog(resolver)
|
||||
|
||||
@@ -45,32 +45,34 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
|
||||
}
|
||||
}
|
||||
|
||||
prog := digestsCatalogingProgress(int64(len(locations)))
|
||||
prog := catalogingProgress(int64(len(locations)))
|
||||
for _, location := range locations {
|
||||
prog.Increment()
|
||||
prog.AtomicStage.Set(location.Path())
|
||||
|
||||
result, err := i.catalogLocation(resolver, location)
|
||||
|
||||
if errors.Is(err, ErrUndigestableFile) {
|
||||
continue
|
||||
}
|
||||
|
||||
prog.AtomicStage.Set(location.Path())
|
||||
|
||||
if internal.IsErrPathPermission(err) {
|
||||
log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
prog.SetError(err)
|
||||
return nil, fmt.Errorf("failed to process file %q: %w", location.RealPath, err)
|
||||
}
|
||||
|
||||
prog.Increment()
|
||||
|
||||
results[location.Coordinates] = result
|
||||
}
|
||||
|
||||
log.Debugf("file digests cataloger processed %d files", prog.Current())
|
||||
|
||||
prog.AtomicStage.Set(fmt.Sprintf("%s digests", humanize.Comma(prog.Current())))
|
||||
prog.AtomicStage.Set(fmt.Sprintf("%s files", humanize.Comma(prog.Current())))
|
||||
prog.SetCompleted()
|
||||
|
||||
return results, nil
|
||||
@@ -101,13 +103,12 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
|
||||
return digests, nil
|
||||
}
|
||||
|
||||
func digestsCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
|
||||
func catalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "Catalog file digests",
|
||||
WhileRunning: "Cataloging file digests",
|
||||
OnSuccess: "Cataloged file digests",
|
||||
Default: "File digests",
|
||||
},
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
|
||||
return bus.StartCatalogerTask(info, locations, "")
|
||||
|
||||
@@ -43,16 +43,18 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
|
||||
}()
|
||||
}
|
||||
|
||||
prog := metadataCatalogingProgress(int64(len(locations)))
|
||||
prog := catalogingProgress(-1)
|
||||
for location := range locations {
|
||||
prog.Increment()
|
||||
prog.AtomicStage.Set(location.Path())
|
||||
|
||||
metadata, err := resolver.FileMetadataByLocation(location)
|
||||
if err != nil {
|
||||
prog.SetError(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prog.Increment()
|
||||
|
||||
results[location.Coordinates] = metadata
|
||||
}
|
||||
|
||||
@@ -64,13 +66,12 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func metadataCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
|
||||
func catalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "Catalog file metadata",
|
||||
WhileRunning: "Cataloging file metadata",
|
||||
OnSuccess: "Cataloged file metadata",
|
||||
Default: "File metadata",
|
||||
},
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
|
||||
return bus.StartCatalogerTask(info, locations, "")
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
package file
|
||||
|
||||
const (
|
||||
NoFilesSelection Selection = "none"
|
||||
FilesOwnedByPackageSelection Selection = "owned-by-package"
|
||||
AllFilesSelection Selection = "all"
|
||||
)
|
||||
|
||||
type Selection string
|
||||
-90
@@ -17,103 +17,13 @@ Similar to the cataloging process, Linux distribution identification is also per
|
||||
package syft
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/wagoodman/go-partybus"
|
||||
|
||||
"github.com/anchore/go-logger"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/linux"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
// CatalogPackages takes an inventory of packages from the given image from a particular perspective
|
||||
// (e.g. squashed source, all-layers source). Returns the discovered set of packages, the identified Linux
|
||||
// distribution, and the source object used to wrap the data source.
|
||||
func CatalogPackages(src source.Source, cfg cataloger.Config) (*pkg.Collection, []artifact.Relationship, *linux.Release, error) {
|
||||
resolver, err := src.FileResolver(cfg.Search.Scope)
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err)
|
||||
}
|
||||
|
||||
// find the distro
|
||||
release := linux.IdentifyRelease(resolver)
|
||||
if release != nil {
|
||||
log.Infof("identified distro: %s", release.String())
|
||||
} else {
|
||||
log.Info("could not identify distro")
|
||||
}
|
||||
|
||||
// if the catalogers have been configured, use them regardless of input type
|
||||
var catalogers []pkg.Cataloger
|
||||
if len(cfg.Catalogers) > 0 {
|
||||
catalogers = cataloger.AllCatalogers(cfg)
|
||||
} else {
|
||||
// otherwise conditionally use the correct set of loggers based on the input type (container image or directory)
|
||||
|
||||
// TODO: this is bad, we should not be using the concrete type to determine the cataloger set
|
||||
// instead this should be a caller concern (pass the catalogers you want to use). The SBOM build PR will do this.
|
||||
switch src.(type) {
|
||||
case *source.StereoscopeImageSource:
|
||||
log.Info("cataloging an image")
|
||||
catalogers = cataloger.ImageCatalogers(cfg)
|
||||
case *source.FileSource:
|
||||
log.Info("cataloging a file")
|
||||
catalogers = cataloger.AllCatalogers(cfg)
|
||||
case *source.DirectorySource:
|
||||
log.Info("cataloging a directory")
|
||||
catalogers = cataloger.DirectoryCatalogers(cfg)
|
||||
default:
|
||||
return nil, nil, nil, fmt.Errorf("unsupported source type: %T", src)
|
||||
}
|
||||
}
|
||||
|
||||
catalog, relationships, err := cataloger.Catalog(resolver, release, cfg.Parallelism, catalogers...)
|
||||
|
||||
// apply exclusions to the package catalog
|
||||
// default config value for this is true
|
||||
// https://github.com/anchore/syft/issues/931
|
||||
if cfg.ExcludeBinaryOverlapByOwnership {
|
||||
for _, r := range relationships {
|
||||
if cataloger.ExcludeBinaryByFileOwnershipOverlap(r, catalog) {
|
||||
catalog.Delete(r.To.ID())
|
||||
relationships = removeRelationshipsByID(relationships, r.To.ID())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// no need to consider source relationships for os -> binary exclusions
|
||||
relationships = append(relationships, newSourceRelationshipsFromCatalog(src, catalog)...)
|
||||
return catalog, relationships, release, err
|
||||
}
|
||||
|
||||
func removeRelationshipsByID(relationships []artifact.Relationship, id artifact.ID) []artifact.Relationship {
|
||||
var filtered []artifact.Relationship
|
||||
for _, r := range relationships {
|
||||
if r.To.ID() != id && r.From.ID() != id {
|
||||
filtered = append(filtered, r)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func newSourceRelationshipsFromCatalog(src source.Source, c *pkg.Collection) []artifact.Relationship {
|
||||
relationships := make([]artifact.Relationship, 0) // Should we pre-allocate this by giving catalog a Len() method?
|
||||
for p := range c.Enumerate() {
|
||||
relationships = append(relationships, artifact.Relationship{
|
||||
From: src,
|
||||
To: p,
|
||||
Type: artifact.ContainsRelationship,
|
||||
})
|
||||
}
|
||||
|
||||
return relationships
|
||||
}
|
||||
|
||||
// SetLogger sets the logger object used for all syft logging calls.
|
||||
func SetLogger(logger logger.Logger) {
|
||||
log.Set(logger)
|
||||
|
||||
+2
-1
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/relationship"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
@@ -34,7 +35,7 @@ func Test_removeRelationshipsByID(t *testing.T) {
|
||||
|
||||
for _, r := range relationships {
|
||||
if r.From.ID() == "1" || r.From.ID() == "2" {
|
||||
relationships = removeRelationshipsByID(relationships, r.From.ID())
|
||||
relationships = relationship.RemoveRelationshipsByID(relationships, r.From.ID())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
const catalogerName = "binary-cataloger"
|
||||
|
||||
type CatalogerConfig struct {
|
||||
Classifiers []Classifier
|
||||
Classifiers []Classifier `yaml:"classifiers" json:"classifiers" mapstructure:"classifiers"`
|
||||
}
|
||||
|
||||
func DefaultCatalogerConfig() CatalogerConfig {
|
||||
|
||||
@@ -23,25 +23,25 @@ import (
|
||||
// Classifier is a generic package classifier that can be used to match a package definition
|
||||
// to a file that meets the given content criteria of the EvidenceMatcher.
|
||||
type Classifier struct {
|
||||
Class string
|
||||
Class string `json:"class"`
|
||||
|
||||
// FileGlob is a selector to narrow down file inspection using the **/glob* syntax
|
||||
FileGlob string
|
||||
FileGlob string `json:"fileGlob"`
|
||||
|
||||
// EvidenceMatcher is what will be used to match against the file in the source
|
||||
// location. If the matcher returns a package, the file will be considered a candidate.
|
||||
EvidenceMatcher EvidenceMatcher
|
||||
EvidenceMatcher EvidenceMatcher `json:"-"`
|
||||
|
||||
// Information below is used to specify the Package information when returned
|
||||
|
||||
// Package is the name to use for the package
|
||||
Package string
|
||||
Package string `json:"package"`
|
||||
|
||||
// PURL is the Package URL to use when generating a package
|
||||
PURL packageurl.PackageURL
|
||||
PURL packageurl.PackageURL `json:"purl"`
|
||||
|
||||
// CPEs are the specific CPEs we want to include for this binary with updated version information
|
||||
CPEs []cpe.CPE
|
||||
CPEs []cpe.CPE `json:"cpes"`
|
||||
}
|
||||
|
||||
// EvidenceMatcher is a function called to catalog Packages that match some sort of evidence
|
||||
|
||||
@@ -1,229 +0,0 @@
|
||||
package cataloger
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/wagoodman/go-progress"
|
||||
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/linux"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common/cpe"
|
||||
)
|
||||
|
||||
// Monitor provides progress-related data for observing the progress of a Catalog() call (published on the event bus).
|
||||
type Monitor struct {
|
||||
FilesProcessed progress.Monitorable // the number of files selected and contents analyzed from all registered catalogers
|
||||
PackagesDiscovered progress.Monitorable // the number of packages discovered from all registered catalogers
|
||||
}
|
||||
|
||||
// catalogResult provides the result of running a single cataloger against source
|
||||
type catalogResult struct {
|
||||
Packages []pkg.Package
|
||||
Relationships []artifact.Relationship
|
||||
// Discovered may sometimes be more than len(packages)
|
||||
Discovered int64
|
||||
Error error
|
||||
}
|
||||
|
||||
func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerResult *catalogResult, err error) {
|
||||
// handle individual cataloger panics
|
||||
defer func() {
|
||||
if e := recover(); e != nil {
|
||||
err = fmt.Errorf("%v at:\n%s", e, string(debug.Stack()))
|
||||
}
|
||||
}()
|
||||
|
||||
catalogerResult = new(catalogResult)
|
||||
|
||||
// find packages from the underlying raw data
|
||||
log.WithFields("cataloger", cataloger.Name()).Trace("cataloging started")
|
||||
packages, relationships, err := cataloger.Catalog(resolver)
|
||||
if err != nil {
|
||||
log.WithFields("cataloger", cataloger.Name()).Warn("error while cataloging")
|
||||
return catalogerResult, err
|
||||
}
|
||||
|
||||
catalogedPackages := len(packages)
|
||||
|
||||
log.WithFields("cataloger", cataloger.Name()).Debugf("discovered %d packages", catalogedPackages)
|
||||
catalogerResult.Discovered = int64(catalogedPackages)
|
||||
|
||||
for _, p := range packages {
|
||||
// generate CPEs (note: this is excluded from package ID, so is safe to mutate)
|
||||
// we might have binary classified CPE already with the package so we want to append here
|
||||
|
||||
dictionaryCPE, ok := cpe.DictionaryFind(p)
|
||||
if ok {
|
||||
log.Debugf("used CPE dictionary to find CPE for %s package %q: %s", p.Type, p.Name, dictionaryCPE.BindToFmtString())
|
||||
p.CPEs = append(p.CPEs, dictionaryCPE)
|
||||
} else {
|
||||
p.CPEs = append(p.CPEs, cpe.Generate(p)...)
|
||||
}
|
||||
|
||||
// if we were not able to identify the language we have an opportunity
|
||||
// to try and get this value from the PURL. Worst case we assert that
|
||||
// we could not identify the language at either stage and set UnknownLanguage
|
||||
if p.Language == "" {
|
||||
p.Language = pkg.LanguageFromPURL(p.PURL)
|
||||
}
|
||||
|
||||
// create file-to-package relationships for files owned by the package
|
||||
owningRelationships, err := packageFileOwnershipRelationships(p, resolver)
|
||||
if err != nil {
|
||||
log.WithFields("cataloger", cataloger.Name(), "package", p.Name, "error", err).Warnf("unable to create any package-file relationships")
|
||||
} else {
|
||||
catalogerResult.Relationships = append(catalogerResult.Relationships, owningRelationships...)
|
||||
}
|
||||
catalogerResult.Packages = append(catalogerResult.Packages, p)
|
||||
}
|
||||
|
||||
catalogerResult.Relationships = append(catalogerResult.Relationships, relationships...)
|
||||
log.WithFields("cataloger", cataloger.Name()).Trace("cataloging complete")
|
||||
return catalogerResult, err
|
||||
}
|
||||
|
||||
// Catalog a given source (container image or filesystem) with the given catalogers, returning all discovered packages.
|
||||
// In order to efficiently retrieve contents from a underlying container image the content fetch requests are
|
||||
// done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single
|
||||
// request.
|
||||
//
|
||||
//nolint:funlen
|
||||
func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, catalogers ...pkg.Cataloger) (*pkg.Collection, []artifact.Relationship, error) {
|
||||
catalog := pkg.NewCollection()
|
||||
var allRelationships []artifact.Relationship
|
||||
|
||||
prog := monitorPackageCatalogingTask()
|
||||
|
||||
// perform analysis, accumulating errors for each failed analysis
|
||||
var errs error
|
||||
|
||||
nCatalogers := len(catalogers)
|
||||
|
||||
// we do not need more parallelism than there are `catalogers`.
|
||||
parallelism = int(math.Min(float64(nCatalogers), math.Max(1.0, float64(parallelism))))
|
||||
log.WithFields("parallelism", parallelism, "catalogers", nCatalogers).Debug("cataloging packages")
|
||||
|
||||
jobs := make(chan pkg.Cataloger, nCatalogers)
|
||||
results := make(chan *catalogResult, nCatalogers)
|
||||
|
||||
waitGroup := sync.WaitGroup{}
|
||||
|
||||
var totalPackagesDiscovered int64
|
||||
|
||||
for i := 0; i < parallelism; i++ {
|
||||
waitGroup.Add(1)
|
||||
|
||||
go func() {
|
||||
defer waitGroup.Done()
|
||||
|
||||
// wait for / get the next cataloger job available.
|
||||
for cataloger := range jobs {
|
||||
result, err := runCataloger(cataloger, resolver)
|
||||
|
||||
// ensure we set the error to be aggregated
|
||||
result.Error = err
|
||||
|
||||
prog.Add(result.Discovered)
|
||||
totalPackagesDiscovered += result.Discovered
|
||||
count := humanize.Comma(totalPackagesDiscovered)
|
||||
prog.AtomicStage.Set(fmt.Sprintf("%s packages", count))
|
||||
|
||||
results <- result
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Enqueue the jobs
|
||||
for _, cataloger := range catalogers {
|
||||
jobs <- cataloger
|
||||
}
|
||||
close(jobs)
|
||||
|
||||
// Wait for the jobs to finish
|
||||
waitGroup.Wait()
|
||||
close(results)
|
||||
|
||||
// collect the results
|
||||
for result := range results {
|
||||
if result.Error != nil {
|
||||
errs = multierror.Append(errs, result.Error)
|
||||
}
|
||||
for _, p := range result.Packages {
|
||||
catalog.Add(p)
|
||||
}
|
||||
allRelationships = append(allRelationships, result.Relationships...)
|
||||
}
|
||||
|
||||
allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...)
|
||||
|
||||
if errs != nil {
|
||||
prog.SetError(errs)
|
||||
} else {
|
||||
prog.SetCompleted()
|
||||
}
|
||||
|
||||
return catalog, allRelationships, errs
|
||||
}
|
||||
|
||||
func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver) ([]artifact.Relationship, error) {
|
||||
fileOwner, ok := p.Metadata.(pkg.FileOwner)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
locations := map[artifact.ID]file.Location{}
|
||||
|
||||
for _, path := range fileOwner.OwnedFiles() {
|
||||
pathRefs, err := resolver.FilesByPath(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to find path for path=%q: %w", path, err)
|
||||
}
|
||||
|
||||
if len(pathRefs) == 0 {
|
||||
// ideally we want to warn users about missing files from a package, however, it is very common for
|
||||
// container image authors to delete files that are not needed in order to keep image sizes small. Adding
|
||||
// a warning here would be needlessly noisy (even for popular base images).
|
||||
continue
|
||||
}
|
||||
|
||||
for _, ref := range pathRefs {
|
||||
if oldRef, ok := locations[ref.Coordinates.ID()]; ok {
|
||||
log.Debugf("found path duplicate of %s", oldRef.RealPath)
|
||||
}
|
||||
locations[ref.Coordinates.ID()] = ref
|
||||
}
|
||||
}
|
||||
|
||||
var relationships []artifact.Relationship
|
||||
for _, location := range locations {
|
||||
relationships = append(relationships, artifact.Relationship{
|
||||
From: p,
|
||||
To: location.Coordinates,
|
||||
Type: artifact.ContainsRelationship,
|
||||
})
|
||||
}
|
||||
return relationships, nil
|
||||
}
|
||||
|
||||
func monitorPackageCatalogingTask() *monitor.CatalogerTaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "Catalog packages",
|
||||
WhileRunning: "Cataloging packages",
|
||||
OnSuccess: "Cataloged packages",
|
||||
},
|
||||
HideOnSuccess: false,
|
||||
}
|
||||
|
||||
return bus.StartCatalogerTask(info, -1, "")
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package cataloger
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/linux"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
func Test_CatalogPanicHandling(t *testing.T) {
|
||||
catalog, relationships, err := Catalog(
|
||||
file.NewMockResolverForPaths(),
|
||||
&linux.Release{},
|
||||
1,
|
||||
panickingCataloger{},
|
||||
returningCataloger{},
|
||||
)
|
||||
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "catalog_test.go")
|
||||
require.Len(t, catalog.Sorted(), 2)
|
||||
require.Len(t, relationships, 1)
|
||||
}
|
||||
|
||||
type panickingCataloger struct{}
|
||||
|
||||
func (p panickingCataloger) Name() string {
|
||||
return "panicking-cataloger"
|
||||
}
|
||||
|
||||
func (p panickingCataloger) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
panic("something bad happened")
|
||||
}
|
||||
|
||||
var _ pkg.Cataloger = (*panickingCataloger)(nil)
|
||||
|
||||
type returningCataloger struct{}
|
||||
|
||||
func (p returningCataloger) Name() string {
|
||||
return "returning-cataloger"
|
||||
}
|
||||
|
||||
func (p returningCataloger) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
pkg1 := pkg.Package{
|
||||
Name: "package-1",
|
||||
Version: "1.0",
|
||||
}
|
||||
pkg1.SetID()
|
||||
pkg2 := pkg.Package{
|
||||
Name: "package-2",
|
||||
Version: "2.0",
|
||||
}
|
||||
pkg2.SetID()
|
||||
return []pkg.Package{pkg1, pkg2}, []artifact.Relationship{
|
||||
{
|
||||
From: pkg1,
|
||||
To: pkg2,
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
var _ pkg.Cataloger = (*returningCataloger)(nil)
|
||||
@@ -1,212 +0,0 @@
|
||||
/*
|
||||
Package cataloger provides the ability to process files from a container image or file system and discover packages
|
||||
(gems, wheels, jars, rpms, debs, etc). Specifically, this package contains both a catalog function to utilize all
|
||||
catalogers defined in child packages as well as the interface definition to implement a cataloger.
|
||||
*/
|
||||
package cataloger
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/alpine"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/arch"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/binary"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/cpp"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/dart"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/debian"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/dotnet"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/elixir"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/erlang"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/gentoo"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/githubactions"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/golang"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/haskell"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/javascript"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/kernel"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/nix"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/php"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/python"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/r"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/redhat"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/ruby"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/rust"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/sbom"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/swift"
|
||||
)
|
||||
|
||||
const AllCatalogersPattern = "all"
|
||||
|
||||
// ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
|
||||
func ImageCatalogers(cfg Config) []pkg.Cataloger {
|
||||
return filterCatalogers([]pkg.Cataloger{
|
||||
arch.NewDBCataloger(),
|
||||
alpine.NewDBCataloger(),
|
||||
binary.NewCataloger(cfg.Binary),
|
||||
cpp.NewConanInfoCataloger(),
|
||||
debian.NewDBCataloger(),
|
||||
dotnet.NewDotnetPortableExecutableCataloger(),
|
||||
golang.NewGoModuleBinaryCataloger(cfg.Golang),
|
||||
java.NewArchiveCataloger(cfg.JavaConfig()),
|
||||
java.NewNativeImageCataloger(),
|
||||
javascript.NewPackageCataloger(),
|
||||
nix.NewStoreCataloger(),
|
||||
php.NewComposerInstalledCataloger(),
|
||||
gentoo.NewPortageCataloger(),
|
||||
python.NewInstalledPackageCataloger(),
|
||||
r.NewPackageCataloger(),
|
||||
redhat.NewDBCataloger(),
|
||||
ruby.NewInstalledGemSpecCataloger(),
|
||||
sbom.NewCataloger(),
|
||||
}, cfg.Catalogers)
|
||||
}
|
||||
|
||||
// DirectoryCatalogers returns a slice of locally implemented catalogers that are fit for detecting packages from index files (and select installations)
|
||||
func DirectoryCatalogers(cfg Config) []pkg.Cataloger {
|
||||
return filterCatalogers([]pkg.Cataloger{
|
||||
arch.NewDBCataloger(),
|
||||
alpine.NewDBCataloger(),
|
||||
binary.NewCataloger(cfg.Binary),
|
||||
cpp.NewConanCataloger(),
|
||||
dart.NewPubspecLockCataloger(),
|
||||
debian.NewDBCataloger(),
|
||||
dotnet.NewDotnetDepsCataloger(),
|
||||
dotnet.NewDotnetPortableExecutableCataloger(),
|
||||
elixir.NewMixLockCataloger(),
|
||||
erlang.NewRebarLockCataloger(),
|
||||
githubactions.NewActionUsageCataloger(),
|
||||
githubactions.NewWorkflowUsageCataloger(),
|
||||
golang.NewGoModuleFileCataloger(cfg.Golang),
|
||||
golang.NewGoModuleBinaryCataloger(cfg.Golang),
|
||||
haskell.NewHackageCataloger(),
|
||||
java.NewArchiveCataloger(cfg.JavaConfig()),
|
||||
java.NewGradleLockfileCataloger(),
|
||||
java.NewPomCataloger(cfg.JavaConfig()),
|
||||
java.NewNativeImageCataloger(),
|
||||
javascript.NewLockCataloger(cfg.Javascript),
|
||||
nix.NewStoreCataloger(),
|
||||
php.NewComposerLockCataloger(),
|
||||
gentoo.NewPortageCataloger(),
|
||||
python.NewPackageCataloger(cfg.Python),
|
||||
python.NewInstalledPackageCataloger(),
|
||||
redhat.NewArchiveCataloger(),
|
||||
redhat.NewDBCataloger(),
|
||||
ruby.NewGemFileLockCataloger(),
|
||||
ruby.NewGemSpecCataloger(),
|
||||
rust.NewCargoLockCataloger(),
|
||||
sbom.NewCataloger(),
|
||||
swift.NewCocoapodsCataloger(),
|
||||
swift.NewSwiftPackageManagerCataloger(),
|
||||
}, cfg.Catalogers)
|
||||
}
|
||||
|
||||
// AllCatalogers returns all implemented catalogers
|
||||
func AllCatalogers(cfg Config) []pkg.Cataloger {
|
||||
return filterCatalogers([]pkg.Cataloger{
|
||||
arch.NewDBCataloger(),
|
||||
alpine.NewDBCataloger(),
|
||||
binary.NewCataloger(cfg.Binary),
|
||||
cpp.NewConanCataloger(),
|
||||
dart.NewPubspecLockCataloger(),
|
||||
debian.NewDBCataloger(),
|
||||
dotnet.NewDotnetDepsCataloger(),
|
||||
dotnet.NewDotnetPortableExecutableCataloger(),
|
||||
elixir.NewMixLockCataloger(),
|
||||
erlang.NewRebarLockCataloger(),
|
||||
githubactions.NewActionUsageCataloger(),
|
||||
githubactions.NewWorkflowUsageCataloger(),
|
||||
golang.NewGoModuleFileCataloger(cfg.Golang),
|
||||
golang.NewGoModuleBinaryCataloger(cfg.Golang),
|
||||
haskell.NewHackageCataloger(),
|
||||
java.NewArchiveCataloger(cfg.JavaConfig()),
|
||||
java.NewGradleLockfileCataloger(),
|
||||
java.NewPomCataloger(cfg.JavaConfig()),
|
||||
java.NewNativeImageCataloger(),
|
||||
javascript.NewLockCataloger(cfg.Javascript),
|
||||
javascript.NewPackageCataloger(),
|
||||
kernel.NewLinuxKernelCataloger(cfg.LinuxKernel),
|
||||
nix.NewStoreCataloger(),
|
||||
php.NewComposerInstalledCataloger(),
|
||||
php.NewComposerLockCataloger(),
|
||||
gentoo.NewPortageCataloger(),
|
||||
python.NewPackageCataloger(cfg.Python),
|
||||
python.NewInstalledPackageCataloger(),
|
||||
r.NewPackageCataloger(),
|
||||
redhat.NewArchiveCataloger(),
|
||||
redhat.NewDBCataloger(),
|
||||
ruby.NewGemFileLockCataloger(),
|
||||
ruby.NewGemSpecCataloger(),
|
||||
ruby.NewInstalledGemSpecCataloger(),
|
||||
rust.NewAuditBinaryCataloger(),
|
||||
rust.NewCargoLockCataloger(),
|
||||
sbom.NewCataloger(),
|
||||
swift.NewCocoapodsCataloger(),
|
||||
swift.NewSwiftPackageManagerCataloger(),
|
||||
}, cfg.Catalogers)
|
||||
}
|
||||
|
||||
func RequestedAllCatalogers(cfg Config) bool {
|
||||
for _, enableCatalogerPattern := range cfg.Catalogers {
|
||||
if enableCatalogerPattern == AllCatalogersPattern {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func filterCatalogers(catalogers []pkg.Cataloger, enabledCatalogerPatterns []string) []pkg.Cataloger {
|
||||
// if cataloger is not set, all applicable catalogers are enabled by default
|
||||
if len(enabledCatalogerPatterns) == 0 {
|
||||
return catalogers
|
||||
}
|
||||
for _, enableCatalogerPattern := range enabledCatalogerPatterns {
|
||||
if enableCatalogerPattern == AllCatalogersPattern {
|
||||
return catalogers
|
||||
}
|
||||
}
|
||||
var keepCatalogers []pkg.Cataloger
|
||||
for _, cataloger := range catalogers {
|
||||
if contains(enabledCatalogerPatterns, cataloger.Name()) {
|
||||
keepCatalogers = append(keepCatalogers, cataloger)
|
||||
continue
|
||||
}
|
||||
log.Infof("skipping cataloger %q", cataloger.Name())
|
||||
}
|
||||
return keepCatalogers
|
||||
}
|
||||
|
||||
func contains(enabledPartial []string, catalogerName string) bool {
|
||||
catalogerName = strings.TrimSuffix(catalogerName, "-cataloger")
|
||||
for _, partial := range enabledPartial {
|
||||
partial = strings.TrimSuffix(partial, "-cataloger")
|
||||
if partial == "" {
|
||||
continue
|
||||
}
|
||||
if hasFullWord(partial, catalogerName) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasFullWord(targetPhrase, candidate string) bool {
|
||||
if targetPhrase == "cataloger" || targetPhrase == "" {
|
||||
return false
|
||||
}
|
||||
start := strings.Index(candidate, targetPhrase)
|
||||
if start == -1 {
|
||||
return false
|
||||
}
|
||||
|
||||
if start > 0 && candidate[start-1] != '-' {
|
||||
return false
|
||||
}
|
||||
|
||||
end := start + len(targetPhrase)
|
||||
if end < len(candidate) && candidate[end] != '-' {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -1,342 +0,0 @@
|
||||
package cataloger
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
var _ pkg.Cataloger = (*dummy)(nil)
|
||||
|
||||
type dummy struct {
|
||||
name string
|
||||
}
|
||||
|
||||
func (d dummy) Name() string {
|
||||
return d.name
|
||||
}
|
||||
|
||||
func (d dummy) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
panic("not implemented")
|
||||
}
|
||||
|
||||
func Test_filterCatalogers(t *testing.T) {
|
||||
largeCatalogerList := []string{
|
||||
"alpm-db-cataloger",
|
||||
"apkdb-cataloger",
|
||||
"binary-cataloger",
|
||||
"conan-cataloger",
|
||||
"dartlang-lock-cataloger",
|
||||
"dpkg-db-cataloger",
|
||||
"dotnet-deps-cataloger",
|
||||
"elixir-mix-lock-cataloger",
|
||||
"erlang-rebar-lock-cataloger",
|
||||
"go-module-file-cataloger",
|
||||
"go-module-binary-cataloger",
|
||||
"haskell-cataloger",
|
||||
"graalvm-native-image-cataloger",
|
||||
"java-cataloger",
|
||||
"java-pom-cataloger",
|
||||
"javascript-package-cataloger",
|
||||
"javascript-lock-cataloger",
|
||||
"php-composer-installed-cataloger",
|
||||
"php-composer-lock-cataloger",
|
||||
"portage-cataloger",
|
||||
"python-package-cataloger",
|
||||
"python-installed-package-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
"rpm-archive-cataloger",
|
||||
"ruby-gemfile-cataloger",
|
||||
"ruby-installed-gemspec-cataloger",
|
||||
"rust-cargo-lock-cataloger",
|
||||
"cargo-auditable-binary-cataloger",
|
||||
"sbom-cataloger",
|
||||
"cocoapods-cataloger",
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
patterns []string
|
||||
catalogers []string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "no filtering",
|
||||
patterns: nil,
|
||||
catalogers: []string{
|
||||
"ruby-installed-gemspec-cataloger",
|
||||
"python-installed-package-cataloger",
|
||||
"php-composer-installed-cataloger",
|
||||
"javascript-package-cataloger",
|
||||
"dpkg-db-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
"java-cataloger",
|
||||
"apkdb-cataloger",
|
||||
"go-module-binary-cataloger",
|
||||
},
|
||||
want: []string{
|
||||
"ruby-installed-gemspec-cataloger",
|
||||
"python-installed-package-cataloger",
|
||||
"php-composer-installed-cataloger",
|
||||
"javascript-package-cataloger",
|
||||
"dpkg-db-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
"java-cataloger",
|
||||
"apkdb-cataloger",
|
||||
"go-module-binary-cataloger",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "exact name match",
|
||||
patterns: []string{
|
||||
"rpm-db-cataloger",
|
||||
"javascript-package-cataloger",
|
||||
},
|
||||
catalogers: []string{
|
||||
"ruby-installed-gemspec-cataloger",
|
||||
"python-installed-package-cataloger",
|
||||
"php-composer-installed-cataloger",
|
||||
"javascript-package-cataloger",
|
||||
"dpkg-db-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
"java-cataloger",
|
||||
"apkdb-cataloger",
|
||||
"go-module-binary-cataloger",
|
||||
},
|
||||
want: []string{
|
||||
"javascript-package-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "partial name match",
|
||||
patterns: []string{
|
||||
"ruby",
|
||||
"installed",
|
||||
},
|
||||
catalogers: []string{
|
||||
"ruby-installed-gemspec-cataloger",
|
||||
"ruby-gemfile-cataloger",
|
||||
"python-installed-package-cataloger",
|
||||
"php-composer-installed-cataloger",
|
||||
"javascript-package-cataloger",
|
||||
"dpkg-db-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
"java-cataloger",
|
||||
"apkdb-cataloger",
|
||||
"go-module-binary-cataloger",
|
||||
},
|
||||
want: []string{
|
||||
"php-composer-installed-cataloger",
|
||||
"python-installed-package-cataloger",
|
||||
"ruby-installed-gemspec-cataloger",
|
||||
"ruby-gemfile-cataloger",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ignore 'cataloger' keyword",
|
||||
patterns: []string{
|
||||
"cataloger",
|
||||
},
|
||||
catalogers: []string{
|
||||
"ruby-installed-gemspec-cataloger",
|
||||
"ruby-gemfile-cataloger",
|
||||
"python-installed-package-cataloger",
|
||||
"php-composer-installed-cataloger",
|
||||
"javascript-package-cataloger",
|
||||
"dpkg-db-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
"java-cataloger",
|
||||
"apkdb-cataloger",
|
||||
"go-module-binary-cataloger",
|
||||
},
|
||||
want: []string{},
|
||||
},
|
||||
{
|
||||
name: "only some patterns match",
|
||||
patterns: []string{
|
||||
"cataloger",
|
||||
"go-module",
|
||||
},
|
||||
catalogers: []string{
|
||||
"ruby-installed-gemspec-cataloger",
|
||||
"ruby-gemfile-cataloger",
|
||||
"python-installed-package-cataloger",
|
||||
"php-composer-installed-cataloger",
|
||||
"javascript-package-cataloger",
|
||||
"dpkg-db-cataloger",
|
||||
"rpm-db-cataloger",
|
||||
"java-cataloger",
|
||||
"apkdb-cataloger",
|
||||
"go-module-binary-cataloger",
|
||||
},
|
||||
want: []string{
|
||||
"go-module-binary-cataloger",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "don't cross match ecosystems with matching prefix",
|
||||
patterns: []string{
|
||||
"java-cataloger",
|
||||
},
|
||||
catalogers: []string{
|
||||
"javascript-package-cataloger",
|
||||
"java-cataloger",
|
||||
},
|
||||
want: []string{
|
||||
"java-cataloger",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "don't cross match ecosystems with short, common name",
|
||||
patterns: []string{
|
||||
"go",
|
||||
},
|
||||
catalogers: largeCatalogerList,
|
||||
want: []string{
|
||||
"go-module-file-cataloger",
|
||||
"go-module-binary-cataloger",
|
||||
//"rust-cargo-lock-cataloger", // with naive "contains" matching
|
||||
//"cargo-auditable-binary-cataloger", // with naive "contains" matching
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ignore partial matches",
|
||||
patterns: []string{
|
||||
"mod",
|
||||
},
|
||||
catalogers: largeCatalogerList,
|
||||
want: []string{
|
||||
// important! these are NOT found
|
||||
//"go-module-file-cataloger",
|
||||
//"go-module-binary-cataloger",
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var catalogers []pkg.Cataloger
|
||||
for _, n := range tt.catalogers {
|
||||
catalogers = append(catalogers, dummy{name: n})
|
||||
}
|
||||
got := filterCatalogers(catalogers, tt.patterns)
|
||||
var gotNames []string
|
||||
for _, g := range got {
|
||||
gotNames = append(gotNames, g.Name())
|
||||
}
|
||||
assert.ElementsMatch(t, tt.want, gotNames)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_contains(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
enabledCatalogers []string
|
||||
catalogerName string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "keep exact match",
|
||||
enabledCatalogers: []string{
|
||||
"php-composer-installed-cataloger",
|
||||
},
|
||||
catalogerName: "php-composer-installed-cataloger",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "match substring",
|
||||
enabledCatalogers: []string{
|
||||
"python",
|
||||
},
|
||||
catalogerName: "python-installed-package-cataloger",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "dont match on 'cataloger'",
|
||||
enabledCatalogers: []string{
|
||||
"cataloger",
|
||||
},
|
||||
catalogerName: "python-installed-package-cataloger",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, contains(tt.enabledCatalogers, tt.catalogerName))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_hasFullWord(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
targetPhrase string
|
||||
candidate string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "exact match",
|
||||
targetPhrase: "php-composer-installed-cataloger",
|
||||
candidate: "php-composer-installed-cataloger",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "partial, full word match",
|
||||
targetPhrase: "composer",
|
||||
candidate: "php-composer-installed-cataloger",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "partial, full, multi-word match",
|
||||
targetPhrase: "php-composer",
|
||||
candidate: "php-composer-installed-cataloger",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "prefix match",
|
||||
targetPhrase: "php",
|
||||
candidate: "php-composer-installed-cataloger",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "postfix match with -cataloger suffix",
|
||||
targetPhrase: "installed",
|
||||
candidate: "php-composer-installed-cataloger",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "postfix match",
|
||||
targetPhrase: "installed",
|
||||
candidate: "php-composer-installed",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "ignore cataloger keyword",
|
||||
targetPhrase: "cataloger",
|
||||
candidate: "php-composer-installed-cataloger",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "ignore partial match",
|
||||
targetPhrase: "hp",
|
||||
candidate: "php-composer-installed-cataloger",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "ignore empty string",
|
||||
targetPhrase: "",
|
||||
candidate: "php-composer-installed-cataloger",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equalf(t, tt.want, hasFullWord(tt.targetPhrase, tt.candidate), "hasFullWord(%v, %v)", tt.targetPhrase, tt.candidate)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package cataloger
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/binary"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/golang"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/javascript"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/kernel"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/python"
|
||||
)
|
||||
|
||||
// TODO: these field naming vs helper function naming schemes are inconsistent.
|
||||
type Config struct {
|
||||
Search SearchConfig
|
||||
Golang golang.CatalogerConfig
|
||||
LinuxKernel kernel.LinuxKernelCatalogerConfig
|
||||
Python python.CatalogerConfig
|
||||
Java java.ArchiveCatalogerConfig
|
||||
Javascript javascript.CatalogerConfig
|
||||
Binary binary.CatalogerConfig
|
||||
Catalogers []string
|
||||
Parallelism int
|
||||
ExcludeBinaryOverlapByOwnership bool
|
||||
}
|
||||
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
Search: DefaultSearchConfig(),
|
||||
Parallelism: 1,
|
||||
LinuxKernel: kernel.DefaultLinuxCatalogerConfig(),
|
||||
Python: python.DefaultCatalogerConfig(),
|
||||
Java: java.DefaultArchiveCatalogerConfig(),
|
||||
Javascript: javascript.DefaultCatalogerConfig(),
|
||||
Binary: binary.DefaultCatalogerConfig(),
|
||||
ExcludeBinaryOverlapByOwnership: true,
|
||||
}
|
||||
}
|
||||
|
||||
// JavaConfig merges relevant config values from Config to return a java.Config struct.
|
||||
// Values like IncludeUnindexedArchives and IncludeIndexedArchives are used across catalogers
|
||||
// and are not specific to Java requiring this merge.
|
||||
func (c Config) JavaConfig() java.ArchiveCatalogerConfig {
|
||||
return java.ArchiveCatalogerConfig{
|
||||
ArchiveSearchConfig: cataloging.ArchiveSearchConfig{
|
||||
IncludeUnindexedArchives: c.Search.IncludeUnindexedArchives,
|
||||
IncludeIndexedArchives: c.Search.IncludeIndexedArchives,
|
||||
},
|
||||
UseNetwork: c.Java.UseNetwork,
|
||||
MavenBaseURL: c.Java.MavenBaseURL,
|
||||
MaxParentRecursiveDepth: c.Java.MaxParentRecursiveDepth,
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"sort"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/relationship"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
@@ -122,7 +123,7 @@ func parseDotnetDeps(_ file.Resolver, _ *generic.Environment, reader file.Locati
|
||||
// sort the relationships for deterministic output
|
||||
// TODO: ideally this would be replaced with artifact.SortRelationships when one exists and is type agnostic.
|
||||
// this will only consider package-to-package relationships.
|
||||
pkg.SortRelationships(relationships)
|
||||
relationship.Sort(relationships)
|
||||
|
||||
return pkgs, relationships, nil
|
||||
}
|
||||
|
||||
@@ -18,13 +18,18 @@ import (
|
||||
|
||||
var versionCandidateGroups = regexp.MustCompile(`(?P<version>\d+(\.\d+)?(\.\d+)?)(?P<candidate>\w*)`)
|
||||
|
||||
const (
|
||||
modFileCatalogerName = "go-module-file-cataloger"
|
||||
binaryCatalogerName = "go-module-binary-cataloger"
|
||||
)
|
||||
|
||||
// NewGoModuleFileCataloger returns a new cataloger object that searches within go.mod files.
|
||||
func NewGoModuleFileCataloger(opts CatalogerConfig) pkg.Cataloger {
|
||||
c := goModCataloger{
|
||||
licenses: newGoLicenses(opts),
|
||||
licenses: newGoLicenses(modFileCatalogerName, opts),
|
||||
}
|
||||
return &progressingCataloger{
|
||||
cataloger: generic.NewCataloger("go-module-file-cataloger").
|
||||
cataloger: generic.NewCataloger(modFileCatalogerName).
|
||||
WithParserByGlobs(c.parseGoModFile, "**/go.mod"),
|
||||
}
|
||||
}
|
||||
@@ -32,10 +37,10 @@ func NewGoModuleFileCataloger(opts CatalogerConfig) pkg.Cataloger {
|
||||
// NewGoModuleBinaryCataloger returns a new cataloger object that searches within binaries built by the go compiler.
|
||||
func NewGoModuleBinaryCataloger(opts CatalogerConfig) pkg.Cataloger {
|
||||
c := goBinaryCataloger{
|
||||
licenses: newGoLicenses(opts),
|
||||
licenses: newGoLicenses(binaryCatalogerName, opts),
|
||||
}
|
||||
return &progressingCataloger{
|
||||
cataloger: generic.NewCataloger("go-module-binary-cataloger").
|
||||
cataloger: generic.NewCataloger(binaryCatalogerName).
|
||||
WithParserByMimeTypes(c.parseGoBinary, internal.ExecutableMIMETypeSet.List()...),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,13 +30,15 @@ import (
|
||||
)
|
||||
|
||||
type goLicenses struct {
|
||||
catalogerName string
|
||||
opts CatalogerConfig
|
||||
localModCacheResolver file.WritableResolver
|
||||
lowerLicenseFileNames *strset.Set
|
||||
}
|
||||
|
||||
func newGoLicenses(opts CatalogerConfig) goLicenses {
|
||||
func newGoLicenses(catalogerName string, opts CatalogerConfig) goLicenses {
|
||||
return goLicenses{
|
||||
catalogerName: catalogerName,
|
||||
opts: opts,
|
||||
localModCacheResolver: modCacheResolver(opts.LocalModCacheDir),
|
||||
lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...),
|
||||
@@ -125,6 +127,7 @@ func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([]
|
||||
OnSuccess: "Downloaded go mod",
|
||||
},
|
||||
HideOnSuccess: true,
|
||||
ParentID: c.catalogerName,
|
||||
}, -1, "")
|
||||
|
||||
fsys, err := getModule(prog, proxies, moduleName, moduleVersion)
|
||||
|
||||
@@ -67,6 +67,7 @@ func Test_LocalLicenseSearch(t *testing.T) {
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
l := newGoLicenses(
|
||||
"",
|
||||
CatalogerConfig{
|
||||
SearchLocalModCacheLicenses: true,
|
||||
LocalModCacheDir: path.Join(wd, "test-fixtures", "licenses", "pkg", "mod"),
|
||||
@@ -154,11 +155,14 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
l := newGoLicenses(CatalogerConfig{
|
||||
SearchRemoteLicenses: true,
|
||||
Proxies: []string{server.URL},
|
||||
LocalModCacheDir: modDir,
|
||||
})
|
||||
l := newGoLicenses(
|
||||
"",
|
||||
CatalogerConfig{
|
||||
SearchRemoteLicenses: true,
|
||||
Proxies: []string{server.URL},
|
||||
LocalModCacheDir: modDir,
|
||||
},
|
||||
)
|
||||
|
||||
licenses, err := l.getLicenses(fileresolver.Empty{}, test.name, test.version)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
"github.com/anchore/syft/internal/relationship"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/linux"
|
||||
@@ -346,8 +347,8 @@ func (p *CatalogTester) assertPkgs(t *testing.T, pkgs []pkg.Package, relationshi
|
||||
opts = append(opts, cmp.Reporter(&r))
|
||||
|
||||
// order should not matter
|
||||
pkg.SortRelationships(p.expectedRelationships)
|
||||
pkg.SortRelationships(relationships)
|
||||
relationship.Sort(p.expectedRelationships)
|
||||
relationship.Sort(relationships)
|
||||
|
||||
if diff := cmp.Diff(p.expectedRelationships, relationships, opts...); diff != "" {
|
||||
t.Log("Specific Differences:\n" + r.String())
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
package cataloger
|
||||
|
||||
import (
|
||||
"slices"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
var (
|
||||
osCatalogerTypes = []pkg.Type{
|
||||
pkg.AlpmPkg,
|
||||
pkg.ApkPkg,
|
||||
pkg.DebPkg,
|
||||
pkg.NixPkg,
|
||||
pkg.PortagePkg,
|
||||
pkg.RpmPkg,
|
||||
}
|
||||
binaryCatalogerTypes = []pkg.Type{
|
||||
pkg.BinaryPkg,
|
||||
}
|
||||
)
|
||||
|
||||
// ExcludeBinaryByFileOwnershipOverlap will remove packages from a collection given the following properties are true
|
||||
// 1) the relationship between packages is OwnershipByFileOverlap
|
||||
// 2) the parent is an "os" package
|
||||
// 3) the child is a synthetic package generated by the binary cataloger
|
||||
// 4) the package names are identical
|
||||
// This was implemented as a way to help resolve: https://github.com/anchore/syft/issues/931
|
||||
func ExcludeBinaryByFileOwnershipOverlap(r artifact.Relationship, c *pkg.Collection) bool {
|
||||
if artifact.OwnershipByFileOverlapRelationship != r.Type {
|
||||
return false
|
||||
}
|
||||
|
||||
parent := c.Package(r.From.ID())
|
||||
if parent == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
parentInExclusion := slices.Contains(osCatalogerTypes, parent.Type)
|
||||
if !parentInExclusion {
|
||||
return false
|
||||
}
|
||||
|
||||
child := c.Package(r.To.ID())
|
||||
if child == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return slices.Contains(binaryCatalogerTypes, child.Type)
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package cataloger
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
func TestExclude(t *testing.T) {
|
||||
packageA := pkg.Package{Name: "package-a", Type: pkg.ApkPkg}
|
||||
packageB := pkg.Package{Name: "package-a", Type: pkg.PythonPkg}
|
||||
packageC := pkg.Package{Name: "package-a", Type: pkg.BinaryPkg}
|
||||
packageD := pkg.Package{Name: "package-d", Type: pkg.BinaryPkg}
|
||||
for _, p := range []*pkg.Package{&packageA, &packageB, &packageC, &packageD} {
|
||||
p := p
|
||||
p.SetID()
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
relationship artifact.Relationship
|
||||
packages *pkg.Collection
|
||||
shouldExclude bool
|
||||
}{
|
||||
{
|
||||
name: "no exclusions from os -> python",
|
||||
relationship: artifact.Relationship{
|
||||
Type: artifact.OwnershipByFileOverlapRelationship,
|
||||
From: packageA,
|
||||
To: packageB,
|
||||
},
|
||||
packages: pkg.NewCollection(packageA, packageB),
|
||||
shouldExclude: false,
|
||||
},
|
||||
{
|
||||
name: "exclusions from os -> binary",
|
||||
relationship: artifact.Relationship{
|
||||
Type: artifact.OwnershipByFileOverlapRelationship,
|
||||
From: packageA,
|
||||
To: packageC,
|
||||
},
|
||||
packages: pkg.NewCollection(packageA, packageC),
|
||||
shouldExclude: true,
|
||||
},
|
||||
{
|
||||
name: "no exclusions from python -> binary",
|
||||
relationship: artifact.Relationship{
|
||||
Type: artifact.OwnershipByFileOverlapRelationship,
|
||||
From: packageB,
|
||||
To: packageC,
|
||||
},
|
||||
packages: pkg.NewCollection(packageB, packageC),
|
||||
shouldExclude: false,
|
||||
},
|
||||
{
|
||||
name: "no exclusions for different package names",
|
||||
relationship: artifact.Relationship{
|
||||
Type: artifact.OwnershipByFileOverlapRelationship,
|
||||
From: packageA,
|
||||
To: packageD,
|
||||
},
|
||||
packages: pkg.NewCollection(packageA, packageD),
|
||||
shouldExclude: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if !ExcludeBinaryByFileOwnershipOverlap(test.relationship, test.packages) && test.shouldExclude {
|
||||
t.Errorf("expected to exclude relationship %+v", test.relationship)
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
package cataloger
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
type SearchConfig struct {
|
||||
IncludeIndexedArchives bool
|
||||
IncludeUnindexedArchives bool
|
||||
Scope source.Scope
|
||||
}
|
||||
|
||||
func DefaultSearchConfig() SearchConfig {
|
||||
return SearchConfig{
|
||||
IncludeIndexedArchives: true,
|
||||
IncludeUnindexedArchives: false,
|
||||
Scope: source.SquashedScope,
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package pkg
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
)
|
||||
|
||||
func NewRelationships(catalog *Collection) []artifact.Relationship {
|
||||
rels := RelationshipsByFileOwnership(catalog)
|
||||
rels = append(rels, RelationshipsEvidentBy(catalog)...)
|
||||
return rels
|
||||
}
|
||||
|
||||
// SortRelationships takes a set of package-to-package relationships and sorts them in a stable order by name and version.
|
||||
// Note: this does not consider package-to-other, other-to-package, or other-to-other relationships.
|
||||
// TODO: ideally this should be replaced with a more type-agnostic sort function that resides in the artifact package.
|
||||
func SortRelationships(rels []artifact.Relationship) {
|
||||
sort.SliceStable(rels, func(i, j int) bool {
|
||||
return relationshipLess(rels[i], rels[j])
|
||||
})
|
||||
}
|
||||
|
||||
func relationshipLess(i, j artifact.Relationship) bool {
|
||||
iFrom, ok1 := i.From.(Package)
|
||||
iTo, ok2 := i.To.(Package)
|
||||
jFrom, ok3 := j.From.(Package)
|
||||
jTo, ok4 := j.To.(Package)
|
||||
|
||||
if !(ok1 && ok2 && ok3 && ok4) {
|
||||
return false
|
||||
}
|
||||
|
||||
if iFrom.Name != jFrom.Name {
|
||||
return iFrom.Name < jFrom.Name
|
||||
}
|
||||
if iFrom.Version != jFrom.Version {
|
||||
return iFrom.Version < jFrom.Version
|
||||
}
|
||||
if iTo.Name != jTo.Name {
|
||||
return iTo.Name < jTo.Name
|
||||
}
|
||||
if iTo.Version != jTo.Version {
|
||||
return iTo.Version < jTo.Version
|
||||
}
|
||||
return i.Type < j.Type
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package pkg
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
)
|
||||
|
||||
// AltRpmDBGlob allows db matches against new locations introduced in fedora:{36,37}
|
||||
// See https://github.com/anchore/syft/issues/1077 for larger context
|
||||
const AltRpmDBGlob = "**/rpm/{Packages,Packages.db,rpmdb.sqlite}"
|
||||
|
||||
var globsForbiddenFromBeingOwned = []string{
|
||||
// any OS DBs should automatically be ignored to prevent cyclic issues (e.g. the "rpm" RPM owns the path to the
|
||||
// RPM DB, so if not ignored that package would own all other packages on the system).
|
||||
ApkDBGlob,
|
||||
DpkgDBGlob,
|
||||
RpmDBGlob,
|
||||
AltRpmDBGlob,
|
||||
// DEB packages share common copyright info between, this does not mean that sharing these paths implies ownership.
|
||||
"/usr/share/doc/**/copyright",
|
||||
}
|
||||
|
||||
type ownershipByFilesMetadata struct {
|
||||
Files []string `json:"files"`
|
||||
}
|
||||
|
||||
// RelationshipsByFileOwnership creates a package-to-package relationship based on discovering which packages have
|
||||
// evidence locations that overlap with ownership claim from another package's package manager metadata.
|
||||
func RelationshipsByFileOwnership(catalog *Collection) []artifact.Relationship {
|
||||
var relationships = findOwnershipByFilesRelationships(catalog)
|
||||
|
||||
var edges []artifact.Relationship
|
||||
for parentID, children := range relationships {
|
||||
for childID, files := range children {
|
||||
fs := files.List()
|
||||
sort.Strings(fs)
|
||||
edges = append(edges, artifact.Relationship{
|
||||
From: catalog.byID[parentID],
|
||||
To: catalog.byID[childID],
|
||||
Type: artifact.OwnershipByFileOverlapRelationship,
|
||||
Data: ownershipByFilesMetadata{
|
||||
Files: fs,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return edges
|
||||
}
|
||||
|
||||
// findOwnershipByFilesRelationships find overlaps in file ownership with a file that defines another package. Specifically, a .Location.Path of
|
||||
// a package is found to be owned by another (from the owner's .Metadata.Files[]).
|
||||
func findOwnershipByFilesRelationships(catalog *Collection) map[artifact.ID]map[artifact.ID]*strset.Set {
|
||||
var relationships = make(map[artifact.ID]map[artifact.ID]*strset.Set)
|
||||
|
||||
if catalog == nil {
|
||||
return relationships
|
||||
}
|
||||
|
||||
for _, candidateOwnerPkg := range catalog.Sorted() {
|
||||
id := candidateOwnerPkg.ID()
|
||||
if candidateOwnerPkg.Metadata == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// check to see if this is a file owner
|
||||
pkgFileOwner, ok := candidateOwnerPkg.Metadata.(FileOwner)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, ownedFilePath := range pkgFileOwner.OwnedFiles() {
|
||||
if matchesAny(ownedFilePath, globsForbiddenFromBeingOwned) {
|
||||
// we skip over known exceptions to file ownership, such as the RPM package owning
|
||||
// the RPM DB path, otherwise the RPM package would "own" all RPMs, which is not intended
|
||||
continue
|
||||
}
|
||||
|
||||
// look for package(s) in the catalog that may be owned by this package and mark the relationship
|
||||
for _, subPackage := range catalog.PackagesByPath(ownedFilePath) {
|
||||
subID := subPackage.ID()
|
||||
if subID == id {
|
||||
continue
|
||||
}
|
||||
if _, exists := relationships[id]; !exists {
|
||||
relationships[id] = make(map[artifact.ID]*strset.Set)
|
||||
}
|
||||
|
||||
if _, exists := relationships[id][subID]; !exists {
|
||||
relationships[id][subID] = strset.New()
|
||||
}
|
||||
relationships[id][subID].Add(ownedFilePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return relationships
|
||||
}
|
||||
|
||||
func matchesAny(s string, globs []string) bool {
|
||||
for _, g := range globs {
|
||||
matches, err := doublestar.Match(g, s)
|
||||
if err != nil {
|
||||
log.Errorf("failed to match glob=%q : %+v", g, err)
|
||||
}
|
||||
if matches {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
package pkg
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
func TestOwnershipByFilesRelationship(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(t testing.TB) ([]Package, []artifact.Relationship)
|
||||
}{
|
||||
{
|
||||
name: "owns-by-real-path",
|
||||
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
|
||||
parent := Package{
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewVirtualLocation("/a/path", "/another/path"),
|
||||
file.NewVirtualLocation("/b/path", "/bee/path"),
|
||||
),
|
||||
Type: RpmPkg,
|
||||
Metadata: RpmDBEntry{
|
||||
Files: []RpmFileRecord{
|
||||
{Path: "/owning/path/1"},
|
||||
{Path: "/owning/path/2"},
|
||||
{Path: "/d/path"},
|
||||
},
|
||||
},
|
||||
}
|
||||
parent.SetID()
|
||||
|
||||
child := Package{
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewVirtualLocation("/c/path", "/another/path"),
|
||||
file.NewVirtualLocation("/d/path", "/another/path"),
|
||||
),
|
||||
Type: NpmPkg,
|
||||
}
|
||||
child.SetID()
|
||||
|
||||
relationship := artifact.Relationship{
|
||||
From: parent,
|
||||
To: child,
|
||||
Type: artifact.OwnershipByFileOverlapRelationship,
|
||||
Data: ownershipByFilesMetadata{
|
||||
Files: []string{
|
||||
"/d/path",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return []Package{parent, child}, []artifact.Relationship{relationship}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "owns-by-virtual-path",
|
||||
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
|
||||
parent := Package{
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewVirtualLocation("/a/path", "/some/other/path"),
|
||||
file.NewVirtualLocation("/b/path", "/bee/path"),
|
||||
),
|
||||
Type: RpmPkg,
|
||||
Metadata: RpmDBEntry{
|
||||
Files: []RpmFileRecord{
|
||||
{Path: "/owning/path/1"},
|
||||
{Path: "/owning/path/2"},
|
||||
{Path: "/another/path"},
|
||||
},
|
||||
},
|
||||
}
|
||||
parent.SetID()
|
||||
|
||||
child := Package{
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewVirtualLocation("/c/path", "/another/path"),
|
||||
file.NewLocation("/d/path"),
|
||||
),
|
||||
Type: NpmPkg,
|
||||
}
|
||||
child.SetID()
|
||||
|
||||
relationship := artifact.Relationship{
|
||||
From: parent,
|
||||
To: child,
|
||||
Type: artifact.OwnershipByFileOverlapRelationship,
|
||||
Data: ownershipByFilesMetadata{
|
||||
Files: []string{
|
||||
"/another/path",
|
||||
},
|
||||
},
|
||||
}
|
||||
return []Package{parent, child}, []artifact.Relationship{relationship}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ignore-empty-path",
|
||||
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
|
||||
parent := Package{
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewVirtualLocation("/a/path", "/some/other/path"),
|
||||
file.NewVirtualLocation("/b/path", "/bee/path"),
|
||||
),
|
||||
Type: RpmPkg,
|
||||
Metadata: RpmDBEntry{
|
||||
Files: []RpmFileRecord{
|
||||
{Path: "/owning/path/1"},
|
||||
{Path: "/owning/path/2"},
|
||||
{Path: ""},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
parent.SetID()
|
||||
|
||||
child := Package{
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewVirtualLocation("/c/path", "/another/path"),
|
||||
file.NewLocation("/d/path"),
|
||||
),
|
||||
Type: NpmPkg,
|
||||
}
|
||||
|
||||
child.SetID()
|
||||
|
||||
return []Package{parent, child}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
pkgs, expectedRelations := test.setup(t)
|
||||
c := NewCollection(pkgs...)
|
||||
relationships := RelationshipsByFileOwnership(c)
|
||||
|
||||
assert.Len(t, relationships, len(expectedRelations))
|
||||
for idx, expectedRelationship := range expectedRelations {
|
||||
actualRelationship := relationships[idx]
|
||||
assert.Equal(t, expectedRelationship.From.ID(), actualRelationship.From.ID())
|
||||
assert.Equal(t, expectedRelationship.To.ID(), actualRelationship.To.ID())
|
||||
assert.Equal(t, expectedRelationship.Type, actualRelationship.Type)
|
||||
assert.Equal(t, expectedRelationship.Data, actualRelationship.Data)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package pkg
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
)
|
||||
|
||||
func RelationshipsEvidentBy(catalog *Collection) []artifact.Relationship {
|
||||
var edges []artifact.Relationship
|
||||
for _, p := range catalog.Sorted() {
|
||||
for _, l := range p.Locations.ToSlice() {
|
||||
if v, exists := l.Annotations[EvidenceAnnotationKey]; !exists || v != PrimaryEvidenceAnnotation {
|
||||
// skip non-primary evidence from being expressed as a relationship.
|
||||
// note: this may be configurable in the future.
|
||||
continue
|
||||
}
|
||||
edges = append(edges, artifact.Relationship{
|
||||
From: p,
|
||||
To: l.Coordinates,
|
||||
Type: artifact.EvidentByRelationship,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return edges
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
package pkg
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
func TestRelationshipsEvidentBy(t *testing.T) {
|
||||
|
||||
c := NewCollection()
|
||||
|
||||
coordA := file.Coordinates{
|
||||
RealPath: "/somewhere/real",
|
||||
FileSystemID: "abc",
|
||||
}
|
||||
coordC := file.Coordinates{
|
||||
RealPath: "/somewhere/real",
|
||||
FileSystemID: "abc",
|
||||
}
|
||||
coordD := file.Coordinates{
|
||||
RealPath: "/somewhere/real",
|
||||
FileSystemID: "abc",
|
||||
}
|
||||
pkgA := Package{
|
||||
Locations: file.NewLocationSet(
|
||||
// added!
|
||||
file.NewLocationFromCoordinates(coordA).WithAnnotation(EvidenceAnnotationKey, PrimaryEvidenceAnnotation),
|
||||
// ignored...
|
||||
file.NewLocationFromCoordinates(coordC).WithAnnotation(EvidenceAnnotationKey, SupportingEvidenceAnnotation),
|
||||
file.NewLocationFromCoordinates(coordD),
|
||||
),
|
||||
}
|
||||
pkgA.SetID()
|
||||
c.Add(pkgA)
|
||||
|
||||
coordB := file.Coordinates{
|
||||
RealPath: "/somewhere-else/real",
|
||||
FileSystemID: "def",
|
||||
}
|
||||
pkgB := Package{
|
||||
Locations: file.NewLocationSet(
|
||||
// added!
|
||||
file.NewLocationFromCoordinates(coordB).WithAnnotation(EvidenceAnnotationKey, PrimaryEvidenceAnnotation),
|
||||
),
|
||||
}
|
||||
pkgB.SetID()
|
||||
c.Add(pkgB)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
catalog *Collection
|
||||
want []artifact.Relationship
|
||||
}{
|
||||
{
|
||||
name: "go case",
|
||||
catalog: c,
|
||||
want: []artifact.Relationship{
|
||||
{
|
||||
From: pkgB,
|
||||
To: coordB,
|
||||
Type: artifact.EvidentByRelationship,
|
||||
},
|
||||
{
|
||||
From: pkgA,
|
||||
To: coordA,
|
||||
Type: artifact.EvidentByRelationship,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
actual := RelationshipsEvidentBy(tt.catalog)
|
||||
require.Len(t, actual, len(tt.want))
|
||||
for i := range actual {
|
||||
assert.Equal(t, tt.want[i].From.ID(), actual[i].From.ID(), "from mismatch at index %d", i)
|
||||
assert.Equal(t, tt.want[i].To.ID(), actual[i].To.ID(), "to mismatch at index %d", i)
|
||||
assert.Equal(t, tt.want[i].Type, actual[i].Type, "type mismatch at index %d", i)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user