Replace core SBOM-creation API with builder pattern (#1383)

* remove existing cataloging API

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add file cataloging config

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add package cataloging config

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add configs for cross-cutting concerns

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename CLI option configs to not require import aliases later

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update all nested structs for the Catalog struct

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update Catalog cli options

- add new cataloger selection options (selection and default)
- remove the excludeBinaryOverlapByOwnership
- deprecate "catalogers" flag
- add new javascript configuration

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* migrate relationship capabilities to separate internal package

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor golang cataloger to use configuration options when creating packages

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* create internal object to facilitate reading from and writing to an SBOM

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* create a command-like object (task) to facilitate partial SBOM creation

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add cataloger selection capability

- be able to parse string expressions into a set of resolved actions against sets
- be able to use expressions to select/add/remove tasks to/from the final set of tasks to run

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add package, file, and environment related tasks

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update existing file catalogers to use nested UI elements

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add CreateSBOMConfig that drives the SBOM creation process

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* capture SBOM creation info as a struct

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add CreateSBOM() function

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix tests

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update docs with SBOM selection help + breaking changes

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix multiple override default inputs

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix deprecation flag printing to stdout

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor cataloger selection description to separate object

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* keep expression errors and show specific suggestions only

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address additional review feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address more review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* addressed additional PR review feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix file selection references

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove guess language data generation option

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tests for coordinatesForSelection

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename relationship attributes

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add descriptions to relationships config fields

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* improve documentation around configuration options

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add explicit errors around legacy config entries

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2024-01-12 17:39:13 -05:00
committed by GitHub
parent 1bffc6ef0c
commit b0ab75fd89
126 changed files with 5740 additions and 1841 deletions
+134
View File
@@ -0,0 +1,134 @@
package relationship
import (
"sort"
"github.com/bmatcuk/doublestar/v4"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
)
// altRpmDBGlob allows db matches against new locations introduced in fedora:{36,37}
// See https://github.com/anchore/syft/issues/1077 for larger context
const altRpmDBGlob = "**/rpm/{Packages,Packages.db,rpmdb.sqlite}"
var globsForbiddenFromBeingOwned = []string{
// any OS DBs should automatically be ignored to prevent cyclic issues (e.g. the "rpm" RPM owns the path to the
// RPM DB, so if not ignored that package would own all other packages on the system).
pkg.ApkDBGlob,
pkg.DpkgDBGlob,
pkg.RpmDBGlob,
altRpmDBGlob,
// DEB packages share common copyright info between, this does not mean that sharing these paths implies ownership.
"/usr/share/doc/**/copyright",
}
type ownershipByFilesMetadata struct {
Files []string `json:"files"`
}
func byFileOwnershipOverlapWorker(accessor sbomsync.Accessor) {
var relationships []artifact.Relationship
accessor.ReadFromSBOM(func(s *sbom.SBOM) {
relationships = byFileOwnershipOverlap(s.Artifacts.Packages)
})
accessor.WriteToSBOM(func(s *sbom.SBOM) {
s.Relationships = append(s.Relationships, relationships...)
})
}
// byFileOwnershipOverlap creates a package-to-package relationship based on discovering which packages have
// evidence locations that overlap with ownership claim from another package's package manager metadata.
func byFileOwnershipOverlap(catalog *pkg.Collection) []artifact.Relationship {
var relationships = findOwnershipByFilesRelationships(catalog)
var edges []artifact.Relationship
for parentID, children := range relationships {
for childID, files := range children {
fs := files.List()
sort.Strings(fs)
parent := catalog.Package(parentID) // TODO: this is potentially expensive
child := catalog.Package(childID) // TODO: this is potentially expensive
edges = append(edges, artifact.Relationship{
From: parent,
To: child,
Type: artifact.OwnershipByFileOverlapRelationship,
Data: ownershipByFilesMetadata{
Files: fs,
},
})
}
}
return edges
}
// findOwnershipByFilesRelationships find overlaps in file ownership with a file that defines another package. Specifically, a .Location.Path of
// a package is found to be owned by another (from the owner's .Metadata.Files[]).
func findOwnershipByFilesRelationships(catalog *pkg.Collection) map[artifact.ID]map[artifact.ID]*strset.Set {
var relationships = make(map[artifact.ID]map[artifact.ID]*strset.Set)
if catalog == nil {
return relationships
}
for _, candidateOwnerPkg := range catalog.Sorted() {
id := candidateOwnerPkg.ID()
if candidateOwnerPkg.Metadata == nil {
continue
}
// check to see if this is a file owner
pkgFileOwner, ok := candidateOwnerPkg.Metadata.(pkg.FileOwner)
if !ok {
continue
}
for _, ownedFilePath := range pkgFileOwner.OwnedFiles() {
if matchesAny(ownedFilePath, globsForbiddenFromBeingOwned) {
// we skip over known exceptions to file ownership, such as the RPM package owning
// the RPM DB path, otherwise the RPM package would "own" all RPMs, which is not intended
continue
}
// look for package(s) in the catalog that may be owned by this package and mark the relationship
for _, subPackage := range catalog.PackagesByPath(ownedFilePath) {
subID := subPackage.ID()
if subID == id {
continue
}
if _, exists := relationships[id]; !exists {
relationships[id] = make(map[artifact.ID]*strset.Set)
}
if _, exists := relationships[id][subID]; !exists {
relationships[id][subID] = strset.New()
}
relationships[id][subID].Add(ownedFilePath)
}
}
}
return relationships
}
func matchesAny(s string, globs []string) bool {
for _, g := range globs {
matches, err := doublestar.Match(g, s)
if err != nil {
log.Errorf("failed to match glob=%q : %+v", g, err)
}
if matches {
return true
}
}
return false
}
@@ -0,0 +1,153 @@
package relationship
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
)
func TestOwnershipByFilesRelationship(t *testing.T) {
tests := []struct {
name string
setup func(t testing.TB) ([]pkg.Package, []artifact.Relationship)
}{
{
name: "owns-by-real-path",
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
parent := pkg.Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/a/path", "/another/path"),
file.NewVirtualLocation("/b/path", "/bee/path"),
),
Type: pkg.RpmPkg,
Metadata: pkg.RpmDBEntry{
Files: []pkg.RpmFileRecord{
{Path: "/owning/path/1"},
{Path: "/owning/path/2"},
{Path: "/d/path"},
},
},
}
parent.SetID()
child := pkg.Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/c/path", "/another/path"),
file.NewVirtualLocation("/d/path", "/another/path"),
),
Type: pkg.NpmPkg,
}
child.SetID()
relationship := artifact.Relationship{
From: parent,
To: child,
Type: artifact.OwnershipByFileOverlapRelationship,
Data: ownershipByFilesMetadata{
Files: []string{
"/d/path",
},
},
}
return []pkg.Package{parent, child}, []artifact.Relationship{relationship}
},
},
{
name: "owns-by-virtual-path",
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
parent := pkg.Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/a/path", "/some/other/path"),
file.NewVirtualLocation("/b/path", "/bee/path"),
),
Type: pkg.RpmPkg,
Metadata: pkg.RpmDBEntry{
Files: []pkg.RpmFileRecord{
{Path: "/owning/path/1"},
{Path: "/owning/path/2"},
{Path: "/another/path"},
},
},
}
parent.SetID()
child := pkg.Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/c/path", "/another/path"),
file.NewLocation("/d/path"),
),
Type: pkg.NpmPkg,
}
child.SetID()
relationship := artifact.Relationship{
From: parent,
To: child,
Type: artifact.OwnershipByFileOverlapRelationship,
Data: ownershipByFilesMetadata{
Files: []string{
"/another/path",
},
},
}
return []pkg.Package{parent, child}, []artifact.Relationship{relationship}
},
},
{
name: "ignore-empty-path",
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
parent := pkg.Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/a/path", "/some/other/path"),
file.NewVirtualLocation("/b/path", "/bee/path"),
),
Type: pkg.RpmPkg,
Metadata: pkg.RpmDBEntry{
Files: []pkg.RpmFileRecord{
{Path: "/owning/path/1"},
{Path: "/owning/path/2"},
{Path: ""},
},
},
}
parent.SetID()
child := pkg.Package{
Locations: file.NewLocationSet(
file.NewVirtualLocation("/c/path", "/another/path"),
file.NewLocation("/d/path"),
),
Type: pkg.NpmPkg,
}
child.SetID()
return []pkg.Package{parent, child}, nil
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pkgs, expectedRelations := test.setup(t)
c := pkg.NewCollection(pkgs...)
relationships := byFileOwnershipOverlap(c)
assert.Len(t, relationships, len(expectedRelations))
for idx, expectedRelationship := range expectedRelations {
actualRelationship := relationships[idx]
assert.Equal(t, expectedRelationship.From.ID(), actualRelationship.From.ID())
assert.Equal(t, expectedRelationship.To.ID(), actualRelationship.To.ID())
assert.Equal(t, expectedRelationship.Type, actualRelationship.Type)
assert.Equal(t, expectedRelationship.Data, actualRelationship.Data)
}
})
}
}
+26
View File
@@ -0,0 +1,26 @@
package relationship
import (
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
)
func evidentBy(catalog *pkg.Collection) []artifact.Relationship {
var edges []artifact.Relationship
for _, p := range catalog.Sorted() {
for _, l := range p.Locations.ToSlice() {
if v, exists := l.Annotations[pkg.EvidenceAnnotationKey]; !exists || v != pkg.PrimaryEvidenceAnnotation {
// skip non-primary evidence from being expressed as a relationship.
// note: this may be configurable in the future.
continue
}
edges = append(edges, artifact.Relationship{
From: p,
To: l.Coordinates,
Type: artifact.EvidentByRelationship,
})
}
}
return edges
}
+88
View File
@@ -0,0 +1,88 @@
package relationship
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
)
func TestRelationshipsEvidentBy(t *testing.T) {
c := pkg.NewCollection()
coordA := file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
}
coordC := file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
}
coordD := file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
}
pkgA := pkg.Package{
Locations: file.NewLocationSet(
// added!
file.NewLocationFromCoordinates(coordA).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
// ignored...
file.NewLocationFromCoordinates(coordC).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation),
file.NewLocationFromCoordinates(coordD),
),
}
pkgA.SetID()
c.Add(pkgA)
coordB := file.Coordinates{
RealPath: "/somewhere-else/real",
FileSystemID: "def",
}
pkgB := pkg.Package{
Locations: file.NewLocationSet(
// added!
file.NewLocationFromCoordinates(coordB).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
),
}
pkgB.SetID()
c.Add(pkgB)
tests := []struct {
name string
catalog *pkg.Collection
want []artifact.Relationship
}{
{
name: "go case",
catalog: c,
want: []artifact.Relationship{
{
From: pkgB,
To: coordB,
Type: artifact.EvidentByRelationship,
},
{
From: pkgA,
To: coordA,
Type: artifact.EvidentByRelationship,
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual := evidentBy(tt.catalog)
require.Len(t, actual, len(tt.want))
for i := range actual {
assert.Equal(t, tt.want[i].From.ID(), actual[i].From.ID(), "from mismatch at index %d", i)
assert.Equal(t, tt.want[i].To.ID(), actual[i].To.ID(), "to mismatch at index %d", i)
assert.Equal(t, tt.want[i].Type, actual[i].Type, "type mismatch at index %d", i)
}
})
}
}
@@ -0,0 +1,64 @@
package relationship
import (
"slices"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
)
var (
osCatalogerTypes = []pkg.Type{
pkg.AlpmPkg,
pkg.ApkPkg,
pkg.DebPkg,
pkg.NixPkg,
pkg.PortagePkg,
pkg.RpmPkg,
}
binaryCatalogerTypes = []pkg.Type{
pkg.BinaryPkg,
}
)
func excludeBinariesByFileOwnershipOverlap(accessor sbomsync.Accessor) {
accessor.WriteToSBOM(func(s *sbom.SBOM) {
for _, r := range s.Relationships {
if excludeBinaryByFileOwnershipOverlap(r, s.Artifacts.Packages) {
s.Artifacts.Packages.Delete(r.To.ID())
s.Relationships = RemoveRelationshipsByID(s.Relationships, r.To.ID())
}
}
})
}
// excludeBinaryByFileOwnershipOverlap will remove packages from a collection given the following properties are true
// 1) the relationship between packages is OwnershipByFileOverlap
// 2) the parent is an "os" package
// 3) the child is a synthetic package generated by the binary cataloger
// 4) the package names are identical
// This was implemented as a way to help resolve: https://github.com/anchore/syft/issues/931
func excludeBinaryByFileOwnershipOverlap(r artifact.Relationship, c *pkg.Collection) bool {
if artifact.OwnershipByFileOverlapRelationship != r.Type {
return false
}
parent := c.Package(r.From.ID())
if parent == nil {
return false
}
parentInExclusion := slices.Contains(osCatalogerTypes, parent.Type)
if !parentInExclusion {
return false
}
child := c.Package(r.To.ID())
if child == nil {
return false
}
return slices.Contains(binaryCatalogerTypes, child.Type)
}
@@ -0,0 +1,76 @@
package relationship
import (
"testing"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
)
func TestExclude(t *testing.T) {
packageA := pkg.Package{Name: "package-a", Type: pkg.ApkPkg}
packageB := pkg.Package{Name: "package-a", Type: pkg.PythonPkg}
packageC := pkg.Package{Name: "package-a", Type: pkg.BinaryPkg}
packageD := pkg.Package{Name: "package-d", Type: pkg.BinaryPkg}
for _, p := range []*pkg.Package{&packageA, &packageB, &packageC, &packageD} {
p := p
p.SetID()
}
tests := []struct {
name string
relationship artifact.Relationship
packages *pkg.Collection
shouldExclude bool
}{
{
name: "no exclusions from os -> python",
relationship: artifact.Relationship{
Type: artifact.OwnershipByFileOverlapRelationship,
From: packageA,
To: packageB,
},
packages: pkg.NewCollection(packageA, packageB),
shouldExclude: false,
},
{
name: "exclusions from os -> binary",
relationship: artifact.Relationship{
Type: artifact.OwnershipByFileOverlapRelationship,
From: packageA,
To: packageC,
},
packages: pkg.NewCollection(packageA, packageC),
shouldExclude: true,
},
{
name: "no exclusions from python -> binary",
relationship: artifact.Relationship{
Type: artifact.OwnershipByFileOverlapRelationship,
From: packageB,
To: packageC,
},
packages: pkg.NewCollection(packageB, packageC),
shouldExclude: false,
},
{
name: "no exclusions for different package names",
relationship: artifact.Relationship{
Type: artifact.OwnershipByFileOverlapRelationship,
From: packageA,
To: packageD,
},
packages: pkg.NewCollection(packageA, packageD),
shouldExclude: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if !excludeBinaryByFileOwnershipOverlap(test.relationship, test.packages) && test.shouldExclude {
t.Errorf("expected to exclude relationship %+v", test.relationship)
}
})
}
}
+37
View File
@@ -0,0 +1,37 @@
package relationship
import (
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/sbom"
)
func Finalize(builder sbomsync.Builder, cfg cataloging.RelationshipsConfig, src artifact.Identifiable) {
accessor := builder.(sbomsync.Accessor)
// add relationships showing packages that are evident by a file which is owned by another package (package-to-package)
if cfg.PackageFileOwnershipOverlap {
byFileOwnershipOverlapWorker(accessor)
}
// conditionally remove binary packages based on file ownership overlap relationships found
// https://github.com/anchore/syft/issues/931
if cfg.ExcludeBinaryPackagesWithFileOwnershipOverlap {
excludeBinariesByFileOwnershipOverlap(accessor)
}
// add source "contains package" relationship (source-to-package)
var sourceRelationships []artifact.Relationship
accessor.ReadFromSBOM(func(s *sbom.SBOM) {
sourceRelationships = toSource(src, s.Artifacts.Packages)
})
builder.AddRelationships(sourceRelationships...)
// add evident-by relationships (package-to-file)
var evidentByRelationships []artifact.Relationship
accessor.ReadFromSBOM(func(s *sbom.SBOM) {
evidentByRelationships = evidentBy(s.Artifacts.Packages)
})
builder.AddRelationships(evidentByRelationships...)
}
+14
View File
@@ -0,0 +1,14 @@
package relationship
import "github.com/anchore/syft/syft/artifact"
// TODO: put under test...
func RemoveRelationshipsByID(relationships []artifact.Relationship, id artifact.ID) []artifact.Relationship {
var filtered []artifact.Relationship
for _, r := range relationships {
if r.To.ID() != id && r.From.ID() != id {
filtered = append(filtered, r)
}
}
return filtered
}
+42
View File
@@ -0,0 +1,42 @@
package relationship
import (
"sort"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
)
// Sort takes a set of package-to-package relationships and sorts them in a stable order by name and version.
// Note: this does not consider package-to-other, other-to-package, or other-to-other relationships.
// TODO: ideally this should be replaced with a more type-agnostic sort function that resides in the artifact package.
func Sort(rels []artifact.Relationship) {
sort.SliceStable(rels, func(i, j int) bool {
return less(rels[i], rels[j])
})
}
func less(i, j artifact.Relationship) bool {
iFrom, ok1 := i.From.(pkg.Package)
iTo, ok2 := i.To.(pkg.Package)
jFrom, ok3 := j.From.(pkg.Package)
jTo, ok4 := j.To.(pkg.Package)
if !(ok1 && ok2 && ok3 && ok4) {
return false
}
if iFrom.Name != jFrom.Name {
return iFrom.Name < jFrom.Name
}
if iFrom.Version != jFrom.Version {
return iFrom.Version < jFrom.Version
}
if iTo.Name != jTo.Name {
return iTo.Name < jTo.Name
}
if iTo.Version != jTo.Version {
return iTo.Version < jTo.Version
}
return i.Type < j.Type
}
+19
View File
@@ -0,0 +1,19 @@
package relationship
import (
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
)
func toSource(src artifact.Identifiable, c *pkg.Collection) []artifact.Relationship {
relationships := make([]artifact.Relationship, 0) // Should we pre-allocate this by giving catalog a Len() method?
for p := range c.Enumerate() {
relationships = append(relationships, artifact.Relationship{
From: src,
To: p,
Type: artifact.ContainsRelationship,
})
}
return relationships
}