Replace core SBOM-creation API with builder pattern (#1383)

* remove existing cataloging API Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add file cataloging config Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add package cataloging config Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add configs for cross-cutting concerns Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename CLI option configs to not require import aliases later Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update all nested structs for the Catalog struct Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update Catalog cli options - add new cataloger selection options (selection and default) - remove the excludeBinaryOverlapByOwnership - deprecate "catalogers" flag - add new javascript configuration Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * migrate relationship capabilities to separate internal package Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * refactor golang cataloger to use configuration options when creating packages Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * create internal object to facilitate reading from and writing to an SBOM Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * create a command-like object (task) to facilitate partial SBOM creation Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add cataloger selection capability - be able to parse string expressions into a set of resolved actions against sets - be able to use expressions to select/add/remove tasks to/from the final set of tasks to run Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add package, file, and environment related tasks Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update existing file catalogers to use nested UI elements Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add CreateSBOMConfig that drives the SBOM creation process Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * capture SBOM creation info as a struct Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add CreateSBOM() function Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix tests Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update docs with SBOM selection help + breaking changes Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix multiple override default inputs Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix deprecation flag printing to stdout Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * refactor cataloger selection description to separate object Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address review comments Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * keep expression errors and show specific suggestions only Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address additional review feedback Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address more review comments Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * addressed additional PR review feedback Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix file selection references Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * remove guess language data generation option Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add tests for coordinatesForSelection Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename relationship attributes Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add descriptions to relationships config fields Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * improve documentation around configuration options Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add explicit errors around legacy config entries Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
2026-11-02 07:26:34 +01:00 · 2024-01-12 17:39:13 -05:00
parent 1bffc6ef0c
commit b0ab75fd89
126 changed files with 5740 additions and 1841 deletions
@@ -0,0 +1,502 @@
+package syft
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/scylladb/go-set/strset"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/internal/task"
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/cataloging"
+	"github.com/anchore/syft/syft/cataloging/filecataloging"
+	"github.com/anchore/syft/syft/cataloging/pkgcataloging"
+	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+var _ pkg.Cataloger = (*dummyCataloger)(nil)
+
+type dummyCataloger struct {
+	name string
+}
+
+func newDummyCataloger(name string) pkg.Cataloger {
+	return dummyCataloger{name: name}
+}
+
+func (d dummyCataloger) Name() string {
+	return d.name
+}
+
+func (d dummyCataloger) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
+	return nil, nil, nil
+}
+
+func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
+	pkgIntersect := func(intersect ...string) []string {
+		var sets []*strset.Set
+		for _, s := range intersect {
+			sets = append(sets, strset.New(pkgCatalogerNamesWithTagOrName(t, s)...))
+		}
+
+		intersectSet := strset.Intersection(sets...)
+
+		slice := intersectSet.List()
+
+		sort.Strings(slice)
+
+		return slice
+	}
+
+	addTo := func(slice []string, add ...string) []string {
+		slice = append(slice, add...)
+		sort.Strings(slice)
+		return slice
+	}
+
+	imgSrc := source.Description{
+		Metadata: source.StereoscopeImageSourceMetadata{},
+	}
+
+	dirSrc := source.Description{
+		Metadata: source.DirectorySourceMetadata{},
+	}
+
+	fileSrc := source.Description{
+		Metadata: source.FileSourceMetadata{},
+	}
+
+	tests := []struct {
+		name          string
+		src           source.Description
+		cfg           *CreateSBOMConfig
+		wantTaskNames [][]string
+		wantManifest  *catalogerManifest
+		wantErr       require.ErrorAssertionFunc
+	}{
+		{
+			name: "default catalogers for image source",
+			src:  imgSrc,
+			cfg:  DefaultCreateSBOMConfig(),
+			wantTaskNames: [][]string{
+				environmentCatalogerNames(),
+				pkgCatalogerNamesWithTagOrName(t, "image"),
+				fileCatalogerNames(true, true),
+				relationshipCatalogerNames(),
+			},
+			wantManifest: &catalogerManifest{
+				Requested: pkgcataloging.SelectionRequest{
+					DefaultNamesOrTags: []string{"image"},
+				},
+				Used: pkgCatalogerNamesWithTagOrName(t, "image"),
+			},
+			wantErr: require.NoError,
+		},
+		{
+			name: "default catalogers for directory source",
+			src:  dirSrc,
+			cfg:  DefaultCreateSBOMConfig(),
+			wantTaskNames: [][]string{
+				environmentCatalogerNames(),
+				pkgCatalogerNamesWithTagOrName(t, "directory"),
+				fileCatalogerNames(true, true),
+				relationshipCatalogerNames(),
+			},
+			wantManifest: &catalogerManifest{
+				Requested: pkgcataloging.SelectionRequest{
+					DefaultNamesOrTags: []string{"directory"},
+				},
+				Used: pkgCatalogerNamesWithTagOrName(t, "directory"),
+			},
+			wantErr: require.NoError,
+		},
+		{
+			// note, the file source acts like a directory scan
+			name: "default catalogers for file source",
+			src:  fileSrc,
+			cfg:  DefaultCreateSBOMConfig(),
+			wantTaskNames: [][]string{
+				environmentCatalogerNames(),
+				pkgCatalogerNamesWithTagOrName(t, "directory"),
+				fileCatalogerNames(true, true),
+				relationshipCatalogerNames(),
+			},
+			wantManifest: &catalogerManifest{
+				Requested: pkgcataloging.SelectionRequest{
+					DefaultNamesOrTags: []string{"directory"},
+				},
+				Used: pkgCatalogerNamesWithTagOrName(t, "directory"),
+			},
+			wantErr: require.NoError,
+		},
+		{
+			name: "no file digest cataloger",
+			src:  imgSrc,
+			cfg:  DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithHashers(nil)),
+			wantTaskNames: [][]string{
+				environmentCatalogerNames(),
+				pkgCatalogerNamesWithTagOrName(t, "image"),
+				fileCatalogerNames(false, true), // note: the digest cataloger is not included
+				relationshipCatalogerNames(),
+			},
+			wantManifest: &catalogerManifest{
+				Requested: pkgcataloging.SelectionRequest{
+					DefaultNamesOrTags: []string{"image"},
+				},
+				Used: pkgCatalogerNamesWithTagOrName(t, "image"),
+			},
+			wantErr: require.NoError,
+		},
+		{
+			name: "select no file catalogers",
+			src:  imgSrc,
+			cfg:  DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.NoFilesSelection)),
+			wantTaskNames: [][]string{
+				environmentCatalogerNames(),
+				pkgCatalogerNamesWithTagOrName(t, "image"),
+				// note: there are no file catalogers in their own group
+				relationshipCatalogerNames(),
+			},
+			wantManifest: &catalogerManifest{
+				Requested: pkgcataloging.SelectionRequest{
+					DefaultNamesOrTags: []string{"image"},
+				},
+				Used: pkgCatalogerNamesWithTagOrName(t, "image"),
+			},
+			wantErr: require.NoError,
+		},
+		{
+			name: "select all file catalogers",
+			src:  imgSrc,
+			cfg:  DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)),
+			wantTaskNames: [][]string{
+				environmentCatalogerNames(),
+				// note: there is a single group of catalogers for pkgs and files
+				append(
+					pkgCatalogerNamesWithTagOrName(t, "image"),
+					fileCatalogerNames(true, true)...,
+				),
+				relationshipCatalogerNames(),
+			},
+			wantManifest: &catalogerManifest{
+				Requested: pkgcataloging.SelectionRequest{
+					DefaultNamesOrTags: []string{"image"},
+				},
+				Used: pkgCatalogerNamesWithTagOrName(t, "image"),
+			},
+			wantErr: require.NoError,
+		},
+		{
+			name: "user-provided persistent cataloger is always run (image)",
+			src:  imgSrc,
+			cfg: DefaultCreateSBOMConfig().WithCatalogers(
+				pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")),
+			),
+			wantTaskNames: [][]string{
+				environmentCatalogerNames(),
+				addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"),
+				fileCatalogerNames(true, true),
+				relationshipCatalogerNames(),
+			},
+			wantManifest: &catalogerManifest{
+				Requested: pkgcataloging.SelectionRequest{
+					DefaultNamesOrTags: []string{"image"},
+				},
+				Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"),
+			},
+			wantErr: require.NoError,
+		},
+		{
+			name: "user-provided persistent cataloger is always run (directory)",
+			src:  dirSrc,
+			cfg: DefaultCreateSBOMConfig().WithCatalogers(
+				pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")),
+			),
+			wantTaskNames: [][]string{
+				environmentCatalogerNames(),
+				addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"),
+				fileCatalogerNames(true, true),
+				relationshipCatalogerNames(),
+			},
+			wantManifest: &catalogerManifest{
+				Requested: pkgcataloging.SelectionRequest{
+					DefaultNamesOrTags: []string{"directory"},
+				},
+				Used: addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"),
+			},
+			wantErr: require.NoError,
+		},
+		{
+			name: "user-provided persistent cataloger is always run (user selection does not affect this)",
+			src:  imgSrc,
+			cfg: DefaultCreateSBOMConfig().WithCatalogers(
+				pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")),
+			).WithCatalogerSelection(pkgcataloging.NewSelectionRequest().WithSubSelections("javascript")),
+			wantTaskNames: [][]string{
+				environmentCatalogerNames(),
+				addTo(pkgIntersect("image", "javascript"), "persistent"),
+				fileCatalogerNames(true, true),
+				relationshipCatalogerNames(),
+			},
+			wantManifest: &catalogerManifest{
+				Requested: pkgcataloging.SelectionRequest{
+					DefaultNamesOrTags: []string{"image"},
+					SubSelectTags:      []string{"javascript"},
+				},
+				Used: addTo(pkgIntersect("image", "javascript"), "persistent"),
+			},
+			wantErr: require.NoError,
+		},
+		{
+			name: "user-provided cataloger runs when selected",
+			src:  imgSrc,
+			cfg: DefaultCreateSBOMConfig().WithCatalogers(
+				pkgcataloging.NewCatalogerReference(newDummyCataloger("user-provided"), []string{"image"}),
+			),
+			wantTaskNames: [][]string{
+				environmentCatalogerNames(),
+				addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"),
+				fileCatalogerNames(true, true),
+				relationshipCatalogerNames(),
+			},
+			wantManifest: &catalogerManifest{
+				Requested: pkgcataloging.SelectionRequest{
+					DefaultNamesOrTags: []string{"image"},
+				},
+				Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"),
+			},
+			wantErr: require.NoError,
+		},
+		{
+			name: "user-provided cataloger NOT run when NOT selected",
+			src:  imgSrc,
+			cfg: DefaultCreateSBOMConfig().WithCatalogers(
+				pkgcataloging.NewCatalogerReference(newDummyCataloger("user-provided"), []string{"bogus-selector-will-never-be-used"}),
+			),
+			wantTaskNames: [][]string{
+				environmentCatalogerNames(),
+				pkgCatalogerNamesWithTagOrName(t, "image"),
+				fileCatalogerNames(true, true),
+				relationshipCatalogerNames(),
+			},
+			wantManifest: &catalogerManifest{
+				Requested: pkgcataloging.SelectionRequest{
+					DefaultNamesOrTags: []string{"image"},
+				},
+				Used: pkgCatalogerNamesWithTagOrName(t, "image"),
+			},
+			wantErr: require.NoError,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.wantErr == nil {
+				tt.wantErr = require.NoError
+			}
+
+			// sanity check
+			require.NotEmpty(t, tt.wantTaskNames)
+			for _, group := range tt.wantTaskNames {
+				require.NotEmpty(t, group)
+			}
+
+			// test the subject
+			gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src)
+			tt.wantErr(t, err)
+			if err != nil {
+				return
+			}
+
+			gotNames := taskGroupNames(gotTasks)
+
+			if d := cmp.Diff(
+				tt.wantTaskNames,
+				gotNames,
+				// order within a group does not matter
+				cmpopts.SortSlices(func(a, b string) bool {
+					return a < b
+				}),
+			); d != "" {
+				t.Errorf("mismatched task group names (-want +got):\n%s", d)
+			}
+
+			if d := cmp.Diff(tt.wantManifest, gotManifest); d != "" {
+				t.Errorf("mismatched cataloger manifest (-want +got):\n%s", d)
+			}
+		})
+	}
+}
+
+func pkgCatalogerNamesWithTagOrName(t *testing.T, token string) []string {
+	var names []string
+	cfg := task.DefaultCatalogingFactoryConfig()
+	for _, factory := range task.DefaultPackageTaskFactories() {
+		cat := factory(cfg)
+
+		name := cat.Name()
+
+		if selector, ok := cat.(task.Selector); ok {
+			if selector.HasAllSelectors(token) {
+				names = append(names, name)
+				continue
+			}
+		}
+		if name == token {
+			names = append(names, name)
+		}
+	}
+
+	// these thresholds are arbitrary but should be large enough to catch any major changes
+	switch token {
+	case "image":
+		require.Greater(t, len(names), 18, "minimum cataloger sanity check failed token")
+	case "directory":
+		require.Greater(t, len(names), 25, "minimum cataloger sanity check failed token")
+	default:
+		require.Greater(t, len(names), 0, "minimum cataloger sanity check failed token")
+	}
+
+	sort.Strings(names)
+	return names
+}
+
+func fileCatalogerNames(digest, metadata bool) []string {
+	var names []string
+	if digest {
+		names = append(names, "file-digest-cataloger")
+	}
+	if metadata {
+		names = append(names, "file-metadata-cataloger")
+	}
+	return names
+}
+
+func relationshipCatalogerNames() []string {
+	return []string{"relationships-cataloger"}
+}
+
+func environmentCatalogerNames() []string {
+	return []string{"environment-cataloger"}
+}
+
+func taskGroupNames(groups [][]task.Task) [][]string {
+	var names [][]string
+	for _, group := range groups {
+		var groupNames []string
+		for _, tsk := range group {
+			groupNames = append(groupNames, tsk.Name())
+		}
+		names = append(names, groupNames)
+	}
+	return names
+}
+
+func Test_replaceDefaultTagReferences(t *testing.T) {
+
+	tests := []struct {
+		name string
+		lst  []string
+		want []string
+	}{
+		{
+			name: "no default tag",
+			lst:  []string{"foo", "bar"},
+			want: []string{"foo", "bar"},
+		},
+		{
+			name: "replace default tag",
+			lst:  []string{"foo", "default", "bar"},
+			want: []string{"foo", "replacement", "bar"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, replaceDefaultTagReferences("replacement", tt.lst))
+		})
+	}
+}
+
+func Test_findDefaultTag(t *testing.T) {
+
+	tests := []struct {
+		name    string
+		src     source.Description
+		want    string
+		wantErr require.ErrorAssertionFunc
+	}{
+		{
+			name: "image",
+			src: source.Description{
+				Metadata: source.StereoscopeImageSourceMetadata{},
+			},
+			want: pkgcataloging.ImageTag,
+		},
+		{
+			name: "directory",
+			src: source.Description{
+				Metadata: source.DirectorySourceMetadata{},
+			},
+			want: pkgcataloging.DirectoryTag,
+		},
+		{
+			name: "file",
+			src: source.Description{
+				Metadata: source.FileSourceMetadata{},
+			},
+			want: pkgcataloging.DirectoryTag, // not a mistake...
+		},
+		{
+			name: "unknown",
+			src: source.Description{
+				Metadata: struct{}{},
+			},
+			wantErr: require.Error,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.wantErr == nil {
+				tt.wantErr = require.NoError
+			}
+			got, err := findDefaultTag(tt.src)
+			tt.wantErr(t, err)
+			if err != nil {
+				return
+			}
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestCreateSBOMConfig_validate(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     *CreateSBOMConfig
+		wantErr assert.ErrorAssertionFunc
+	}{
+		{
+			name: "incompatible ExcludeBinaryPackagesWithFileOwnershipOverlap selection",
+			cfg: DefaultCreateSBOMConfig().
+				WithRelationshipsConfig(
+					cataloging.DefaultRelationshipsConfig().
+						WithExcludeBinaryPackagesWithFileOwnershipOverlap(true).
+						WithPackageFileOwnershipOverlap(false),
+				),
+			wantErr: assert.Error,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.wantErr == nil {
+				tt.wantErr = assert.NoError
+			}
+			tt.wantErr(t, tt.cfg.validate())
+		})
+	}
+}