Replace core SBOM-creation API with builder pattern (#1383)

* remove existing cataloging API Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add file cataloging config Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add package cataloging config Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add configs for cross-cutting concerns Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename CLI option configs to not require import aliases later Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update all nested structs for the Catalog struct Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update Catalog cli options - add new cataloger selection options (selection and default) - remove the excludeBinaryOverlapByOwnership - deprecate "catalogers" flag - add new javascript configuration Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * migrate relationship capabilities to separate internal package Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * refactor golang cataloger to use configuration options when creating packages Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * create internal object to facilitate reading from and writing to an SBOM Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * create a command-like object (task) to facilitate partial SBOM creation Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add cataloger selection capability - be able to parse string expressions into a set of resolved actions against sets - be able to use expressions to select/add/remove tasks to/from the final set of tasks to run Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add package, file, and environment related tasks Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update existing file catalogers to use nested UI elements Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add CreateSBOMConfig that drives the SBOM creation process Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * capture SBOM creation info as a struct Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add CreateSBOM() function Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix tests Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * update docs with SBOM selection help + breaking changes Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix multiple override default inputs Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix deprecation flag printing to stdout Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * refactor cataloger selection description to separate object Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address review comments Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * keep expression errors and show specific suggestions only Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address additional review feedback Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * address more review comments Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * addressed additional PR review feedback Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * fix file selection references Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * remove guess language data generation option Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add tests for coordinatesForSelection Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * rename relationship attributes Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add descriptions to relationships config fields Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * improve documentation around configuration options Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * add explicit errors around legacy config entries Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
2026-10-31 22:46:36 +01:00 · 2024-01-12 17:39:13 -05:00
parent 1bffc6ef0c
commit b0ab75fd89
126 changed files with 5740 additions and 1841 deletions
@@ -6,21 +6,40 @@ import (
 	"fmt"
 	"io"

+	"github.com/dustin/go-humanize"
+
 	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/bus"
+	intFile "github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/event/monitor"
 	"github.com/anchore/syft/syft/file"
 )

+type Config struct {
+	// Globs are the file patterns that must be matched for a file to be considered for cataloging.
+	Globs []string `yaml:"globs" json:"globs" mapstructure:"globs"`
+
+	// SkipFilesAboveSize is the maximum file size (in bytes) to allow to be considered while cataloging. If the file is larger than this size it will be skipped.
+	SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"`
+}
+
 type Cataloger struct {
 	globs                     []string
 	skipFilesAboveSizeInBytes int64
 }

-func NewCataloger(globs []string, skipFilesAboveSize int64) (*Cataloger, error) {
+func DefaultConfig() Config {
+	return Config{
+		SkipFilesAboveSize: 250 * intFile.KB,
+	}
+}
+
+func NewCataloger(cfg Config) *Cataloger {
 	return &Cataloger{
-		globs:                     globs,
-		skipFilesAboveSizeInBytes: skipFilesAboveSize,
-	}, nil
+		globs:                     cfg.Globs,
+		skipFilesAboveSizeInBytes: cfg.SkipFilesAboveSize,
+	}
 }

 func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string, error) {
@@ -31,9 +50,15 @@ func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string
 	if err != nil {
 		return nil, err
 	}
+
+	prog := catalogingProgress(int64(len(locations)))
+
 	for _, location := range locations {
+		prog.AtomicStage.Set(location.Path())
+
 		metadata, err := resolver.FileMetadataByLocation(location)
 		if err != nil {
+			prog.SetError(err)
 			return nil, err
 		}

@@ -47,12 +72,20 @@ func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string
 			continue
 		}
 		if err != nil {
+			prog.SetError(err)
 			return nil, err
 		}
+
+		prog.Increment()
+
 		results[location.Coordinates] = result
 	}
+
 	log.Debugf("file contents cataloger processed %d files", len(results))

+	prog.AtomicStage.Set(fmt.Sprintf("%s files", humanize.Comma(prog.Current())))
+	prog.SetCompleted()
+
 	return results, nil
 }

@@ -66,7 +99,7 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
 	buf := &bytes.Buffer{}
 	encoder := base64.NewEncoder(base64.StdEncoding, buf)
 	if _, err = io.Copy(encoder, contentReader); err != nil {
-		return "", internal.ErrPath{Context: "contents-cataloger", Path: location.RealPath, Err: err}
+		return "", internal.ErrPath{Context: "content-cataloger", Path: location.RealPath, Err: err}
 	}
 	// note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes
 	if err := encoder.Close(); err != nil {
@@ -75,3 +108,14 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati

 	return buf.String(), nil
 }
+
+func catalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
+	info := monitor.GenericTask{
+		Title: monitor.Title{
+			Default: "File contents",
+		},
+		ParentID: monitor.TopLevelCatalogingTaskID,
+	}
+
+	return bus.StartCatalogerTask(info, locations, "")
+}
@@ -67,8 +67,10 @@ func TestContentsCataloger(t *testing.T) {

 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			c, err := NewCataloger(test.globs, test.maxSize)
-			assert.NoError(t, err)
+			c := NewCataloger(Config{
+				Globs:              test.globs,
+				SkipFilesAboveSize: test.maxSize,
+			})

 			resolver := file.NewMockResolverForPaths(test.files...)
 			actual, err := c.Catalog(resolver)
@@ -45,32 +45,34 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
 		}
 	}

-	prog := digestsCatalogingProgress(int64(len(locations)))
+	prog := catalogingProgress(int64(len(locations)))
 	for _, location := range locations {
-		prog.Increment()
-		prog.AtomicStage.Set(location.Path())
-
 		result, err := i.catalogLocation(resolver, location)

 		if errors.Is(err, ErrUndigestableFile) {
 			continue
 		}

+		prog.AtomicStage.Set(location.Path())
+
 		if internal.IsErrPathPermission(err) {
 			log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
 			continue
 		}

 		if err != nil {
-			return nil, err
+			prog.SetError(err)
+			return nil, fmt.Errorf("failed to process file %q: %w", location.RealPath, err)
 		}
+
 		prog.Increment()
+
 		results[location.Coordinates] = result
 	}

 	log.Debugf("file digests cataloger processed %d files", prog.Current())

-	prog.AtomicStage.Set(fmt.Sprintf("%s digests", humanize.Comma(prog.Current())))
+	prog.AtomicStage.Set(fmt.Sprintf("%s files", humanize.Comma(prog.Current())))
 	prog.SetCompleted()

 	return results, nil
@@ -101,13 +103,12 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
 	return digests, nil
 }

-func digestsCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
+func catalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
 	info := monitor.GenericTask{
 		Title: monitor.Title{
-			Default:      "Catalog file digests",
-			WhileRunning: "Cataloging file digests",
-			OnSuccess:    "Cataloged file digests",
+			Default: "File digests",
 		},
+		ParentID: monitor.TopLevelCatalogingTaskID,
 	}

 	return bus.StartCatalogerTask(info, locations, "")
@@ -43,16 +43,18 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
 		}()
 	}

-	prog := metadataCatalogingProgress(int64(len(locations)))
+	prog := catalogingProgress(-1)
 	for location := range locations {
-		prog.Increment()
 		prog.AtomicStage.Set(location.Path())

 		metadata, err := resolver.FileMetadataByLocation(location)
 		if err != nil {
+			prog.SetError(err)
 			return nil, err
 		}

+		prog.Increment()
+
 		results[location.Coordinates] = metadata
 	}

@@ -64,13 +66,12 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
 	return results, nil
 }

-func metadataCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
+func catalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
 	info := monitor.GenericTask{
 		Title: monitor.Title{
-			Default:      "Catalog file metadata",
-			WhileRunning: "Cataloging file metadata",
-			OnSuccess:    "Cataloged file metadata",
+			Default: "File metadata",
 		},
+		ParentID: monitor.TopLevelCatalogingTaskID,
 	}

 	return bus.StartCatalogerTask(info, locations, "")
@@ -0,0 +1,9 @@
+package file
+
+const (
+	NoFilesSelection             Selection = "none"
+	FilesOwnedByPackageSelection Selection = "owned-by-package"
+	AllFilesSelection            Selection = "all"
+)
+
+type Selection string