Refactor source API (#1846)

* refactor source API and syft json source block Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update source detection and format test utils Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * generate list of all source metadata types Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * extract base and root normalization into helper functions Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * preserve syftjson model package name import ref Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * alias should not be a pointer Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <alex.goodman@anchore.com> Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
2026-10-30 14:02:26 +01:00 · 2023-06-30 14:19:16 +00:00
parent 608dbded06
commit 4da3be864f
126 changed files with 7384 additions and 3190 deletions
@@ -12,6 +12,7 @@ import (
 	"golang.org/x/exp/slices"

 	"github.com/anchore/stereoscope"
+	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/cmd/syft/cli/eventloop"
 	"github.com/anchore/syft/cmd/syft/cli/options"
 	"github.com/anchore/syft/cmd/syft/cli/packages"
@@ -34,17 +35,8 @@ func Run(_ context.Context, app *config.Application, args []string) error {
 		return err
 	}

-	// could be an image or a directory, with or without a scheme
-	// TODO: validate that source is image
+	// note: must be a container image
 	userInput := args[0]
-	si, err := source.ParseInputWithNameVersion(userInput, app.Platform, app.SourceName, app.SourceVersion, app.DefaultImagePullSource)
-	if err != nil {
-		return fmt.Errorf("could not generate source input for packages command: %w", err)
-	}
-
-	if si.Scheme != source.ImageScheme {
-		return fmt.Errorf("attestations are only supported for oci images at this time")
-	}

 	eventBus := partybus.NewBus()
 	stereoscope.SetBus(eventBus)
@@ -52,7 +44,7 @@ func Run(_ context.Context, app *config.Application, args []string) error {
 	subscription := eventBus.Subscribe()

 	return eventloop.EventLoop(
-		execWorker(app, *si),
+		execWorker(app, userInput),
 		eventloop.SetupSignals(),
 		subscription,
 		stereoscope.Cleanup,
@@ -60,13 +52,48 @@ func Run(_ context.Context, app *config.Application, args []string) error {
 	)
 }

-func buildSBOM(app *config.Application, si source.Input, errs chan error) (*sbom.SBOM, error) {
-	src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions)
-	if cleanup != nil {
-		defer cleanup()
+func buildSBOM(app *config.Application, userInput string, errs chan error) (*sbom.SBOM, error) {
+	cfg := source.DetectConfig{
+		DefaultImageSource: app.DefaultImagePullSource,
+	}
+	detection, err := source.Detect(userInput, cfg)
+	if err != nil {
+		return nil, fmt.Errorf("could not deteremine source: %w", err)
+	}
+
+	if detection.IsContainerImage() {
+		return nil, fmt.Errorf("attestations are only supported for oci images at this time")
+	}
+
+	var platform *image.Platform
+
+	if app.Platform != "" {
+		platform, err = image.NewPlatform(app.Platform)
+		if err != nil {
+			return nil, fmt.Errorf("invalid platform: %w", err)
+		}
+	}
+
+	src, err := detection.NewSource(
+		source.DetectionSourceConfig{
+			Alias: source.Alias{
+				Name:    app.SourceName,
+				Version: app.SourceVersion,
+			},
+			RegistryOptions: app.Registry.ToOptions(),
+			Platform:        platform,
+			Exclude: source.ExcludeConfig{
+				Paths: app.Exclusions,
+			},
+			DigestAlgorithms: nil,
+		},
+	)
+
+	if src != nil {
+		defer src.Close()
 	}
 	if err != nil {
-		return nil, fmt.Errorf("failed to construct source from user input %q: %w", si.UserInput, err)
+		return nil, fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
 	}

 	s, err := packages.GenerateSBOM(src, errs, app)
@@ -75,20 +102,20 @@ func buildSBOM(app *config.Application, si source.Input, errs chan error) (*sbom
 	}

 	if s == nil {
-		return nil, fmt.Errorf("no SBOM produced for %q", si.UserInput)
+		return nil, fmt.Errorf("no SBOM produced for %q", userInput)
 	}

 	return s, nil
 }

 //nolint:funlen
-func execWorker(app *config.Application, si source.Input) <-chan error {
+func execWorker(app *config.Application, userInput string) <-chan error {
 	errs := make(chan error)
 	go func() {
 		defer close(errs)
 		defer bus.Publish(partybus.Event{Type: event.Exit})

-		s, err := buildSBOM(app, si, errs)
+		s, err := buildSBOM(app, userInput, errs)
 		if err != nil {
 			errs <- fmt.Errorf("unable to build SBOM: %w", err)
 			return
@@ -136,7 +163,7 @@ func execWorker(app *config.Application, si source.Input) <-chan error {
 			predicateType = "custom"
 		}

-		args := []string{"attest", si.UserInput, "--predicate", f.Name(), "--type", predicateType}
+		args := []string{"attest", userInput, "--predicate", f.Name(), "--type", predicateType}
 		if app.Attest.Key != "" {
 			args = append(args, "--key", app.Attest.Key)
 		}
@@ -16,7 +16,7 @@ import (
 	"github.com/anchore/syft/syft/source"
 )

-type Task func(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error)
+type Task func(*sbom.Artifacts, source.Source) ([]artifact.Relationship, error)

 func Tasks(app *config.Application) ([]Task, error) {
 	var tasks []Task
@@ -48,7 +48,7 @@ func generateCatalogPackagesTask(app *config.Application) (Task, error) {
 		return nil, nil
 	}

-	task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
+	task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
 		packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, app.ToCatalogerConfig())

 		results.Packages = packageCatalog
@@ -67,7 +67,7 @@ func generateCatalogFileMetadataTask(app *config.Application) (Task, error) {

 	metadataCataloger := filemetadata.NewCataloger()

-	task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
+	task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
 		resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
 		if err != nil {
 			return nil, err
@@ -110,7 +110,7 @@ func generateCatalogFileDigestsTask(app *config.Application) (Task, error) {

 	digestsCataloger := filedigest.NewCataloger(hashes)

-	task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
+	task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
 		resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
 		if err != nil {
 			return nil, err
@@ -142,7 +142,7 @@ func generateCatalogSecretsTask(app *config.Application) (Task, error) {
 		return nil, err
 	}

-	task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
+	task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
 		resolver, err := src.FileResolver(app.Secrets.Cataloger.ScopeOpt)
 		if err != nil {
 			return nil, err
@@ -169,7 +169,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) {
 		return nil, err
 	}

-	task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
+	task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) {
 		resolver, err := src.FileResolver(app.FileContents.Cataloger.ScopeOpt)
 		if err != nil {
 			return nil, err
@@ -186,7 +186,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) {
 	return task, nil
 }

-func RunTask(t Task, a *sbom.Artifacts, src *source.Source, c chan<- artifact.Relationship, errs chan<- error) {
+func RunTask(t Task, a *sbom.Artifacts, src source.Source, c chan<- artifact.Relationship, errs chan<- error) {
 	defer close(c)

 	relationships, err := t(a, src)
@@ -7,6 +7,7 @@ import (
 	"github.com/wagoodman/go-partybus"

 	"github.com/anchore/stereoscope"
+	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/cmd/syft/cli/eventloop"
 	"github.com/anchore/syft/cmd/syft/cli/options"
 	"github.com/anchore/syft/internal"
@@ -35,10 +36,6 @@ func Run(_ context.Context, app *config.Application, args []string) error {

 	// could be an image or a directory, with or without a scheme
 	userInput := args[0]
-	si, err := source.ParseInputWithNameVersion(userInput, app.Platform, app.SourceName, app.SourceVersion, app.DefaultImagePullSource)
-	if err != nil {
-		return fmt.Errorf("could not generate source input for packages command: %w", err)
-	}

 	eventBus := partybus.NewBus()
 	stereoscope.SetBus(eventBus)
@@ -46,7 +43,7 @@ func Run(_ context.Context, app *config.Application, args []string) error {
 	subscription := eventBus.Subscribe()

 	return eventloop.EventLoop(
-		execWorker(app, *si, writer),
+		execWorker(app, userInput, writer),
 		eventloop.SetupSignals(),
 		subscription,
 		stereoscope.Cleanup,
@@ -54,17 +51,52 @@ func Run(_ context.Context, app *config.Application, args []string) error {
 	)
 }

-func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-chan error {
+func execWorker(app *config.Application, userInput string, writer sbom.Writer) <-chan error {
 	errs := make(chan error)
 	go func() {
 		defer close(errs)

-		src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions)
-		if cleanup != nil {
-			defer cleanup()
+		detection, err := source.Detect(
+			userInput,
+			source.DetectConfig{
+				DefaultImageSource: app.DefaultImagePullSource,
+			},
+		)
+		if err != nil {
+			errs <- fmt.Errorf("could not deteremine source: %w", err)
+			return
+		}
+
+		var platform *image.Platform
+
+		if app.Platform != "" {
+			platform, err = image.NewPlatform(app.Platform)
+			if err != nil {
+				errs <- fmt.Errorf("invalid platform: %w", err)
+				return
+			}
+		}
+
+		src, err := detection.NewSource(
+			source.DetectionSourceConfig{
+				Alias: source.Alias{
+					Name:    app.SourceName,
+					Version: app.SourceVersion,
+				},
+				RegistryOptions: app.Registry.ToOptions(),
+				Platform:        platform,
+				Exclude: source.ExcludeConfig{
+					Paths: app.Exclusions,
+				},
+				DigestAlgorithms: nil,
+			},
+		)
+
+		if src != nil {
+			defer src.Close()
 		}
 		if err != nil {
-			errs <- fmt.Errorf("failed to construct source from user input %q: %w", si.UserInput, err)
+			errs <- fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
 			return
 		}

@@ -75,7 +107,7 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-
 		}

 		if s == nil {
-			errs <- fmt.Errorf("no SBOM produced for %q", si.UserInput)
+			errs <- fmt.Errorf("no SBOM produced for %q", userInput)
 		}

 		bus.Publish(partybus.Event{
@@ -86,14 +118,14 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-
 	return errs
 }

-func GenerateSBOM(src *source.Source, errs chan error, app *config.Application) (*sbom.SBOM, error) {
+func GenerateSBOM(src source.Source, errs chan error, app *config.Application) (*sbom.SBOM, error) {
 	tasks, err := eventloop.Tasks(app)
 	if err != nil {
 		return nil, err
 	}

 	s := sbom.SBOM{
-		Source: src.Metadata,
+		Source: src.Describe(),
 		Descriptor: sbom.Descriptor{
 			Name:          internal.ApplicationName,
 			Version:       version.FromBuild().Version,
@@ -106,7 +138,7 @@ func GenerateSBOM(src *source.Source, errs chan error, app *config.Application)
 	return &s, nil
 }

-func buildRelationships(s *sbom.SBOM, src *source.Source, tasks []eventloop.Task, errs chan error) {
+func buildRelationships(s *sbom.SBOM, src source.Source, tasks []eventloop.Task, errs chan error) {
 	var relationships []<-chan artifact.Relationship
 	for _, task := range tasks {
 		c := make(chan artifact.Relationship)
@@ -9,6 +9,7 @@ import (
 	"github.com/wagoodman/go-partybus"

 	"github.com/anchore/stereoscope"
+	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/cmd/syft/cli/eventloop"
 	"github.com/anchore/syft/cmd/syft/cli/options"
 	"github.com/anchore/syft/cmd/syft/cli/packages"
@@ -38,10 +39,6 @@ func Run(_ context.Context, app *config.Application, args []string) error {
 	}()

 	userInput := args[0]
-	si, err := source.ParseInputWithNameVersion(userInput, app.Platform, app.SourceName, app.SourceVersion, app.DefaultImagePullSource)
-	if err != nil {
-		return fmt.Errorf("could not generate source input for packages command: %w", err)
-	}

 	eventBus := partybus.NewBus()
 	stereoscope.SetBus(eventBus)
@@ -49,7 +46,7 @@ func Run(_ context.Context, app *config.Application, args []string) error {
 	subscription := eventBus.Subscribe()

 	return eventloop.EventLoop(
-		execWorker(app, *si, writer),
+		execWorker(app, userInput, writer),
 		eventloop.SetupSignals(),
 		subscription,
 		stereoscope.Cleanup,
@@ -57,7 +54,8 @@ func Run(_ context.Context, app *config.Application, args []string) error {
 	)
 }

-func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-chan error {
+//nolint:funlen
+func execWorker(app *config.Application, userInput string, writer sbom.Writer) <-chan error {
 	errs := make(chan error)
 	go func() {
 		defer close(errs)
@@ -72,17 +70,52 @@ func execWorker(app *config.Application, si source.Input, writer sbom.Writer) <-
 			return
 		}

-		src, cleanup, err := source.New(si, app.Registry.ToOptions(), app.Exclusions)
+		detection, err := source.Detect(
+			userInput,
+			source.DetectConfig{
+				DefaultImageSource: app.DefaultImagePullSource,
+			},
+		)
 		if err != nil {
-			errs <- err
+			errs <- fmt.Errorf("could not deteremine source: %w", err)
 			return
 		}
-		if cleanup != nil {
-			defer cleanup()
+
+		var platform *image.Platform
+
+		if app.Platform != "" {
+			platform, err = image.NewPlatform(app.Platform)
+			if err != nil {
+				errs <- fmt.Errorf("invalid platform: %w", err)
+				return
+			}
+		}
+
+		src, err := detection.NewSource(
+			source.DetectionSourceConfig{
+				Alias: source.Alias{
+					Name:    app.SourceName,
+					Version: app.SourceVersion,
+				},
+				RegistryOptions: app.Registry.ToOptions(),
+				Platform:        platform,
+				Exclude: source.ExcludeConfig{
+					Paths: app.Exclusions,
+				},
+				DigestAlgorithms: nil,
+			},
+		)
+
+		if src != nil {
+			defer src.Close()
+		}
+		if err != nil {
+			errs <- fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
+			return
 		}

 		s := sbom.SBOM{
-			Source: src.Metadata,
+			Source: src.Describe(),
 			Descriptor: sbom.Descriptor{
 				Name:          internal.ApplicationName,
 				Version:       version.FromBuild().Version,