promote catalog task pattern to all commands (#636)

Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>

cmd/cmd.go

@@ -48,7 +48,13 @@ func initCmdAliasBindings() {
 		panic(err)
 	}
 
-	if activeCmd == packagesCmd || activeCmd == rootCmd {
+	// enable all cataloger by default if power-user command is run
+	if activeCmd == powerUserCmd {
+		config.PowerUserCatalogerEnabledDefault()
+	}
+
+	switch activeCmd {
+	case packagesCmd, rootCmd:
 		// note: we need to lazily bind config options since they are shared between both the root command
 		// and the packages command. Otherwise there will be global viper state that is in contention.
 		// See for more details: https://github.com/spf13/viper/issues/233 . Additionally, the bindings must occur BEFORE
@@ -58,7 +64,7 @@ func initCmdAliasBindings() {
 		if err = bindPackagesConfigOptions(activeCmd.Flags()); err != nil {
 			panic(err)
 		}
-	} else {
+	default:
 		// even though the root command or packages command is NOT being run, we still need default bindings
 		// such that application config parsing passes.
 		if err = bindPackagesConfigOptions(packagesCmd.Flags()); err != nil {

cmd/packages.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"sync"
 
 	"github.com/anchore/stereoscope"
 	"github.com/anchore/syft/internal"
@@ -13,7 +14,7 @@ import (
 	"github.com/anchore/syft/internal/formats"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/ui"
-	"github.com/anchore/syft/syft"
+	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/event"
 	"github.com/anchore/syft/syft/format"
 	"github.com/anchore/syft/syft/sbom"
@@ -238,6 +239,12 @@ func packagesExecWorker(userInput string) <-chan error {
 	go func() {
 		defer close(errs)
 
+		tasks, err := tasks()
+		if err != nil {
+			errs <- err
+			return
+		}
+
 		f := formats.ByOption(packagesPresenterOpt)
 		if f == nil {
 			errs <- fmt.Errorf("unknown format: %s", packagesPresenterOpt)
@@ -255,23 +262,24 @@ func packagesExecWorker(userInput string) <-chan error {
 			defer cleanup()
 		}
 
-		catalog, relationships, d, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt)
+		s := sbom.SBOM{
-		if err != nil {
+			Source: src.Metadata,
-			errs <- fmt.Errorf("failed to catalog input: %w", err)
-			return
 		}
 
-		sbomResult := sbom.SBOM{
+		var relationships []<-chan artifact.Relationship
-			Artifacts: sbom.Artifacts{
+		for _, task := range tasks {
-				PackageCatalog: catalog,
+			c := make(chan artifact.Relationship)
-				Distro:         d,
+			relationships = append(relationships, c)
-			},
+
-			Relationships: relationships,
+			go runTask(task, &s.Artifacts, src, c, errs)
-			Source:        src.Metadata,
+		}
+
+		for relationship := range mergeRelationships(relationships...) {
+			s.Relationships = append(s.Relationships, relationship)
 		}
 
 		if appConfig.Anchore.Host != "" {
-			if err := runPackageSbomUpload(src, sbomResult); err != nil {
+			if err := runPackageSbomUpload(src, s); err != nil {
 				errs <- err
 				return
 			}
@@ -279,12 +287,33 @@ func packagesExecWorker(userInput string) <-chan error {
 
 		bus.Publish(partybus.Event{
 			Type:  event.PresenterReady,
-			Value: f.Presenter(sbomResult),
+			Value: f.Presenter(s),
 		})
 	}()
 	return errs
 }
 
+func mergeRelationships(cs ...<-chan artifact.Relationship) <-chan artifact.Relationship {
+	var wg sync.WaitGroup
+	var relationships = make(chan artifact.Relationship)
+
+	wg.Add(len(cs))
+	for _, c := range cs {
+		go func(c <-chan artifact.Relationship) {
+			for n := range c {
+				relationships <- n
+			}
+			wg.Done()
+		}(c)
+	}
+
+	go func() {
+		wg.Wait()
+		close(relationships)
+	}()
+	return relationships
+}
+
 func runPackageSbomUpload(src *source.Source, s sbom.SBOM) error {
 	log.Infof("uploading results to %s", appConfig.Anchore.Host)
 

cmd/power_user.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"sync"
+	"os"
 
 	"github.com/anchore/syft/syft/artifact"
+	"github.com/gookit/color"
 
 	"github.com/anchore/syft/syft/sbom"
 
@@ -23,6 +24,8 @@ import (
 
 const powerUserExample = `  {{.appName}} {{.command}} <image>
 
+  DEPRECATED - THIS COMMAND WILL BE REMOVED in v1.0.0
+
   Only image sources are supported (e.g. docker: , docker-archive: , oci: , etc.), the directory source (dir:) is not supported.
 
   All behavior is controlled via application configuration and environment variables (see https://github.com/anchore/syft#configuration)
@@ -76,6 +79,10 @@ func powerUserExec(_ *cobra.Command, args []string) error {
 		if err := closer(); err != nil {
 			log.Warnf("unable to write to report destination: %+v", err)
 		}
+
+		// inform user at end of run that command will be removed
+		deprecated := color.Style{color.Red, color.OpBold}.Sprint("DEPRECATED: This command will be removed in v1.0.0")
+		fmt.Fprintln(os.Stderr, deprecated)
 	}()
 
 	if err != nil {
@@ -95,7 +102,11 @@ func powerUserExecWorker(userInput string) <-chan error {
 	go func() {
 		defer close(errs)
 
-		tasks, err := powerUserTasks()
+		appConfig.Secrets.Cataloger.Enabled = true
+		appConfig.FileMetadata.Cataloger.Enabled = true
+		appConfig.FileContents.Cataloger.Enabled = true
+		appConfig.FileClassification.Cataloger.Enabled = true
+		tasks, err := tasks()
 		if err != nil {
 			errs <- err
 			return
@@ -116,15 +127,15 @@ func powerUserExecWorker(userInput string) <-chan error {
 			Source: src.Metadata,
 		}
 
-		var results []<-chan artifact.Relationship
+		var relationships []<-chan artifact.Relationship
 		for _, task := range tasks {
 			c := make(chan artifact.Relationship)
-			results = append(results, c)
+			relationships = append(relationships, c)
 
 			go runTask(task, &s.Artifacts, src, c, errs)
 		}
 
-		for relationship := range mergeResults(results...) {
+		for relationship := range mergeRelationships(relationships...) {
 			s.Relationships = append(s.Relationships, relationship)
 		}
 
@@ -133,40 +144,6 @@ func powerUserExecWorker(userInput string) <-chan error {
 			Value: poweruser.NewJSONPresenter(s, *appConfig),
 		})
 	}()
+
 	return errs
 }
-
-func runTask(t powerUserTask, a *sbom.Artifacts, src *source.Source, c chan<- artifact.Relationship, errs chan<- error) {
-	defer close(c)
-
-	relationships, err := t(a, src)
-	if err != nil {
-		errs <- err
-		return
-	}
-
-	for _, relationship := range relationships {
-		c <- relationship
-	}
-}
-
-func mergeResults(cs ...<-chan artifact.Relationship) <-chan artifact.Relationship {
-	var wg sync.WaitGroup
-	var results = make(chan artifact.Relationship)
-
-	wg.Add(len(cs))
-	for _, c := range cs {
-		go func(c <-chan artifact.Relationship) {
-			for n := range c {
-				results <- n
-			}
-			wg.Done()
-		}(c)
-	}
-
-	go func() {
-		wg.Wait()
-		close(results)
-	}()
-	return results
-}

cmd/tasks.go

@@ -4,27 +4,25 @@ import (
 	"crypto"
 	"fmt"
 
-	"github.com/anchore/syft/syft/artifact"
-
-	"github.com/anchore/syft/syft/sbom"
-
 	"github.com/anchore/syft/syft"
+	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/sbom"
 	"github.com/anchore/syft/syft/source"
 )
 
-type powerUserTask func(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error)
+type task func(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error)
 
-func powerUserTasks() ([]powerUserTask, error) {
+func tasks() ([]task, error) {
-	var tasks []powerUserTask
+	var tasks []task
 
-	generators := []func() (powerUserTask, error){
+	generators := []func() (task, error){
-		catalogPackagesTask,
+		generateCatalogPackagesTask,
-		catalogFileMetadataTask,
+		generateCatalogFileMetadataTask,
-		catalogFileDigestsTask,
+		generateCatalogFileDigestsTask,
-		catalogSecretsTask,
+		generateCatalogSecretsTask,
-		catalogFileClassificationsTask,
+		generateCatalogFileClassificationsTask,
-		catalogContentsTask,
+		generateCatalogContentsTask,
 	}
 
 	for _, generator := range generators {
@@ -32,6 +30,7 @@ func powerUserTasks() ([]powerUserTask, error) {
 		if err != nil {
 			return nil, err
 		}
+
 		if task != nil {
 			tasks = append(tasks, task)
 		}
@@ -40,7 +39,7 @@ func powerUserTasks() ([]powerUserTask, error) {
 	return tasks, nil
 }
 
-func catalogPackagesTask() (powerUserTask, error) {
+func generateCatalogPackagesTask() (task, error) {
 	if !appConfig.Package.Cataloger.Enabled {
 		return nil, nil
 	}
@@ -60,7 +59,7 @@ func catalogPackagesTask() (powerUserTask, error) {
 	return task, nil
 }
 
-func catalogFileMetadataTask() (powerUserTask, error) {
+func generateCatalogFileMetadataTask() (task, error) {
 	if !appConfig.FileMetadata.Cataloger.Enabled {
 		return nil, nil
 	}
@@ -84,7 +83,7 @@ func catalogFileMetadataTask() (powerUserTask, error) {
 	return task, nil
 }
 
-func catalogFileDigestsTask() (powerUserTask, error) {
+func generateCatalogFileDigestsTask() (task, error) {
 	if !appConfig.FileMetadata.Cataloger.Enabled {
 		return nil, nil
 	}
@@ -130,7 +129,7 @@ func catalogFileDigestsTask() (powerUserTask, error) {
 	return task, nil
 }
 
-func catalogSecretsTask() (powerUserTask, error) {
+func generateCatalogSecretsTask() (task, error) {
 	if !appConfig.Secrets.Cataloger.Enabled {
 		return nil, nil
 	}
@@ -162,7 +161,7 @@ func catalogSecretsTask() (powerUserTask, error) {
 	return task, nil
 }
 
-func catalogFileClassificationsTask() (powerUserTask, error) {
+func generateCatalogFileClassificationsTask() (task, error) {
 	if !appConfig.FileClassification.Cataloger.Enabled {
 		return nil, nil
 	}
@@ -190,7 +189,7 @@ func catalogFileClassificationsTask() (powerUserTask, error) {
 	return task, nil
 }
 
-func catalogContentsTask() (powerUserTask, error) {
+func generateCatalogContentsTask() (task, error) {
 	if !appConfig.FileContents.Cataloger.Enabled {
 		return nil, nil
 	}
@@ -216,3 +215,17 @@ func catalogContentsTask() (powerUserTask, error) {
 
 	return task, nil
 }
+
+func runTask(t task, a *sbom.Artifacts, src *source.Source, c chan<- artifact.Relationship, errs chan<- error) {
+	defer close(c)
+
+	relationships, err := t(a, src)
+	if err != nil {
+		errs <- err
+		return
+	}
+
+	for _, relationship := range relationships {
+		c <- relationship
+	}
+}

internal/config/application.go

@@ -17,6 +17,8 @@ import (
 
 var ErrApplicationConfigNotFound = fmt.Errorf("application config not found")
 
+var catalogerEnabledDefault = false
+
 type defaultValueLoader interface {
 	loadDefaultValues(*viper.Viper)
 }
@@ -44,6 +46,11 @@ type Application struct {
 	Registry           registry           `yaml:"registry" json:"registry" mapstructure:"registry"`
 }
 
+// PowerUserCatalogerEnabledDefault switches all catalogers to be enabled when running power-user command
+func PowerUserCatalogerEnabledDefault() {
+	catalogerEnabledDefault = true
+}
+
 func newApplicationConfig(v *viper.Viper, cliOpts CliOnlyOptions) *Application {
 	config := &Application{
 		CliOptions: cliOpts,

internal/config/file_classification.go

@@ -10,7 +10,7 @@ type fileClassification struct {
 }
 
 func (cfg fileClassification) loadDefaultValues(v *viper.Viper) {
-	v.SetDefault("file-classification.cataloger.enabled", true)
+	v.SetDefault("file-classification.cataloger.enabled", catalogerEnabledDefault)
 	v.SetDefault("file-classification.cataloger.scope", source.SquashedScope)
 }
 

internal/config/file_contents.go

@@ -13,7 +13,7 @@ type fileContents struct {
 }
 
 func (cfg fileContents) loadDefaultValues(v *viper.Viper) {
-	v.SetDefault("file-contents.cataloger.enabled", true)
+	v.SetDefault("file-contents.cataloger.enabled", catalogerEnabledDefault)
 	v.SetDefault("file-contents.cataloger.scope", source.SquashedScope)
 	v.SetDefault("file-contents.skip-files-above-size", 1*file.MB)
 	v.SetDefault("file-contents.globs", []string{})

internal/config/file_metadata.go

@@ -11,7 +11,7 @@ type FileMetadata struct {
 }
 
 func (cfg FileMetadata) loadDefaultValues(v *viper.Viper) {
-	v.SetDefault("file-metadata.cataloger.enabled", true)
+	v.SetDefault("file-metadata.cataloger.enabled", catalogerEnabledDefault)
 	v.SetDefault("file-metadata.cataloger.scope", source.SquashedScope)
 	v.SetDefault("file-metadata.digests", []string{"sha256"})
 }

internal/config/secrets.go

@@ -15,7 +15,7 @@ type secrets struct {
 }
 
 func (cfg secrets) loadDefaultValues(v *viper.Viper) {
-	v.SetDefault("secrets.cataloger.enabled", true)
+	v.SetDefault("secrets.cataloger.enabled", catalogerEnabledDefault)
 	v.SetDefault("secrets.cataloger.scope", source.AllLayersScope)
 	v.SetDefault("secrets.reveal-values", false)
 	v.SetDefault("secrets.skip-files-above-size", 1*file.MB)

internal/ui/select.go

@@ -11,8 +11,6 @@ import (
 	"golang.org/x/term"
 )
 
-// TODO: build tags to exclude options from windows
-
 // Select is responsible for determining the specific UI function given select user option, the current platform
 // config values, and environment status (such as a TTY being present). The first UI in the returned slice of UIs
 // is intended to be used and the UIs that follow are meant to be attempted only in a fallback posture when there