feat: multi-level configuration and profiles (#3337)

Signed-off-by: Keith Zantow <kzantow@gmail.com>

cmd/syft/internal/clio_setup_config.go

@@ -20,18 +20,18 @@ func AppClioSetupConfig(id clio.Identification, out io.Writer) *clio.SetupConfig
 		WithConfigInRootHelp().   // --help on the root command renders the full application config in the help text
 		WithUIConstructor(
 			// select a UI based on the logging configuration and state of stdin (if stdin is a tty)
-			func(cfg clio.Config) ([]clio.UI, error) {
+			func(cfg clio.Config) (*clio.UICollection, error) {
 				noUI := ui.None(out, cfg.Log.Quiet)
 				if !cfg.Log.AllowUI(os.Stdin) || cfg.Log.Quiet {
-					return []clio.UI{noUI}, nil
+					return clio.NewUICollection(noUI), nil
 				}
 
-				return []clio.UI{
+				return clio.NewUICollection(
 					ui.New(out, cfg.Log.Quiet,
 						ui2.New(ui2.DefaultHandlerConfig()),
 					),
 					noUI,
-				}, nil
+				), nil
 			},
 		).
 		WithInitializers(

cmd/syft/internal/commands/scan.go

@@ -13,6 +13,7 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/anchore/clio"
+	"github.com/anchore/fangs"
 	"github.com/anchore/go-collections"
 	"github.com/anchore/stereoscope"
 	"github.com/anchore/stereoscope/pkg/image"
@@ -109,7 +110,7 @@ func (o *scanOptions) PostLoad() error {
 }
 
 func (o *scanOptions) validateLegacyOptionsNotUsed() error {
-	if o.Config.ConfigFile == "" {
+	if len(fangs.Flatten(o.Config.ConfigFile)) == 0 {
 		return nil
 	}
 
@@ -121,32 +122,33 @@ func (o *scanOptions) validateLegacyOptionsNotUsed() error {
 		File                            any     `yaml:"file" json:"file" mapstructure:"file"`
 	}
 
-	by, err := os.ReadFile(o.Config.ConfigFile)
+	for _, f := range fangs.Flatten(o.Config.ConfigFile) {
-	if err != nil {
+		by, err := os.ReadFile(f)
-		return fmt.Errorf("unable to read config file during validations %q: %w", o.Config.ConfigFile, err)
+		if err != nil {
-	}
+			return fmt.Errorf("unable to read config file during validations %q: %w", f, err)
+		}
 
-	var legacy legacyConfig
+		var legacy legacyConfig
-	if err := yaml.Unmarshal(by, &legacy); err != nil {
+		if err := yaml.Unmarshal(by, &legacy); err != nil {
-		return fmt.Errorf("unable to parse config file during validations %q: %w", o.Config.ConfigFile, err)
+			return fmt.Errorf("unable to parse config file during validations %q: %w", f, err)
-	}
+		}
 
-	if legacy.DefaultImagePullSource != nil {
+		if legacy.DefaultImagePullSource != nil {
-		return fmt.Errorf("the config file option 'default-image-pull-source' has been removed, please use 'source.image.default-pull-source' instead")
+			return fmt.Errorf("the config file option 'default-image-pull-source' has been removed, please use 'source.image.default-pull-source' instead")
-	}
+		}
 
-	if legacy.ExcludeBinaryOverlapByOwnership != nil {
+		if legacy.ExcludeBinaryOverlapByOwnership != nil {
-		return fmt.Errorf("the config file option 'exclude-binary-overlap-by-ownership' has been removed, please use 'package.exclude-binary-overlap-by-ownership' instead")
+			return fmt.Errorf("the config file option 'exclude-binary-overlap-by-ownership' has been removed, please use 'package.exclude-binary-overlap-by-ownership' instead")
-	}
+		}
 
-	if legacy.BasePath != nil {
+		if legacy.BasePath != nil {
-		return fmt.Errorf("the config file option 'base-path' has been removed, please use 'source.base-path' instead")
+			return fmt.Errorf("the config file option 'base-path' has been removed, please use 'source.base-path' instead")
-	}
+		}
 
-	if legacy.File != nil && reflect.TypeOf(legacy.File).Kind() == reflect.String {
+		if legacy.File != nil && reflect.TypeOf(legacy.File).Kind() == reflect.String {
-		return fmt.Errorf("the config file option 'file' has been removed, please use 'outputs' instead")
+			return fmt.Errorf("the config file option 'file' has been removed, please use 'outputs' instead")
+		}
 	}
-
 	return nil
 }
 

cmd/syft/internal/options/config.go

@@ -2,11 +2,11 @@ package options
 
 import "github.com/anchore/fangs"
 
-// Config holds a reference to the specific config file that was used to load application configuration
+// Config holds a reference to the specific config file(s) that were used to load application configuration
 type Config struct {
 	ConfigFile string `yaml:"config" json:"config" mapstructure:"config"`
 }
 
 func (cfg *Config) DescribeFields(descriptions fangs.FieldDescriptionSet) {
-	descriptions.Add(&cfg.ConfigFile, "the configuration file that was used to load application configuration")
+	descriptions.Add(&cfg.ConfigFile, "the configuration file(s) used to load application configuration")
 }

go.mod

@@ -11,8 +11,8 @@ require (
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/acobaugh/osrelease v0.1.0
 	github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9
-	github.com/anchore/clio v0.0.0-20240522144804-d81e109008aa
+	github.com/anchore/clio v0.0.0-20241015191535-f538a9016e10
-	github.com/anchore/fangs v0.0.0-20240903175602-e716ef12c23d
+	github.com/anchore/fangs v0.0.0-20241014201141-b6e4b3469f10
 	github.com/anchore/go-collections v0.0.0-20240216171411-9321230ce537
 	github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a
 	github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb

go.sum

@@ -97,10 +97,10 @@ github.com/anchore/archiver/v3 v3.5.2 h1:Bjemm2NzuRhmHy3m0lRe5tNoClB9A4zYyDV58Pa
 github.com/anchore/archiver/v3 v3.5.2/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4=
 github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 h1:p0ZIe0htYOX284Y4axJaGBvXHU0VCCzLN5Wf5XbKStU=
 github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9/go.mod h1:3ZsFB9tzW3vl4gEiUeuSOMDnwroWxIxJelOOHUp8dSw=
-github.com/anchore/clio v0.0.0-20240522144804-d81e109008aa h1:pwlAn4O9SBUnlgfa69YcqIynbUyobLVFYu8HxSoCffA=
+github.com/anchore/clio v0.0.0-20241015191535-f538a9016e10 h1:3xmanFdoQEH0REvPA+gLm3Km0/981F4z2a/7ADTlv8k=
-github.com/anchore/clio v0.0.0-20240522144804-d81e109008aa/go.mod h1:nD3H5uIvjxlfmakOBgtyFQbk5Zjp3l538kxfpHPslzI=
+github.com/anchore/clio v0.0.0-20241015191535-f538a9016e10/go.mod h1:h6Ly2hlKjQoPtI3rA8oB5afSmB/XimhcY55xbuW4Dwo=
-github.com/anchore/fangs v0.0.0-20240903175602-e716ef12c23d h1:ZD4wdCBgJJzJybjTUIEiiupLF7B9H3WLuBTjspBO2Mc=
+github.com/anchore/fangs v0.0.0-20241014201141-b6e4b3469f10 h1:w+HibE+e/heP6ysADh7sWxg5LhYdVqrpB1A4Hmgjyx8=
-github.com/anchore/fangs v0.0.0-20240903175602-e716ef12c23d/go.mod h1:Xh4ObY3fmoMzOEVXwDtS1uK44JC7+nRD0n29/1KYFYg=
+github.com/anchore/fangs v0.0.0-20241014201141-b6e4b3469f10/go.mod h1:s0L1//Sxn6Rq0Dcxx+dmT/RRmD9HhsaJjJkPUJHLJLM=
 github.com/anchore/go-collections v0.0.0-20240216171411-9321230ce537 h1:GjNGuwK5jWjJMyVppBjYS54eOiiSNv4Ba869k4wh72Q=
 github.com/anchore/go-collections v0.0.0-20240216171411-9321230ce537/go.mod h1:1aiktV46ATCkuVg0O573ZrH56BUawTECPETbZyBcqT8=
 github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a h1:nJ2G8zWKASyVClGVgG7sfM5mwoZlZ2zYpIzN2OhjWkw=

test/cli/config_test.go

@@ -0,0 +1,211 @@
+package cli
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func Test_configLoading(t *testing.T) {
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+	defer func() { require.NoError(t, os.Chdir(cwd)) }()
+
+	configsDir := filepath.Join(cwd, "test-fixtures", "configs")
+	path := func(path string) string {
+		return filepath.Join(configsDir, filepath.Join(strings.Split(path, "/")...))
+	}
+
+	type creds struct {
+		Authority string `yaml:"authority"`
+	}
+
+	type registry struct {
+		Credentials []creds `yaml:"auth"`
+	}
+
+	type config struct {
+		Registry registry `yaml:"registry"`
+	}
+
+	tests := []struct {
+		name     string
+		home     string
+		cwd      string
+		args     []string
+		expected []creds
+		err      string
+	}{
+		{
+			name: "single explicit config",
+			home: configsDir,
+			cwd:  cwd,
+			args: []string{
+				"-c",
+				path("dir1/.syft.yaml"),
+			},
+			expected: []creds{
+				{
+					Authority: "dir1-authority",
+				},
+			},
+		},
+		{
+			name: "multiple explicit config",
+			home: configsDir,
+			cwd:  cwd,
+			args: []string{
+				"-c",
+				path("dir1/.syft.yaml"),
+				"-c",
+				path("dir2/.syft.yaml"),
+			},
+			expected: []creds{
+				{
+					Authority: "dir1-authority",
+				},
+				{
+					Authority: "dir2-authority",
+				},
+			},
+		},
+		{
+			name: "empty profile override",
+			home: configsDir,
+			cwd:  cwd,
+			args: []string{
+				"-c",
+				path("dir1/.syft.yaml"),
+				"-c",
+				path("dir2/.syft.yaml"),
+				"--profile",
+				"no-auth",
+			},
+			expected: []creds{},
+		},
+		{
+			name: "no profiles defined",
+			home: configsDir,
+			cwd:  configsDir,
+			args: []string{
+				"--profile",
+				"invalid",
+			},
+			err: "not found in any configuration files",
+		},
+		{
+			name: "invalid profile name",
+			home: configsDir,
+			cwd:  cwd,
+			args: []string{
+				"-c",
+				path("dir1/.syft.yaml"),
+				"-c",
+				path("dir2/.syft.yaml"),
+				"--profile",
+				"alt",
+			},
+			err: "profile not found",
+		},
+		{
+			name: "explicit with profile override",
+			home: configsDir,
+			cwd:  cwd,
+			args: []string{
+				"-c",
+				path("dir1/.syft.yaml"),
+				"-c",
+				path("dir2/.syft.yaml"),
+				"--profile",
+				"alt-auth",
+			},
+			expected: []creds{
+				{
+					Authority: "dir1-alt-authority", // dir1 is still first
+				},
+				{
+					Authority: "dir2-alt-authority",
+				},
+			},
+		},
+		{
+			name: "single in cwd",
+			home: configsDir,
+			cwd:  path("dir2"),
+			args: []string{},
+			expected: []creds{
+				{
+					Authority: "dir2-authority",
+				},
+			},
+		},
+		{
+			name: "single in home",
+			home: path("dir2"),
+			cwd:  configsDir,
+			args: []string{},
+			expected: []creds{
+				{
+					Authority: "dir2-authority",
+				},
+			},
+		},
+		{
+			name: "inherited in cwd",
+			home: path("dir1"),
+			cwd:  path("dir2"),
+			args: []string{},
+			expected: []creds{
+				{
+					Authority: "dir2-authority", // dir2 is in cwd, giving higher priority
+				},
+				{
+					Authority: "dir1-authority", // home has "lower priority and should be after"
+				},
+			},
+		},
+		{
+			name: "inherited profile override",
+			home: path("dir1"),
+			cwd:  path("dir2"),
+			args: []string{
+				"--profile",
+				"alt-auth",
+			},
+			expected: []creds{
+				{
+					Authority: "dir2-alt-authority", // dir2 is in cwd, giving higher priority
+				},
+				{
+					Authority: "dir1-alt-authority", // dir1 is home, lower priority
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require.NoError(t, os.Chdir(test.cwd))
+			defer func() { require.NoError(t, os.Chdir(cwd)) }()
+			env := map[string]string{
+				"HOME":            test.home,
+				"XDG_CONFIG_HOME": test.home,
+			}
+			_, stdout, stderr := runSyft(t, env, append([]string{"config", "--load"}, test.args...)...)
+			if test.err != "" {
+				require.Contains(t, stderr, test.err)
+				return
+			} else {
+				require.Empty(t, stderr)
+			}
+			got := config{}
+			err = yaml.NewDecoder(strings.NewReader(stdout)).Decode(&got)
+			require.NoError(t, err)
+			require.Equal(t, test.expected, got.Registry.Credentials)
+		})
+	}
+}

test/cli/test-fixtures/configs/dir1/.syft.yaml

@@ -0,0 +1,13 @@
+registry:
+  auth:
+    - authority: dir1-authority
+
+profiles:
+  no-auth:
+    registry:
+      auth: []
+
+  alt-auth:
+    registry:
+      auth:
+        - authority: dir1-alt-authority

test/cli/test-fixtures/configs/dir2/.syft.yaml

@@ -0,0 +1,9 @@
+registry:
+  auth:
+    - authority: dir2-authority
+
+profiles:
+  alt-auth:
+    registry:
+      auth:
+        - authority: dir2-alt-authority