oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-02 23:46:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add secrets search capability (#367)

Browse Source 
* add initial secrets cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update ETUI elements with new catalogers (file metadata, digests, and secrets)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update secrets cataloger to read full contents into memory for searching

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype of parallelization secret regex search

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype with single aggregated regex

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype for secret search line-by-line

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype hybrid secrets search

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add secrets cataloger with line strategy

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* adjust verbiage towards SearchResults instead of Secrets + add tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update json schema with secrets cataloger results

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* address PR comments

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update readme with secrets config options

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* ensure file catalogers call AllLocations once

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2021-04-01 21:34:15 +00:00
committed by GitHub
parent 557ad8be49
commit 9ec09add67
53 changed files with 2549 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/config/anchore.go
+6
View File
@@ -1,5 +1,7 @@
package config
import "github.com/spf13/viper"
type anchore struct {
// upload options
Host string `yaml:"host" json:"host" mapstructure:"host"` // -H , hostname of the engine/enterprise instance to upload to (setting this value enables upload)
@@ -11,3 +13,7 @@ type anchore struct {
Dockerfile string `yaml:"dockerfile" json:"dockerfile" mapstructure:"dockerfile"` // -d , dockerfile to attach for upload
OverwriteExistingImage bool `yaml:"overwrite-existing-image" json:"overwrite-existing-image" mapstructure:"overwrite-existing-image"` // --overwrite-existing-image , if any of the SBOM components have already been uploaded this flag will ensure they are overwritten with the current upload
}
func (cfg anchore) loadDefaultValues(v *viper.Viper) {
v.SetDefault("anchore.path", "")
}
internal/config/application.go
+48 -28
View File
@@ -4,10 +4,9 @@ import (
"errors"
"fmt"
"path"
"reflect"
"strings"
"github.com/anchore/syft/syft/source"
"github.com/adrg/xdg"
"github.com/anchore/syft/internal"
"github.com/mitchellh/go-homedir"
@@ -18,6 +17,14 @@ import (
var ErrApplicationConfigNotFound = fmt.Errorf("application config not found")
type defaultValueLoader interface {
loadDefaultValues(*viper.Viper)
}
type parser interface {
parseConfigValues() error
}
// Application is the main syft application configuration.
type Application struct {
ConfigPath string `yaml:",omitempty" json:"configPath"` // the location where the application config was read from (either from -c or discovered while loading)
@@ -30,34 +37,56 @@ type Application struct {
Anchore anchore `yaml:"anchore" json:"anchore" mapstructure:"anchore"` // options for interacting with Anchore Engine/Enterprise
Package Packages `yaml:"package" json:"package" mapstructure:"package"`
FileMetadata FileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"`
Secrets Secrets `yaml:"secrets" json:"secrets" mapstructure:"secrets"`
}
func newApplicationConfig(v *viper.Viper, cliOpts CliOnlyOptions) *Application {
config := &Application{
CliOptions: cliOpts,
}
config.loadDefaultValues(v)
return config
}
// LoadApplicationConfig populates the given viper object with application configuration discovered on disk
func LoadApplicationConfig(v *viper.Viper, cliOpts CliOnlyOptions) (*Application, error) {
// the user may not have a config, and this is OK, we can use the default config + default cobra cli values instead
setNonCliDefaultAppConfigValues(v)
config := newApplicationConfig(v, cliOpts)
if err := readConfig(v, cliOpts.ConfigPath); err != nil && !errors.Is(err, ErrApplicationConfigNotFound) {
return nil, err
}
config := &Application{
CliOptions: cliOpts,
}
if err := v.Unmarshal(config); err != nil {
return nil, fmt.Errorf("unable to parse config: %w", err)
}
config.ConfigPath = v.ConfigFileUsed()
if err := config.build(); err != nil {
if err := config.parseConfigValues(); err != nil {
return nil, fmt.Errorf("invalid application config: %w", err)
}
return config, nil
}
// init loads the default configuration values into the viper instance (before the config values are read and parsed).
func (cfg Application) loadDefaultValues(v *viper.Viper) {
// set the default values for primitive fields in this struct
v.SetDefault("check-for-app-update", true)
// for each field in the configuration struct, see if the field implements the defaultValueLoader interface and invoke it if it does
value := reflect.ValueOf(cfg)
for i := 0; i < value.NumField(); i++ {
// note: the defaultValueLoader method receiver is NOT a pointer receiver.
if loadable, ok := value.Field(i).Interface().(defaultValueLoader); ok {
// the field implements defaultValueLoader, call it
loadable.loadDefaultValues(v)
}
}
}
// build inflates simple config values into syft native objects (or other complex objects) after the config is fully read in.
func (cfg *Application) build() error {
func (cfg *Application) parseConfigValues() error {
if cfg.Quiet {
// TODO: this is bad: quiet option trumps all other logging options
// we should be able to quiet the console logging and leave file logging alone...
@@ -92,12 +121,16 @@ func (cfg *Application) build() error {
return fmt.Errorf("cannot provide dockerfile option without enabling upload")
}
for _, builder := range []func() error{
cfg.Package.build,
cfg.FileMetadata.build,
} {
if err := builder(); err != nil {
return err
// for each field in the configuration struct, see if the field implements the parser interface
// note: the app config is a pointer, so we need to grab the elements explicitly (to traverse the address)
value := reflect.ValueOf(cfg).Elem()
for i := 0; i < value.NumField(); i++ {
// note: since the interface method of parser is a pointer receiver we need to get the value of the field as a pointer.
if parsable, ok := value.Field(i).Addr().Interface().(parser); ok {
// the field implements parser, call it
if err := parsable.parseConfigValues(); err != nil {
return err
}
}
}
@@ -181,16 +214,3 @@ func readConfig(v *viper.Viper, configPath string) error {
return ErrApplicationConfigNotFound
}
// setNonCliDefaultAppConfigValues ensures that there are sane defaults for values that do not have CLI equivalent options (where there would already be a default value)
func setNonCliDefaultAppConfigValues(v *viper.Viper) {
v.SetDefault("anchore.path", "")
v.SetDefault("log.structured", false)
v.SetDefault("check-for-app-update", true)
v.SetDefault("dev.profile-cpu", false)
v.SetDefault("dev.profile-mem", false)
v.SetDefault("package.cataloger.enabled", true)
v.SetDefault("file-metadata.cataloger.enabled", true)
v.SetDefault("file-metadata.cataloger.scope", source.SquashedScope)
v.SetDefault("file-metadata.digests", []string{"sha256"})
}
internal/config/cataloger_options.go
+1 -1
View File
@@ -12,7 +12,7 @@ type catalogerOptions struct {
ScopeOpt source.Scope `yaml:"-" json:"-"`
}
func (cfg *catalogerOptions) build() error {
func (cfg *catalogerOptions) parseConfigValues() error {
scopeOption := source.ParseScope(cfg.Scope)
if scopeOption == source.UnknownScope {
return fmt.Errorf("bad scope value %q", cfg.Scope)
internal/config/development.go
+7
View File
@@ -1,6 +1,13 @@
package config
import "github.com/spf13/viper"
type Development struct {
ProfileCPU bool `yaml:"profile-cpu" json:"profile-cpu" mapstructure:"profile-cpu"`
ProfileMem bool `yaml:"profile-mem" json:"profile-mem" mapstructure:"profile-mem"`
}
func (cfg Development) loadDefaultValues(v *viper.Viper) {
v.SetDefault("dev.profile-cpu", false)
v.SetDefault("dev.profile-mem", false)
}
internal/config/file_metadata.go
+13 -2
View File
@@ -1,10 +1,21 @@
package config
import (
"github.com/anchore/syft/syft/source"
"github.com/spf13/viper"
)
type FileMetadata struct {
Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
Digests []string `yaml:"digests" json:"digests" mapstructure:"digests"`
}
func (cfg *FileMetadata) build() error {
return cfg.Cataloger.build()
func (cfg FileMetadata) loadDefaultValues(v *viper.Viper) {
v.SetDefault("file-metadata.cataloger.enabled", true)
v.SetDefault("file-metadata.cataloger.scope", source.SquashedScope)
v.SetDefault("file-metadata.digests", []string{"sha256"})
}
func (cfg *FileMetadata) parseConfigValues() error {
return cfg.Cataloger.parseConfigValues()
}
internal/config/logging.go
+8 -1
View File
@@ -1,6 +1,9 @@
package config
import "github.com/sirupsen/logrus"
import (
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// logging contains all logging-related configuration options available to the user via the application config.
type logging struct {
@@ -9,3 +12,7 @@ type logging struct {
Level string `yaml:"level" json:"level" mapstructure:"level"` // the log level string hint
FileLocation string `yaml:"file" json:"file-location" mapstructure:"file"` // the file path to write logs to
}
func (cfg logging) loadDefaultValues(v *viper.Viper) {
v.SetDefault("log.structured", false)
}
internal/config/packages.go
+8 -2
View File
@@ -1,9 +1,15 @@
package config
import "github.com/spf13/viper"
type Packages struct {
Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
}
func (cfg *Packages) build() error {
return cfg.Cataloger.build()
func (cfg Packages) loadDefaultValues(v *viper.Viper) {
v.SetDefault("package.cataloger.enabled", true)
}
func (cfg *Packages) parseConfigValues() error {
return cfg.Cataloger.parseConfigValues()
}
internal/config/secrets.go
+28
View File
@@ -0,0 +1,28 @@
package config
import (
"github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft/source"
"github.com/spf13/viper"
)
type Secrets struct {
Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
AdditionalPatterns map[string]string `yaml:"additional-patterns" json:"additional-patterns" mapstructure:"additional-patterns"`
ExcludePatternNames []string `yaml:"exclude-pattern-names" json:"exclude-pattern-names" mapstructure:"exclude-pattern-names"`
RevealValues bool `yaml:"reveal-values" json:"reveal-values" mapstructure:"reveal-values"`
SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"`
}
func (cfg Secrets) loadDefaultValues(v *viper.Viper) {
v.SetDefault("secrets.cataloger.enabled", true)
v.SetDefault("secrets.cataloger.scope", source.AllLayersScope)
v.SetDefault("secrets.reveal-values", false)
v.SetDefault("secrets.skip-files-above-size", 1*file.MB)
v.SetDefault("secrets.additional-patterns", map[string]string{})
v.SetDefault("secrets.exclude-pattern-names", []string{})
}
func (cfg *Secrets) parseConfigValues() error {
return cfg.Cataloger.parseConfigValues()
}