From 9ec09add676a96c6d4f61cbe11415868be38c30c Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 1 Apr 2021 17:34:15 -0400 Subject: [PATCH] Add secrets search capability (#367) * add initial secrets cataloger Signed-off-by: Alex Goodman * update ETUI elements with new catalogers (file metadata, digests, and secrets) Signed-off-by: Alex Goodman * update secrets cataloger to read full contents into memory for searching Signed-off-by: Alex Goodman * quick prototype of parallelization secret regex search Signed-off-by: Alex Goodman * quick prototype with single aggregated regex Signed-off-by: Alex Goodman * quick prototype for secret search line-by-line Signed-off-by: Alex Goodman * quick prototype hybrid secrets search Signed-off-by: Alex Goodman * add secrets cataloger with line strategy Signed-off-by: Alex Goodman * adjust verbiage towards SearchResults instead of Secrets + add tests Signed-off-by: Alex Goodman * update json schema with secrets cataloger results Signed-off-by: Alex Goodman * address PR comments Signed-off-by: Alex Goodman * update readme with secrets config options Signed-off-by: Alex Goodman * ensure file catalogers call AllLocations once Signed-off-by: Alex Goodman --- README.md | 31 + cmd/power_user.go | 16 +- cmd/power_user_tasks.go | 37 +- internal/config/anchore.go | 6 + internal/config/application.go | 76 +- internal/config/cataloger_options.go | 2 +- internal/config/development.go | 7 + internal/config/file_metadata.go | 15 +- internal/config/logging.go | 9 +- internal/config/packages.go | 10 +- internal/config/secrets.go | 28 + internal/constants.go | 2 +- .../snapshot/TestJSONDirsPresenter.golden | 4 +- .../snapshot/TestJSONImgsPresenter.golden | 18 +- .../stereoscope-fixture-image-simple.golden | Bin 16896 -> 16896 bytes internal/presenter/poweruser/json_document.go | 4 +- .../poweruser/json_document_config.go | 1 + internal/presenter/poweruser/json_secrets.go | 32 + .../snapshot/TestJSONPresenter.golden | 14 +- schema/json/schema-1.0.5.json | 890 ++++++++++++++++++ syft/event/event.go | 12 +- syft/event/parsers/parsers.go | 43 +- syft/file/digest_cataloger.go | 36 + syft/file/generate_search_patterns.go | 56 ++ syft/file/generate_search_patterns_test.go | 125 +++ syft/file/metadata_cataloger.go | 34 + syft/file/metadata_cataloger_test.go | 3 +- syft/file/newline_counter.go | 39 + syft/file/newline_counter_test.go | 35 + syft/file/search_result.go | 18 + syft/file/secrets_cataloger.go | 150 +++ syft/file/secrets_cataloger_test.go | 444 +++++++++ syft/file/secrets_search_by_line_strategy.go | 134 +++ .../test-fixtures/secrets/default/api-key.txt | 14 + .../test-fixtures/secrets/default/aws.env | 3 + .../test-fixtures/secrets/default/aws.ini | 4 + .../secrets/default/docker-config.json | 10 + .../secrets/default/not-docker-config.json | 4 + .../default/private-key-false-positive.pem | 1 + .../secrets/default/private-key-openssl.pem | 9 + .../secrets/default/private-key.pem | 10 + .../secrets/default/private-keys.pem | 16 + syft/file/test-fixtures/secrets/multiple.txt | 6 + syft/file/test-fixtures/secrets/simple.txt | 4 + syft/lib.go | 2 +- syft/pkg/cataloger/catalog.go | 2 +- syft/source/file_metadata.go | 2 + syft/source/mock_resolver.go | 21 +- test/cli/power_user_cmd_test.go | 29 +- .../test-fixtures/image-secrets/Dockerfile | 2 + .../test-fixtures/image-secrets/api-key.txt | 1 + ui/event_handlers.go | 139 ++- ui/handler.go | 11 +- 53 files changed, 2549 insertions(+), 72 deletions(-) create mode 100644 internal/config/secrets.go create mode 100644 internal/presenter/poweruser/json_secrets.go create mode 100644 schema/json/schema-1.0.5.json create mode 100644 syft/file/generate_search_patterns.go create mode 100644 syft/file/generate_search_patterns_test.go create mode 100644 syft/file/newline_counter.go create mode 100644 syft/file/newline_counter_test.go create mode 100644 syft/file/search_result.go create mode 100644 syft/file/secrets_cataloger.go create mode 100644 syft/file/secrets_cataloger_test.go create mode 100644 syft/file/secrets_search_by_line_strategy.go create mode 100644 syft/file/test-fixtures/secrets/default/api-key.txt create mode 100644 syft/file/test-fixtures/secrets/default/aws.env create mode 100644 syft/file/test-fixtures/secrets/default/aws.ini create mode 100644 syft/file/test-fixtures/secrets/default/docker-config.json create mode 100644 syft/file/test-fixtures/secrets/default/not-docker-config.json create mode 100644 syft/file/test-fixtures/secrets/default/private-key-false-positive.pem create mode 100644 syft/file/test-fixtures/secrets/default/private-key-openssl.pem create mode 100644 syft/file/test-fixtures/secrets/default/private-key.pem create mode 100644 syft/file/test-fixtures/secrets/default/private-keys.pem create mode 100644 syft/file/test-fixtures/secrets/multiple.txt create mode 100644 syft/file/test-fixtures/secrets/simple.txt create mode 100644 test/cli/test-fixtures/image-secrets/Dockerfile create mode 100644 test/cli/test-fixtures/image-secrets/api-key.txt diff --git a/README.md b/README.md index a16433c2d..b90157efe 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,37 @@ file-metadata: # SYFT_FILE_METADATA_DIGESTS env var digests: ["sha256"] +# cataloging secrets is exposed through the power-user subcommand +secrets: + cataloger: + # enable/disable cataloging of secrets + # SYFT_SECRETS_CATALOGER_ENABLED env var + enabled: true + + # the search space to look for secrets (options: all-layers, squashed) + # SYFT_SECRETS_CATALOGER_SCOPE env var + scope: "all-layers" + + # show extracted secret values in the final JSON report + # SYFT_SECRETS_REVEAL_VALUES env var + reveal-values: false + + # skip searching a file entirely if it is above the given size (default = 10MB; unit = bytes) + # SYFT_SECRETS_SKIP_FILES_ABOVE_SIZE env var + skip-files-above-size: 10485760 + + # name-regex pairs to consider when searching files for secrets. Note: the regex must match single line patterns + # but may also have OPTIONAL multiline capture groups. Regexes with a named capture group of "value" will + # use the entire regex to match, but the secret value will be assumed to be entirely contained within the + # "value" named capture group. + additional-patterns: {} + + # names to exclude from the secrets search, valid values are: "aws-access-key", "aws-secret-key", "pem-private-key", + # "docker-config-auth", and "generic-api-key". Note: this does not consider any names introduced in the + # "secrets.additional-patterns" config option. + # SYFT_SECRETS_EXCLUDE_PATTERN_NAMES env var + exclude-pattern-names: [] + log: # use structured logging # same as SYFT_LOG_STRUCTURED env var diff --git a/cmd/power_user.go b/cmd/power_user.go index 962470eba..d44d73c68 100644 --- a/cmd/power_user.go +++ b/cmd/power_user.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "sync" "github.com/anchore/syft/internal" @@ -97,13 +98,20 @@ func powerUserExecWorker(userInput string) <-chan error { ApplicationConfig: *appConfig, } + wg := &sync.WaitGroup{} for _, task := range tasks { - if err = task(&analysisResults, src); err != nil { - errs <- err - return - } + wg.Add(1) + go func(task powerUserTask) { + defer wg.Done() + if err = task(&analysisResults, src); err != nil { + errs <- err + return + } + }(task) } + wg.Wait() + bus.Publish(partybus.Event{ Type: event.PresenterReady, Value: poweruser.NewJSONPresenter(analysisResults), diff --git a/cmd/power_user_tasks.go b/cmd/power_user_tasks.go index a33036cac..d6510922e 100644 --- a/cmd/power_user_tasks.go +++ b/cmd/power_user_tasks.go @@ -18,7 +18,8 @@ func powerUserTasks() ([]powerUserTask, error) { generators := []func() (powerUserTask, error){ catalogPackagesTask, catalogFileMetadataTask, - catalogFileDigestTask, + catalogFileDigestsTask, + catalogSecretsTask, } for _, generator := range generators { @@ -78,7 +79,7 @@ func catalogFileMetadataTask() (powerUserTask, error) { return task, nil } -func catalogFileDigestTask() (powerUserTask, error) { +func catalogFileDigestsTask() (powerUserTask, error) { if !appConfig.FileMetadata.Cataloger.Enabled { return nil, nil } @@ -123,3 +124,35 @@ func catalogFileDigestTask() (powerUserTask, error) { return task, nil } + +func catalogSecretsTask() (powerUserTask, error) { + if !appConfig.Secrets.Cataloger.Enabled { + return nil, nil + } + + patterns, err := file.GenerateSearchPatterns(file.DefaultSecretsPatterns, appConfig.Secrets.AdditionalPatterns, appConfig.Secrets.ExcludePatternNames) + if err != nil { + return nil, err + } + + secretsCataloger, err := file.NewSecretsCataloger(patterns, appConfig.Secrets.RevealValues, appConfig.Secrets.SkipFilesAboveSize) + if err != nil { + return nil, err + } + + task := func(results *poweruser.JSONDocumentConfig, src source.Source) error { + resolver, err := src.FileResolver(appConfig.Secrets.Cataloger.ScopeOpt) + if err != nil { + return err + } + + result, err := secretsCataloger.Catalog(resolver) + if err != nil { + return err + } + results.Secrets = result + return nil + } + + return task, nil +} diff --git a/internal/config/anchore.go b/internal/config/anchore.go index 7b363baa0..430c3834b 100644 --- a/internal/config/anchore.go +++ b/internal/config/anchore.go @@ -1,5 +1,7 @@ package config +import "github.com/spf13/viper" + type anchore struct { // upload options Host string `yaml:"host" json:"host" mapstructure:"host"` // -H , hostname of the engine/enterprise instance to upload to (setting this value enables upload) @@ -11,3 +13,7 @@ type anchore struct { Dockerfile string `yaml:"dockerfile" json:"dockerfile" mapstructure:"dockerfile"` // -d , dockerfile to attach for upload OverwriteExistingImage bool `yaml:"overwrite-existing-image" json:"overwrite-existing-image" mapstructure:"overwrite-existing-image"` // --overwrite-existing-image , if any of the SBOM components have already been uploaded this flag will ensure they are overwritten with the current upload } + +func (cfg anchore) loadDefaultValues(v *viper.Viper) { + v.SetDefault("anchore.path", "") +} diff --git a/internal/config/application.go b/internal/config/application.go index 24c97d1a8..0f1ea7b5a 100644 --- a/internal/config/application.go +++ b/internal/config/application.go @@ -4,10 +4,9 @@ import ( "errors" "fmt" "path" + "reflect" "strings" - "github.com/anchore/syft/syft/source" - "github.com/adrg/xdg" "github.com/anchore/syft/internal" "github.com/mitchellh/go-homedir" @@ -18,6 +17,14 @@ import ( var ErrApplicationConfigNotFound = fmt.Errorf("application config not found") +type defaultValueLoader interface { + loadDefaultValues(*viper.Viper) +} + +type parser interface { + parseConfigValues() error +} + // Application is the main syft application configuration. type Application struct { ConfigPath string `yaml:",omitempty" json:"configPath"` // the location where the application config was read from (either from -c or discovered while loading) @@ -30,34 +37,56 @@ type Application struct { Anchore anchore `yaml:"anchore" json:"anchore" mapstructure:"anchore"` // options for interacting with Anchore Engine/Enterprise Package Packages `yaml:"package" json:"package" mapstructure:"package"` FileMetadata FileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"` + Secrets Secrets `yaml:"secrets" json:"secrets" mapstructure:"secrets"` +} + +func newApplicationConfig(v *viper.Viper, cliOpts CliOnlyOptions) *Application { + config := &Application{ + CliOptions: cliOpts, + } + config.loadDefaultValues(v) + return config } // LoadApplicationConfig populates the given viper object with application configuration discovered on disk func LoadApplicationConfig(v *viper.Viper, cliOpts CliOnlyOptions) (*Application, error) { // the user may not have a config, and this is OK, we can use the default config + default cobra cli values instead - setNonCliDefaultAppConfigValues(v) + config := newApplicationConfig(v, cliOpts) + if err := readConfig(v, cliOpts.ConfigPath); err != nil && !errors.Is(err, ErrApplicationConfigNotFound) { return nil, err } - config := &Application{ - CliOptions: cliOpts, - } - if err := v.Unmarshal(config); err != nil { return nil, fmt.Errorf("unable to parse config: %w", err) } config.ConfigPath = v.ConfigFileUsed() - if err := config.build(); err != nil { + if err := config.parseConfigValues(); err != nil { return nil, fmt.Errorf("invalid application config: %w", err) } return config, nil } +// init loads the default configuration values into the viper instance (before the config values are read and parsed). +func (cfg Application) loadDefaultValues(v *viper.Viper) { + // set the default values for primitive fields in this struct + v.SetDefault("check-for-app-update", true) + + // for each field in the configuration struct, see if the field implements the defaultValueLoader interface and invoke it if it does + value := reflect.ValueOf(cfg) + for i := 0; i < value.NumField(); i++ { + // note: the defaultValueLoader method receiver is NOT a pointer receiver. + if loadable, ok := value.Field(i).Interface().(defaultValueLoader); ok { + // the field implements defaultValueLoader, call it + loadable.loadDefaultValues(v) + } + } +} + // build inflates simple config values into syft native objects (or other complex objects) after the config is fully read in. -func (cfg *Application) build() error { +func (cfg *Application) parseConfigValues() error { if cfg.Quiet { // TODO: this is bad: quiet option trumps all other logging options // we should be able to quiet the console logging and leave file logging alone... @@ -92,12 +121,16 @@ func (cfg *Application) build() error { return fmt.Errorf("cannot provide dockerfile option without enabling upload") } - for _, builder := range []func() error{ - cfg.Package.build, - cfg.FileMetadata.build, - } { - if err := builder(); err != nil { - return err + // for each field in the configuration struct, see if the field implements the parser interface + // note: the app config is a pointer, so we need to grab the elements explicitly (to traverse the address) + value := reflect.ValueOf(cfg).Elem() + for i := 0; i < value.NumField(); i++ { + // note: since the interface method of parser is a pointer receiver we need to get the value of the field as a pointer. + if parsable, ok := value.Field(i).Addr().Interface().(parser); ok { + // the field implements parser, call it + if err := parsable.parseConfigValues(); err != nil { + return err + } } } @@ -181,16 +214,3 @@ func readConfig(v *viper.Viper, configPath string) error { return ErrApplicationConfigNotFound } - -// setNonCliDefaultAppConfigValues ensures that there are sane defaults for values that do not have CLI equivalent options (where there would already be a default value) -func setNonCliDefaultAppConfigValues(v *viper.Viper) { - v.SetDefault("anchore.path", "") - v.SetDefault("log.structured", false) - v.SetDefault("check-for-app-update", true) - v.SetDefault("dev.profile-cpu", false) - v.SetDefault("dev.profile-mem", false) - v.SetDefault("package.cataloger.enabled", true) - v.SetDefault("file-metadata.cataloger.enabled", true) - v.SetDefault("file-metadata.cataloger.scope", source.SquashedScope) - v.SetDefault("file-metadata.digests", []string{"sha256"}) -} diff --git a/internal/config/cataloger_options.go b/internal/config/cataloger_options.go index 72e4c0977..2b78bb0c5 100644 --- a/internal/config/cataloger_options.go +++ b/internal/config/cataloger_options.go @@ -12,7 +12,7 @@ type catalogerOptions struct { ScopeOpt source.Scope `yaml:"-" json:"-"` } -func (cfg *catalogerOptions) build() error { +func (cfg *catalogerOptions) parseConfigValues() error { scopeOption := source.ParseScope(cfg.Scope) if scopeOption == source.UnknownScope { return fmt.Errorf("bad scope value %q", cfg.Scope) diff --git a/internal/config/development.go b/internal/config/development.go index dbb38ff08..ece8faea8 100644 --- a/internal/config/development.go +++ b/internal/config/development.go @@ -1,6 +1,13 @@ package config +import "github.com/spf13/viper" + type Development struct { ProfileCPU bool `yaml:"profile-cpu" json:"profile-cpu" mapstructure:"profile-cpu"` ProfileMem bool `yaml:"profile-mem" json:"profile-mem" mapstructure:"profile-mem"` } + +func (cfg Development) loadDefaultValues(v *viper.Viper) { + v.SetDefault("dev.profile-cpu", false) + v.SetDefault("dev.profile-mem", false) +} diff --git a/internal/config/file_metadata.go b/internal/config/file_metadata.go index 274fce24b..9d67d1012 100644 --- a/internal/config/file_metadata.go +++ b/internal/config/file_metadata.go @@ -1,10 +1,21 @@ package config +import ( + "github.com/anchore/syft/syft/source" + "github.com/spf13/viper" +) + type FileMetadata struct { Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"` Digests []string `yaml:"digests" json:"digests" mapstructure:"digests"` } -func (cfg *FileMetadata) build() error { - return cfg.Cataloger.build() +func (cfg FileMetadata) loadDefaultValues(v *viper.Viper) { + v.SetDefault("file-metadata.cataloger.enabled", true) + v.SetDefault("file-metadata.cataloger.scope", source.SquashedScope) + v.SetDefault("file-metadata.digests", []string{"sha256"}) +} + +func (cfg *FileMetadata) parseConfigValues() error { + return cfg.Cataloger.parseConfigValues() } diff --git a/internal/config/logging.go b/internal/config/logging.go index 53139419a..26f59ef64 100644 --- a/internal/config/logging.go +++ b/internal/config/logging.go @@ -1,6 +1,9 @@ package config -import "github.com/sirupsen/logrus" +import ( + "github.com/sirupsen/logrus" + "github.com/spf13/viper" +) // logging contains all logging-related configuration options available to the user via the application config. type logging struct { @@ -9,3 +12,7 @@ type logging struct { Level string `yaml:"level" json:"level" mapstructure:"level"` // the log level string hint FileLocation string `yaml:"file" json:"file-location" mapstructure:"file"` // the file path to write logs to } + +func (cfg logging) loadDefaultValues(v *viper.Viper) { + v.SetDefault("log.structured", false) +} diff --git a/internal/config/packages.go b/internal/config/packages.go index 8193ab04c..e3aa0e03c 100644 --- a/internal/config/packages.go +++ b/internal/config/packages.go @@ -1,9 +1,15 @@ package config +import "github.com/spf13/viper" + type Packages struct { Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"` } -func (cfg *Packages) build() error { - return cfg.Cataloger.build() +func (cfg Packages) loadDefaultValues(v *viper.Viper) { + v.SetDefault("package.cataloger.enabled", true) +} + +func (cfg *Packages) parseConfigValues() error { + return cfg.Cataloger.parseConfigValues() } diff --git a/internal/config/secrets.go b/internal/config/secrets.go new file mode 100644 index 000000000..54ee17dc7 --- /dev/null +++ b/internal/config/secrets.go @@ -0,0 +1,28 @@ +package config + +import ( + "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/syft/source" + "github.com/spf13/viper" +) + +type Secrets struct { + Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"` + AdditionalPatterns map[string]string `yaml:"additional-patterns" json:"additional-patterns" mapstructure:"additional-patterns"` + ExcludePatternNames []string `yaml:"exclude-pattern-names" json:"exclude-pattern-names" mapstructure:"exclude-pattern-names"` + RevealValues bool `yaml:"reveal-values" json:"reveal-values" mapstructure:"reveal-values"` + SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"` +} + +func (cfg Secrets) loadDefaultValues(v *viper.Viper) { + v.SetDefault("secrets.cataloger.enabled", true) + v.SetDefault("secrets.cataloger.scope", source.AllLayersScope) + v.SetDefault("secrets.reveal-values", false) + v.SetDefault("secrets.skip-files-above-size", 1*file.MB) + v.SetDefault("secrets.additional-patterns", map[string]string{}) + v.SetDefault("secrets.exclude-pattern-names", []string{}) +} + +func (cfg *Secrets) parseConfigValues() error { + return cfg.Cataloger.parseConfigValues() +} diff --git a/internal/constants.go b/internal/constants.go index b027f8502..40280bcf2 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -6,5 +6,5 @@ const ( // JSONSchemaVersion is the current schema version output by the JSON presenter // This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment. - JSONSchemaVersion = "1.0.4" + JSONSchemaVersion = "1.0.5" ) diff --git a/internal/presenter/packages/test-fixtures/snapshot/TestJSONDirsPresenter.golden b/internal/presenter/packages/test-fixtures/snapshot/TestJSONDirsPresenter.golden index b43c0e029..22592df9f 100644 --- a/internal/presenter/packages/test-fixtures/snapshot/TestJSONDirsPresenter.golden +++ b/internal/presenter/packages/test-fixtures/snapshot/TestJSONDirsPresenter.golden @@ -75,7 +75,7 @@ "version": "[not provided]" }, "schema": { - "version": "1.0.4", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.4.json" + "version": "1.0.5", + "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.5.json" } } diff --git a/internal/presenter/packages/test-fixtures/snapshot/TestJSONImgsPresenter.golden b/internal/presenter/packages/test-fixtures/snapshot/TestJSONImgsPresenter.golden index 6fb8ab011..444f0bbea 100644 --- a/internal/presenter/packages/test-fixtures/snapshot/TestJSONImgsPresenter.golden +++ b/internal/presenter/packages/test-fixtures/snapshot/TestJSONImgsPresenter.golden @@ -9,7 +9,7 @@ "locations": [ { "path": "/somefile-1.txt", - "layerID": "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b" + "layerID": "sha256:6c376352c0537f4483e4033e332d7a4ab9433db68c54c297a834d36719aeb6c9" } ], "licenses": [ @@ -40,7 +40,7 @@ "locations": [ { "path": "/somefile-2.txt", - "layerID": "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703" + "layerID": "sha256:fc8218a8142ee4952bb8d9b96b3e9838322e9e6eae6477136bcad8fd768949b7" } ], "licenses": [], @@ -67,7 +67,7 @@ "type": "image", "target": { "userInput": "user-image-input", - "imageID": "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0", + "imageID": "sha256:1f9cb9dc477f7482856f88ed40c38e260db0526d7a0dad5a0be566bfedde929b", "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", "mediaType": "application/vnd.docker.distribution.manifest.v2+json", "tags": [ @@ -77,17 +77,17 @@ "layers": [ { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b", + "digest": "sha256:6c376352c0537f4483e4033e332d7a4ab9433db68c54c297a834d36719aeb6c9", "size": 22 }, { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "digest": "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703", + "digest": "sha256:fc8218a8142ee4952bb8d9b96b3e9838322e9e6eae6477136bcad8fd768949b7", "size": 16 } ], - "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjoxNTg2LCJkaWdlc3QiOiJzaGEyNTY6YzJiNDZiNGViMDYyOTY5MzNiN2NmMDcyMjY4Mzk2NGU5ZWNiZDkzMjY1YjllZjZhZTk2NDJlMzk1MmFmYmJhMCJ9LCJsYXllcnMiOlt7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLCJzaXplIjoyMDQ4LCJkaWdlc3QiOiJzaGEyNTY6M2RlMTZjNWI4NjU5YTJlOGQ4ODhiOGRlZDg0MjdiZTdhNTY4NmEzYzhjNGU0ZGQzMGRlMjBmMzYyODI3Mjg1YiJ9LHsibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjozNjZhM2Y1NjUzZTM0NjczYjg3NTg5MWIwMjE2NDc0NDBkMDEyN2MyZWYwNDFlM2IxYTIyZGEyYTdkNGYzNzAzIn1dfQ==", - "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJIb3N0bmFtZSI6IiIsIkRvbWFpbm5hbWUiOiIiLCJVc2VyIjoiIiwiQXR0YWNoU3RkaW4iOmZhbHNlLCJBdHRhY2hTdGRvdXQiOmZhbHNlLCJBdHRhY2hTdGRlcnIiOmZhbHNlLCJUdHkiOmZhbHNlLCJPcGVuU3RkaW4iOmZhbHNlLCJTdGRpbk9uY2UiOmZhbHNlLCJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpudWxsLCJJbWFnZSI6InNoYTI1NjpkYWMyMTUwMzhjMDUwZTM1NzMwNTVlZmU4YTkwM2NkMWY5YmJkZmU0ZjlhZTlkODk5OTFjNTljY2M2OTA1MmU1IiwiVm9sdW1lcyI6bnVsbCwiV29ya2luZ0RpciI6IiIsIkVudHJ5cG9pbnQiOm51bGwsIk9uQnVpbGQiOm51bGwsIkxhYmVscyI6bnVsbH0sImNvbnRhaW5lcl9jb25maWciOnsiSG9zdG5hbWUiOiIiLCJEb21haW5uYW1lIjoiIiwiVXNlciI6IiIsIkF0dGFjaFN0ZGluIjpmYWxzZSwiQXR0YWNoU3Rkb3V0IjpmYWxzZSwiQXR0YWNoU3RkZXJyIjpmYWxzZSwiVHR5IjpmYWxzZSwiT3BlblN0ZGluIjpmYWxzZSwiU3RkaW5PbmNlIjpmYWxzZSwiRW52IjpbIlBBVEg9L3Vzci9sb2NhbC9zYmluOi91c3IvbG9jYWwvYmluOi91c3Ivc2JpbjovdXNyL2Jpbjovc2JpbjovYmluIl0sIkNtZCI6WyIvYmluL3NoIiwiLWMiLCIjKG5vcCkgQUREIGZpbGU6ZGYzYjc0NGY1NGE5YjE2YjliOWFlZDQwZTNlOThkOWNhMmI0OWY1YTc3ZDlmYThhOTc2OTBkN2JhZjU4ODgyMCBpbiAvc29tZWZpbGUtMi50eHQgIl0sIkltYWdlIjoic2hhMjU2OmRhYzIxNTAzOGMwNTBlMzU3MzA1NWVmZThhOTAzY2QxZjliYmRmZTRmOWFlOWQ4OTk5MWM1OWNjYzY5MDUyZTUiLCJWb2x1bWVzIjpudWxsLCJXb3JraW5nRGlyIjoiIiwiRW50cnlwb2ludCI6bnVsbCwiT25CdWlsZCI6bnVsbCwiTGFiZWxzIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMDMtMjNUMTg6MTU6NTguODcyMjg5OFoiLCJkb2NrZXJfdmVyc2lvbiI6IjIwLjEwLjIiLCJoaXN0b3J5IjpbeyJjcmVhdGVkIjoiMjAyMS0wMy0yM1QxODoxNTo1OC42MTc3OTU2WiIsImNyZWF0ZWRfYnkiOiIvYmluL3NoIC1jICMobm9wKSBBREQgZmlsZTphYzMyZGEyM2Q1MWU4MDFmMDJmOTI0MTIzZWQzMDk5MGViM2YwZmVjMWI5ZWQ0ZjBiMDZjMjRlODhiOWMzNjk1IGluIC9zb21lZmlsZS0xLnR4dCAifSx7ImNyZWF0ZWQiOiIyMDIxLTAzLTIzVDE4OjE1OjU4Ljg3MjI4OThaIiwiY3JlYXRlZF9ieSI6Ii9iaW4vc2ggLWMgIyhub3ApIEFERCBmaWxlOmRmM2I3NDRmNTRhOWIxNmI5YjlhZWQ0MGUzZTk4ZDljYTJiNDlmNWE3N2Q5ZmE4YTk3NjkwZDdiYWY1ODg4MjAgaW4gL3NvbWVmaWxlLTIudHh0ICJ9XSwib3MiOiJsaW51eCIsInJvb3RmcyI6eyJ0eXBlIjoibGF5ZXJzIiwiZGlmZl9pZHMiOlsic2hhMjU2OjNkZTE2YzViODY1OWEyZThkODg4YjhkZWQ4NDI3YmU3YTU2ODZhM2M4YzRlNGRkMzBkZTIwZjM2MjgyNzI4NWIiLCJzaGEyNTY6MzY2YTNmNTY1M2UzNDY3M2I4NzU4OTFiMDIxNjQ3NDQwZDAxMjdjMmVmMDQxZTNiMWEyMmRhMmE3ZDRmMzcwMyJdfX0=", + "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjoxNTg2LCJkaWdlc3QiOiJzaGEyNTY6MWY5Y2I5ZGM0NzdmNzQ4Mjg1NmY4OGVkNDBjMzhlMjYwZGIwNTI2ZDdhMGRhZDVhMGJlNTY2YmZlZGRlOTI5YiJ9LCJsYXllcnMiOlt7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLCJzaXplIjoyMDQ4LCJkaWdlc3QiOiJzaGEyNTY6NmMzNzYzNTJjMDUzN2Y0NDgzZTQwMzNlMzMyZDdhNGFiOTQzM2RiNjhjNTRjMjk3YTgzNGQzNjcxOWFlYjZjOSJ9LHsibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjpmYzgyMThhODE0MmVlNDk1MmJiOGQ5Yjk2YjNlOTgzODMyMmU5ZTZlYWU2NDc3MTM2YmNhZDhmZDc2ODk0OWI3In1dfQ==", + "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJIb3N0bmFtZSI6IiIsIkRvbWFpbm5hbWUiOiIiLCJVc2VyIjoiIiwiQXR0YWNoU3RkaW4iOmZhbHNlLCJBdHRhY2hTdGRvdXQiOmZhbHNlLCJBdHRhY2hTdGRlcnIiOmZhbHNlLCJUdHkiOmZhbHNlLCJPcGVuU3RkaW4iOmZhbHNlLCJTdGRpbk9uY2UiOmZhbHNlLCJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpudWxsLCJJbWFnZSI6InNoYTI1NjoyOWQ1YjFjOTkyNjg0MzgwYjQ3NTEyMjliMmNjN2E4MzdkOTBmOWQ1OTJhYmIxZjAyZGYzZGRkMGQ3OWFjMDkxIiwiVm9sdW1lcyI6bnVsbCwiV29ya2luZ0RpciI6IiIsIkVudHJ5cG9pbnQiOm51bGwsIk9uQnVpbGQiOm51bGwsIkxhYmVscyI6bnVsbH0sImNvbnRhaW5lcl9jb25maWciOnsiSG9zdG5hbWUiOiIiLCJEb21haW5uYW1lIjoiIiwiVXNlciI6IiIsIkF0dGFjaFN0ZGluIjpmYWxzZSwiQXR0YWNoU3Rkb3V0IjpmYWxzZSwiQXR0YWNoU3RkZXJyIjpmYWxzZSwiVHR5IjpmYWxzZSwiT3BlblN0ZGluIjpmYWxzZSwiU3RkaW5PbmNlIjpmYWxzZSwiRW52IjpbIlBBVEg9L3Vzci9sb2NhbC9zYmluOi91c3IvbG9jYWwvYmluOi91c3Ivc2JpbjovdXNyL2Jpbjovc2JpbjovYmluIl0sIkNtZCI6WyIvYmluL3NoIiwiLWMiLCIjKG5vcCkgQUREIGZpbGU6ZGYzYjc0NGY1NGE5YjE2YjliOWFlZDQwZTNlOThkOWNhMmI0OWY1YTc3ZDlmYThhOTc2OTBkN2JhZjU4ODgyMCBpbiAvc29tZWZpbGUtMi50eHQgIl0sIkltYWdlIjoic2hhMjU2OjI5ZDViMWM5OTI2ODQzODBiNDc1MTIyOWIyY2M3YTgzN2Q5MGY5ZDU5MmFiYjFmMDJkZjNkZGQwZDc5YWMwOTEiLCJWb2x1bWVzIjpudWxsLCJXb3JraW5nRGlyIjoiIiwiRW50cnlwb2ludCI6bnVsbCwiT25CdWlsZCI6bnVsbCwiTGFiZWxzIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMDQtMDFUMTI6NDg6MzIuMjYzNjAzMVoiLCJkb2NrZXJfdmVyc2lvbiI6IjIwLjEwLjIiLCJoaXN0b3J5IjpbeyJjcmVhdGVkIjoiMjAyMS0wNC0wMVQxMjo0ODozMi4wODY3MTY2WiIsImNyZWF0ZWRfYnkiOiIvYmluL3NoIC1jICMobm9wKSBBREQgZmlsZTphYzMyZGEyM2Q1MWU4MDFmMDJmOTI0MTIzZWQzMDk5MGViM2YwZmVjMWI5ZWQ0ZjBiMDZjMjRlODhiOWMzNjk1IGluIC9zb21lZmlsZS0xLnR4dCAifSx7ImNyZWF0ZWQiOiIyMDIxLTA0LTAxVDEyOjQ4OjMyLjI2MzYwMzFaIiwiY3JlYXRlZF9ieSI6Ii9iaW4vc2ggLWMgIyhub3ApIEFERCBmaWxlOmRmM2I3NDRmNTRhOWIxNmI5YjlhZWQ0MGUzZTk4ZDljYTJiNDlmNWE3N2Q5ZmE4YTk3NjkwZDdiYWY1ODg4MjAgaW4gL3NvbWVmaWxlLTIudHh0ICJ9XSwib3MiOiJsaW51eCIsInJvb3RmcyI6eyJ0eXBlIjoibGF5ZXJzIiwiZGlmZl9pZHMiOlsic2hhMjU2OjZjMzc2MzUyYzA1MzdmNDQ4M2U0MDMzZTMzMmQ3YTRhYjk0MzNkYjY4YzU0YzI5N2E4MzRkMzY3MTlhZWI2YzkiLCJzaGEyNTY6ZmM4MjE4YTgxNDJlZTQ5NTJiYjhkOWI5NmIzZTk4MzgzMjJlOWU2ZWFlNjQ3NzEzNmJjYWQ4ZmQ3Njg5NDliNyJdfX0=", "repoDigests": [], "scope": "Squashed" } @@ -102,7 +102,7 @@ "version": "[not provided]" }, "schema": { - "version": "1.0.4", - "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.4.json" + "version": "1.0.5", + "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.5.json" } } diff --git a/internal/presenter/packages/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden b/internal/presenter/packages/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden index 10739912e2dde8dde068e7331a59d19f8a62b18d..b6759cac62be2f99be64e5380b3296f714e9a102 100644 GIT binary patch literal 16896 zcmeHO+iu%N5Y=j6K(hZsmr2w|AsI^g0UW2Y++{3Wn5G z+TDliDsM;EUu33or+E`xGpj;2%FM(ImhEM(EMk6=jV`k1W!)5J(l)~RgK}c>;^z3z zI#i36CrxAA>FdVlMK+3NT!+2v`dm3{K0X;LxOCa`s5!shKb(eQRnGkAuy8PlW&dZz zhir75{qf}J`LDxST@A;jGvi@x^I~-C`1efrFo)elE)CFa0M62mP3j!wA5)lCj8{(H0ArzIn2eYj zQL;}26)?O04T$;Dy$93z5UM&a!A+t6K1TZlnoje&DJyWoM7|e3yG@ zJHDEPgribWP8{M?M#dBk@V2CZ0;bY1V>P4HTcI3hj_7t?VxEGI!1}Fl8l*0#v*;AT z$|%f82uyQgE%=4i!nPhl6(uC-Ac6@376h0I>x@^?OQAH=R%UN6FL(4}O1Iew0!o+& z{v5D&k_3Vc7gi$SDN_nwYlsE|Yd$Zc5IS(~6x9e}tQq6dIv0lz7$rgsNzU)C`>u}L zmDB3_4@*g4{l_05yK|TCeErv1|8JhXdi~puWND7qJ|My|KsW!ad zIr7=mP?DUlfIw&e?;ig{_>b@UzrZQ}--$I`WO*AlbTxyUPymEsEfEADZ483YX#O{k z70TjUid+}L_2HMwyIx**Rs3)sxa_wa+9$U)?|=5u`j22vd3yhU63`7c zT(ceXn0kYPZ;4u?e}sr==YJv~rjT6rAij`7B>mrlaBI5_&^~kWtbc0iK2VBHD4Kd} zw@ZQ(0ttbHKtkY=AP}uXz`?_hQw}l70+7D}ICv>|0RE2~_@*L+hi@np2z(5$XiPZ} z!&qw01=|Y$2Olwj5={Kx4s4{e{gnyL( zgJk0Wc49M2{;#VU04Fvz6@-qP<`UQBGqeF~%7#hVFe@oi_awP|cR5qvNLJe?k`TWl zF}{=8_v_!J>%_ku{+}RB6aTvl>$v9s=h=?ix#N2Nf1dxd%Kzh}|LtgFRq7(s^_uwq zakVuaB?J-z34w&b{}q9WDe@TV<{@=lUy8xH_dn-!B^S8;{-5%{z7*}xyAFyyo-QlB zGQ5deKwyGig=u+YP8KB$O{hXyyK))^F+XdI%m#T|V>PJr$rNgK6h}fhjuF&)cuypY zP`{vq)&bJSp}0ywE7L)e7#)0Zl1Mra>FO$?K#1eEt9oWD{Rw?mmcPo4_x8TXFLC_wALg$*i5FT9bON2_EK#yUa-Na-Xu z0N@$nT8{_oKbVbZDF@{8v-;oO9Z*4-5}Z)3280o!#9%1bi|(FcUN^47Ds`2#`fy$4 z?I^$OYX4VHU%dSF;J1%em(bRpV%6?fVbmO7u|K7Z4Tha<^07)?#eL)Ur~5n6@|Cvx z{|Bal&i()Ix-8Z_)4Kkz7+vZAI9=fQzmt~vjrafaERWeF>)H&my)0DV8fb-4tI3E^ z{2mfAS$zL$6O={c@&c+ic$MEf1UKP7YN#$_9yN^%Coh|r z7uh7aX$^bX^}TZ5e0UQo%(_f@*qmKoADlvQSI#1IPz0>RGX2xyT{bD^)9GIJc!p1O zdHL(i9WN{Ts+`VeP+!ITSypfJ;`m9vtnJgHsm@Nzyuf;0<~=BWndj4MQO})+=`#Dp zURL5|HqDFq`|M&(U)pqZSHq8|?hLB^b`5p!XHEYXAFltX(z@^e_bPOQ5p_9(luuzq z_nY_41_j>|wZ{GgSN@mdKV?dDJs^~}+l7Aozoo-%T{b6qJ!~7m@Wj=_c8G?eskc>H zFSpk~uYq0zy#^kN2Aq<{5FUe}F7POnR3=8HZFE*4w(~kL6Szr4X$68tt-&UmbYzKZ z1_Wk@jpa@{rYeF<{$L@ZwE?#z(sWK|K0f?W0HNq|8N`F_x~NV zn&xE(r}I!@L-dulW#b`OvT} z{fdLuCyq%qi7qfI2{(aAl#S=bL`q120tpY1ChL7nAQF~tqp{Y~AZ-W%OOp&F)_lV_ zHJ0HmQs#G1)p?1$IAdg=l6}@TtgnGQZ?cu#Mb{73WUdM)c>|%DS1n?9v#11`>pukJ zI~aQ(LJ?gD{%!GpMf&-F7wzNj_`aPI*Yp3S|Icmy-`jsX##oWMIQwDO;Q#xT=swtM zpw~dJfnEduQw;>%1S&7U6UD5u9Jh=i5!_%a!|?zBECe4dXG(etN#Pbw2?pFs=92f0 z?04rs9E)DzKa`-mXwQFk(B{swd$wNvPWfTNjn+luEApb-B+KZD*L@CzH@=r^cGO@pd{;bjI10m67KvxeEIP)cq+ke5geA)a zWgH?W))K(CP)LG6ah8c95uA7H*U^BCnXI^d^rOca$x09gjfUw{eV;)-f;<@vJuyq7h7c z&<@Y+mE*w#0V1M8BQTPs$u_bVz`^DPoR){~_;Ltqpn|dv`1 m*YwbKk pos { + break + } + last = nlPos + } + return last +} diff --git a/syft/file/newline_counter_test.go b/syft/file/newline_counter_test.go new file mode 100644 index 000000000..24282bceb --- /dev/null +++ b/syft/file/newline_counter_test.go @@ -0,0 +1,35 @@ +package file + +import ( + "bufio" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLineCounter_ReadRune(t *testing.T) { + counter := &newlineCounter{RuneReader: bufio.NewReader(strings.NewReader("hi\nwhat's the weather like today?\ndunno...\n"))} + var err error + for err == nil { + _, _, err = counter.ReadRune() + } + if err != io.EOF { + t.Fatalf("should have gotten an eof, got %+v", err) + } + assert.Equal(t, 3, len(counter.newLines), "bad line count") + assert.Equal(t, []int64{3, 34, 43}, counter.newLines, "bad line positions") +} + +func TestLineCounter_newlinesBefore(t *testing.T) { + counter := &newlineCounter{RuneReader: bufio.NewReader(strings.NewReader("hi\nwhat's the weather like today?\ndunno...\n"))} + var err error + for err == nil { + _, _, err = counter.ReadRune() + } + if err != io.EOF { + t.Fatalf("should have gotten an eof, got %+v", err) + } + assert.Equal(t, 1, counter.newlinesBefore(10), "bad line count") +} diff --git a/syft/file/search_result.go b/syft/file/search_result.go new file mode 100644 index 000000000..dc12d14df --- /dev/null +++ b/syft/file/search_result.go @@ -0,0 +1,18 @@ +package file + +import ( + "fmt" +) + +type SearchResult struct { + Classification string `json:"classification"` + LineNumber int64 `json:"lineNumber"` + LineOffset int64 `json:"lineOffset"` + SeekPosition int64 `json:"seekPosition"` + Length int64 `json:"length"` + Value string `json:"value,omitempty"` +} + +func (s SearchResult) String() string { + return fmt.Sprintf("SearchResult(classification=%q seek=%q length=%q)", s.Classification, s.SeekPosition, s.Length) +} diff --git a/syft/file/secrets_cataloger.go b/syft/file/secrets_cataloger.go new file mode 100644 index 000000000..c83ef1167 --- /dev/null +++ b/syft/file/secrets_cataloger.go @@ -0,0 +1,150 @@ +package file + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "regexp" + "sort" + + "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/source" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" +) + +var DefaultSecretsPatterns = map[string]string{ + "aws-access-key": `(?i)aws_access_key_id["'=:\s]*?(?P(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})`, + "aws-secret-key": `(?i)aws_secret_access_key["'=:\s]*?(?P[0-9a-zA-Z/+]{40})`, + "pem-private-key": `-----BEGIN (\S+ )?PRIVATE KEY(\sBLOCK)?-----((?P(\n.*?)+)-----END (\S+ )?PRIVATE KEY(\sBLOCK)?-----)?`, + "docker-config-auth": `"auths"((.*\n)*.*?"auth"\s*:\s*"(?P[^"]+)")?`, + "generic-api-key": `(?i)api(-|_)?key["'=:\s]*?(?P[A-Z0-9]{20,60})["']?(\s|$)`, +} + +type SecretsCataloger struct { + patterns map[string]*regexp.Regexp + revealValues bool + skipFilesAboveSize int64 +} + +func NewSecretsCataloger(patterns map[string]*regexp.Regexp, revealValues bool, maxFileSize int64) (*SecretsCataloger, error) { + return &SecretsCataloger{ + patterns: patterns, + revealValues: revealValues, + skipFilesAboveSize: maxFileSize, + }, nil +} + +func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Location][]SearchResult, error) { + results := make(map[source.Location][]SearchResult) + var locations []source.Location + for location := range resolver.AllLocations() { + locations = append(locations, location) + } + stage, prog, secretsDiscovered := secretsCatalogingProgress(int64(len(locations))) + for _, location := range locations { + stage.Current = location.RealPath + result, err := i.catalogLocation(resolver, location) + if err != nil { + return nil, err + } + if len(result) > 0 { + secretsDiscovered.N += int64(len(result)) + results[location] = result + } + prog.N++ + } + log.Debugf("secrets cataloger discovered %d secrets", secretsDiscovered.N) + prog.SetCompleted() + return results, nil +} + +func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]SearchResult, error) { + metadata, err := resolver.FileMetadataByLocation(location) + if err != nil { + return nil, err + } + + if i.skipFilesAboveSize > 0 && metadata.Size > i.skipFilesAboveSize { + return nil, nil + } + + // TODO: in the future we can swap out search strategies here + secrets, err := catalogLocationByLine(resolver, location, i.patterns) + if err != nil { + return nil, err + } + + if i.revealValues { + for idx, secret := range secrets { + value, err := extractValue(resolver, location, secret.SeekPosition, secret.Length) + if err != nil { + return nil, err + } + secrets[idx].Value = value + } + } + + // sort by the start location of each secret as it appears in the location + sort.SliceStable(secrets, func(i, j int) bool { + return secrets[i].SeekPosition < secrets[j].SeekPosition + }) + + return secrets, nil +} + +func extractValue(resolver source.FileResolver, location source.Location, start, length int64) (string, error) { + readCloser, err := resolver.FileContentsByLocation(location) + if err != nil { + return "", fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) + } + defer readCloser.Close() + + n, err := io.CopyN(ioutil.Discard, readCloser, start) + if err != nil { + return "", fmt.Errorf("unable to read contents for location=%q : %w", location, err) + } + if n != start { + return "", fmt.Errorf("unexpected seek location for location=%q : %d != %d", location, n, start) + } + + var buf bytes.Buffer + n, err = io.CopyN(&buf, readCloser, length) + if err != nil { + return "", fmt.Errorf("unable to read secret value for location=%q : %w", location, err) + } + if n != length { + return "", fmt.Errorf("unexpected secret length for location=%q : %d != %d", location, n, length) + } + + return buf.String(), nil +} + +type SecretsMonitor struct { + progress.Stager + SecretsDiscovered progress.Monitorable + progress.Progressable +} + +func secretsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual, *progress.Manual) { + stage := &progress.Stage{} + secretsDiscovered := &progress.Manual{} + prog := &progress.Manual{ + Total: locations, + } + + bus.Publish(partybus.Event{ + Type: event.SecretsCatalogerStarted, + Source: secretsDiscovered, + Value: SecretsMonitor{ + Stager: progress.Stager(stage), + SecretsDiscovered: secretsDiscovered, + Progressable: prog, + }, + }) + + return stage, prog, secretsDiscovered +} diff --git a/syft/file/secrets_cataloger_test.go b/syft/file/secrets_cataloger_test.go new file mode 100644 index 000000000..14ac42808 --- /dev/null +++ b/syft/file/secrets_cataloger_test.go @@ -0,0 +1,444 @@ +package file + +import ( + "regexp" + "testing" + + "github.com/anchore/syft/internal/file" + + "github.com/anchore/syft/syft/source" + + "github.com/stretchr/testify/assert" +) + +func TestSecretsCataloger(t *testing.T) { + tests := []struct { + name string + fixture string + reveal bool + maxSize int64 + patterns map[string]string + expected []SearchResult + constructorErr bool + catalogErr bool + }{ + { + name: "go-case-find-and-reveal", + fixture: "test-fixtures/secrets/simple.txt", + reveal: true, + patterns: map[string]string{ + "simple-secret-key": `^secret_key=.*`, + }, + expected: []SearchResult{ + { + Classification: "simple-secret-key", + LineNumber: 2, + LineOffset: 0, + SeekPosition: 34, + Length: 21, + Value: "secret_key=clear_text", + }, + }, + }, + { + name: "dont-reveal-secret-value", + fixture: "test-fixtures/secrets/simple.txt", + reveal: false, + patterns: map[string]string{ + "simple-secret-key": `^secret_key=.*`, + }, + expected: []SearchResult{ + { + Classification: "simple-secret-key", + LineNumber: 2, + LineOffset: 0, + SeekPosition: 34, + Length: 21, + Value: "", + }, + }, + }, + { + name: "reveal-named-capture-group", + fixture: "test-fixtures/secrets/simple.txt", + reveal: true, + patterns: map[string]string{ + "simple-secret-key": `^secret_key=(?P.*)`, + }, + expected: []SearchResult{ + { + Classification: "simple-secret-key", + LineNumber: 2, + LineOffset: 11, + SeekPosition: 45, + Length: 10, + Value: "clear_text", + }, + }, + }, + { + name: "multiple-secret-instances", + fixture: "test-fixtures/secrets/multiple.txt", + reveal: true, + patterns: map[string]string{ + "simple-secret-key": `secret_key=.*`, + }, + expected: []SearchResult{ + { + Classification: "simple-secret-key", + LineNumber: 1, + LineOffset: 0, + SeekPosition: 0, + Length: 22, + Value: "secret_key=clear_text1", + }, + { + Classification: "simple-secret-key", + LineNumber: 3, + LineOffset: 0, + SeekPosition: 57, + Length: 22, + Value: "secret_key=clear_text2", + }, + { + Classification: "simple-secret-key", + LineNumber: 4, + // note: this test captures a line offset case + LineOffset: 1, + SeekPosition: 81, + Length: 22, + Value: "secret_key=clear_text3", + }, + { + Classification: "simple-secret-key", + LineNumber: 6, + LineOffset: 0, + SeekPosition: 139, + Length: 22, + Value: "secret_key=clear_text4", + }, + }, + }, + { + name: "multiple-secret-instances-with-capture-group", + fixture: "test-fixtures/secrets/multiple.txt", + reveal: true, + patterns: map[string]string{ + "simple-secret-key": `secret_key=(?P.*)`, + }, + expected: []SearchResult{ + { + Classification: "simple-secret-key", + LineNumber: 1, + // note: value capture group location + LineOffset: 11, + SeekPosition: 11, + Length: 11, + Value: "clear_text1", + }, + { + Classification: "simple-secret-key", + LineNumber: 3, + LineOffset: 11, + SeekPosition: 68, + Length: 11, + Value: "clear_text2", + }, + { + Classification: "simple-secret-key", + LineNumber: 4, + // note: value capture group location + offset + LineOffset: 12, + SeekPosition: 92, + Length: 11, + Value: "clear_text3", + }, + { + Classification: "simple-secret-key", + LineNumber: 6, + LineOffset: 11, + SeekPosition: 150, + Length: 11, + Value: "clear_text4", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + regexObjs := make(map[string]*regexp.Regexp) + for name, pattern := range test.patterns { + // always assume given patterns should be multiline + obj, err := regexp.Compile(`` + pattern) + if err != nil { + t.Fatalf("unable to parse regex: %+v", err) + } + regexObjs[name] = obj + } + + c, err := NewSecretsCataloger(regexObjs, test.reveal, test.maxSize) + if err != nil && !test.constructorErr { + t.Fatalf("could not create cataloger (but should have been able to): %+v", err) + } else if err == nil && test.constructorErr { + t.Fatalf("expected constructor error but did not get one") + } else if test.constructorErr && err != nil { + return + } + + resolver := source.NewMockResolverForPaths(test.fixture) + + actualResults, err := c.Catalog(resolver) + if err != nil && !test.catalogErr { + t.Fatalf("could not catalog (but should have been able to): %+v", err) + } else if err == nil && test.catalogErr { + t.Fatalf("expected catalog error but did not get one") + } else if test.catalogErr && err != nil { + return + } + + loc := source.NewLocation(test.fixture) + if _, exists := actualResults[loc]; !exists { + t.Fatalf("could not find location=%q in results", loc) + } + + assert.Equal(t, test.expected, actualResults[loc], "mismatched secrets") + }) + } +} + +func TestSecretsCataloger_DefaultSecrets(t *testing.T) { + regexObjs, err := GenerateSearchPatterns(DefaultSecretsPatterns, nil, nil) + if err != nil { + t.Fatalf("unable to get patterns: %+v", err) + } + + tests := []struct { + fixture string + expected []SearchResult + }{ + { + fixture: "test-fixtures/secrets/default/aws.env", + expected: []SearchResult{ + { + Classification: "aws-access-key", + LineNumber: 2, + LineOffset: 25, + SeekPosition: 64, + Length: 20, + Value: "AKIAIOSFODNN7EXAMPLE", + }, + { + Classification: "aws-secret-key", + LineNumber: 3, + LineOffset: 29, + SeekPosition: 114, + Length: 40, + Value: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + }, + { + fixture: "test-fixtures/secrets/default/aws.ini", + expected: []SearchResult{ + { + Classification: "aws-access-key", + LineNumber: 3, + LineOffset: 18, + SeekPosition: 67, + Length: 20, + Value: "AKIAIOSFODNN7EXAMPLE", + }, + { + Classification: "aws-secret-key", + LineNumber: 4, + LineOffset: 22, + SeekPosition: 110, + Length: 40, + Value: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + }, + }, + { + fixture: "test-fixtures/secrets/default/private-key.pem", + expected: []SearchResult{ + { + Classification: "pem-private-key", + LineNumber: 2, + LineOffset: 27, + SeekPosition: 66, + Length: 351, + Value: ` +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG +cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD +VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD +bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF +BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh +z3P668YfhUbKdRF6S42Cg6zn +`, + }, + }, + }, + { + fixture: "test-fixtures/secrets/default/private-key-openssl.pem", + expected: []SearchResult{ + { + Classification: "pem-private-key", + LineNumber: 2, + LineOffset: 35, + SeekPosition: 74, + Length: 351, + Value: ` +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG +cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD +VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD +bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF +BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh +z3P668YfhUbKdRF6S42Cg6zn +`, + }, + }, + }, + { + // note: this test proves that the PEM regex matches the smallest possible match + // since the test catches two adjacent secrets + fixture: "test-fixtures/secrets/default/private-keys.pem", + expected: []SearchResult{ + { + Classification: "pem-private-key", + LineNumber: 1, + LineOffset: 35, + SeekPosition: 35, + Length: 351, + Value: ` +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG +cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD +VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD +bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF +BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh +z3P668YfhUbKdRF6S42Cg6zn +`, + }, + { + Classification: "pem-private-key", + LineNumber: 9, + LineOffset: 35, + SeekPosition: 455, + Length: 351, + Value: ` +MIIEvgTHISISNOTAREALKEYoIBAQDBj08DBj08DBj08DBj08DBj08DBsp5++4an3 +cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgY5 +VQQDDBcqLmF3cy10ZXN0SISNOTAREALKEYoIBAQDBj08DfffKoZIhvcNAQEBBQA7 +bml6SISNOTAREALKEYoIBAQDBj08DdssBggrBgEFBQcBAQSBkzCBkDBNBggrBgE8 +BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmd1 +j4f668YfhUbKdRF6S6734856 +`, + }, + }, + }, + { + fixture: "test-fixtures/secrets/default/private-key-false-positive.pem", + expected: nil, + }, + { + // this test represents: + // 1. a docker config + // 2. a named capture group with the correct line number and line offset case + // 3. the named capture group is in a different line than the match start, and both the match start and the capture group have different line offsets + fixture: "test-fixtures/secrets/default/docker-config.json", + expected: []SearchResult{ + { + Classification: "docker-config-auth", + LineNumber: 5, + LineOffset: 15, + SeekPosition: 100, + Length: 10, + Value: "tOpsyKreTz", + }, + }, + }, + { + fixture: "test-fixtures/secrets/default/not-docker-config.json", + expected: nil, + }, + { + fixture: "test-fixtures/secrets/default/api-key.txt", + expected: []SearchResult{ + { + Classification: "generic-api-key", + LineNumber: 2, + LineOffset: 7, + SeekPosition: 33, + Length: 20, + Value: "12345A7a901b34567890", + }, + { + Classification: "generic-api-key", + LineNumber: 3, + LineOffset: 9, + SeekPosition: 63, + Length: 30, + Value: "12345A7a901b345678901234567890", + }, + { + Classification: "generic-api-key", + LineNumber: 4, + LineOffset: 10, + SeekPosition: 104, + Length: 40, + Value: "12345A7a901b3456789012345678901234567890", + }, + { + Classification: "generic-api-key", + LineNumber: 5, + LineOffset: 10, + SeekPosition: 156, + Length: 50, + Value: "12345A7a901b34567890123456789012345678901234567890", + }, + { + Classification: "generic-api-key", + LineNumber: 6, + LineOffset: 16, + SeekPosition: 224, + Length: 60, + Value: "12345A7a901b345678901234567890123456789012345678901234567890", + }, + { + Classification: "generic-api-key", + LineNumber: 14, + LineOffset: 8, + SeekPosition: 502, + Length: 20, + Value: "11111111111111111111", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.fixture, func(t *testing.T) { + + c, err := NewSecretsCataloger(regexObjs, true, 10*file.MB) + if err != nil { + t.Fatalf("could not create cataloger: %+v", err) + } + + resolver := source.NewMockResolverForPaths(test.fixture) + + actualResults, err := c.Catalog(resolver) + if err != nil { + t.Fatalf("could not catalog: %+v", err) + } + + loc := source.NewLocation(test.fixture) + if _, exists := actualResults[loc]; !exists && test.expected != nil { + t.Fatalf("could not find location=%q in results", loc) + } else if !exists && test.expected == nil { + return + } + + assert.Equal(t, test.expected, actualResults[loc], "mismatched secrets") + }) + } +} diff --git a/syft/file/secrets_search_by_line_strategy.go b/syft/file/secrets_search_by_line_strategy.go new file mode 100644 index 000000000..4ee93a24d --- /dev/null +++ b/syft/file/secrets_search_by_line_strategy.go @@ -0,0 +1,134 @@ +package file + +import ( + "bufio" + "errors" + "fmt" + "io" + "io/ioutil" + "regexp" + + "github.com/anchore/syft/syft/source" +) + +func catalogLocationByLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp) ([]SearchResult, error) { + readCloser, err := resolver.FileContentsByLocation(location) + if err != nil { + return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) + } + defer readCloser.Close() + + var scanner = bufio.NewReader(readCloser) + var position int64 + var allSecrets []SearchResult + var lineNo int64 + var readErr error + for !errors.Is(readErr, io.EOF) { + lineNo++ + var line []byte + // TODO: we're at risk of large memory usage for very long lines + line, readErr = scanner.ReadBytes('\n') + if readErr != nil && readErr != io.EOF { + return nil, readErr + } + + lineSecrets, err := searchForSecretsWithinLine(resolver, location, patterns, line, lineNo, position) + if err != nil { + return nil, err + } + position += int64(len(line)) + allSecrets = append(allSecrets, lineSecrets...) + } + + return allSecrets, nil +} + +func searchForSecretsWithinLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp, line []byte, lineNo int64, position int64) ([]SearchResult, error) { + var secrets []SearchResult + for name, pattern := range patterns { + matches := pattern.FindAllIndex(line, -1) + for i, match := range matches { + if i%2 == 1 { + // FindAllIndex returns pairs of numbers for each match, we are only interested in the starting (first) + // position in each pair. + continue + } + + lineOffset := int64(match[0]) + seekLocation := position + lineOffset + reader, err := readerAtPosition(resolver, location, seekLocation) + if err != nil { + return nil, err + } + + secret := extractSecretFromPosition(reader, name, pattern, lineNo, lineOffset, seekLocation) + if secret != nil { + secrets = append(secrets, *secret) + } + } + } + + return secrets, nil +} + +func readerAtPosition(resolver source.FileResolver, location source.Location, seekPosition int64) (io.ReadCloser, error) { + readCloser, err := resolver.FileContentsByLocation(location) + if err != nil { + return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) + } + if seekPosition > 0 { + n, err := io.CopyN(ioutil.Discard, readCloser, seekPosition) + if err != nil { + return nil, fmt.Errorf("unable to read contents for location=%q while searching for secrets: %w", location, err) + } + if n != seekPosition { + return nil, fmt.Errorf("unexpected seek location for location=%q while searching for secrets: %d != %d", location, n, seekPosition) + } + } + return readCloser, nil +} + +func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *regexp.Regexp, lineNo, lineOffset, seekPosition int64) *SearchResult { + reader := &newlineCounter{RuneReader: bufio.NewReader(readCloser)} + positions := pattern.FindReaderSubmatchIndex(reader) + if len(positions) == 0 { + // no matches found + return nil + } + + index := pattern.SubexpIndex("value") + var indexOffset int + if index != -1 { + // there is a capture group, use the capture group selection as the secret value. To do this we want to + // use the position at the discovered offset. Note: all positions come in pairs, so you will need to adjust + // the offset accordingly (multiply by 2). + indexOffset = index * 2 + } + // get the start and stop of the secret value. Note: this covers both when there is a capture group + // and when there is not a capture group (full value match) + start, stop := int64(positions[indexOffset]), int64(positions[indexOffset+1]) + + if start < 0 || stop < 0 { + // no match location found. This can happen when there is a value capture group specified by the user + // and there was a match on the overall regex, but not for the capture group (which is possible if the capture + // group is optional). + return nil + } + + // lineNoOfSecret are the number of lines which occur before the start of the secret value + var lineNoOfSecret = lineNo + int64(reader.newlinesBefore(start)) + // lineOffsetOfSecret are the number of bytes that occur after the last newline but before the secret value. + var lineOffsetOfSecret = start - reader.newlinePositionBefore(start) + if lineNoOfSecret == lineNo { + // the secret value starts in the same line as the overall match, so we must consider that line offset + lineOffsetOfSecret += lineOffset + } + + return &SearchResult{ + Classification: name, + SeekPosition: start + seekPosition, + Length: stop - start, + LineNumber: lineNoOfSecret, + LineOffset: lineOffsetOfSecret, + } +} diff --git a/syft/file/test-fixtures/secrets/default/api-key.txt b/syft/file/test-fixtures/secrets/default/api-key.txt new file mode 100644 index 000000000..63cb62cac --- /dev/null +++ b/syft/file/test-fixtures/secrets/default/api-key.txt @@ -0,0 +1,14 @@ +# these should be matches +apikey=12345A7a901b34567890 +api_key =12345A7a901b345678901234567890 +API-KEY= '12345A7a901b3456789012345678901234567890' +API-key: "12345A7a901b34567890123456789012345678901234567890" +some_ApI-kEy = "12345A7a901b345678901234567890123456789012345678901234567890" + +# these should be non matches +api_key = "toolong12345A7a901b345678901234567890123456789012345678901234567890" +api_key = "tooshort" +not_api_k3y = "badkeyname12345A7a901b34567890" + +# value at EOF should match +api_key=11111111111111111111 \ No newline at end of file diff --git a/syft/file/test-fixtures/secrets/default/aws.env b/syft/file/test-fixtures/secrets/default/aws.env new file mode 100644 index 000000000..7e5888a38 --- /dev/null +++ b/syft/file/test-fixtures/secrets/default/aws.env @@ -0,0 +1,3 @@ +# note: these are NOT real credentials +export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \ No newline at end of file diff --git a/syft/file/test-fixtures/secrets/default/aws.ini b/syft/file/test-fixtures/secrets/default/aws.ini new file mode 100644 index 000000000..0413424f6 --- /dev/null +++ b/syft/file/test-fixtures/secrets/default/aws.ini @@ -0,0 +1,4 @@ +# note: these are NOT real credentials +[default] +aws_access_key_id=AKIAIOSFODNN7EXAMPLE +aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \ No newline at end of file diff --git a/syft/file/test-fixtures/secrets/default/docker-config.json b/syft/file/test-fixtures/secrets/default/docker-config.json new file mode 100644 index 000000000..2a239cdaa --- /dev/null +++ b/syft/file/test-fixtures/secrets/default/docker-config.json @@ -0,0 +1,10 @@ +{ + "experimental" : "disabled", + "auths" : { + "https://index.docker.io/v1/" : { + "auth": "tOpsyKreTz" + } + }, + "stackOrchestrator" : "swarm", + "credsStore" : "desktop" +} diff --git a/syft/file/test-fixtures/secrets/default/not-docker-config.json b/syft/file/test-fixtures/secrets/default/not-docker-config.json new file mode 100644 index 000000000..c6baa32f1 --- /dev/null +++ b/syft/file/test-fixtures/secrets/default/not-docker-config.json @@ -0,0 +1,4 @@ +{ + "endpoint" : "http://somewhere", + "auth" : "basic" +} diff --git a/syft/file/test-fixtures/secrets/default/private-key-false-positive.pem b/syft/file/test-fixtures/secrets/default/private-key-false-positive.pem new file mode 100644 index 000000000..2cc142e3c --- /dev/null +++ b/syft/file/test-fixtures/secrets/default/private-key-false-positive.pem @@ -0,0 +1 @@ +-----BEGIN OPENSSL PRIVATE KEY----- \ No newline at end of file diff --git a/syft/file/test-fixtures/secrets/default/private-key-openssl.pem b/syft/file/test-fixtures/secrets/default/private-key-openssl.pem new file mode 100644 index 000000000..48e4d2361 --- /dev/null +++ b/syft/file/test-fixtures/secrets/default/private-key-openssl.pem @@ -0,0 +1,9 @@ +# note: this is NOT a real private key +-----BEGIN OPENSSL PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG +cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD +VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD +bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF +BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh +z3P668YfhUbKdRF6S42Cg6zn +-----END OPENSSL PRIVATE KEY----- \ No newline at end of file diff --git a/syft/file/test-fixtures/secrets/default/private-key.pem b/syft/file/test-fixtures/secrets/default/private-key.pem new file mode 100644 index 000000000..e13aa1955 --- /dev/null +++ b/syft/file/test-fixtures/secrets/default/private-key.pem @@ -0,0 +1,10 @@ +# note: this is NOT a real private key +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG +cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD +VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD +bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF +BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh +z3P668YfhUbKdRF6S42Cg6zn +-----END PRIVATE KEY----- +other embedded text \ No newline at end of file diff --git a/syft/file/test-fixtures/secrets/default/private-keys.pem b/syft/file/test-fixtures/secrets/default/private-keys.pem new file mode 100644 index 000000000..27c045365 --- /dev/null +++ b/syft/file/test-fixtures/secrets/default/private-keys.pem @@ -0,0 +1,16 @@ +-----BEGIN OPENSSL PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG +cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD +VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD +bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF +BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh +z3P668YfhUbKdRF6S42Cg6zn +-----END OPENSSL PRIVATE KEY----- +-----BEGIN OPENSSL PRIVATE KEY----- +MIIEvgTHISISNOTAREALKEYoIBAQDBj08DBj08DBj08DBj08DBj08DBsp5++4an3 +cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgY5 +VQQDDBcqLmF3cy10ZXN0SISNOTAREALKEYoIBAQDBj08DfffKoZIhvcNAQEBBQA7 +bml6SISNOTAREALKEYoIBAQDBj08DdssBggrBgEFBQcBAQSBkzCBkDBNBggrBgE8 +BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmd1 +j4f668YfhUbKdRF6S6734856 +-----END OPENSSL PRIVATE KEY----- \ No newline at end of file diff --git a/syft/file/test-fixtures/secrets/multiple.txt b/syft/file/test-fixtures/secrets/multiple.txt new file mode 100644 index 000000000..3c50205d1 --- /dev/null +++ b/syft/file/test-fixtures/secrets/multiple.txt @@ -0,0 +1,6 @@ +secret_key=clear_text1 +other text that should be ignored +secret_key=clear_text2 + secret_key=clear_text3 +also things that should be ignored +secret_key=clear_text4 \ No newline at end of file diff --git a/syft/file/test-fixtures/secrets/simple.txt b/syft/file/test-fixtures/secrets/simple.txt new file mode 100644 index 000000000..67fc2b755 --- /dev/null +++ b/syft/file/test-fixtures/secrets/simple.txt @@ -0,0 +1,4 @@ +other text that should be ignored +secret_key=clear_text +---secret_key=clear_text +also things that should be ignored \ No newline at end of file diff --git a/syft/lib.go b/syft/lib.go index 9b9bfd965..ed9a86ef6 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -35,7 +35,7 @@ import ( func CatalogPackages(src source.Source, scope source.Scope) (*pkg.Catalog, *distro.Distro, error) { resolver, err := src.FileResolver(scope) if err != nil { - return nil, nil, fmt.Errorf("unable to determine FileResolver while cataloging packages: %w", err) + return nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err) } // find the distro diff --git a/syft/pkg/cataloger/catalog.go b/syft/pkg/cataloger/catalog.go index 962388844..27eedd3c6 100644 --- a/syft/pkg/cataloger/catalog.go +++ b/syft/pkg/cataloger/catalog.go @@ -54,7 +54,7 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers catalogedPackages := len(packages) - log.Debugf("cataloger '%s' discovered '%d' packages", theCataloger.Name(), catalogedPackages) + log.Debugf("package cataloger %q discovered %d packages", theCataloger.Name(), catalogedPackages) packagesDiscovered.N += int64(catalogedPackages) for _, p := range packages { diff --git a/syft/source/file_metadata.go b/syft/source/file_metadata.go index 9140810d2..f08be9421 100644 --- a/syft/source/file_metadata.go +++ b/syft/source/file_metadata.go @@ -12,6 +12,7 @@ type FileMetadata struct { UserID int GroupID int LinkDestination string + Size int64 } func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata, error) { @@ -26,5 +27,6 @@ func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata, UserID: entry.Metadata.UserID, GroupID: entry.Metadata.GroupID, LinkDestination: entry.Metadata.Linkname, + Size: entry.Metadata.Size, }, nil } diff --git a/syft/source/mock_resolver.go b/syft/source/mock_resolver.go index b574ccd14..d97278722 100644 --- a/syft/source/mock_resolver.go +++ b/syft/source/mock_resolver.go @@ -118,6 +118,23 @@ func (r MockResolver) AllLocations() <-chan Location { return results } -func (r MockResolver) FileMetadataByLocation(Location) (FileMetadata, error) { - panic("not implemented") +func (r MockResolver) FileMetadataByLocation(l Location) (FileMetadata, error) { + info, err := os.Stat(l.RealPath) + if err != nil { + return FileMetadata{}, err + } + + // other types not supported + ty := RegularFile + if info.IsDir() { + ty = Directory + } + + return FileMetadata{ + Mode: info.Mode(), + Type: ty, + UserID: 0, // not supported + GroupID: 0, // not supported + Size: info.Size(), + }, nil } diff --git a/test/cli/power_user_cmd_test.go b/test/cli/power_user_cmd_test.go index 5bc16a759..3246c20c4 100644 --- a/test/cli/power_user_cmd_test.go +++ b/test/cli/power_user_cmd_test.go @@ -6,8 +6,6 @@ import ( ) func TestPowerUserCmdFlags(t *testing.T) { - request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage") - tests := []struct { name string args []string @@ -16,14 +14,14 @@ func TestPowerUserCmdFlags(t *testing.T) { }{ { name: "json-output-flag-fails", - args: []string{"power-user", "-o", "json", request}, + args: []string{"power-user", "-o", "json", "docker-archive:" + getFixtureImage(t, "image-pkg-coverage")}, assertions: []traitAssertion{ assertFailingReturnCode, }, }, { - name: "default-results", - args: []string{"power-user", request}, + name: "default-results-w-pkg-coverage", + args: []string{"power-user", "docker-archive:" + getFixtureImage(t, "image-pkg-coverage")}, assertions: []traitAssertion{ assertNotInOutput(" command is deprecated"), // only the root command should be deprecated assertInOutput(`"type": "RegularFile"`), // proof of file-metadata data @@ -32,6 +30,27 @@ func TestPowerUserCmdFlags(t *testing.T) { assertSuccessfulReturnCode, }, }, + { + name: "defaut-secrets-results-w-reveal-values", + env: map[string]string{ + "SYFT_SECRETS_REVEAL_VALUES": "true", + }, + args: []string{"power-user", "docker-archive:" + getFixtureImage(t, "image-secrets")}, + assertions: []traitAssertion{ + assertInOutput(`"classification": "generic-api-key"`), // proof of the secrets cataloger finding something + assertInOutput(`"12345A7a901b345678901234567890123456789012345678901234567890"`), // proof of the secrets cataloger finding the api key + assertSuccessfulReturnCode, + }, + }, + { + name: "default-secret-results-dont-reveal-values", + args: []string{"power-user", "docker-archive:" + getFixtureImage(t, "image-secrets")}, + assertions: []traitAssertion{ + assertInOutput(`"classification": "generic-api-key"`), // proof of the secrets cataloger finding something + assertNotInOutput(`"12345A7a901b345678901234567890123456789012345678901234567890"`), // proof of the secrets cataloger finding the api key + assertSuccessfulReturnCode, + }, + }, } for _, test := range tests { diff --git a/test/cli/test-fixtures/image-secrets/Dockerfile b/test/cli/test-fixtures/image-secrets/Dockerfile new file mode 100644 index 000000000..28a283a0b --- /dev/null +++ b/test/cli/test-fixtures/image-secrets/Dockerfile @@ -0,0 +1,2 @@ +FROM scratch +ADD api-key.txt . \ No newline at end of file diff --git a/test/cli/test-fixtures/image-secrets/api-key.txt b/test/cli/test-fixtures/image-secrets/api-key.txt new file mode 100644 index 000000000..6cbbd99e3 --- /dev/null +++ b/test/cli/test-fixtures/image-secrets/api-key.txt @@ -0,0 +1 @@ +some_ApI-kEy = "12345A7a901b345678901234567890123456789012345678901234567890" diff --git a/ui/event_handlers.go b/ui/event_handlers.go index 103a4dd95..1b1e2c0fe 100644 --- a/ui/event_handlers.go +++ b/ui/event_handlers.go @@ -25,7 +25,7 @@ const maxBarWidth = 50 const statusSet = common.SpinnerDotSet // SpinnerCircleOutlineSet const completedStatus = "✔" // "●" const tileFormat = color.Bold -const statusTitleTemplate = " %s %-28s " +const statusTitleTemplate = " %s %-31s " const interval = 150 * time.Millisecond var ( @@ -270,7 +270,7 @@ func ReadImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event // PackageCatalogerStartedHandler periodically writes catalog statistics to a single line. func PackageCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { - monitor, err := syftEventParsers.ParseCatalogerStarted(event) + monitor, err := syftEventParsers.ParsePackageCatalogerStarted(event) if err != nil { return fmt.Errorf("bad %s event: %w", event.Type, err) } @@ -284,7 +284,7 @@ func PackageCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event _, spinner := startProcess() stream := progress.StreamMonitors(ctx, []progress.Monitorable{monitor.FilesProcessed, monitor.PackagesDiscovered}, interval) - title := tileFormat.Sprint("Cataloging image") + title := tileFormat.Sprint("Cataloging packages") formatFn := func(p int64) { spin := color.Magenta.Sprint(spinner.Next()) @@ -301,7 +301,7 @@ func PackageCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event } spin := color.Green.Sprint(completedStatus) - title = tileFormat.Sprint("Cataloged image") + title = tileFormat.Sprint("Cataloged packages") auxInfo := auxInfoFormat.Sprintf("[%d packages]", monitor.PackagesDiscovered.Current()) _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo)) }() @@ -309,6 +309,137 @@ func PackageCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event return nil } +// SecretsCatalogerStartedHandler shows the intermittent secrets searching progress. +// nolint:dupl +func SecretsCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { + prog, err := syftEventParsers.ParseSecretsCatalogingStarted(event) + if err != nil { + return fmt.Errorf("bad %s event: %w", event.Type, err) + } + + line, err := fr.Append() + if err != nil { + return err + } + wg.Add(1) + + formatter, spinner := startProcess() + stream := progress.Stream(ctx, prog, interval) + title := tileFormat.Sprint("Cataloging secrets") + + formatFn := func(p progress.Progress) { + progStr, err := formatter.Format(p) + spin := color.Magenta.Sprint(spinner.Next()) + if err != nil { + _, _ = io.WriteString(line, fmt.Sprintf("Error: %+v", err)) + } else { + auxInfo := auxInfoFormat.Sprintf("[%d secrets]", prog.SecretsDiscovered.Current()) + _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s %s", spin, title, progStr, auxInfo)) + } + } + + go func() { + defer wg.Done() + + formatFn(progress.Progress{}) + for p := range stream { + formatFn(p) + } + + spin := color.Green.Sprint(completedStatus) + title = tileFormat.Sprint("Cataloged secrets") + auxInfo := auxInfoFormat.Sprintf("[%d secrets]", prog.SecretsDiscovered.Current()) + _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo)) + }() + return err +} + +// FileMetadataCatalogerStartedHandler shows the intermittent secrets searching progress. +// nolint:dupl +func FileMetadataCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { + prog, err := syftEventParsers.ParseFileMetadataCatalogingStarted(event) + if err != nil { + return fmt.Errorf("bad %s event: %w", event.Type, err) + } + + line, err := fr.Append() + if err != nil { + return err + } + wg.Add(1) + + formatter, spinner := startProcess() + stream := progress.Stream(ctx, prog, interval) + title := tileFormat.Sprint("Cataloging file metadata") + + formatFn := func(p progress.Progress) { + progStr, err := formatter.Format(p) + spin := color.Magenta.Sprint(spinner.Next()) + if err != nil { + _, _ = io.WriteString(line, fmt.Sprintf("Error: %+v", err)) + } else { + _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, progStr)) + } + } + + go func() { + defer wg.Done() + + formatFn(progress.Progress{}) + for p := range stream { + formatFn(p) + } + + spin := color.Green.Sprint(completedStatus) + title = tileFormat.Sprint("Cataloged file metadata") + _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title)) + }() + return err +} + +// FileMetadataCatalogerStartedHandler shows the intermittent secrets searching progress. +// nolint:dupl +func FileDigestsCatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { + prog, err := syftEventParsers.ParseFileDigestsCatalogingStarted(event) + if err != nil { + return fmt.Errorf("bad %s event: %w", event.Type, err) + } + + line, err := fr.Append() + if err != nil { + return err + } + wg.Add(1) + + formatter, spinner := startProcess() + stream := progress.Stream(ctx, prog, interval) + title := tileFormat.Sprint("Cataloging file digests") + + formatFn := func(p progress.Progress) { + progStr, err := formatter.Format(p) + spin := color.Magenta.Sprint(spinner.Next()) + if err != nil { + _, _ = io.WriteString(line, fmt.Sprintf("Error: %+v", err)) + } else { + _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, progStr)) + } + } + + go func() { + defer wg.Done() + + formatFn(progress.Progress{}) + for p := range stream { + formatFn(p) + } + + spin := color.Green.Sprint(completedStatus) + title = tileFormat.Sprint("Cataloged file digests") + _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title)) + }() + return err +} + // ImportStartedHandler shows the intermittent upload progress to Anchore Enterprise. // nolint:dupl func ImportStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { diff --git a/ui/handler.go b/ui/handler.go index edf2e5ead..a1f920727 100644 --- a/ui/handler.go +++ b/ui/handler.go @@ -27,7 +27,7 @@ func NewHandler() *Handler { // RespondsTo indicates if the handler is capable of handling the given event. func (r *Handler) RespondsTo(event partybus.Event) bool { switch event.Type { - case stereoscopeEvent.PullDockerImage, stereoscopeEvent.ReadImage, stereoscopeEvent.FetchImage, syftEvent.PackageCatalogerStarted, syftEvent.ImportStarted: + case stereoscopeEvent.PullDockerImage, stereoscopeEvent.ReadImage, stereoscopeEvent.FetchImage, syftEvent.PackageCatalogerStarted, syftEvent.SecretsCatalogerStarted, syftEvent.FileDigestsCatalogerStarted, syftEvent.FileMetadataCatalogerStarted, syftEvent.ImportStarted: return true default: return false @@ -49,6 +49,15 @@ func (r *Handler) Handle(ctx context.Context, fr *frame.Frame, event partybus.Ev case syftEvent.PackageCatalogerStarted: return PackageCatalogerStartedHandler(ctx, fr, event, wg) + case syftEvent.SecretsCatalogerStarted: + return SecretsCatalogerStartedHandler(ctx, fr, event, wg) + + case syftEvent.FileDigestsCatalogerStarted: + return FileDigestsCatalogerStartedHandler(ctx, fr, event, wg) + + case syftEvent.FileMetadataCatalogerStarted: + return FileMetadataCatalogerStartedHandler(ctx, fr, event, wg) + case syftEvent.ImportStarted: return ImportStartedHandler(ctx, fr, event, wg) }