From 5565bdef0c77aa7e2adb28b74b192989bdb39750 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 20 Nov 2023 10:44:28 -0500 Subject: [PATCH] Remove the power-user command and related catalogers (#2306) * remove the power-user command Signed-off-by: Alex Goodman * remove secrets + classifier catalogers Signed-off-by: Alex Goodman * bump json schema Signed-off-by: Alex Goodman * regenerate json schema Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- DEVELOPING.md | 2 - README.md | 37 +- cmd/syft/cli/cli.go | 6 +- cmd/syft/cli/commands/packages.go | 4 + cmd/syft/cli/commands/poweruser.go | 161 -- cmd/syft/cli/eventloop/tasks.go | 34 - cmd/syft/cli/options/catalog.go | 38 +- cmd/syft/cli/options/file_classification.go | 17 - cmd/syft/cli/options/secrets.go | 23 - cmd/syft/cli/ui/handle_secrets_cataloger.go | 57 - .../cli/ui/handle_secrets_cataloger_test.go | 96 - cmd/syft/cli/ui/handler.go | 3 - internal/constants.go | 2 +- schema/json/README.md | 2 +- schema/json/schema-13.0.0.json | 2150 +++++++++++++++++ syft/event/event.go | 4 - syft/event/parsers/parsers.go | 14 - syft/file/cataloger/filecontent/cataloger.go | 2 - syft/file/cataloger/secrets/cataloger.go | 158 -- syft/file/cataloger/secrets/cataloger_test.go | 443 ---- .../secrets/generate_search_patterns.go | 56 - .../secrets/generate_search_patterns_test.go | 125 - .../file/cataloger/secrets/newline_counter.go | 39 - .../cataloger/secrets/newline_counter_test.go | 35 - .../secrets_search_by_line_strategy.go | 135 -- .../test-fixtures/secrets/default/api-key.txt | 14 - .../test-fixtures/secrets/default/aws.env | 3 - .../test-fixtures/secrets/default/aws.ini | 4 - .../secrets/default/docker-config.json | 10 - .../secrets/default/not-docker-config.json | 4 - .../default/private-key-false-positive.pem | 1 - .../secrets/default/private-key-openssl.pem | 9 - .../secrets/default/private-key.pem | 10 - .../secrets/default/private-keys.pem | 16 - .../test-fixtures/secrets/multiple.txt | 6 - .../secrets/test-fixtures/secrets/simple.txt | 4 - syft/format/syftjson/model/document.go | 11 +- syft/format/syftjson/to_format_model.go | 17 - syft/sbom/sbom.go | 1 - test/cli/json_schema_test.go | 5 - test/cli/power_user_cmd_test.go | 101 - 41 files changed, 2182 insertions(+), 1677 deletions(-) delete mode 100644 cmd/syft/cli/commands/poweruser.go delete mode 100644 cmd/syft/cli/options/file_classification.go delete mode 100644 cmd/syft/cli/options/secrets.go delete mode 100644 cmd/syft/cli/ui/handle_secrets_cataloger.go delete mode 100644 cmd/syft/cli/ui/handle_secrets_cataloger_test.go create mode 100644 schema/json/schema-13.0.0.json delete mode 100644 syft/file/cataloger/secrets/cataloger.go delete mode 100644 syft/file/cataloger/secrets/cataloger_test.go delete mode 100644 syft/file/cataloger/secrets/generate_search_patterns.go delete mode 100644 syft/file/cataloger/secrets/generate_search_patterns_test.go delete mode 100644 syft/file/cataloger/secrets/newline_counter.go delete mode 100644 syft/file/cataloger/secrets/newline_counter_test.go delete mode 100644 syft/file/cataloger/secrets/secrets_search_by_line_strategy.go delete mode 100644 syft/file/cataloger/secrets/test-fixtures/secrets/default/api-key.txt delete mode 100644 syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.env delete mode 100644 syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.ini delete mode 100644 syft/file/cataloger/secrets/test-fixtures/secrets/default/docker-config.json delete mode 100644 syft/file/cataloger/secrets/test-fixtures/secrets/default/not-docker-config.json delete mode 100644 syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-false-positive.pem delete mode 100644 syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-openssl.pem delete mode 100644 syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key.pem delete mode 100644 syft/file/cataloger/secrets/test-fixtures/secrets/default/private-keys.pem delete mode 100644 syft/file/cataloger/secrets/test-fixtures/secrets/multiple.txt delete mode 100644 syft/file/cataloger/secrets/test-fixtures/secrets/simple.txt delete mode 100644 test/cli/power_user_cmd_test.go diff --git a/DEVELOPING.md b/DEVELOPING.md index f3b8edfbd..064759cfb 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -84,8 +84,6 @@ They are registered in `syft/cli/commands/go`. │ ├── options/ │ ├── packages/ │ ├── packages.go - │ ├── poweruser/ - │ ├── poweruser.go │ └── version.go └── main.go ``` diff --git a/README.md b/README.md index b9a13b963..bcc8876d6 100644 --- a/README.md +++ b/README.md @@ -656,14 +656,13 @@ python: # when given an arbitrary constraint will be used (even if that version may not be available/published). guess-unpinned-requirements: false -# cataloging file contents is exposed through the power-user subcommand file-contents: cataloger: - # enable/disable cataloging of secrets + # enable/disable cataloging of file contents # SYFT_FILE_CONTENTS_CATALOGER_ENABLED env var enabled: true - # the search space to look for secrets (options: all-layers, squashed) + # the search space to look for file contents (options: all-layers, squashed) # SYFT_FILE_CONTENTS_CATALOGER_SCOPE env var scope: "squashed" @@ -675,7 +674,6 @@ file-contents: # SYFT_FILE_CONTENTS_GLOBS env var globs: [] -# cataloging file metadata is exposed through the power-user subcommand file-metadata: cataloger: # enable/disable cataloging of file metadata @@ -693,37 +691,6 @@ file-metadata: # maximum number of workers used to process the list of package catalogers in parallel parallelism: 1 -# cataloging secrets is exposed through the power-user subcommand -secrets: - cataloger: - # enable/disable cataloging of secrets - # SYFT_SECRETS_CATALOGER_ENABLED env var - enabled: true - - # the search space to look for secrets (options: all-layers, squashed) - # SYFT_SECRETS_CATALOGER_SCOPE env var - scope: "all-layers" - - # show extracted secret values in the final JSON report - # SYFT_SECRETS_REVEAL_VALUES env var - reveal-values: false - - # skip searching a file entirely if it is above the given size (default = 1MB; unit = bytes) - # SYFT_SECRETS_SKIP_FILES_ABOVE_SIZE env var - skip-files-above-size: 1048576 - - # name-regex pairs to consider when searching files for secrets. Note: the regex must match single line patterns - # but may also have OPTIONAL multiline capture groups. Regexes with a named capture group of "value" will - # use the entire regex to match, but the secret value will be assumed to be entirely contained within the - # "value" named capture group. - additional-patterns: {} - - # names to exclude from the secrets search, valid values are: "aws-access-key", "aws-secret-key", "pem-private-key", - # "docker-config-auth", and "generic-api-key". Note: this does not consider any names introduced in the - # "secrets.additional-patterns" config option. - # SYFT_SECRETS_EXCLUDE_PATTERN_NAMES env var - exclude-pattern-names: [] - # options that apply to all scan sources source: # alias name for the source diff --git a/cmd/syft/cli/cli.go b/cmd/syft/cli/cli.go index d95c7ecd2..2c7f44dc3 100644 --- a/cmd/syft/cli/cli.go +++ b/cmd/syft/cli/cli.go @@ -17,9 +17,8 @@ import ( "github.com/anchore/syft/internal/redact" ) -// Application constructs the `syft packages` command, aliases the root command to `syft packages`, -// and constructs the `syft power-user` command. It is also responsible for -// organizing flag usage and injecting the application config for each command. +// Application constructs the `syft packages` command and aliases the root command to `syft packages`. +// It is also responsible for organizing flag usage and injecting the application config for each command. // It also constructs the syft attest command and the syft version command. // `RunE` is the earliest that the complete application configuration can be loaded. func Application(id clio.Identification) clio.Application { @@ -86,7 +85,6 @@ func create(id clio.Identification, out io.Writer) (clio.Application, *cobra.Com // add sub-commands rootCmd.AddCommand( packagesCmd, - commands.PowerUser(app), commands.Attest(app), commands.Convert(app), clio.VersionCommand(id), diff --git a/cmd/syft/cli/commands/packages.go b/cmd/syft/cli/commands/packages.go index adc291c32..1e239fb7a 100644 --- a/cmd/syft/cli/commands/packages.go +++ b/cmd/syft/cli/commands/packages.go @@ -12,6 +12,7 @@ import ( "github.com/anchore/syft/cmd/syft/cli/options" "github.com/anchore/syft/cmd/syft/internal/ui" "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" @@ -157,6 +158,9 @@ func runPackages(id clio.Identification, opts *packagesOptions, userInput string ) if err != nil { + if userInput == "power-user" { + bus.Notify("Note: the 'power-user' command has been removed.") + } return fmt.Errorf("failed to construct source from user input %q: %w", userInput, err) } diff --git a/cmd/syft/cli/commands/poweruser.go b/cmd/syft/cli/commands/poweruser.go deleted file mode 100644 index 6c3718497..000000000 --- a/cmd/syft/cli/commands/poweruser.go +++ /dev/null @@ -1,161 +0,0 @@ -package commands - -import ( - "fmt" - "os" - - "github.com/gookit/color" - "github.com/hashicorp/go-multierror" - "github.com/spf13/cobra" - - "github.com/anchore/clio" - "github.com/anchore/stereoscope/pkg/image" - "github.com/anchore/syft/cmd/syft/cli/eventloop" - "github.com/anchore/syft/cmd/syft/cli/options" - "github.com/anchore/syft/cmd/syft/internal/ui" - "github.com/anchore/syft/internal" - "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/format/syftjson" - "github.com/anchore/syft/syft/sbom" - "github.com/anchore/syft/syft/source" -) - -const powerUserExample = ` {{.appName}} {{.command}} - DEPRECATED - THIS COMMAND WILL BE REMOVED in v1.0.0 - Template outputs are not supported. - All behavior is controlled via application configuration and environment variables (see https://github.com/anchore/syft#configuration) -` - -type powerUserOptions struct { - options.Config `yaml:",inline" mapstructure:",squash"` - options.OutputFile `yaml:",inline" mapstructure:",squash"` - options.UpdateCheck `yaml:",inline" mapstructure:",squash"` - options.Catalog `yaml:",inline" mapstructure:",squash"` -} - -func PowerUser(app clio.Application) *cobra.Command { - id := app.ID() - - pkgs := options.DefaultCatalog() - pkgs.Secrets.Cataloger.Enabled = true - pkgs.FileMetadata.Cataloger.Enabled = true - pkgs.FileContents.Cataloger.Enabled = true - pkgs.FileClassification.Cataloger.Enabled = true - opts := &powerUserOptions{ - Catalog: pkgs, - OutputFile: options.OutputFile{ // nolint:staticcheck - Enabled: true, - }, - } - - return app.SetupCommand(&cobra.Command{ - Use: "power-user [IMAGE]", - Short: "Run bulk operations on container images", - Example: internal.Tprintf(powerUserExample, map[string]interface{}{ - "appName": id.Name, - "command": "power-user", - }), - Args: validatePackagesArgs, - Hidden: true, - PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck), - RunE: func(cmd *cobra.Command, args []string) error { - restoreStdout := ui.CaptureStdoutToTraceLog() - defer restoreStdout() - - return runPowerUser(id, opts, args[0]) - }, - }, opts) -} - -//nolint:funlen -func runPowerUser(id clio.Identification, opts *powerUserOptions, userInput string) error { - writer, err := opts.SBOMWriter(syftjson.NewFormatEncoder()) - if err != nil { - return err - } - defer func() { - // inform user at end of run that command will be removed - deprecated := color.Style{color.Red, color.OpBold}.Sprint("DEPRECATED: This command will be removed in v1.0.0") - fmt.Fprintln(os.Stderr, deprecated) - }() - - tasks, err := eventloop.Tasks(&opts.Catalog) - if err != nil { - return err - } - - detection, err := source.Detect( - userInput, - source.DetectConfig{ - DefaultImageSource: opts.DefaultImagePullSource, - }, - ) - if err != nil { - return fmt.Errorf("could not deteremine source: %w", err) - } - - var platform *image.Platform - - if opts.Platform != "" { - platform, err = image.NewPlatform(opts.Platform) - if err != nil { - return fmt.Errorf("invalid platform: %w", err) - } - } - - src, err := detection.NewSource( - source.DetectionSourceConfig{ - Alias: source.Alias{ - Name: opts.Source.Name, - Version: opts.Source.Version, - }, - RegistryOptions: opts.Registry.ToOptions(), - Platform: platform, - Exclude: source.ExcludeConfig{ - Paths: opts.Exclusions, - }, - DigestAlgorithms: nil, - BasePath: opts.BasePath, - }, - ) - - if src != nil { - defer src.Close() - } - if err != nil { - return fmt.Errorf("failed to construct source from user input %q: %w", userInput, err) - } - - s := sbom.SBOM{ - Source: src.Describe(), - Descriptor: sbom.Descriptor{ - Name: id.Name, - Version: id.Version, - Configuration: opts, - }, - } - - var errs error - var relationships []<-chan artifact.Relationship - for _, task := range tasks { - c := make(chan artifact.Relationship) - relationships = append(relationships, c) - - go func(task eventloop.Task) { - err := eventloop.RunTask(task, &s.Artifacts, src, c) - errs = multierror.Append(errs, err) - }(task) - } - - if errs != nil { - return errs - } - - s.Relationships = append(s.Relationships, mergeRelationships(relationships...)...) - - if err := writer.Write(s); err != nil { - return fmt.Errorf("failed to write sbom: %w", err) - } - - return nil -} diff --git a/cmd/syft/cli/eventloop/tasks.go b/cmd/syft/cli/eventloop/tasks.go index feabca573..8cfb68503 100644 --- a/cmd/syft/cli/eventloop/tasks.go +++ b/cmd/syft/cli/eventloop/tasks.go @@ -8,7 +8,6 @@ import ( "github.com/anchore/syft/syft/file/cataloger/filecontent" "github.com/anchore/syft/syft/file/cataloger/filedigest" "github.com/anchore/syft/syft/file/cataloger/filemetadata" - "github.com/anchore/syft/syft/file/cataloger/secrets" "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/source" ) @@ -22,7 +21,6 @@ func Tasks(opts *options.Catalog) ([]Task, error) { generateCatalogPackagesTask, generateCatalogFileMetadataTask, generateCatalogFileDigestsTask, - generateCatalogSecretsTask, generateCatalogContentsTask, } @@ -110,38 +108,6 @@ func generateCatalogFileDigestsTask(opts *options.Catalog) (Task, error) { return task, nil } -func generateCatalogSecretsTask(opts *options.Catalog) (Task, error) { - if !opts.Secrets.Cataloger.Enabled { - return nil, nil - } - - patterns, err := secrets.GenerateSearchPatterns(secrets.DefaultSecretsPatterns, opts.Secrets.AdditionalPatterns, opts.Secrets.ExcludePatternNames) - if err != nil { - return nil, err - } - - secretsCataloger, err := secrets.NewCataloger(patterns, opts.Secrets.RevealValues, opts.Secrets.SkipFilesAboveSize) //nolint:staticcheck - if err != nil { - return nil, err - } - - task := func(results *sbom.Artifacts, src source.Source) ([]artifact.Relationship, error) { - resolver, err := src.FileResolver(opts.Secrets.Cataloger.GetScope()) - if err != nil { - return nil, err - } - - result, err := secretsCataloger.Catalog(resolver) - if err != nil { - return nil, err - } - results.Secrets = result - return nil, nil - } - - return task, nil -} - func generateCatalogContentsTask(opts *options.Catalog) (Task, error) { if !opts.FileContents.Cataloger.Enabled { return nil, nil diff --git a/cmd/syft/cli/options/catalog.go b/cmd/syft/cli/options/catalog.go index d8b9f8e59..f7c468d7d 100644 --- a/cmd/syft/cli/options/catalog.go +++ b/cmd/syft/cli/options/catalog.go @@ -21,25 +21,23 @@ import ( ) type Catalog struct { - Catalogers []string `yaml:"catalogers" json:"catalogers" mapstructure:"catalogers"` - Package pkg `yaml:"package" json:"package" mapstructure:"package"` - Golang golang `yaml:"golang" json:"golang" mapstructure:"golang"` - Java java `yaml:"java" json:"java" mapstructure:"java"` - LinuxKernel linuxKernel `yaml:"linux-kernel" json:"linux-kernel" mapstructure:"linux-kernel"` - Python python `yaml:"python" json:"python" mapstructure:"python"` - FileMetadata fileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"` - FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"` - FileContents fileContents `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"` - Secrets secrets `yaml:"secrets" json:"secrets" mapstructure:"secrets"` - Registry registry `yaml:"registry" json:"registry" mapstructure:"registry"` - Exclusions []string `yaml:"exclude" json:"exclude" mapstructure:"exclude"` - Platform string `yaml:"platform" json:"platform" mapstructure:"platform"` - Name string `yaml:"name" json:"name" mapstructure:"name"` - Source sourceCfg `yaml:"source" json:"source" mapstructure:"source"` - Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel - DefaultImagePullSource string `yaml:"default-image-pull-source" json:"default-image-pull-source" mapstructure:"default-image-pull-source"` // specify default image pull source - BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths - ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files + Catalogers []string `yaml:"catalogers" json:"catalogers" mapstructure:"catalogers"` + Package pkg `yaml:"package" json:"package" mapstructure:"package"` + Golang golang `yaml:"golang" json:"golang" mapstructure:"golang"` + Java java `yaml:"java" json:"java" mapstructure:"java"` + LinuxKernel linuxKernel `yaml:"linux-kernel" json:"linux-kernel" mapstructure:"linux-kernel"` + Python python `yaml:"python" json:"python" mapstructure:"python"` + FileMetadata fileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"` + FileContents fileContents `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"` + Registry registry `yaml:"registry" json:"registry" mapstructure:"registry"` + Exclusions []string `yaml:"exclude" json:"exclude" mapstructure:"exclude"` + Platform string `yaml:"platform" json:"platform" mapstructure:"platform"` + Name string `yaml:"name" json:"name" mapstructure:"name"` + Source sourceCfg `yaml:"source" json:"source" mapstructure:"source"` + Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel + DefaultImagePullSource string `yaml:"default-image-pull-source" json:"default-image-pull-source" mapstructure:"default-image-pull-source"` // specify default image pull source + BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths + ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files } var _ interface { @@ -52,9 +50,7 @@ func DefaultCatalog() Catalog { Package: defaultPkg(), LinuxKernel: defaultLinuxKernel(), FileMetadata: defaultFileMetadata(), - FileClassification: defaultFileClassification(), FileContents: defaultFileContents(), - Secrets: defaultSecrets(), Source: defaultSourceCfg(), Parallelism: 1, ExcludeBinaryOverlapByOwnership: true, diff --git a/cmd/syft/cli/options/file_classification.go b/cmd/syft/cli/options/file_classification.go deleted file mode 100644 index 9f1abcdfb..000000000 --- a/cmd/syft/cli/options/file_classification.go +++ /dev/null @@ -1,17 +0,0 @@ -package options - -import ( - "github.com/anchore/syft/syft/source" -) - -type fileClassification struct { - Cataloger scope `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"` -} - -func defaultFileClassification() fileClassification { - return fileClassification{ - Cataloger: scope{ - Scope: source.SquashedScope.String(), - }, - } -} diff --git a/cmd/syft/cli/options/secrets.go b/cmd/syft/cli/options/secrets.go deleted file mode 100644 index 58693f6e6..000000000 --- a/cmd/syft/cli/options/secrets.go +++ /dev/null @@ -1,23 +0,0 @@ -package options - -import ( - "github.com/anchore/syft/internal/file" - "github.com/anchore/syft/syft/source" -) - -type secrets struct { - Cataloger scope `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"` - AdditionalPatterns map[string]string `yaml:"additional-patterns" json:"additional-patterns" mapstructure:"additional-patterns"` - ExcludePatternNames []string `yaml:"exclude-pattern-names" json:"exclude-pattern-names" mapstructure:"exclude-pattern-names"` - RevealValues bool `yaml:"reveal-values" json:"reveal-values" mapstructure:"reveal-values"` - SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"` -} - -func defaultSecrets() secrets { - return secrets{ - Cataloger: scope{ - Scope: source.AllLayersScope.String(), - }, - SkipFilesAboveSize: 1 * file.MB, - } -} diff --git a/cmd/syft/cli/ui/handle_secrets_cataloger.go b/cmd/syft/cli/ui/handle_secrets_cataloger.go deleted file mode 100644 index 95b96454b..000000000 --- a/cmd/syft/cli/ui/handle_secrets_cataloger.go +++ /dev/null @@ -1,57 +0,0 @@ -package ui - -import ( - "fmt" - - tea "github.com/charmbracelet/bubbletea" - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" - - "github.com/anchore/bubbly/bubbles/taskprogress" - "github.com/anchore/syft/internal/log" - syftEventParsers "github.com/anchore/syft/syft/event/parsers" - "github.com/anchore/syft/syft/file/cataloger/secrets" -) - -var _ progress.StagedProgressable = (*secretsCatalogerProgressAdapter)(nil) - -// Deprecated: will be removed in syft 1.0 -type secretsCatalogerProgressAdapter struct { - *secrets.Monitor -} - -// Deprecated: will be removed in syft 1.0 -func newSecretsCatalogerProgressAdapter(monitor *secrets.Monitor) secretsCatalogerProgressAdapter { - return secretsCatalogerProgressAdapter{ - Monitor: monitor, - } -} - -func (s secretsCatalogerProgressAdapter) Stage() string { - return fmt.Sprintf("%d secrets", s.Monitor.SecretsDiscovered.Current()) -} - -// Deprecated: will be removed in syft 1.0 -func (m *Handler) handleSecretsCatalogerStarted(e partybus.Event) []tea.Model { - mon, err := syftEventParsers.ParseSecretsCatalogingStarted(e) - if err != nil { - log.WithFields("error", err).Warn("unable to parse event") - return nil - } - - tsk := m.newTaskProgress( - taskprogress.Title{ - Default: "Catalog secrets", - Running: "Cataloging secrets", - Success: "Cataloged secrets", - }, - - taskprogress.WithStagedProgressable( - newSecretsCatalogerProgressAdapter(mon), - ), - ) - - tsk.HideStageOnSuccess = false - - return []tea.Model{tsk} -} diff --git a/cmd/syft/cli/ui/handle_secrets_cataloger_test.go b/cmd/syft/cli/ui/handle_secrets_cataloger_test.go deleted file mode 100644 index 3a04cbce5..000000000 --- a/cmd/syft/cli/ui/handle_secrets_cataloger_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package ui - -import ( - "testing" - "time" - - tea "github.com/charmbracelet/bubbletea" - "github.com/gkampitakis/go-snaps/snaps" - "github.com/stretchr/testify/require" - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" - - "github.com/anchore/bubbly/bubbles/taskprogress" - syftEvent "github.com/anchore/syft/syft/event" - "github.com/anchore/syft/syft/file/cataloger/secrets" -) - -func TestHandler_handleSecretsCatalogerStarted(t *testing.T) { - - tests := []struct { - name string - eventFn func(*testing.T) partybus.Event - iterations int - }{ - { - name: "cataloging in progress", - eventFn: func(t *testing.T) partybus.Event { - stage := &progress.Stage{ - Current: "current", - } - secretsDiscovered := progress.NewManual(-1) - secretsDiscovered.Set(64) - prog := progress.NewManual(72) - prog.Set(50) - - return partybus.Event{ - Type: syftEvent.SecretsCatalogerStarted, - Source: secretsDiscovered, - Value: secrets.Monitor{ - Stager: progress.Stager(stage), - SecretsDiscovered: secretsDiscovered, - Progressable: prog, - }, - } - }, - }, - { - name: "cataloging complete", - eventFn: func(t *testing.T) partybus.Event { - stage := &progress.Stage{ - Current: "current", - } - secretsDiscovered := progress.NewManual(-1) - secretsDiscovered.Set(64) - prog := progress.NewManual(72) - prog.Set(72) - prog.SetCompleted() - - return partybus.Event{ - Type: syftEvent.SecretsCatalogerStarted, - Source: secretsDiscovered, - Value: secrets.Monitor{ - Stager: progress.Stager(stage), - SecretsDiscovered: secretsDiscovered, - Progressable: prog, - }, - } - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - event := tt.eventFn(t) - handler := New(DefaultHandlerConfig()) - handler.WindowSize = tea.WindowSizeMsg{ - Width: 100, - Height: 80, - } - - models := handler.Handle(event) - require.Len(t, models, 1) - model := models[0] - - tsk, ok := model.(taskprogress.Model) - require.True(t, ok) - - got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ - Time: time.Now(), - Sequence: tsk.Sequence(), - ID: tsk.ID(), - }) - t.Log(got) - snaps.MatchSnapshot(t, got) - }) - } -} diff --git a/cmd/syft/cli/ui/handler.go b/cmd/syft/cli/ui/handler.go index b2054467f..1a0e64d10 100644 --- a/cmd/syft/cli/ui/handler.go +++ b/cmd/syft/cli/ui/handler.go @@ -58,9 +58,6 @@ func New(cfg HandlerConfig) *Handler { syftEvent.FileIndexingStarted: h.handleFileIndexingStarted, syftEvent.AttestationStarted: h.handleAttestationStarted, syftEvent.CatalogerTaskStarted: h.handleCatalogerTaskStarted, - - // deprecated - syftEvent.SecretsCatalogerStarted: h.handleSecretsCatalogerStarted, }) return h diff --git a/internal/constants.go b/internal/constants.go index ba9730a76..78981d388 100644 --- a/internal/constants.go +++ b/internal/constants.go @@ -3,5 +3,5 @@ package internal const ( // JSONSchemaVersion is the current schema version output by the JSON encoder // This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment. - JSONSchemaVersion = "12.0.1" + JSONSchemaVersion = "13.0.0" ) diff --git a/schema/json/README.md b/schema/json/README.md index 7de19b64d..99032613d 100644 --- a/schema/json/README.md +++ b/schema/json/README.md @@ -1,6 +1,6 @@ # JSON Schema -This is the JSON schema for output from the JSON presenters (`syft packages -o json` and `syft power-user `). The required inputs for defining the JSON schema are as follows: +This is the JSON schema for output from the JSON presenters (`syft packages -o json`). The required inputs for defining the JSON schema are as follows: - the value of `internal.JSONSchemaVersion` that governs the schema filename - the `Document` struct definition within `github.com/anchore/syft/syft/formats/syftjson/model/document.go` that governs the overall document shape diff --git a/schema/json/schema-13.0.0.json b/schema/json/schema-13.0.0.json new file mode 100644 index 000000000..b3beeee6a --- /dev/null +++ b/schema/json/schema-13.0.0.json @@ -0,0 +1,2150 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "anchore.io/schema/syft/json/13.0.0/document", + "$ref": "#/$defs/Document", + "$defs": { + "AlpmDbEntry": { + "properties": { + "basepackage": { + "type": "string" + }, + "package": { + "type": "string" + }, + "version": { + "type": "string" + }, + "description": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "packager": { + "type": "string" + }, + "url": { + "type": "string" + }, + "validation": { + "type": "string" + }, + "reason": { + "type": "integer" + }, + "files": { + "items": { + "$ref": "#/$defs/AlpmFileRecord" + }, + "type": "array" + }, + "backup": { + "items": { + "$ref": "#/$defs/AlpmFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "basepackage", + "package", + "version", + "description", + "architecture", + "size", + "packager", + "url", + "validation", + "reason", + "files", + "backup" + ] + }, + "AlpmFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "type": { + "type": "string" + }, + "uid": { + "type": "string" + }, + "gid": { + "type": "string" + }, + "time": { + "type": "string", + "format": "date-time" + }, + "size": { + "type": "string" + }, + "link": { + "type": "string" + }, + "digest": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + } + }, + "type": "object" + }, + "ApkDbEntry": { + "properties": { + "package": { + "type": "string" + }, + "originPackage": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "version": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "url": { + "type": "string" + }, + "description": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "installedSize": { + "type": "integer" + }, + "pullDependencies": { + "items": { + "type": "string" + }, + "type": "array" + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array" + }, + "pullChecksum": { + "type": "string" + }, + "gitCommitOfApkPort": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/ApkFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "package", + "originPackage", + "maintainer", + "version", + "architecture", + "url", + "description", + "size", + "installedSize", + "pullDependencies", + "provides", + "pullChecksum", + "gitCommitOfApkPort", + "files" + ] + }, + "ApkFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "ownerUid": { + "type": "string" + }, + "ownerGid": { + "type": "string" + }, + "permissions": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "BinarySignature": { + "properties": { + "matches": { + "items": { + "$ref": "#/$defs/ClassifierMatch" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "matches" + ] + }, + "CConanFileEntry": { + "properties": { + "ref": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ref" + ] + }, + "CConanInfoEntry": { + "properties": { + "ref": { + "type": "string" + }, + "package_id": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ref" + ] + }, + "CConanLockEntry": { + "properties": { + "ref": { + "type": "string" + }, + "package_id": { + "type": "string" + }, + "prev": { + "type": "string" + }, + "requires": { + "items": { + "type": "string" + }, + "type": "array" + }, + "build_requires": { + "items": { + "type": "string" + }, + "type": "array" + }, + "py_requires": { + "items": { + "type": "string" + }, + "type": "array" + }, + "options": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "path": { + "type": "string" + }, + "context": { + "type": "string" + } + }, + "type": "object", + "required": [ + "ref" + ] + }, + "ClassifierMatch": { + "properties": { + "classifier": { + "type": "string" + }, + "location": { + "$ref": "#/$defs/Location" + } + }, + "type": "object", + "required": [ + "classifier", + "location" + ] + }, + "CocoaPodfileLockEntry": { + "properties": { + "checksum": { + "type": "string" + } + }, + "type": "object", + "required": [ + "checksum" + ] + }, + "Coordinates": { + "properties": { + "path": { + "type": "string" + }, + "layerID": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "DartPubspecLockEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "hosted_url": { + "type": "string" + }, + "vcs_url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "Descriptor": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "configuration": true + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "Digest": { + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "algorithm", + "value" + ] + }, + "Document": { + "properties": { + "artifacts": { + "items": { + "$ref": "#/$defs/Package" + }, + "type": "array" + }, + "artifactRelationships": { + "items": { + "$ref": "#/$defs/Relationship" + }, + "type": "array" + }, + "files": { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" + }, + "source": { + "$ref": "#/$defs/Source" + }, + "distro": { + "$ref": "#/$defs/LinuxRelease" + }, + "descriptor": { + "$ref": "#/$defs/Descriptor" + }, + "schema": { + "$ref": "#/$defs/Schema" + } + }, + "type": "object", + "required": [ + "artifacts", + "artifactRelationships", + "source", + "distro", + "descriptor", + "schema" + ] + }, + "DotnetDepsEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "path": { + "type": "string" + }, + "sha512": { + "type": "string" + }, + "hashPath": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "path", + "sha512", + "hashPath" + ] + }, + "DotnetPortableExecutableEntry": { + "properties": { + "assemblyVersion": { + "type": "string" + }, + "legalCopyright": { + "type": "string" + }, + "comments": { + "type": "string" + }, + "internalName": { + "type": "string" + }, + "companyName": { + "type": "string" + }, + "productName": { + "type": "string" + }, + "productVersion": { + "type": "string" + } + }, + "type": "object", + "required": [ + "assemblyVersion", + "legalCopyright", + "companyName", + "productName", + "productVersion" + ] + }, + "DpkgDbEntry": { + "properties": { + "package": { + "type": "string" + }, + "source": { + "type": "string" + }, + "version": { + "type": "string" + }, + "sourceVersion": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "installedSize": { + "type": "integer" + }, + "provides": { + "items": { + "type": "string" + }, + "type": "array" + }, + "depends": { + "items": { + "type": "string" + }, + "type": "array" + }, + "preDepends": { + "items": { + "type": "string" + }, + "type": "array" + }, + "files": { + "items": { + "$ref": "#/$defs/DpkgFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "package", + "source", + "version", + "sourceVersion", + "architecture", + "maintainer", + "installedSize", + "files" + ] + }, + "DpkgFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + }, + "isConfigFile": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "path", + "isConfigFile" + ] + }, + "ElixirMixLockEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "pkgHash": { + "type": "string" + }, + "pkgHashExt": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "pkgHash", + "pkgHashExt" + ] + }, + "ErlangRebarLockEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "pkgHash": { + "type": "string" + }, + "pkgHashExt": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "pkgHash", + "pkgHashExt" + ] + }, + "File": { + "properties": { + "id": { + "type": "string" + }, + "location": { + "$ref": "#/$defs/Coordinates" + }, + "metadata": { + "$ref": "#/$defs/FileMetadataEntry" + }, + "contents": { + "type": "string" + }, + "digests": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + }, + "licenses": { + "items": { + "$ref": "#/$defs/FileLicense" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "id", + "location" + ] + }, + "FileLicense": { + "properties": { + "value": { + "type": "string" + }, + "spdxExpression": { + "type": "string" + }, + "type": { + "type": "string" + }, + "evidence": { + "$ref": "#/$defs/FileLicenseEvidence" + } + }, + "type": "object", + "required": [ + "value", + "spdxExpression", + "type" + ] + }, + "FileLicenseEvidence": { + "properties": { + "confidence": { + "type": "integer" + }, + "offset": { + "type": "integer" + }, + "extent": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "confidence", + "offset", + "extent" + ] + }, + "FileMetadataEntry": { + "properties": { + "mode": { + "type": "integer" + }, + "type": { + "type": "string" + }, + "linkDestination": { + "type": "string" + }, + "userID": { + "type": "integer" + }, + "groupID": { + "type": "integer" + }, + "mimeType": { + "type": "string" + }, + "size": { + "type": "integer" + } + }, + "type": "object", + "required": [ + "mode", + "type", + "userID", + "groupID", + "mimeType", + "size" + ] + }, + "GoModuleBuildinfoEntry": { + "properties": { + "goBuildSettings": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "goCompiledVersion": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "h1Digest": { + "type": "string" + }, + "mainModule": { + "type": "string" + }, + "goCryptoSettings": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "goCompiledVersion", + "architecture" + ] + }, + "GoModuleEntry": { + "properties": { + "h1Digest": { + "type": "string" + } + }, + "type": "object" + }, + "HaskellHackageStackEntry": { + "properties": { + "pkgHash": { + "type": "string" + } + }, + "type": "object" + }, + "HaskellHackageStackLockEntry": { + "properties": { + "pkgHash": { + "type": "string" + }, + "snapshotURL": { + "type": "string" + } + }, + "type": "object" + }, + "IDLikes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "JavaArchive": { + "properties": { + "virtualPath": { + "type": "string" + }, + "manifest": { + "$ref": "#/$defs/JavaManifest" + }, + "pomProperties": { + "$ref": "#/$defs/JavaPomProperties" + }, + "pomProject": { + "$ref": "#/$defs/JavaPomProject" + }, + "digest": { + "items": { + "$ref": "#/$defs/Digest" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "virtualPath" + ] + }, + "JavaManifest": { + "properties": { + "main": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "namedSections": { + "patternProperties": { + ".*": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "JavaPomParent": { + "properties": { + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "type": "object", + "required": [ + "groupId", + "artifactId", + "version" + ] + }, + "JavaPomProject": { + "properties": { + "path": { + "type": "string" + }, + "parent": { + "$ref": "#/$defs/JavaPomParent" + }, + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path", + "groupId", + "artifactId", + "version", + "name" + ] + }, + "JavaPomProperties": { + "properties": { + "path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "groupId": { + "type": "string" + }, + "artifactId": { + "type": "string" + }, + "version": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "extraFields": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "path", + "name", + "groupId", + "artifactId", + "version" + ] + }, + "JavascriptNpmPackage": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "author": { + "type": "string" + }, + "homepage": { + "type": "string" + }, + "description": { + "type": "string" + }, + "url": { + "type": "string" + }, + "private": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "name", + "version", + "author", + "homepage", + "description", + "url", + "private" + ] + }, + "JavascriptNpmPackageLockEntry": { + "properties": { + "resolved": { + "type": "string" + }, + "integrity": { + "type": "string" + } + }, + "type": "object", + "required": [ + "resolved", + "integrity" + ] + }, + "License": { + "properties": { + "value": { + "type": "string" + }, + "spdxExpression": { + "type": "string" + }, + "type": { + "type": "string" + }, + "urls": { + "items": { + "type": "string" + }, + "type": "array" + }, + "locations": { + "items": { + "$ref": "#/$defs/Location" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "value", + "spdxExpression", + "type", + "urls", + "locations" + ] + }, + "LinuxKernelArchive": { + "properties": { + "name": { + "type": "string" + }, + "architecture": { + "type": "string" + }, + "version": { + "type": "string" + }, + "extendedVersion": { + "type": "string" + }, + "buildTime": { + "type": "string" + }, + "author": { + "type": "string" + }, + "format": { + "type": "string" + }, + "rwRootFS": { + "type": "boolean" + }, + "swapDevice": { + "type": "integer" + }, + "rootDevice": { + "type": "integer" + }, + "videoMode": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "architecture", + "version" + ] + }, + "LinuxKernelModule": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "sourceVersion": { + "type": "string" + }, + "path": { + "type": "string" + }, + "description": { + "type": "string" + }, + "author": { + "type": "string" + }, + "license": { + "type": "string" + }, + "kernelVersion": { + "type": "string" + }, + "versionMagic": { + "type": "string" + }, + "parameters": { + "patternProperties": { + ".*": { + "$ref": "#/$defs/LinuxKernelModuleParameter" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "LinuxKernelModuleParameter": { + "properties": { + "type": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "type": "object" + }, + "LinuxRelease": { + "properties": { + "prettyName": { + "type": "string" + }, + "name": { + "type": "string" + }, + "id": { + "type": "string" + }, + "idLike": { + "$ref": "#/$defs/IDLikes" + }, + "version": { + "type": "string" + }, + "versionID": { + "type": "string" + }, + "versionCodename": { + "type": "string" + }, + "buildID": { + "type": "string" + }, + "imageID": { + "type": "string" + }, + "imageVersion": { + "type": "string" + }, + "variant": { + "type": "string" + }, + "variantID": { + "type": "string" + }, + "homeURL": { + "type": "string" + }, + "supportURL": { + "type": "string" + }, + "bugReportURL": { + "type": "string" + }, + "privacyPolicyURL": { + "type": "string" + }, + "cpeName": { + "type": "string" + }, + "supportEnd": { + "type": "string" + } + }, + "type": "object" + }, + "Location": { + "properties": { + "path": { + "type": "string" + }, + "layerID": { + "type": "string" + }, + "accessPath": { + "type": "string" + }, + "annotations": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object", + "required": [ + "path", + "accessPath" + ] + }, + "MicrosoftKbPatch": { + "properties": { + "product_id": { + "type": "string" + }, + "kb": { + "type": "string" + } + }, + "type": "object", + "required": [ + "product_id", + "kb" + ] + }, + "NixStoreEntry": { + "properties": { + "outputHash": { + "type": "string" + }, + "output": { + "type": "string" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "outputHash", + "files" + ] + }, + "Package": { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "type": { + "type": "string" + }, + "foundBy": { + "type": "string" + }, + "locations": { + "items": { + "$ref": "#/$defs/Location" + }, + "type": "array" + }, + "licenses": { + "$ref": "#/$defs/licenses" + }, + "language": { + "type": "string" + }, + "cpes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "purl": { + "type": "string" + }, + "metadataType": { + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#/$defs/AlpmDbEntry" + }, + { + "$ref": "#/$defs/ApkDbEntry" + }, + { + "$ref": "#/$defs/BinarySignature" + }, + { + "$ref": "#/$defs/CConanFileEntry" + }, + { + "$ref": "#/$defs/CConanInfoEntry" + }, + { + "$ref": "#/$defs/CConanLockEntry" + }, + { + "$ref": "#/$defs/CocoaPodfileLockEntry" + }, + { + "$ref": "#/$defs/DartPubspecLockEntry" + }, + { + "$ref": "#/$defs/DotnetDepsEntry" + }, + { + "$ref": "#/$defs/DotnetPortableExecutableEntry" + }, + { + "$ref": "#/$defs/DpkgDbEntry" + }, + { + "$ref": "#/$defs/ElixirMixLockEntry" + }, + { + "$ref": "#/$defs/ErlangRebarLockEntry" + }, + { + "$ref": "#/$defs/GoModuleBuildinfoEntry" + }, + { + "$ref": "#/$defs/GoModuleEntry" + }, + { + "$ref": "#/$defs/HaskellHackageStackEntry" + }, + { + "$ref": "#/$defs/HaskellHackageStackLockEntry" + }, + { + "$ref": "#/$defs/JavaArchive" + }, + { + "$ref": "#/$defs/JavascriptNpmPackage" + }, + { + "$ref": "#/$defs/JavascriptNpmPackageLockEntry" + }, + { + "$ref": "#/$defs/LinuxKernelArchive" + }, + { + "$ref": "#/$defs/LinuxKernelModule" + }, + { + "$ref": "#/$defs/MicrosoftKbPatch" + }, + { + "$ref": "#/$defs/NixStoreEntry" + }, + { + "$ref": "#/$defs/PhpComposerInstalledEntry" + }, + { + "$ref": "#/$defs/PhpComposerLockEntry" + }, + { + "$ref": "#/$defs/PortageDbEntry" + }, + { + "$ref": "#/$defs/PythonPackage" + }, + { + "$ref": "#/$defs/PythonPipRequirementsEntry" + }, + { + "$ref": "#/$defs/PythonPipfileLockEntry" + }, + { + "$ref": "#/$defs/RDescription" + }, + { + "$ref": "#/$defs/RpmArchive" + }, + { + "$ref": "#/$defs/RpmDbEntry" + }, + { + "$ref": "#/$defs/RubyGemspec" + }, + { + "$ref": "#/$defs/RustCargoAuditEntry" + }, + { + "$ref": "#/$defs/RustCargoLockEntry" + }, + { + "$ref": "#/$defs/SwiftPackageManagerLockEntry" + } + ] + } + }, + "type": "object", + "required": [ + "id", + "name", + "version", + "type", + "foundBy", + "locations", + "licenses", + "language", + "cpes", + "purl" + ] + }, + "PhpComposerAuthors": { + "properties": { + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "homepage": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "PhpComposerExternalReference": { + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": "string" + }, + "reference": { + "type": "string" + }, + "shasum": { + "type": "string" + } + }, + "type": "object", + "required": [ + "type", + "url", + "reference" + ] + }, + "PhpComposerInstalledEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "dist": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "require": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "provide": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "require-dev": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "suggest": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "license": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": { + "type": "string" + }, + "notification-url": { + "type": "string" + }, + "bin": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "$ref": "#/$defs/PhpComposerAuthors" + }, + "type": "array" + }, + "description": { + "type": "string" + }, + "homepage": { + "type": "string" + }, + "keywords": { + "items": { + "type": "string" + }, + "type": "array" + }, + "time": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "dist" + ] + }, + "PhpComposerLockEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "dist": { + "$ref": "#/$defs/PhpComposerExternalReference" + }, + "require": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "provide": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "require-dev": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "suggest": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + }, + "license": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": { + "type": "string" + }, + "notification-url": { + "type": "string" + }, + "bin": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "$ref": "#/$defs/PhpComposerAuthors" + }, + "type": "array" + }, + "description": { + "type": "string" + }, + "homepage": { + "type": "string" + }, + "keywords": { + "items": { + "type": "string" + }, + "type": "array" + }, + "time": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "dist" + ] + }, + "PortageDbEntry": { + "properties": { + "installedSize": { + "type": "integer" + }, + "files": { + "items": { + "$ref": "#/$defs/PortageFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "installedSize", + "files" + ] + }, + "PortageFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/Digest" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "PythonDirectURLOriginInfo": { + "properties": { + "url": { + "type": "string" + }, + "commitId": { + "type": "string" + }, + "vcs": { + "type": "string" + } + }, + "type": "object", + "required": [ + "url" + ] + }, + "PythonFileDigest": { + "properties": { + "algorithm": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "algorithm", + "value" + ] + }, + "PythonFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "digest": { + "$ref": "#/$defs/PythonFileDigest" + }, + "size": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path" + ] + }, + "PythonPackage": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "author": { + "type": "string" + }, + "authorEmail": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/PythonFileRecord" + }, + "type": "array" + }, + "sitePackagesRootPath": { + "type": "string" + }, + "topLevelPackages": { + "items": { + "type": "string" + }, + "type": "array" + }, + "directUrlOrigin": { + "$ref": "#/$defs/PythonDirectURLOriginInfo" + } + }, + "type": "object", + "required": [ + "name", + "version", + "author", + "authorEmail", + "platform", + "sitePackagesRootPath" + ] + }, + "PythonPipRequirementsEntry": { + "properties": { + "name": { + "type": "string" + }, + "extras": { + "items": { + "type": "string" + }, + "type": "array" + }, + "versionConstraint": { + "type": "string" + }, + "url": { + "type": "string" + }, + "markers": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "versionConstraint" + ] + }, + "PythonPipfileLockEntry": { + "properties": { + "hashes": { + "items": { + "type": "string" + }, + "type": "array" + }, + "index": { + "type": "string" + } + }, + "type": "object", + "required": [ + "hashes", + "index" + ] + }, + "RDescription": { + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "author": { + "type": "string" + }, + "maintainer": { + "type": "string" + }, + "url": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repository": { + "type": "string" + }, + "built": { + "type": "string" + }, + "needsCompilation": { + "type": "boolean" + }, + "imports": { + "items": { + "type": "string" + }, + "type": "array" + }, + "depends": { + "items": { + "type": "string" + }, + "type": "array" + }, + "suggests": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "Relationship": { + "properties": { + "parent": { + "type": "string" + }, + "child": { + "type": "string" + }, + "type": { + "type": "string" + }, + "metadata": true + }, + "type": "object", + "required": [ + "parent", + "child", + "type" + ] + }, + "RpmArchive": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "epoch": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "architecture": { + "type": "string" + }, + "release": { + "type": "string" + }, + "sourceRpm": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "vendor": { + "type": "string" + }, + "modularityLabel": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/RpmFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "version", + "epoch", + "architecture", + "release", + "sourceRpm", + "size", + "vendor", + "modularityLabel", + "files" + ] + }, + "RpmDbEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "epoch": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "architecture": { + "type": "string" + }, + "release": { + "type": "string" + }, + "sourceRpm": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "vendor": { + "type": "string" + }, + "modularityLabel": { + "type": "string" + }, + "files": { + "items": { + "$ref": "#/$defs/RpmFileRecord" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "version", + "epoch", + "architecture", + "release", + "sourceRpm", + "size", + "vendor", + "modularityLabel", + "files" + ] + }, + "RpmFileRecord": { + "properties": { + "path": { + "type": "string" + }, + "mode": { + "type": "integer" + }, + "size": { + "type": "integer" + }, + "digest": { + "$ref": "#/$defs/Digest" + }, + "userName": { + "type": "string" + }, + "groupName": { + "type": "string" + }, + "flags": { + "type": "string" + } + }, + "type": "object", + "required": [ + "path", + "mode", + "size", + "digest", + "userName", + "groupName", + "flags" + ] + }, + "RubyGemspec": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "files": { + "items": { + "type": "string" + }, + "type": "array" + }, + "authors": { + "items": { + "type": "string" + }, + "type": "array" + }, + "homepage": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version" + ] + }, + "RustCargoAuditEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source" + ] + }, + "RustCargoLockEntry": { + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "source": { + "type": "string" + }, + "checksum": { + "type": "string" + }, + "dependencies": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "version", + "source", + "checksum", + "dependencies" + ] + }, + "Schema": { + "properties": { + "version": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "type": "object", + "required": [ + "version", + "url" + ] + }, + "Source": { + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "type": { + "type": "string" + }, + "metadata": true + }, + "type": "object", + "required": [ + "id", + "name", + "version", + "type", + "metadata" + ] + }, + "SwiftPackageManagerLockEntry": { + "properties": { + "revision": { + "type": "string" + } + }, + "type": "object", + "required": [ + "revision" + ] + }, + "licenses": { + "items": { + "$ref": "#/$defs/License" + }, + "type": "array" + } + } +} diff --git a/syft/event/event.go b/syft/event/event.go index c095e9eef..b18ff9dec 100644 --- a/syft/event/event.go +++ b/syft/event/event.go @@ -17,10 +17,6 @@ const ( // PackageCatalogerStarted is a partybus event that occurs when the package cataloging has begun PackageCatalogerStarted partybus.EventType = typePrefix + "-package-cataloger-started-event" - //nolint:gosec - // SecretsCatalogerStarted is a partybus event that occurs when the secrets cataloging has begun - SecretsCatalogerStarted partybus.EventType = typePrefix + "-secrets-cataloger-started-event" - // FileMetadataCatalogerStarted is a partybus event that occurs when the file metadata cataloging has begun FileMetadataCatalogerStarted partybus.EventType = typePrefix + "-file-metadata-cataloger-started-event" diff --git a/syft/event/parsers/parsers.go b/syft/event/parsers/parsers.go index c2a56e17c..5f7f2d7c8 100644 --- a/syft/event/parsers/parsers.go +++ b/syft/event/parsers/parsers.go @@ -12,7 +12,6 @@ import ( "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event/monitor" - "github.com/anchore/syft/syft/file/cataloger/secrets" "github.com/anchore/syft/syft/pkg/cataloger" ) @@ -54,19 +53,6 @@ func ParsePackageCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error) return &monitor, nil } -func ParseSecretsCatalogingStarted(e partybus.Event) (*secrets.Monitor, error) { - if err := checkEventType(e.Type, event.SecretsCatalogerStarted); err != nil { - return nil, err - } - - monitor, ok := e.Value.(secrets.Monitor) - if !ok { - return nil, newPayloadErr(e.Type, "Value", e.Value) - } - - return &monitor, nil -} - func ParseFileMetadataCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) { if err := checkEventType(e.Type, event.FileMetadataCatalogerStarted); err != nil { return nil, err diff --git a/syft/file/cataloger/filecontent/cataloger.go b/syft/file/cataloger/filecontent/cataloger.go index 18ef45151..cc0425372 100644 --- a/syft/file/cataloger/filecontent/cataloger.go +++ b/syft/file/cataloger/filecontent/cataloger.go @@ -11,13 +11,11 @@ import ( "github.com/anchore/syft/syft/file" ) -// Deprecated: will be removed in syft v1.0.0 type Cataloger struct { globs []string skipFilesAboveSizeInBytes int64 } -// Deprecated: will be removed in syft v1.0.0 func NewCataloger(globs []string, skipFilesAboveSize int64) (*Cataloger, error) { return &Cataloger{ globs: globs, diff --git a/syft/file/cataloger/secrets/cataloger.go b/syft/file/cataloger/secrets/cataloger.go deleted file mode 100644 index 1f87fe44e..000000000 --- a/syft/file/cataloger/secrets/cataloger.go +++ /dev/null @@ -1,158 +0,0 @@ -package secrets - -import ( - "bytes" - "fmt" - "io" - "regexp" - "sort" - - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" - - "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/bus" - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/event" - "github.com/anchore/syft/syft/file" - internal2 "github.com/anchore/syft/syft/file/cataloger/internal" -) - -var DefaultSecretsPatterns = map[string]string{ - "aws-access-key": `(?i)aws_access_key_id["'=:\s]*?(?P(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})`, - "aws-secret-key": `(?i)aws_secret_access_key["'=:\s]*?(?P[0-9a-zA-Z/+]{40})`, - "pem-private-key": `-----BEGIN (\S+ )?PRIVATE KEY(\sBLOCK)?-----((?P(\n.*?)+)-----END (\S+ )?PRIVATE KEY(\sBLOCK)?-----)?`, - "docker-config-auth": `"auths"((.*\n)*.*?"auth"\s*:\s*"(?P[^"]+)")?`, - "generic-api-key": `(?i)api(-|_)?key["'=:\s]*?(?P[A-Z0-9]{20,60})["']?(\s|$)`, -} - -// Deprecated: will be removed in syft v1.0.0 -type Cataloger struct { - patterns map[string]*regexp.Regexp - revealValues bool - skipFilesAboveSize int64 -} - -// Deprecated: will be removed in syft v1.0.0 -func NewCataloger(patterns map[string]*regexp.Regexp, revealValues bool, maxFileSize int64) (*Cataloger, error) { - return &Cataloger{ - patterns: patterns, - revealValues: revealValues, - skipFilesAboveSize: maxFileSize, - }, nil -} - -func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates][]file.SearchResult, error) { - results := make(map[file.Coordinates][]file.SearchResult) - locations := internal2.AllRegularFiles(resolver) - stage, prog, secretsDiscovered := secretsCatalogingProgress(int64(len(locations))) - for _, location := range locations { - stage.Current = location.RealPath - result, err := i.catalogLocation(resolver, location) - if internal.IsErrPathPermission(err) { - log.Debugf("secrets cataloger skipping - %+v", err) - continue - } - - if err != nil { - return nil, err - } - if len(result) > 0 { - secretsDiscovered.Add(int64(len(result))) - results[location.Coordinates] = result - } - prog.Increment() - } - log.Debugf("secrets cataloger discovered %d secrets", secretsDiscovered.Current()) - prog.SetCompleted() - return results, nil -} - -func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) ([]file.SearchResult, error) { - metadata, err := resolver.FileMetadataByLocation(location) - if err != nil { - return nil, err - } - - if metadata.Size() == 0 { - return nil, nil - } - - if i.skipFilesAboveSize > 0 && metadata.Size() > i.skipFilesAboveSize { - return nil, nil - } - - // TODO: in the future we can swap out search strategies here - secrets, err := catalogLocationByLine(resolver, location, i.patterns) - if err != nil { - return nil, internal.ErrPath{Context: "secrets-cataloger", Path: location.RealPath, Err: err} - } - - if i.revealValues { - for idx, secret := range secrets { - value, err := extractValue(resolver, location, secret.SeekPosition, secret.Length) - if err != nil { - return nil, err - } - secrets[idx].Value = value - } - } - - // sort by the start location of each secret as it appears in the location - sort.SliceStable(secrets, func(i, j int) bool { - return secrets[i].SeekPosition < secrets[j].SeekPosition - }) - - return secrets, nil -} - -func extractValue(resolver file.Resolver, location file.Location, start, length int64) (string, error) { - readCloser, err := resolver.FileContentsByLocation(location) - if err != nil { - return "", fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) - } - defer internal.CloseAndLogError(readCloser, location.AccessPath) - - n, err := io.CopyN(io.Discard, readCloser, start) - if err != nil { - return "", fmt.Errorf("unable to read contents for location=%q : %w", location, err) - } - if n != start { - return "", fmt.Errorf("unexpected seek location for location=%q : %d != %d", location, n, start) - } - - var buf bytes.Buffer - n, err = io.CopyN(&buf, readCloser, length) - if err != nil { - return "", fmt.Errorf("unable to read secret value for location=%q : %w", location, err) - } - if n != length { - return "", fmt.Errorf("unexpected secret length for location=%q : %d != %d", location, n, length) - } - - return buf.String(), nil -} - -type Monitor struct { - progress.Stager - SecretsDiscovered progress.Monitorable - progress.Progressable -} - -func secretsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual, *progress.Manual) { - stage := &progress.Stage{} - secretsDiscovered := &progress.Manual{} - prog := progress.NewManual(locations) - - bus.Publish(partybus.Event{ - Type: event.SecretsCatalogerStarted, - Source: secretsDiscovered, - Value: Monitor{ - Stager: progress.Stager(stage), - SecretsDiscovered: secretsDiscovered, - Progressable: prog, - }, - }) - - return stage, prog, secretsDiscovered -} diff --git a/syft/file/cataloger/secrets/cataloger_test.go b/syft/file/cataloger/secrets/cataloger_test.go deleted file mode 100644 index 2a44417ba..000000000 --- a/syft/file/cataloger/secrets/cataloger_test.go +++ /dev/null @@ -1,443 +0,0 @@ -package secrets - -import ( - "regexp" - "testing" - - "github.com/stretchr/testify/assert" - - intFile "github.com/anchore/syft/internal/file" - "github.com/anchore/syft/syft/file" -) - -func TestSecretsCataloger(t *testing.T) { - tests := []struct { - name string - fixture string - reveal bool - maxSize int64 - patterns map[string]string - expected []file.SearchResult - constructorErr bool - catalogErr bool - }{ - { - name: "go-case-find-and-reveal", - fixture: "test-fixtures/secrets/simple.txt", - reveal: true, - patterns: map[string]string{ - "simple-secret-key": `^secret_key=.*`, - }, - expected: []file.SearchResult{ - { - Classification: "simple-secret-key", - LineNumber: 2, - LineOffset: 0, - SeekPosition: 34, - Length: 21, - Value: "secret_key=clear_text", - }, - }, - }, - { - name: "dont-reveal-secret-value", - fixture: "test-fixtures/secrets/simple.txt", - reveal: false, - patterns: map[string]string{ - "simple-secret-key": `^secret_key=.*`, - }, - expected: []file.SearchResult{ - { - Classification: "simple-secret-key", - LineNumber: 2, - LineOffset: 0, - SeekPosition: 34, - Length: 21, - Value: "", - }, - }, - }, - { - name: "reveal-named-capture-group", - fixture: "test-fixtures/secrets/simple.txt", - reveal: true, - patterns: map[string]string{ - "simple-secret-key": `^secret_key=(?P.*)`, - }, - expected: []file.SearchResult{ - { - Classification: "simple-secret-key", - LineNumber: 2, - LineOffset: 11, - SeekPosition: 45, - Length: 10, - Value: "clear_text", - }, - }, - }, - { - name: "multiple-secret-instances", - fixture: "test-fixtures/secrets/multiple.txt", - reveal: true, - patterns: map[string]string{ - "simple-secret-key": `secret_key=.*`, - }, - expected: []file.SearchResult{ - { - Classification: "simple-secret-key", - LineNumber: 1, - LineOffset: 0, - SeekPosition: 0, - Length: 22, - Value: "secret_key=clear_text1", - }, - { - Classification: "simple-secret-key", - LineNumber: 3, - LineOffset: 0, - SeekPosition: 57, - Length: 22, - Value: "secret_key=clear_text2", - }, - { - Classification: "simple-secret-key", - LineNumber: 4, - // note: this test captures a line offset case - LineOffset: 1, - SeekPosition: 81, - Length: 22, - Value: "secret_key=clear_text3", - }, - { - Classification: "simple-secret-key", - LineNumber: 6, - LineOffset: 0, - SeekPosition: 139, - Length: 22, - Value: "secret_key=clear_text4", - }, - }, - }, - { - name: "multiple-secret-instances-with-capture-group", - fixture: "test-fixtures/secrets/multiple.txt", - reveal: true, - patterns: map[string]string{ - "simple-secret-key": `secret_key=(?P.*)`, - }, - expected: []file.SearchResult{ - { - Classification: "simple-secret-key", - LineNumber: 1, - // note: value capture group location - LineOffset: 11, - SeekPosition: 11, - Length: 11, - Value: "clear_text1", - }, - { - Classification: "simple-secret-key", - LineNumber: 3, - LineOffset: 11, - SeekPosition: 68, - Length: 11, - Value: "clear_text2", - }, - { - Classification: "simple-secret-key", - LineNumber: 4, - // note: value capture group location + offset - LineOffset: 12, - SeekPosition: 92, - Length: 11, - Value: "clear_text3", - }, - { - Classification: "simple-secret-key", - LineNumber: 6, - LineOffset: 11, - SeekPosition: 150, - Length: 11, - Value: "clear_text4", - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - regexObjs := make(map[string]*regexp.Regexp) - for name, pattern := range test.patterns { - // always assume given patterns should be multiline - obj, err := regexp.Compile(`` + pattern) - if err != nil { - t.Fatalf("unable to parse regex: %+v", err) - } - regexObjs[name] = obj - } - - c, err := NewCataloger(regexObjs, test.reveal, test.maxSize) - if err != nil && !test.constructorErr { - t.Fatalf("could not create cataloger (but should have been able to): %+v", err) - } else if err == nil && test.constructorErr { - t.Fatalf("expected constructor error but did not get one") - } else if test.constructorErr && err != nil { - return - } - - resolver := file.NewMockResolverForPaths(test.fixture) - - actualResults, err := c.Catalog(resolver) - if err != nil && !test.catalogErr { - t.Fatalf("could not catalog (but should have been able to): %+v", err) - } else if err == nil && test.catalogErr { - t.Fatalf("expected catalog error but did not get one") - } else if test.catalogErr && err != nil { - return - } - - loc := file.NewLocation(test.fixture) - if _, exists := actualResults[loc.Coordinates]; !exists { - t.Fatalf("could not find location=%q in results", loc) - } - - assert.Equal(t, test.expected, actualResults[loc.Coordinates], "mismatched secrets") - }) - } -} - -func TestSecretsCataloger_DefaultSecrets(t *testing.T) { - regexObjs, err := GenerateSearchPatterns(DefaultSecretsPatterns, nil, nil) - if err != nil { - t.Fatalf("unable to get patterns: %+v", err) - } - - tests := []struct { - fixture string - expected []file.SearchResult - }{ - { - fixture: "test-fixtures/secrets/default/aws.env", - expected: []file.SearchResult{ - { - Classification: "aws-access-key", - LineNumber: 2, - LineOffset: 25, - SeekPosition: 64, - Length: 20, - Value: "AKIAIOSFODNN7EXAMPLE", - }, - { - Classification: "aws-secret-key", - LineNumber: 3, - LineOffset: 29, - SeekPosition: 114, - Length: 40, - Value: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - }, - }, - }, - { - fixture: "test-fixtures/secrets/default/aws.ini", - expected: []file.SearchResult{ - { - Classification: "aws-access-key", - LineNumber: 3, - LineOffset: 18, - SeekPosition: 67, - Length: 20, - Value: "AKIAIOSFODNN7EXAMPLE", - }, - { - Classification: "aws-secret-key", - LineNumber: 4, - LineOffset: 22, - SeekPosition: 110, - Length: 40, - Value: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - }, - }, - }, - { - fixture: "test-fixtures/secrets/default/private-key.pem", - expected: []file.SearchResult{ - { - Classification: "pem-private-key", - LineNumber: 2, - LineOffset: 27, - SeekPosition: 66, - Length: 351, - Value: ` -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG -cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD -VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD -bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF -BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh -z3P668YfhUbKdRF6S42Cg6zn -`, - }, - }, - }, - { - fixture: "test-fixtures/secrets/default/private-key-openssl.pem", - expected: []file.SearchResult{ - { - Classification: "pem-private-key", - LineNumber: 2, - LineOffset: 35, - SeekPosition: 74, - Length: 351, - Value: ` -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG -cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD -VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD -bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF -BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh -z3P668YfhUbKdRF6S42Cg6zn -`, - }, - }, - }, - { - // note: this test proves that the PEM regex matches the smallest possible match - // since the test catches two adjacent secrets - fixture: "test-fixtures/secrets/default/private-keys.pem", - expected: []file.SearchResult{ - { - Classification: "pem-private-key", - LineNumber: 1, - LineOffset: 35, - SeekPosition: 35, - Length: 351, - Value: ` -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG -cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD -VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD -bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF -BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh -z3P668YfhUbKdRF6S42Cg6zn -`, - }, - { - Classification: "pem-private-key", - LineNumber: 9, - LineOffset: 35, - SeekPosition: 455, - Length: 351, - Value: ` -MIIEvgTHISISNOTAREALKEYoIBAQDBj08DBj08DBj08DBj08DBj08DBsp5++4an3 -cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgY5 -VQQDDBcqLmF3cy10ZXN0SISNOTAREALKEYoIBAQDBj08DfffKoZIhvcNAQEBBQA7 -bml6SISNOTAREALKEYoIBAQDBj08DdssBggrBgEFBQcBAQSBkzCBkDBNBggrBgE8 -BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmd1 -j4f668YfhUbKdRF6S6734856 -`, - }, - }, - }, - { - fixture: "test-fixtures/secrets/default/private-key-false-positive.pem", - expected: nil, - }, - { - // this test represents: - // 1. a docker config - // 2. a named capture group with the correct line number and line offset case - // 3. the named capture group is in a different line than the match start, and both the match start and the capture group have different line offsets - fixture: "test-fixtures/secrets/default/docker-config.json", - expected: []file.SearchResult{ - { - Classification: "docker-config-auth", - LineNumber: 5, - LineOffset: 15, - SeekPosition: 100, - Length: 10, - Value: "tOpsyKreTz", - }, - }, - }, - { - fixture: "test-fixtures/secrets/default/not-docker-config.json", - expected: nil, - }, - { - fixture: "test-fixtures/secrets/default/api-key.txt", - expected: []file.SearchResult{ - { - Classification: "generic-api-key", - LineNumber: 2, - LineOffset: 7, - SeekPosition: 33, - Length: 20, - Value: "12345A7a901b34567890", - }, - { - Classification: "generic-api-key", - LineNumber: 3, - LineOffset: 9, - SeekPosition: 63, - Length: 30, - Value: "12345A7a901b345678901234567890", - }, - { - Classification: "generic-api-key", - LineNumber: 4, - LineOffset: 10, - SeekPosition: 104, - Length: 40, - Value: "12345A7a901b3456789012345678901234567890", - }, - { - Classification: "generic-api-key", - LineNumber: 5, - LineOffset: 10, - SeekPosition: 156, - Length: 50, - Value: "12345A7a901b34567890123456789012345678901234567890", - }, - { - Classification: "generic-api-key", - LineNumber: 6, - LineOffset: 16, - SeekPosition: 224, - Length: 60, - Value: "12345A7a901b345678901234567890123456789012345678901234567890", - }, - { - Classification: "generic-api-key", - LineNumber: 14, - LineOffset: 8, - SeekPosition: 502, - Length: 20, - Value: "11111111111111111111", - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.fixture, func(t *testing.T) { - - c, err := NewCataloger(regexObjs, true, 10*intFile.MB) - if err != nil { - t.Fatalf("could not create cataloger: %+v", err) - } - - resolver := file.NewMockResolverForPaths(test.fixture) - - actualResults, err := c.Catalog(resolver) - if err != nil { - t.Fatalf("could not catalog: %+v", err) - } - - loc := file.NewLocation(test.fixture) - if _, exists := actualResults[loc.Coordinates]; !exists && test.expected != nil { - t.Fatalf("could not find location=%q in results", loc) - } else if !exists && test.expected == nil { - return - } - - assert.Equal(t, test.expected, actualResults[loc.Coordinates], "mismatched secrets") - }) - } -} diff --git a/syft/file/cataloger/secrets/generate_search_patterns.go b/syft/file/cataloger/secrets/generate_search_patterns.go deleted file mode 100644 index a46ff483c..000000000 --- a/syft/file/cataloger/secrets/generate_search_patterns.go +++ /dev/null @@ -1,56 +0,0 @@ -package secrets - -import ( - "fmt" - "regexp" - - "github.com/bmatcuk/doublestar/v4" - "github.com/hashicorp/go-multierror" -) - -// GenerateSearchPatterns takes a set of named base patterns, a set of additional named patterns and an name exclusion list and generates a final -// set of regular expressions (indexed by name). The sets are aggregated roughly as such: (base - excluded) + additional. -func GenerateSearchPatterns(basePatterns map[string]string, additionalPatterns map[string]string, excludePatternNames []string) (map[string]*regexp.Regexp, error) { - var regexObjs = make(map[string]*regexp.Regexp) - var errs error - - addFn := func(name, pattern string) { - // always enable multiline search option for extracting secrets with multiline values - obj, err := regexp.Compile(`(?m)` + pattern) - if err != nil { - errs = multierror.Append(errs, fmt.Errorf("unable to parse %q regular expression: %w", name, err)) - } - regexObjs[name] = obj - } - - // add all base cases... unless that base case was asked to be excluded - for name, pattern := range basePatterns { - if !matchesExclusion(excludePatternNames, name) { - addFn(name, pattern) - } - } - - // add all additional cases - for name, pattern := range additionalPatterns { - addFn(name, pattern) - } - - if errs != nil { - return nil, errs - } - - return regexObjs, nil -} - -func matchesExclusion(excludePatternNames []string, name string) bool { - for _, exclude := range excludePatternNames { - matches, err := doublestar.Match(exclude, name) - if err != nil { - return false - } - if matches { - return true - } - } - return false -} diff --git a/syft/file/cataloger/secrets/generate_search_patterns_test.go b/syft/file/cataloger/secrets/generate_search_patterns_test.go deleted file mode 100644 index 37dc3441d..000000000 --- a/syft/file/cataloger/secrets/generate_search_patterns_test.go +++ /dev/null @@ -1,125 +0,0 @@ -package secrets - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestGenerateSearchPatterns(t *testing.T) { - tests := []struct { - name string - base map[string]string - additional map[string]string - exclude []string - expected map[string]string - }{ - { - name: "use-base-set", - base: map[string]string{ - "in-default": `^secret_key=.*`, - }, - expected: map[string]string{ - "in-default": `(?m)^secret_key=.*`, - }, - }, - { - name: "exclude-from-base-set", - base: map[string]string{ - "in-default": `^secret_key=.*`, - "also-in-default": `^also-in-default=.*`, - }, - exclude: []string{"also-in-default"}, - expected: map[string]string{ - "in-default": `(?m)^secret_key=.*`, - }, - }, - { - name: "exclude-multiple-from-base-set", - base: map[string]string{ - "in-default": `^secret_key=.*`, - "also-in-default": `^also-in-default=.*`, - "furthermore-in-default": `^furthermore-in-default=.*`, - }, - exclude: []string{"also-in-default", "furthermore-in-default"}, - expected: map[string]string{ - "in-default": `(?m)^secret_key=.*`, - }, - }, - { - name: "exclude-all", - base: map[string]string{ - "in-default": `^secret_key=.*`, - "also-in-default": `^also-in-default=.*`, - }, - exclude: []string{"*"}, - expected: map[string]string{}, - }, - { - name: "exclude-some", - base: map[string]string{ - "real": `^real=.*`, - "in-default": `^secret_key=.*`, - "also-in-default": `^also-in-default=.*`, - }, - exclude: []string{"*-default"}, - expected: map[string]string{ - "real": `(?m)^real=.*`, - }, - }, - { - name: "additional-pattern-unison", - base: map[string]string{ - "in-default": `^secret_key=.*`, - }, - additional: map[string]string{ - "additional": `^additional=.*`, - }, - expected: map[string]string{ - "in-default": `(?m)^secret_key=.*`, - "additional": `(?m)^additional=.*`, - }, - }, - { - name: "override", - base: map[string]string{ - "in-default": `^secret_key=.*`, - }, - additional: map[string]string{ - "in-default": `^additional=.*`, - }, - expected: map[string]string{ - "in-default": `(?m)^additional=.*`, - }, - }, - { - name: "exclude-and-override", - base: map[string]string{ - "in-default": `^secret_key=.*`, - }, - exclude: []string{"in-default"}, - additional: map[string]string{ - "in-default": `^additional=.*`, - }, - expected: map[string]string{ - "in-default": `(?m)^additional=.*`, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - actualObj, err := GenerateSearchPatterns(test.base, test.additional, test.exclude) - if err != nil { - t.Fatalf("unable to combine: %+v", err) - } - - actual := make(map[string]string) - for n, v := range actualObj { - actual[n] = v.String() - } - - assert.Equal(t, test.expected, actual, "mismatched combination") - }) - } -} diff --git a/syft/file/cataloger/secrets/newline_counter.go b/syft/file/cataloger/secrets/newline_counter.go deleted file mode 100644 index d3c8ef894..000000000 --- a/syft/file/cataloger/secrets/newline_counter.go +++ /dev/null @@ -1,39 +0,0 @@ -package secrets - -import "io" - -type newlineCounter struct { - io.RuneReader - numBytes int64 - newLines []int64 -} - -func (c *newlineCounter) ReadRune() (r rune, size int, err error) { - r, size, err = c.RuneReader.ReadRune() - c.numBytes += int64(size) - if r == '\n' { - c.newLines = append(c.newLines, c.numBytes) - } - return -} - -func (c *newlineCounter) newlinesBefore(pos int64) int { - var result int - for _, nlPos := range c.newLines { - if nlPos <= pos { - result++ - } - } - return result -} - -func (c *newlineCounter) newlinePositionBefore(pos int64) int64 { - var last int64 - for _, nlPos := range c.newLines { - if nlPos > pos { - break - } - last = nlPos - } - return last -} diff --git a/syft/file/cataloger/secrets/newline_counter_test.go b/syft/file/cataloger/secrets/newline_counter_test.go deleted file mode 100644 index 0760e892c..000000000 --- a/syft/file/cataloger/secrets/newline_counter_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package secrets - -import ( - "bufio" - "io" - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestLineCounter_ReadRune(t *testing.T) { - counter := &newlineCounter{RuneReader: bufio.NewReader(strings.NewReader("hi\nwhat's the weather like today?\ndunno...\n"))} - var err error - for err == nil { - _, _, err = counter.ReadRune() - } - if err != io.EOF { - t.Fatalf("should have gotten an eof, got %+v", err) - } - assert.Equal(t, 3, len(counter.newLines), "bad line count") - assert.Equal(t, []int64{3, 34, 43}, counter.newLines, "bad line positions") -} - -func TestLineCounter_newlinesBefore(t *testing.T) { - counter := &newlineCounter{RuneReader: bufio.NewReader(strings.NewReader("hi\nwhat's the weather like today?\ndunno...\n"))} - var err error - for err == nil { - _, _, err = counter.ReadRune() - } - if err != io.EOF { - t.Fatalf("should have gotten an eof, got %+v", err) - } - assert.Equal(t, 1, counter.newlinesBefore(10), "bad line count") -} diff --git a/syft/file/cataloger/secrets/secrets_search_by_line_strategy.go b/syft/file/cataloger/secrets/secrets_search_by_line_strategy.go deleted file mode 100644 index 386801859..000000000 --- a/syft/file/cataloger/secrets/secrets_search_by_line_strategy.go +++ /dev/null @@ -1,135 +0,0 @@ -package secrets - -import ( - "bufio" - "errors" - "fmt" - "io" - "regexp" - - "github.com/anchore/syft/internal" - "github.com/anchore/syft/syft/file" -) - -func catalogLocationByLine(resolver file.Resolver, location file.Location, patterns map[string]*regexp.Regexp) ([]file.SearchResult, error) { - readCloser, err := resolver.FileContentsByLocation(location) - if err != nil { - return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) - } - defer internal.CloseAndLogError(readCloser, location.AccessPath) - - var scanner = bufio.NewReader(readCloser) - var position int64 - var allSecrets []file.SearchResult - var lineNo int64 - var readErr error - for !errors.Is(readErr, io.EOF) { - lineNo++ - var line []byte - // TODO: we're at risk of large memory usage for very long lines - line, readErr = scanner.ReadBytes('\n') - if readErr != nil && readErr != io.EOF { - return nil, readErr - } - - lineSecrets, err := searchForSecretsWithinLine(resolver, location, patterns, line, lineNo, position) - if err != nil { - return nil, err - } - position += int64(len(line)) - allSecrets = append(allSecrets, lineSecrets...) - } - - return allSecrets, nil -} - -func searchForSecretsWithinLine(resolver file.Resolver, location file.Location, patterns map[string]*regexp.Regexp, line []byte, lineNo int64, position int64) ([]file.SearchResult, error) { - var secrets []file.SearchResult - for name, pattern := range patterns { - matches := pattern.FindAllIndex(line, -1) - for i, match := range matches { - if i%2 == 1 { - // FindAllIndex returns pairs of numbers for each match, we are only interested in the starting (first) - // position in each pair. - continue - } - - lineOffset := int64(match[0]) - seekLocation := position + lineOffset - reader, err := readerAtPosition(resolver, location, seekLocation) - if err != nil { - return nil, err - } - - secret := extractSecretFromPosition(reader, name, pattern, lineNo, lineOffset, seekLocation) - if secret != nil { - secrets = append(secrets, *secret) - } - internal.CloseAndLogError(reader, location.AccessPath) - } - } - - return secrets, nil -} - -func readerAtPosition(resolver file.Resolver, location file.Location, seekPosition int64) (io.ReadCloser, error) { - readCloser, err := resolver.FileContentsByLocation(location) - if err != nil { - return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) - } - if seekPosition > 0 { - n, err := io.CopyN(io.Discard, readCloser, seekPosition) - if err != nil { - return nil, fmt.Errorf("unable to read contents for location=%q while searching for secrets: %w", location, err) - } - if n != seekPosition { - return nil, fmt.Errorf("unexpected seek location for location=%q while searching for secrets: %d != %d", location, n, seekPosition) - } - } - return readCloser, nil -} - -func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *regexp.Regexp, lineNo, lineOffset, seekPosition int64) *file.SearchResult { - reader := &newlineCounter{RuneReader: bufio.NewReader(readCloser)} - positions := pattern.FindReaderSubmatchIndex(reader) - if len(positions) == 0 { - // no matches found - return nil - } - - index := pattern.SubexpIndex("value") - var indexOffset int - if index != -1 { - // there is a capture group, use the capture group selection as the secret value. To do this we want to - // use the position at the discovered offset. Note: all positions come in pairs, so you will need to adjust - // the offset accordingly (multiply by 2). - indexOffset = index * 2 - } - // get the start and stop of the secret value. Note: this covers both when there is a capture group - // and when there is not a capture group (full value match) - start, stop := int64(positions[indexOffset]), int64(positions[indexOffset+1]) - - if start < 0 || stop < 0 { - // no match location found. This can happen when there is a value capture group specified by the user - // and there was a match on the overall regex, but not for the capture group (which is possible if the capture - // group is optional). - return nil - } - - // lineNoOfSecret are the number of lines which occur before the start of the secret value - var lineNoOfSecret = lineNo + int64(reader.newlinesBefore(start)) - // lineOffsetOfSecret are the number of bytes that occur after the last newline but before the secret value. - var lineOffsetOfSecret = start - reader.newlinePositionBefore(start) - if lineNoOfSecret == lineNo { - // the secret value starts in the same line as the overall match, so we must consider that line offset - lineOffsetOfSecret += lineOffset - } - - return &file.SearchResult{ - Classification: name, - SeekPosition: start + seekPosition, - Length: stop - start, - LineNumber: lineNoOfSecret, - LineOffset: lineOffsetOfSecret, - } -} diff --git a/syft/file/cataloger/secrets/test-fixtures/secrets/default/api-key.txt b/syft/file/cataloger/secrets/test-fixtures/secrets/default/api-key.txt deleted file mode 100644 index 63cb62cac..000000000 --- a/syft/file/cataloger/secrets/test-fixtures/secrets/default/api-key.txt +++ /dev/null @@ -1,14 +0,0 @@ -# these should be matches -apikey=12345A7a901b34567890 -api_key =12345A7a901b345678901234567890 -API-KEY= '12345A7a901b3456789012345678901234567890' -API-key: "12345A7a901b34567890123456789012345678901234567890" -some_ApI-kEy = "12345A7a901b345678901234567890123456789012345678901234567890" - -# these should be non matches -api_key = "toolong12345A7a901b345678901234567890123456789012345678901234567890" -api_key = "tooshort" -not_api_k3y = "badkeyname12345A7a901b34567890" - -# value at EOF should match -api_key=11111111111111111111 \ No newline at end of file diff --git a/syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.env b/syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.env deleted file mode 100644 index 7e5888a38..000000000 --- a/syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.env +++ /dev/null @@ -1,3 +0,0 @@ -# note: these are NOT real credentials -export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE -export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \ No newline at end of file diff --git a/syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.ini b/syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.ini deleted file mode 100644 index 0413424f6..000000000 --- a/syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.ini +++ /dev/null @@ -1,4 +0,0 @@ -# note: these are NOT real credentials -[default] -aws_access_key_id=AKIAIOSFODNN7EXAMPLE -aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \ No newline at end of file diff --git a/syft/file/cataloger/secrets/test-fixtures/secrets/default/docker-config.json b/syft/file/cataloger/secrets/test-fixtures/secrets/default/docker-config.json deleted file mode 100644 index 2a239cdaa..000000000 --- a/syft/file/cataloger/secrets/test-fixtures/secrets/default/docker-config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "experimental" : "disabled", - "auths" : { - "https://index.docker.io/v1/" : { - "auth": "tOpsyKreTz" - } - }, - "stackOrchestrator" : "swarm", - "credsStore" : "desktop" -} diff --git a/syft/file/cataloger/secrets/test-fixtures/secrets/default/not-docker-config.json b/syft/file/cataloger/secrets/test-fixtures/secrets/default/not-docker-config.json deleted file mode 100644 index c6baa32f1..000000000 --- a/syft/file/cataloger/secrets/test-fixtures/secrets/default/not-docker-config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "endpoint" : "http://somewhere", - "auth" : "basic" -} diff --git a/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-false-positive.pem b/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-false-positive.pem deleted file mode 100644 index 2cc142e3c..000000000 --- a/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-false-positive.pem +++ /dev/null @@ -1 +0,0 @@ ------BEGIN OPENSSL PRIVATE KEY----- \ No newline at end of file diff --git a/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-openssl.pem b/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-openssl.pem deleted file mode 100644 index 48e4d2361..000000000 --- a/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-openssl.pem +++ /dev/null @@ -1,9 +0,0 @@ -# note: this is NOT a real private key ------BEGIN OPENSSL PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG -cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD -VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD -bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF -BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh -z3P668YfhUbKdRF6S42Cg6zn ------END OPENSSL PRIVATE KEY----- \ No newline at end of file diff --git a/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key.pem b/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key.pem deleted file mode 100644 index e13aa1955..000000000 --- a/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key.pem +++ /dev/null @@ -1,10 +0,0 @@ -# note: this is NOT a real private key ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG -cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD -VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD -bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF -BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh -z3P668YfhUbKdRF6S42Cg6zn ------END PRIVATE KEY----- -other embedded text \ No newline at end of file diff --git a/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-keys.pem b/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-keys.pem deleted file mode 100644 index 27c045365..000000000 --- a/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-keys.pem +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN OPENSSL PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG -cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD -VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD -bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF -BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh -z3P668YfhUbKdRF6S42Cg6zn ------END OPENSSL PRIVATE KEY----- ------BEGIN OPENSSL PRIVATE KEY----- -MIIEvgTHISISNOTAREALKEYoIBAQDBj08DBj08DBj08DBj08DBj08DBsp5++4an3 -cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgY5 -VQQDDBcqLmF3cy10ZXN0SISNOTAREALKEYoIBAQDBj08DfffKoZIhvcNAQEBBQA7 -bml6SISNOTAREALKEYoIBAQDBj08DdssBggrBgEFBQcBAQSBkzCBkDBNBggrBgE8 -BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmd1 -j4f668YfhUbKdRF6S6734856 ------END OPENSSL PRIVATE KEY----- \ No newline at end of file diff --git a/syft/file/cataloger/secrets/test-fixtures/secrets/multiple.txt b/syft/file/cataloger/secrets/test-fixtures/secrets/multiple.txt deleted file mode 100644 index 3c50205d1..000000000 --- a/syft/file/cataloger/secrets/test-fixtures/secrets/multiple.txt +++ /dev/null @@ -1,6 +0,0 @@ -secret_key=clear_text1 -other text that should be ignored -secret_key=clear_text2 - secret_key=clear_text3 -also things that should be ignored -secret_key=clear_text4 \ No newline at end of file diff --git a/syft/file/cataloger/secrets/test-fixtures/secrets/simple.txt b/syft/file/cataloger/secrets/test-fixtures/secrets/simple.txt deleted file mode 100644 index 67fc2b755..000000000 --- a/syft/file/cataloger/secrets/test-fixtures/secrets/simple.txt +++ /dev/null @@ -1,4 +0,0 @@ -other text that should be ignored -secret_key=clear_text ----secret_key=clear_text -also things that should be ignored \ No newline at end of file diff --git a/syft/format/syftjson/model/document.go b/syft/format/syftjson/model/document.go index eb41ec79f..a9886ba4b 100644 --- a/syft/format/syftjson/model/document.go +++ b/syft/format/syftjson/model/document.go @@ -4,12 +4,11 @@ package model type Document struct { Artifacts []Package `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog ArtifactRelationships []Relationship `json:"artifactRelationships"` - Files []File `json:"files,omitempty"` // note: must have omitempty - Secrets []Secrets `json:"secrets,omitempty"` // note: must have omitempty - Source Source `json:"source"` // Source represents the original object that was cataloged - Distro LinuxRelease `json:"distro"` // Distro represents the Linux distribution that was detected from the source - Descriptor Descriptor `json:"descriptor"` // Descriptor is a block containing self-describing information about syft - Schema Schema `json:"schema"` // Schema is a block reserved for defining the version for the shape of this JSON document and where to find the schema document to validate the shape + Files []File `json:"files,omitempty"` // note: must have omitempty + Source Source `json:"source"` // Source represents the original object that was cataloged + Distro LinuxRelease `json:"distro"` // Distro represents the Linux distribution that was detected from the source + Descriptor Descriptor `json:"descriptor"` // Descriptor is a block containing self-describing information about syft + Schema Schema `json:"schema"` // Schema is a block reserved for defining the version for the shape of this JSON document and where to find the schema document to validate the shape } // Descriptor describes what created the document as well as surrounding metadata diff --git a/syft/format/syftjson/to_format_model.go b/syft/format/syftjson/to_format_model.go index b07d05021..a05b17bfa 100644 --- a/syft/format/syftjson/to_format_model.go +++ b/syft/format/syftjson/to_format_model.go @@ -38,7 +38,6 @@ func ToFormatModel(s sbom.SBOM, cfg EncoderConfig) model.Document { Artifacts: toPackageModels(s.Artifacts.Packages, cfg), ArtifactRelationships: toRelationshipModel(s.Relationships), Files: toFile(s), - Secrets: toSecrets(s.Artifacts.Secrets), Source: toSourceModel(s.Source), Distro: toLinuxReleaser(s.Artifacts.LinuxDistribution), Descriptor: toDescriptor(s.Descriptor), @@ -83,22 +82,6 @@ func toDescriptor(d sbom.Descriptor) model.Descriptor { } } -func toSecrets(data map[file.Coordinates][]file.SearchResult) []model.Secrets { - results := make([]model.Secrets, 0) - for coordinates, secrets := range data { - results = append(results, model.Secrets{ - Location: coordinates, - Secrets: secrets, - }) - } - - // sort by real path then virtual path to ensure the result is stable across multiple runs - sort.SliceStable(results, func(i, j int) bool { - return results[i].Location.RealPath < results[j].Location.RealPath - }) - return results -} - func toFile(s sbom.SBOM) []model.File { results := make([]model.File, 0) artifacts := s.Artifacts diff --git a/syft/sbom/sbom.go b/syft/sbom/sbom.go index 18058255b..900fe129e 100644 --- a/syft/sbom/sbom.go +++ b/syft/sbom/sbom.go @@ -25,7 +25,6 @@ type Artifacts struct { FileDigests map[file.Coordinates][]file.Digest FileContents map[file.Coordinates]string FileLicenses map[file.Coordinates][]file.License - Secrets map[file.Coordinates][]file.SearchResult LinuxDistribution *linux.Release } diff --git a/test/cli/json_schema_test.go b/test/cli/json_schema_test.go index dd4d19e69..84b822458 100644 --- a/test/cli/json_schema_test.go +++ b/test/cli/json_schema_test.go @@ -36,11 +36,6 @@ func TestJSONSchema(t *testing.T) { args: []string{"-o", "json"}, fixture: imageFixture, }, - { - name: "power-user:image:docker-archive:pkg-coverage", - subcommand: "power-user", - fixture: imageFixture, - }, { name: "packages:dir:pkg-coverage", subcommand: "packages", diff --git a/test/cli/power_user_cmd_test.go b/test/cli/power_user_cmd_test.go deleted file mode 100644 index 757205547..000000000 --- a/test/cli/power_user_cmd_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package cli - -import ( - "testing" -) - -func TestPowerUserCmdFlags(t *testing.T) { - secretsFixture := getFixtureImage(t, "image-secrets") - tests := []struct { - name string - args []string - env map[string]string - assertions []traitAssertion - }{ - { - name: "no-args-shows-help", - args: []string{"power-user"}, - assertions: []traitAssertion{ - assertInOutput("an image/directory argument is required"), // specific error that should be shown - assertInOutput("Run bulk operations on container images"), // excerpt from help description - assertFailingReturnCode, - }, - }, - { - name: "default-results-w-pkg-coverage", - args: []string{"power-user", "docker-archive:" + getFixtureImage(t, "image-pkg-coverage")}, - assertions: []traitAssertion{ - assertNotInOutput(" command is deprecated"), // only the root command should be deprecated - assertInOutput(`"type":"RegularFile"`), // proof of file-metadata data - assertInOutput(`"algorithm":"sha256"`), // proof of file-metadata default digest algorithm of sha256 - assertInOutput(`"metadataType":"apk-db-entry"`), // proof of package artifacts data - assertSuccessfulReturnCode, - }, - }, - { - name: "content-cataloger-wired-up", - args: []string{"power-user", "docker-archive:" + secretsFixture}, - env: map[string]string{ - "SYFT_FILE_CONTENTS_GLOBS": "/api-key.txt", - }, - assertions: []traitAssertion{ - assertInOutput(`"contents":"c29tZV9BcEkta0V5ID0gIjEyMzQ1QTdhOTAxYjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MCIK"`), // proof of the content cataloger - assertSuccessfulReturnCode, - }, - }, - { - name: "default-dir-results-w-pkg-coverage", - args: []string{"power-user", "dir:test-fixtures/image-pkg-coverage"}, - assertions: []traitAssertion{ - assertNotInOutput(" command is deprecated"), // only the root command should be deprecated - assertInOutput(`"type":"RegularFile"`), // proof of file-metadata data - assertInOutput(`"algorithm":"sha256"`), // proof of file-metadata default digest algorithm of sha256 - assertInOutput(`"metadataType":"apk-db-entry"`), // proof of package artifacts data - assertSuccessfulReturnCode, - }, - }, - { - name: "default-secrets-results-w-reveal-values", - env: map[string]string{ - "SYFT_SECRETS_REVEAL_VALUES": "true", - }, - args: []string{"power-user", "docker-archive:" + secretsFixture}, - assertions: []traitAssertion{ - assertInOutput(`"classification":"generic-api-key"`), // proof of the secrets cataloger finding something - assertInOutput(`"12345A7a901b345678901234567890123456789012345678901234567890"`), // proof of the secrets cataloger finding the api key - assertSuccessfulReturnCode, - }, - }, - { - name: "default-secret-results-dont-reveal-values", - args: []string{"power-user", "docker-archive:" + secretsFixture}, - assertions: []traitAssertion{ - assertInOutput(`"classification":"generic-api-key"`), // proof of the secrets cataloger finding something - assertNotInOutput(`"12345A7a901b345678901234567890123456789012345678901234567890"`), // proof of the secrets cataloger finding the api key - assertSuccessfulReturnCode, - }, - }, - { - name: "default-secrets-dir-results-w-reveal-values", - env: map[string]string{ - "SYFT_SECRETS_REVEAL_VALUES": "true", - }, - args: []string{"power-user", "dir:test-fixtures/image-secrets-dir"}, - assertions: []traitAssertion{ - assertInOutput(`"classification":"generic-api-key"`), // proof of the secrets cataloger finding something - assertInOutput(`"12345A7a901b345678901234567890123456789012345678901234567890"`), // proof of the secrets cataloger finding the api key - assertSuccessfulReturnCode, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cmd, stdout, stderr := runSyftSafe(t, test.env, test.args...) - for _, traitFn := range test.assertions { - traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode()) - } - logOutputOnFailure(t, cmd, stdout, stderr) - }) - } -}