oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-02 15:36:41 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add secrets search capability (#367)

Browse Source 
* add initial secrets cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update ETUI elements with new catalogers (file metadata, digests, and secrets)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update secrets cataloger to read full contents into memory for searching

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype of parallelization secret regex search

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype with single aggregated regex

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype for secret search line-by-line

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype hybrid secrets search

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add secrets cataloger with line strategy

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* adjust verbiage towards SearchResults instead of Secrets + add tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update json schema with secrets cataloger results

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* address PR comments

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update readme with secrets config options

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* ensure file catalogers call AllLocations once

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2021-04-01 21:34:15 +00:00
committed by GitHub
parent 557ad8be49
commit 9ec09add67
53 changed files with 2549 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/event/event.go
+11 -1
View File
@@ -11,7 +11,17 @@ const (
AppUpdateAvailable partybus.EventType = "syft-app-update-available"
// PackageCatalogerStarted is a partybus event that occurs when the package cataloging has begun
PackageCatalogerStarted partybus.EventType = "syft-cataloger-started-event"
PackageCatalogerStarted partybus.EventType = "syft-package-cataloger-started-event"
// nolint:gosec
// SecretsCatalogerStarted is a partybus event that occurs when the secrets cataloging has begun
SecretsCatalogerStarted partybus.EventType = "syft-secrets-cataloger-started-event"
// FileMetadataCatalogerStarted is a partybus event that occurs when the file metadata cataloging has begun
FileMetadataCatalogerStarted partybus.EventType = "syft-file-metadata-cataloger-started-event"
// FileDigestsCatalogerStarted is a partybus event that occurs when the file digests cataloging has begun
FileDigestsCatalogerStarted partybus.EventType = "syft-file-digests-cataloger-started-event"
// PresenterReady is a partybus event that occurs when an analysis result is ready for final presentation
PresenterReady partybus.EventType = "syft-presenter-ready-event"
syft/event/parsers/parsers.go
+42 -1
View File
@@ -6,6 +6,8 @@ package parsers
import (
"fmt"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/internal/presenter"
"github.com/wagoodman/go-progress"
@@ -40,7 +42,7 @@ func checkEventType(actual, expected partybus.EventType) error {
return nil
}
func ParseCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error) {
func ParsePackageCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error) {
if err := checkEventType(e.Type, event.PackageCatalogerStarted); err != nil {
return nil, err
}
@@ -53,6 +55,45 @@ func ParseCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error) {
return &monitor, nil
}
func ParseSecretsCatalogingStarted(e partybus.Event) (*file.SecretsMonitor, error) {
if err := checkEventType(e.Type, event.SecretsCatalogerStarted); err != nil {
return nil, err
}
monitor, ok := e.Value.(file.SecretsMonitor)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return &monitor, nil
}
func ParseFileMetadataCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileMetadataCatalogerStarted); err != nil {
return nil, err
}
prog, ok := e.Value.(progress.StagedProgressable)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return prog, nil
}
func ParseFileDigestsCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileDigestsCatalogerStarted); err != nil {
return nil, err
}
prog, ok := e.Value.(progress.StagedProgressable)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return prog, nil
}
func ParsePresenterReady(e partybus.Event) (presenter.Presenter, error) {
if err := checkEventType(e.Type, event.PresenterReady); err != nil {
return nil, err
syft/file/digest_cataloger.go
+36
View File
@@ -7,6 +7,13 @@ import (
"io"
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/syft/event"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/syft/source"
)
@@ -22,13 +29,22 @@ func NewDigestsCataloger(hashes []crypto.Hash) (*DigestsCataloger, error) {
func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Location][]Digest, error) {
results := make(map[source.Location][]Digest)
var locations []source.Location
for location := range resolver.AllLocations() {
locations = append(locations, location)
}
stage, prog := digestsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
result, err := i.catalogLocation(resolver, location)
if err != nil {
return nil, err
}
prog.N++
results[location] = result
}
log.Debugf("file digests cataloger processed %d files", prog.N)
prog.SetCompleted()
return results, nil
}
@@ -78,3 +94,23 @@ func CleanDigestAlgorithmName(name string) string {
lower := strings.ToLower(name)
return strings.Replace(lower, "-", "", -1)
}
func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := &progress.Manual{
Total: locations,
}
bus.Publish(partybus.Event{
Type: event.FileDigestsCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
},
})
return stage, prog
}
syft/file/generate_search_patterns.go
+56
View File
@@ -0,0 +1,56 @@
package file
import (
"fmt"
"regexp"
"github.com/bmatcuk/doublestar/v2"
"github.com/hashicorp/go-multierror"
)
// GenerateSearchPatterns takes a set of named base patterns, a set of additional named patterns and an name exclusion list and generates a final
// set of regular expressions (indexed by name). The sets are aggregated roughly as such: (base - excluded) + additional.
func GenerateSearchPatterns(basePatterns map[string]string, additionalPatterns map[string]string, excludePatternNames []string) (map[string]*regexp.Regexp, error) {
var regexObjs = make(map[string]*regexp.Regexp)
var errs error
addFn := func(name, pattern string) {
// always enable multiline search option for extracting secrets with multiline values
obj, err := regexp.Compile(`(?m)` + pattern)
if err != nil {
errs = multierror.Append(errs, fmt.Errorf("unable to parse %q regular expression: %w", name, err))
}
regexObjs[name] = obj
}
// add all base cases... unless that base case was asked to be excluded
for name, pattern := range basePatterns {
if !matchesExclusion(excludePatternNames, name) {
addFn(name, pattern)
}
}
// add all additional cases
for name, pattern := range additionalPatterns {
addFn(name, pattern)
}
if errs != nil {
return nil, errs
}
return regexObjs, nil
}
func matchesExclusion(excludePatternNames []string, name string) bool {
for _, exclude := range excludePatternNames {
matches, err := doublestar.Match(exclude, name)
if err != nil {
return false
}
if matches {
return true
}
}
return false
}
syft/file/generate_search_patterns_test.go
+125
View File
@@ -0,0 +1,125 @@
package file
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGenerateSearchPatterns(t *testing.T) {
tests := []struct {
name string
base map[string]string
additional map[string]string
exclude []string
expected map[string]string
}{
{
name: "use-base-set",
base: map[string]string{
"in-default": `^secret_key=.*`,
},
expected: map[string]string{
"in-default": `(?m)^secret_key=.*`,
},
},
{
name: "exclude-from-base-set",
base: map[string]string{
"in-default": `^secret_key=.*`,
"also-in-default": `^also-in-default=.*`,
},
exclude: []string{"also-in-default"},
expected: map[string]string{
"in-default": `(?m)^secret_key=.*`,
},
},
{
name: "exclude-multiple-from-base-set",
base: map[string]string{
"in-default": `^secret_key=.*`,
"also-in-default": `^also-in-default=.*`,
"furthermore-in-default": `^furthermore-in-default=.*`,
},
exclude: []string{"also-in-default", "furthermore-in-default"},
expected: map[string]string{
"in-default": `(?m)^secret_key=.*`,
},
},
{
name: "exclude-all",
base: map[string]string{
"in-default": `^secret_key=.*`,
"also-in-default": `^also-in-default=.*`,
},
exclude: []string{"*"},
expected: map[string]string{},
},
{
name: "exclude-some",
base: map[string]string{
"real": `^real=.*`,
"in-default": `^secret_key=.*`,
"also-in-default": `^also-in-default=.*`,
},
exclude: []string{"*-default"},
expected: map[string]string{
"real": `(?m)^real=.*`,
},
},
{
name: "additional-pattern-unison",
base: map[string]string{
"in-default": `^secret_key=.*`,
},
additional: map[string]string{
"additional": `^additional=.*`,
},
expected: map[string]string{
"in-default": `(?m)^secret_key=.*`,
"additional": `(?m)^additional=.*`,
},
},
{
name: "override",
base: map[string]string{
"in-default": `^secret_key=.*`,
},
additional: map[string]string{
"in-default": `^additional=.*`,
},
expected: map[string]string{
"in-default": `(?m)^additional=.*`,
},
},
{
name: "exclude-and-override",
base: map[string]string{
"in-default": `^secret_key=.*`,
},
exclude: []string{"in-default"},
additional: map[string]string{
"in-default": `^additional=.*`,
},
expected: map[string]string{
"in-default": `(?m)^additional=.*`,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actualObj, err := GenerateSearchPatterns(test.base, test.additional, test.exclude)
if err != nil {
t.Fatalf("unable to combine: %+v", err)
}
actual := make(map[string]string)
for n, v := range actualObj {
actual[n] = v.String()
}
assert.Equal(t, test.expected, actual, "mismatched combination")
})
}
}
syft/file/metadata_cataloger.go
+34
View File
@@ -1,7 +1,12 @@
package file
import (
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/source"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
)
type MetadataCataloger struct {
@@ -13,13 +18,42 @@ func NewMetadataCataloger() *MetadataCataloger {
func (i *MetadataCataloger) Catalog(resolver source.FileResolver) (map[source.Location]source.FileMetadata, error) {
results := make(map[source.Location]source.FileMetadata)
var locations []source.Location
for location := range resolver.AllLocations() {
locations = append(locations, location)
}
stage, prog := metadataCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
}
results[location] = metadata
prog.N++
}
log.Debugf("file metadata cataloger processed %d files", prog.N)
prog.SetCompleted()
return results, nil
}
func metadataCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := &progress.Manual{
Total: locations,
}
bus.Publish(partybus.Event{
Type: event.FileMetadataCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
},
})
return stage, prog
}
syft/file/metadata_cataloger_test.go
+2 -1
View File
@@ -53,6 +53,7 @@ func TestFileMetadataCataloger(t *testing.T) {
Type: "RegularFile",
UserID: 1,
GroupID: 2,
Size: 7,
},
},
{
@@ -128,7 +129,7 @@ func TestFileMetadataCataloger(t *testing.T) {
l := source.NewLocationFromImage(test.path, *ref, img)
assert.Equal(t, actual[l], test.expected, "mismatched metadata")
assert.Equal(t, test.expected, actual[l], "mismatched metadata")
})
}
syft/file/newline_counter.go
+39
View File
@@ -0,0 +1,39 @@
package file
import "io"
type newlineCounter struct {
io.RuneReader
numBytes int64
newLines []int64
}
func (c *newlineCounter) ReadRune() (r rune, size int, err error) {
r, size, err = c.RuneReader.ReadRune()
c.numBytes += int64(size)
if r == '\n' {
c.newLines = append(c.newLines, c.numBytes)
}
return
}
func (c *newlineCounter) newlinesBefore(pos int64) int {
var result int
for _, nlPos := range c.newLines {
if nlPos <= pos {
result++
}
}
return result
}
func (c *newlineCounter) newlinePositionBefore(pos int64) int64 {
var last int64
for _, nlPos := range c.newLines {
if nlPos > pos {
break
}
last = nlPos
}
return last
}
syft/file/newline_counter_test.go
+35
View File
@@ -0,0 +1,35 @@
package file
import (
"bufio"
"io"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLineCounter_ReadRune(t *testing.T) {
counter := &newlineCounter{RuneReader: bufio.NewReader(strings.NewReader("hi\nwhat's the weather like today?\ndunno...\n"))}
var err error
for err == nil {
_, _, err = counter.ReadRune()
}
if err != io.EOF {
t.Fatalf("should have gotten an eof, got %+v", err)
}
assert.Equal(t, 3, len(counter.newLines), "bad line count")
assert.Equal(t, []int64{3, 34, 43}, counter.newLines, "bad line positions")
}
func TestLineCounter_newlinesBefore(t *testing.T) {
counter := &newlineCounter{RuneReader: bufio.NewReader(strings.NewReader("hi\nwhat's the weather like today?\ndunno...\n"))}
var err error
for err == nil {
_, _, err = counter.ReadRune()
}
if err != io.EOF {
t.Fatalf("should have gotten an eof, got %+v", err)
}
assert.Equal(t, 1, counter.newlinesBefore(10), "bad line count")
}
syft/file/search_result.go
+18
View File
@@ -0,0 +1,18 @@
package file
import (
"fmt"
)
type SearchResult struct {
Classification string `json:"classification"`
LineNumber int64 `json:"lineNumber"`
LineOffset int64 `json:"lineOffset"`
SeekPosition int64 `json:"seekPosition"`
Length int64 `json:"length"`
Value string `json:"value,omitempty"`
}
func (s SearchResult) String() string {
return fmt.Sprintf("SearchResult(classification=%q seek=%q length=%q)", s.Classification, s.SeekPosition, s.Length)
}
syft/file/secrets_cataloger.go
+150
View File
@@ -0,0 +1,150 @@
package file
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"regexp"
"sort"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/source"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
)
var DefaultSecretsPatterns = map[string]string{
"aws-access-key": `(?i)aws_access_key_id["'=:\s]*?(?P<value>(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})`,
"aws-secret-key": `(?i)aws_secret_access_key["'=:\s]*?(?P<value>[0-9a-zA-Z/+]{40})`,
"pem-private-key": `-----BEGIN (\S+ )?PRIVATE KEY(\sBLOCK)?-----((?P<value>(\n.*?)+)-----END (\S+ )?PRIVATE KEY(\sBLOCK)?-----)?`,
"docker-config-auth": `"auths"((.*\n)*.*?"auth"\s*:\s*"(?P<value>[^"]+)")?`,
"generic-api-key": `(?i)api(-|_)?key["'=:\s]*?(?P<value>[A-Z0-9]{20,60})["']?(\s|$)`,
}
type SecretsCataloger struct {
patterns map[string]*regexp.Regexp
revealValues bool
skipFilesAboveSize int64
}
func NewSecretsCataloger(patterns map[string]*regexp.Regexp, revealValues bool, maxFileSize int64) (*SecretsCataloger, error) {
return &SecretsCataloger{
patterns: patterns,
revealValues: revealValues,
skipFilesAboveSize: maxFileSize,
}, nil
}
func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Location][]SearchResult, error) {
results := make(map[source.Location][]SearchResult)
var locations []source.Location
for location := range resolver.AllLocations() {
locations = append(locations, location)
}
stage, prog, secretsDiscovered := secretsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
result, err := i.catalogLocation(resolver, location)
if err != nil {
return nil, err
}
if len(result) > 0 {
secretsDiscovered.N += int64(len(result))
results[location] = result
}
prog.N++
}
log.Debugf("secrets cataloger discovered %d secrets", secretsDiscovered.N)
prog.SetCompleted()
return results, nil
}
func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]SearchResult, error) {
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
}
if i.skipFilesAboveSize > 0 && metadata.Size > i.skipFilesAboveSize {
return nil, nil
}
// TODO: in the future we can swap out search strategies here
secrets, err := catalogLocationByLine(resolver, location, i.patterns)
if err != nil {
return nil, err
}
if i.revealValues {
for idx, secret := range secrets {
value, err := extractValue(resolver, location, secret.SeekPosition, secret.Length)
if err != nil {
return nil, err
}
secrets[idx].Value = value
}
}
// sort by the start location of each secret as it appears in the location
sort.SliceStable(secrets, func(i, j int) bool {
return secrets[i].SeekPosition < secrets[j].SeekPosition
})
return secrets, nil
}
func extractValue(resolver source.FileResolver, location source.Location, start, length int64) (string, error) {
readCloser, err := resolver.FileContentsByLocation(location)
if err != nil {
return "", fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
}
defer readCloser.Close()
n, err := io.CopyN(ioutil.Discard, readCloser, start)
if err != nil {
return "", fmt.Errorf("unable to read contents for location=%q : %w", location, err)
}
if n != start {
return "", fmt.Errorf("unexpected seek location for location=%q : %d != %d", location, n, start)
}
var buf bytes.Buffer
n, err = io.CopyN(&buf, readCloser, length)
if err != nil {
return "", fmt.Errorf("unable to read secret value for location=%q : %w", location, err)
}
if n != length {
return "", fmt.Errorf("unexpected secret length for location=%q : %d != %d", location, n, length)
}
return buf.String(), nil
}
type SecretsMonitor struct {
progress.Stager
SecretsDiscovered progress.Monitorable
progress.Progressable
}
func secretsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual, *progress.Manual) {
stage := &progress.Stage{}
secretsDiscovered := &progress.Manual{}
prog := &progress.Manual{
Total: locations,
}
bus.Publish(partybus.Event{
Type: event.SecretsCatalogerStarted,
Source: secretsDiscovered,
Value: SecretsMonitor{
Stager: progress.Stager(stage),
SecretsDiscovered: secretsDiscovered,
Progressable: prog,
},
})
return stage, prog, secretsDiscovered
}
syft/file/secrets_cataloger_test.go
+444
View File
@@ -0,0 +1,444 @@
package file
import (
"regexp"
"testing"
"github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft/source"
"github.com/stretchr/testify/assert"
)
func TestSecretsCataloger(t *testing.T) {
tests := []struct {
name string
fixture string
reveal bool
maxSize int64
patterns map[string]string
expected []SearchResult
constructorErr bool
catalogErr bool
}{
{
name: "go-case-find-and-reveal",
fixture: "test-fixtures/secrets/simple.txt",
reveal: true,
patterns: map[string]string{
"simple-secret-key": `^secret_key=.*`,
},
expected: []SearchResult{
{
Classification: "simple-secret-key",
LineNumber: 2,
LineOffset: 0,
SeekPosition: 34,
Length: 21,
Value: "secret_key=clear_text",
},
},
},
{
name: "dont-reveal-secret-value",
fixture: "test-fixtures/secrets/simple.txt",
reveal: false,
patterns: map[string]string{
"simple-secret-key": `^secret_key=.*`,
},
expected: []SearchResult{
{
Classification: "simple-secret-key",
LineNumber: 2,
LineOffset: 0,
SeekPosition: 34,
Length: 21,
Value: "",
},
},
},
{
name: "reveal-named-capture-group",
fixture: "test-fixtures/secrets/simple.txt",
reveal: true,
patterns: map[string]string{
"simple-secret-key": `^secret_key=(?P<value>.*)`,
},
expected: []SearchResult{
{
Classification: "simple-secret-key",
LineNumber: 2,
LineOffset: 11,
SeekPosition: 45,
Length: 10,
Value: "clear_text",
},
},
},
{
name: "multiple-secret-instances",
fixture: "test-fixtures/secrets/multiple.txt",
reveal: true,
patterns: map[string]string{
"simple-secret-key": `secret_key=.*`,
},
expected: []SearchResult{
{
Classification: "simple-secret-key",
LineNumber: 1,
LineOffset: 0,
SeekPosition: 0,
Length: 22,
Value: "secret_key=clear_text1",
},
{
Classification: "simple-secret-key",
LineNumber: 3,
LineOffset: 0,
SeekPosition: 57,
Length: 22,
Value: "secret_key=clear_text2",
},
{
Classification: "simple-secret-key",
LineNumber: 4,
// note: this test captures a line offset case
LineOffset: 1,
SeekPosition: 81,
Length: 22,
Value: "secret_key=clear_text3",
},
{
Classification: "simple-secret-key",
LineNumber: 6,
LineOffset: 0,
SeekPosition: 139,
Length: 22,
Value: "secret_key=clear_text4",
},
},
},
{
name: "multiple-secret-instances-with-capture-group",
fixture: "test-fixtures/secrets/multiple.txt",
reveal: true,
patterns: map[string]string{
"simple-secret-key": `secret_key=(?P<value>.*)`,
},
expected: []SearchResult{
{
Classification: "simple-secret-key",
LineNumber: 1,
// note: value capture group location
LineOffset: 11,
SeekPosition: 11,
Length: 11,
Value: "clear_text1",
},
{
Classification: "simple-secret-key",
LineNumber: 3,
LineOffset: 11,
SeekPosition: 68,
Length: 11,
Value: "clear_text2",
},
{
Classification: "simple-secret-key",
LineNumber: 4,
// note: value capture group location + offset
LineOffset: 12,
SeekPosition: 92,
Length: 11,
Value: "clear_text3",
},
{
Classification: "simple-secret-key",
LineNumber: 6,
LineOffset: 11,
SeekPosition: 150,
Length: 11,
Value: "clear_text4",
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
regexObjs := make(map[string]*regexp.Regexp)
for name, pattern := range test.patterns {
// always assume given patterns should be multiline
obj, err := regexp.Compile(`` + pattern)
if err != nil {
t.Fatalf("unable to parse regex: %+v", err)
}
regexObjs[name] = obj
}
c, err := NewSecretsCataloger(regexObjs, test.reveal, test.maxSize)
if err != nil && !test.constructorErr {
t.Fatalf("could not create cataloger (but should have been able to): %+v", err)
} else if err == nil && test.constructorErr {
t.Fatalf("expected constructor error but did not get one")
} else if test.constructorErr && err != nil {
return
}
resolver := source.NewMockResolverForPaths(test.fixture)
actualResults, err := c.Catalog(resolver)
if err != nil && !test.catalogErr {
t.Fatalf("could not catalog (but should have been able to): %+v", err)
} else if err == nil && test.catalogErr {
t.Fatalf("expected catalog error but did not get one")
} else if test.catalogErr && err != nil {
return
}
loc := source.NewLocation(test.fixture)
if _, exists := actualResults[loc]; !exists {
t.Fatalf("could not find location=%q in results", loc)
}
assert.Equal(t, test.expected, actualResults[loc], "mismatched secrets")
})
}
}
func TestSecretsCataloger_DefaultSecrets(t *testing.T) {
regexObjs, err := GenerateSearchPatterns(DefaultSecretsPatterns, nil, nil)
if err != nil {
t.Fatalf("unable to get patterns: %+v", err)
}
tests := []struct {
fixture string
expected []SearchResult
}{
{
fixture: "test-fixtures/secrets/default/aws.env",
expected: []SearchResult{
{
Classification: "aws-access-key",
LineNumber: 2,
LineOffset: 25,
SeekPosition: 64,
Length: 20,
Value: "AKIAIOSFODNN7EXAMPLE",
},
{
Classification: "aws-secret-key",
LineNumber: 3,
LineOffset: 29,
SeekPosition: 114,
Length: 40,
Value: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
},
},
},
{
fixture: "test-fixtures/secrets/default/aws.ini",
expected: []SearchResult{
{
Classification: "aws-access-key",
LineNumber: 3,
LineOffset: 18,
SeekPosition: 67,
Length: 20,
Value: "AKIAIOSFODNN7EXAMPLE",
},
{
Classification: "aws-secret-key",
LineNumber: 4,
LineOffset: 22,
SeekPosition: 110,
Length: 40,
Value: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
},
},
},
{
fixture: "test-fixtures/secrets/default/private-key.pem",
expected: []SearchResult{
{
Classification: "pem-private-key",
LineNumber: 2,
LineOffset: 27,
SeekPosition: 66,
Length: 351,
Value: `
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
z3P668YfhUbKdRF6S42Cg6zn
`,
},
},
},
{
fixture: "test-fixtures/secrets/default/private-key-openssl.pem",
expected: []SearchResult{
{
Classification: "pem-private-key",
LineNumber: 2,
LineOffset: 35,
SeekPosition: 74,
Length: 351,
Value: `
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
z3P668YfhUbKdRF6S42Cg6zn
`,
},
},
},
{
// note: this test proves that the PEM regex matches the smallest possible match
// since the test catches two adjacent secrets
fixture: "test-fixtures/secrets/default/private-keys.pem",
expected: []SearchResult{
{
Classification: "pem-private-key",
LineNumber: 1,
LineOffset: 35,
SeekPosition: 35,
Length: 351,
Value: `
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
z3P668YfhUbKdRF6S42Cg6zn
`,
},
{
Classification: "pem-private-key",
LineNumber: 9,
LineOffset: 35,
SeekPosition: 455,
Length: 351,
Value: `
MIIEvgTHISISNOTAREALKEYoIBAQDBj08DBj08DBj08DBj08DBj08DBsp5++4an3
cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgY5
VQQDDBcqLmF3cy10ZXN0SISNOTAREALKEYoIBAQDBj08DfffKoZIhvcNAQEBBQA7
bml6SISNOTAREALKEYoIBAQDBj08DdssBggrBgEFBQcBAQSBkzCBkDBNBggrBgE8
BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmd1
j4f668YfhUbKdRF6S6734856
`,
},
},
},
{
fixture: "test-fixtures/secrets/default/private-key-false-positive.pem",
expected: nil,
},
{
// this test represents:
// 1. a docker config
// 2. a named capture group with the correct line number and line offset case
// 3. the named capture group is in a different line than the match start, and both the match start and the capture group have different line offsets
fixture: "test-fixtures/secrets/default/docker-config.json",
expected: []SearchResult{
{
Classification: "docker-config-auth",
LineNumber: 5,
LineOffset: 15,
SeekPosition: 100,
Length: 10,
Value: "tOpsyKreTz",
},
},
},
{
fixture: "test-fixtures/secrets/default/not-docker-config.json",
expected: nil,
},
{
fixture: "test-fixtures/secrets/default/api-key.txt",
expected: []SearchResult{
{
Classification: "generic-api-key",
LineNumber: 2,
LineOffset: 7,
SeekPosition: 33,
Length: 20,
Value: "12345A7a901b34567890",
},
{
Classification: "generic-api-key",
LineNumber: 3,
LineOffset: 9,
SeekPosition: 63,
Length: 30,
Value: "12345A7a901b345678901234567890",
},
{
Classification: "generic-api-key",
LineNumber: 4,
LineOffset: 10,
SeekPosition: 104,
Length: 40,
Value: "12345A7a901b3456789012345678901234567890",
},
{
Classification: "generic-api-key",
LineNumber: 5,
LineOffset: 10,
SeekPosition: 156,
Length: 50,
Value: "12345A7a901b34567890123456789012345678901234567890",
},
{
Classification: "generic-api-key",
LineNumber: 6,
LineOffset: 16,
SeekPosition: 224,
Length: 60,
Value: "12345A7a901b345678901234567890123456789012345678901234567890",
},
{
Classification: "generic-api-key",
LineNumber: 14,
LineOffset: 8,
SeekPosition: 502,
Length: 20,
Value: "11111111111111111111",
},
},
},
}
for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
c, err := NewSecretsCataloger(regexObjs, true, 10*file.MB)
if err != nil {
t.Fatalf("could not create cataloger: %+v", err)
}
resolver := source.NewMockResolverForPaths(test.fixture)
actualResults, err := c.Catalog(resolver)
if err != nil {
t.Fatalf("could not catalog: %+v", err)
}
loc := source.NewLocation(test.fixture)
if _, exists := actualResults[loc]; !exists && test.expected != nil {
t.Fatalf("could not find location=%q in results", loc)
} else if !exists && test.expected == nil {
return
}
assert.Equal(t, test.expected, actualResults[loc], "mismatched secrets")
})
}
}
syft/file/secrets_search_by_line_strategy.go
+134
View File
@@ -0,0 +1,134 @@
package file
import (
"bufio"
"errors"
"fmt"
"io"
"io/ioutil"
"regexp"
"github.com/anchore/syft/syft/source"
)
func catalogLocationByLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp) ([]SearchResult, error) {
readCloser, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
}
defer readCloser.Close()
var scanner = bufio.NewReader(readCloser)
var position int64
var allSecrets []SearchResult
var lineNo int64
var readErr error
for !errors.Is(readErr, io.EOF) {
lineNo++
var line []byte
// TODO: we're at risk of large memory usage for very long lines
line, readErr = scanner.ReadBytes('\n')
if readErr != nil && readErr != io.EOF {
return nil, readErr
}
lineSecrets, err := searchForSecretsWithinLine(resolver, location, patterns, line, lineNo, position)
if err != nil {
return nil, err
}
position += int64(len(line))
allSecrets = append(allSecrets, lineSecrets...)
}
return allSecrets, nil
}
func searchForSecretsWithinLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp, line []byte, lineNo int64, position int64) ([]SearchResult, error) {
var secrets []SearchResult
for name, pattern := range patterns {
matches := pattern.FindAllIndex(line, -1)
for i, match := range matches {
if i%2 == 1 {
// FindAllIndex returns pairs of numbers for each match, we are only interested in the starting (first)
// position in each pair.
continue
}
lineOffset := int64(match[0])
seekLocation := position + lineOffset
reader, err := readerAtPosition(resolver, location, seekLocation)
if err != nil {
return nil, err
}
secret := extractSecretFromPosition(reader, name, pattern, lineNo, lineOffset, seekLocation)
if secret != nil {
secrets = append(secrets, *secret)
}
}
}
return secrets, nil
}
func readerAtPosition(resolver source.FileResolver, location source.Location, seekPosition int64) (io.ReadCloser, error) {
readCloser, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
}
if seekPosition > 0 {
n, err := io.CopyN(ioutil.Discard, readCloser, seekPosition)
if err != nil {
return nil, fmt.Errorf("unable to read contents for location=%q while searching for secrets: %w", location, err)
}
if n != seekPosition {
return nil, fmt.Errorf("unexpected seek location for location=%q while searching for secrets: %d != %d", location, n, seekPosition)
}
}
return readCloser, nil
}
func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *regexp.Regexp, lineNo, lineOffset, seekPosition int64) *SearchResult {
reader := &newlineCounter{RuneReader: bufio.NewReader(readCloser)}
positions := pattern.FindReaderSubmatchIndex(reader)
if len(positions) == 0 {
// no matches found
return nil
}
index := pattern.SubexpIndex("value")
var indexOffset int
if index != -1 {
// there is a capture group, use the capture group selection as the secret value. To do this we want to
// use the position at the discovered offset. Note: all positions come in pairs, so you will need to adjust
// the offset accordingly (multiply by 2).
indexOffset = index * 2
}
// get the start and stop of the secret value. Note: this covers both when there is a capture group
// and when there is not a capture group (full value match)
start, stop := int64(positions[indexOffset]), int64(positions[indexOffset+1])
if start < 0 || stop < 0 {
// no match location found. This can happen when there is a value capture group specified by the user
// and there was a match on the overall regex, but not for the capture group (which is possible if the capture
// group is optional).
return nil
}
// lineNoOfSecret are the number of lines which occur before the start of the secret value
var lineNoOfSecret = lineNo + int64(reader.newlinesBefore(start))
// lineOffsetOfSecret are the number of bytes that occur after the last newline but before the secret value.
var lineOffsetOfSecret = start - reader.newlinePositionBefore(start)
if lineNoOfSecret == lineNo {
// the secret value starts in the same line as the overall match, so we must consider that line offset
lineOffsetOfSecret += lineOffset
}
return &SearchResult{
Classification: name,
SeekPosition: start + seekPosition,
Length: stop - start,
LineNumber: lineNoOfSecret,
LineOffset: lineOffsetOfSecret,
}
}
syft/file/test-fixtures/secrets/default/api-key.txt
+14
View File
@@ -0,0 +1,14 @@
# these should be matches
apikey=12345A7a901b34567890
api_key =12345A7a901b345678901234567890
API-KEY= '12345A7a901b3456789012345678901234567890'
API-key: "12345A7a901b34567890123456789012345678901234567890"
some_ApI-kEy = "12345A7a901b345678901234567890123456789012345678901234567890"
# these should be non matches
api_key = "toolong12345A7a901b345678901234567890123456789012345678901234567890"
api_key = "tooshort"
not_api_k3y = "badkeyname12345A7a901b34567890"
# value at EOF should match
api_key=11111111111111111111
syft/file/test-fixtures/secrets/default/aws.env
+3
View File
@@ -0,0 +1,3 @@
# note: these are NOT real credentials
export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
syft/file/test-fixtures/secrets/default/aws.ini
+4
View File
@@ -0,0 +1,4 @@
# note: these are NOT real credentials
[default]
aws_access_key_id=AKIAIOSFODNN7EXAMPLE
aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
syft/file/test-fixtures/secrets/default/docker-config.json
+10
View File
@@ -0,0 +1,10 @@
{
"experimental" : "disabled",
"auths" : {
"https://index.docker.io/v1/" : {
"auth": "tOpsyKreTz"
}
},
"stackOrchestrator" : "swarm",
"credsStore" : "desktop"
}
syft/file/test-fixtures/secrets/default/not-docker-config.json
+4
View File
@@ -0,0 +1,4 @@
{
"endpoint" : "http://somewhere",
"auth" : "basic"
}
syft/file/test-fixtures/secrets/default/private-key-false-positive.pem
+1
View File
@@ -0,0 +1 @@
-----BEGIN OPENSSL PRIVATE KEY-----
syft/file/test-fixtures/secrets/default/private-key-openssl.pem
+9
View File
@@ -0,0 +1,9 @@
# note: this is NOT a real private key
-----BEGIN OPENSSL PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
z3P668YfhUbKdRF6S42Cg6zn
-----END OPENSSL PRIVATE KEY-----
syft/file/test-fixtures/secrets/default/private-key.pem
+10
View File
@@ -0,0 +1,10 @@
# note: this is NOT a real private key
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
z3P668YfhUbKdRF6S42Cg6zn
-----END PRIVATE KEY-----
other embedded text
syft/file/test-fixtures/secrets/default/private-keys.pem
+16
View File
@@ -0,0 +1,16 @@
-----BEGIN OPENSSL PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
z3P668YfhUbKdRF6S42Cg6zn
-----END OPENSSL PRIVATE KEY-----
-----BEGIN OPENSSL PRIVATE KEY-----
MIIEvgTHISISNOTAREALKEYoIBAQDBj08DBj08DBj08DBj08DBj08DBsp5++4an3
cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgY5
VQQDDBcqLmF3cy10ZXN0SISNOTAREALKEYoIBAQDBj08DfffKoZIhvcNAQEBBQA7
bml6SISNOTAREALKEYoIBAQDBj08DdssBggrBgEFBQcBAQSBkzCBkDBNBggrBgE8
BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmd1
j4f668YfhUbKdRF6S6734856
-----END OPENSSL PRIVATE KEY-----
syft/file/test-fixtures/secrets/multiple.txt
+6
View File
@@ -0,0 +1,6 @@
secret_key=clear_text1
other text that should be ignored
secret_key=clear_text2
secret_key=clear_text3
also things that should be ignored
secret_key=clear_text4
syft/file/test-fixtures/secrets/simple.txt
+4
View File
@@ -0,0 +1,4 @@
other text that should be ignored
secret_key=clear_text
---secret_key=clear_text
also things that should be ignored
syft/lib.go
+1 -1
View File
@@ -35,7 +35,7 @@ import (
func CatalogPackages(src source.Source, scope source.Scope) (*pkg.Catalog, *distro.Distro, error) {
resolver, err := src.FileResolver(scope)
if err != nil {
return nil, nil, fmt.Errorf("unable to determine FileResolver while cataloging packages: %w", err)
return nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err)
}
// find the distro
syft/pkg/cataloger/catalog.go
+1 -1
View File
@@ -54,7 +54,7 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers
catalogedPackages := len(packages)
log.Debugf("cataloger '%s' discovered '%d' packages", theCataloger.Name(), catalogedPackages)
log.Debugf("package cataloger %q discovered %d packages", theCataloger.Name(), catalogedPackages)
packagesDiscovered.N += int64(catalogedPackages)
for _, p := range packages {
syft/source/file_metadata.go
+2
View File
@@ -12,6 +12,7 @@ type FileMetadata struct {
UserID int
GroupID int
LinkDestination string
Size int64
}
func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata, error) {
@@ -26,5 +27,6 @@ func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata,
UserID: entry.Metadata.UserID,
GroupID: entry.Metadata.GroupID,
LinkDestination: entry.Metadata.Linkname,
Size: entry.Metadata.Size,
}, nil
}
syft/source/mock_resolver.go
+19 -2
View File
@@ -118,6 +118,23 @@ func (r MockResolver) AllLocations() <-chan Location {
return results
}
func (r MockResolver) FileMetadataByLocation(Location) (FileMetadata, error) {
panic("not implemented")
func (r MockResolver) FileMetadataByLocation(l Location) (FileMetadata, error) {
info, err := os.Stat(l.RealPath)
if err != nil {
return FileMetadata{}, err
}
// other types not supported
ty := RegularFile
if info.IsDir() {
ty = Directory
}
return FileMetadata{
Mode: info.Mode(),
Type: ty,
UserID: 0, // not supported
GroupID: 0, // not supported
Size: info.Size(),
}, nil
}