Add secrets search capability (#367)

* add initial secrets cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update ETUI elements with new catalogers (file metadata, digests, and secrets) Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update secrets cataloger to read full contents into memory for searching Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * quick prototype of parallelization secret regex search Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * quick prototype with single aggregated regex Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * quick prototype for secret search line-by-line Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * quick prototype hybrid secrets search Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add secrets cataloger with line strategy Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * adjust verbiage towards SearchResults instead of Secrets + add tests Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update json schema with secrets cataloger results Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * address PR comments Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update readme with secrets config options Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * ensure file catalogers call AllLocations once Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2026-11-04 08:26:38 +01:00 · 2021-04-01 21:34:15 +00:00
parent 557ad8be49
commit 9ec09add67
53 changed files with 2549 additions and 72 deletions
@@ -0,0 +1,150 @@
+package file
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"regexp"
+	"sort"
+
+	"github.com/anchore/syft/internal/bus"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/event"
+	"github.com/anchore/syft/syft/source"
+	"github.com/wagoodman/go-partybus"
+	"github.com/wagoodman/go-progress"
+)
+
+var DefaultSecretsPatterns = map[string]string{
+	"aws-access-key":     `(?i)aws_access_key_id["'=:\s]*?(?P<value>(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})`,
+	"aws-secret-key":     `(?i)aws_secret_access_key["'=:\s]*?(?P<value>[0-9a-zA-Z/+]{40})`,
+	"pem-private-key":    `-----BEGIN (\S+ )?PRIVATE KEY(\sBLOCK)?-----((?P<value>(\n.*?)+)-----END (\S+ )?PRIVATE KEY(\sBLOCK)?-----)?`,
+	"docker-config-auth": `"auths"((.*\n)*.*?"auth"\s*:\s*"(?P<value>[^"]+)")?`,
+	"generic-api-key":    `(?i)api(-|_)?key["'=:\s]*?(?P<value>[A-Z0-9]{20,60})["']?(\s|$)`,
+}
+
+type SecretsCataloger struct {
+	patterns           map[string]*regexp.Regexp
+	revealValues       bool
+	skipFilesAboveSize int64
+}
+
+func NewSecretsCataloger(patterns map[string]*regexp.Regexp, revealValues bool, maxFileSize int64) (*SecretsCataloger, error) {
+	return &SecretsCataloger{
+		patterns:           patterns,
+		revealValues:       revealValues,
+		skipFilesAboveSize: maxFileSize,
+	}, nil
+}
+
+func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Location][]SearchResult, error) {
+	results := make(map[source.Location][]SearchResult)
+	var locations []source.Location
+	for location := range resolver.AllLocations() {
+		locations = append(locations, location)
+	}
+	stage, prog, secretsDiscovered := secretsCatalogingProgress(int64(len(locations)))
+	for _, location := range locations {
+		stage.Current = location.RealPath
+		result, err := i.catalogLocation(resolver, location)
+		if err != nil {
+			return nil, err
+		}
+		if len(result) > 0 {
+			secretsDiscovered.N += int64(len(result))
+			results[location] = result
+		}
+		prog.N++
+	}
+	log.Debugf("secrets cataloger discovered %d secrets", secretsDiscovered.N)
+	prog.SetCompleted()
+	return results, nil
+}
+
+func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]SearchResult, error) {
+	metadata, err := resolver.FileMetadataByLocation(location)
+	if err != nil {
+		return nil, err
+	}
+
+	if i.skipFilesAboveSize > 0 && metadata.Size > i.skipFilesAboveSize {
+		return nil, nil
+	}
+
+	// TODO: in the future we can swap out search strategies here
+	secrets, err := catalogLocationByLine(resolver, location, i.patterns)
+	if err != nil {
+		return nil, err
+	}
+
+	if i.revealValues {
+		for idx, secret := range secrets {
+			value, err := extractValue(resolver, location, secret.SeekPosition, secret.Length)
+			if err != nil {
+				return nil, err
+			}
+			secrets[idx].Value = value
+		}
+	}
+
+	// sort by the start location of each secret as it appears in the location
+	sort.SliceStable(secrets, func(i, j int) bool {
+		return secrets[i].SeekPosition < secrets[j].SeekPosition
+	})
+
+	return secrets, nil
+}
+
+func extractValue(resolver source.FileResolver, location source.Location, start, length int64) (string, error) {
+	readCloser, err := resolver.FileContentsByLocation(location)
+	if err != nil {
+		return "", fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
+	}
+	defer readCloser.Close()
+
+	n, err := io.CopyN(ioutil.Discard, readCloser, start)
+	if err != nil {
+		return "", fmt.Errorf("unable to read contents for location=%q : %w", location, err)
+	}
+	if n != start {
+		return "", fmt.Errorf("unexpected seek location for location=%q : %d != %d", location, n, start)
+	}
+
+	var buf bytes.Buffer
+	n, err = io.CopyN(&buf, readCloser, length)
+	if err != nil {
+		return "", fmt.Errorf("unable to read secret value for location=%q : %w", location, err)
+	}
+	if n != length {
+		return "", fmt.Errorf("unexpected secret length for location=%q : %d != %d", location, n, length)
+	}
+
+	return buf.String(), nil
+}
+
+type SecretsMonitor struct {
+	progress.Stager
+	SecretsDiscovered progress.Monitorable
+	progress.Progressable
+}
+
+func secretsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual, *progress.Manual) {
+	stage := &progress.Stage{}
+	secretsDiscovered := &progress.Manual{}
+	prog := &progress.Manual{
+		Total: locations,
+	}
+
+	bus.Publish(partybus.Event{
+		Type:   event.SecretsCatalogerStarted,
+		Source: secretsDiscovered,
+		Value: SecretsMonitor{
+			Stager:            progress.Stager(stage),
+			SecretsDiscovered: secretsDiscovered,
+			Progressable:      prog,
+		},
+	})
+
+	return stage, prog, secretsDiscovered
+}