Add secrets search capability (#367)

* add initial secrets cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update ETUI elements with new catalogers (file metadata, digests, and secrets)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update secrets cataloger to read full contents into memory for searching

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype of parallelization secret regex search

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype with single aggregated regex

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype for secret search line-by-line

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* quick prototype hybrid secrets search

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add secrets cataloger with line strategy

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* adjust verbiage towards SearchResults instead of Secrets + add tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update json schema with secrets cataloger results

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* address PR comments

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update readme with secrets config options

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* ensure file catalogers call AllLocations once

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

internal/presenter/poweruser/json_document.go

@@ -9,7 +9,8 @@ type JSONDocument struct {
 	// here should be optional by supplying "omitempty" on these fields hint to the jsonschema generator to not
 	// require these fields. As an accepted rule in this repo all collections should still be initialized in the
 	// context of being used in a JSON document.
-	FileMetadata []JSONFileMetadata `json:"fileMetadata,omitempty"`
+	FileMetadata []JSONFileMetadata `json:"fileMetadata,omitempty"` // note: must have omitempty
+	Secrets      []JSONSecrets      `json:"secrets,omitempty"`      // note: must have omitempty
 	packages.JSONDocument
 }
 
@@ -27,6 +28,7 @@ func NewJSONDocument(config JSONDocumentConfig) (JSONDocument, error) {
 
 	return JSONDocument{
 		FileMetadata: fileMetadata,
+		Secrets:      NewJSONSecrets(config.Secrets),
 		JSONDocument: pkgsDoc,
 	}, nil
 }

internal/presenter/poweruser/json_document_config.go

@@ -13,6 +13,7 @@ type JSONDocumentConfig struct {
 	PackageCatalog    *pkg.Catalog
 	FileMetadata      map[source.Location]source.FileMetadata
 	FileDigests       map[source.Location][]file.Digest
+	Secrets           map[source.Location][]file.SearchResult
 	Distro            *distro.Distro
 	SourceMetadata    source.Metadata
 }

internal/presenter/poweruser/json_secrets.go

@@ -0,0 +1,32 @@
+package poweruser
+
+import (
+	"sort"
+
+	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/source"
+)
+
+type JSONSecrets struct {
+	Location source.Location     `json:"location"`
+	Secrets  []file.SearchResult `json:"secrets"`
+}
+
+func NewJSONSecrets(data map[source.Location][]file.SearchResult) []JSONSecrets {
+	results := make([]JSONSecrets, 0)
+	for location, secrets := range data {
+		results = append(results, JSONSecrets{
+			Location: location,
+			Secrets:  secrets,
+		})
+	}
+
+	// sort by real path then virtual path to ensure the result is stable across multiple runs
+	sort.SliceStable(results, func(i, j int) bool {
+		if results[i].Location.RealPath != results[j].Location.RealPath {
+			return results[i].Location.VirtualPath < results[j].Location.VirtualPath
+		}
+		return false
+	})
+	return results
+}

internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden

@@ -191,11 +191,21 @@
     "digests": [
      "sha256"
     ]
+   },
+   "secrets": {
+    "cataloger": {
+     "enabled": false,
+     "scope": ""
+    },
+    "additional-patterns": null,
+    "exclude-pattern-names": null,
+    "reveal-values": false,
+    "skip-files-above-size": 0
    }
   }
  },
  "schema": {
-  "version": "1.0.4",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.4.json"
+  "version": "1.0.5",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.5.json"
  }
 }