Merge pull request #370 from anchore/add-file-contents-cataloger

Add file contents cataloger

README.md

@@ -119,6 +119,25 @@ file-classification:
     # SYFT_FILE_CLASSIFICATION_CATALOGER_SCOPE env var
     scope: "squashed"
 
+# cataloging file contents is exposed through the power-user subcommand
+file-contents:
+  cataloger:
+    # enable/disable cataloging of secrets
+    # SYFT_FILE_CONTENTS_CATALOGER_ENABLED env var
+    enabled: true
+
+    # the search space to look for secrets (options: all-layers, squashed)
+    # SYFT_FILE_CONTENTS_CATALOGER_SCOPE env var
+    scope: "squashed"
+
+  # skip searching a file entirely if it is above the given size (default = 1MB; unit = bytes)
+  # SYFT_FILE_CONTENTS_SKIP_FILES_ABOVE_SIZE env var
+  skip-files-above-size: 1048576
+
+  # file globs for the cataloger to match on
+  # SYFT_FILE_CONTENTS_GLOBS env var
+  globs: []
+
 # cataloging file metadata is exposed through the power-user subcommand
 file-metadata:
   cataloger:
@@ -149,9 +168,9 @@ secrets:
   # SYFT_SECRETS_REVEAL_VALUES env var
   reveal-values: false
 
-  # skip searching a file entirely if it is above the given size (default = 10MB; unit = bytes)
+  # skip searching a file entirely if it is above the given size (default = 1MB; unit = bytes)
   # SYFT_SECRETS_SKIP_FILES_ABOVE_SIZE env var
-  skip-files-above-size: 10485760
+  skip-files-above-size: 1048576
 
   # name-regex pairs to consider when searching files for secrets. Note: the regex must match single line patterns
   # but may also have OPTIONAL multiline capture groups. Regexes with a named capture group of "value" will

cmd/power_user_tasks.go

@@ -21,6 +21,7 @@ func powerUserTasks() ([]powerUserTask, error) {
 		catalogFileDigestsTask,
 		catalogSecretsTask,
 		catalogFileClassificationsTask,
+		catalogContentsTask,
 	}
 
 	for _, generator := range generators {
@@ -185,3 +186,30 @@ func catalogFileClassificationsTask() (powerUserTask, error) {
 
 	return task, nil
 }
+
+func catalogContentsTask() (powerUserTask, error) {
+	if !appConfig.FileContents.Cataloger.Enabled {
+		return nil, nil
+	}
+
+	contentsCataloger, err := file.NewContentsCataloger(appConfig.FileContents.Globs, appConfig.FileContents.SkipFilesAboveSize)
+	if err != nil {
+		return nil, err
+	}
+
+	task := func(results *poweruser.JSONDocumentConfig, src source.Source) error {
+		resolver, err := src.FileResolver(appConfig.FileContents.Cataloger.ScopeOpt)
+		if err != nil {
+			return err
+		}
+
+		result, err := contentsCataloger.Catalog(resolver)
+		if err != nil {
+			return err
+		}
+		results.FileContents = result
+		return nil
+	}
+
+	return task, nil
+}

internal/config/application.go

@@ -38,6 +38,7 @@ type Application struct {
 	Package            packages           `yaml:"package" json:"package" mapstructure:"package"`
 	FileMetadata       FileMetadata       `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"`
 	FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"`
+	FileContents       fileContents       `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"`
 	Secrets            secrets            `yaml:"secrets" json:"secrets" mapstructure:"secrets"`
 }
 

internal/config/file_contents.go

@@ -0,0 +1,24 @@
+package config
+
+import (
+	"github.com/anchore/syft/internal/file"
+	"github.com/anchore/syft/syft/source"
+	"github.com/spf13/viper"
+)
+
+type fileContents struct {
+	Cataloger          catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
+	SkipFilesAboveSize int64            `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"`
+	Globs              []string         `yaml:"globs" json:"globs" mapstructure:"globs"`
+}
+
+func (cfg fileContents) loadDefaultValues(v *viper.Viper) {
+	v.SetDefault("file-contents.cataloger.enabled", true)
+	v.SetDefault("file-contents.cataloger.scope", source.SquashedScope)
+	v.SetDefault("file-contents.skip-files-above-size", 1*file.MB)
+	v.SetDefault("file-contents.globs", []string{})
+}
+
+func (cfg *fileContents) parseConfigValues() error {
+	return cfg.Cataloger.parseConfigValues()
+}

internal/presenter/poweruser/json_document.go

@@ -10,6 +10,7 @@ type JSONDocument struct {
 	// require these fields. As an accepted rule in this repo all collections should still be initialized in the
 	// context of being used in a JSON document.
 	FileClassifications []JSONFileClassifications `json:"fileClassifications,omitempty"` // note: must have omitempty
+	FileContents        []JSONFileContents        `json:"fileContents,omitempty"`        // note: must have omitempty
 	FileMetadata        []JSONFileMetadata        `json:"fileMetadata,omitempty"`        // note: must have omitempty
 	Secrets             []JSONSecrets             `json:"secrets,omitempty"`             // note: must have omitempty
 	packages.JSONDocument
@@ -29,6 +30,7 @@ func NewJSONDocument(config JSONDocumentConfig) (JSONDocument, error) {
 
 	return JSONDocument{
 		FileClassifications: NewJSONFileClassifications(config.FileClassifications),
+		FileContents:        NewJSONFileContents(config.FileContents),
 		FileMetadata:        fileMetadata,
 		Secrets:             NewJSONSecrets(config.Secrets),
 		JSONDocument:        pkgsDoc,

internal/presenter/poweruser/json_document_config.go

@@ -14,6 +14,7 @@ type JSONDocumentConfig struct {
 	FileMetadata        map[source.Location]source.FileMetadata
 	FileDigests         map[source.Location][]file.Digest
 	FileClassifications map[source.Location][]file.Classification
+	FileContents        map[source.Location]string
 	Secrets             map[source.Location][]file.SearchResult
 	Distro              *distro.Distro
 	SourceMetadata      source.Metadata

internal/presenter/poweruser/json_file_contents.go

@@ -0,0 +1,31 @@
+package poweruser
+
+import (
+	"sort"
+
+	"github.com/anchore/syft/syft/source"
+)
+
+type JSONFileContents struct {
+	Location source.Location `json:"location"`
+	Contents string          `json:"contents"`
+}
+
+func NewJSONFileContents(data map[source.Location]string) []JSONFileContents {
+	results := make([]JSONFileContents, 0)
+	for location, contents := range data {
+		results = append(results, JSONFileContents{
+			Location: location,
+			Contents: contents,
+		})
+	}
+
+	// sort by real path then virtual path to ensure the result is stable across multiple runs
+	sort.SliceStable(results, func(i, j int) bool {
+		if results[i].Location.RealPath == results[j].Location.RealPath {
+			return results[i].Location.VirtualPath < results[j].Location.VirtualPath
+		}
+		return results[i].Location.RealPath < results[j].Location.RealPath
+	})
+	return results
+}

internal/presenter/poweruser/json_presenter_test.go

@@ -125,6 +125,9 @@ func TestJSONPresenter(t *testing.T) {
 				},
 			},
 		},
+		FileContents: map[source.Location]string{
+			source.NewLocation("/a/place/a"): "the-contents",
+		},
 		Distro: &distro.Distro{
 			Type:       distro.RedHat,
 			RawVersion: "7",

internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden

@@ -1,4 +1,12 @@
 {
+ "fileContents": [
+  {
+   "location": {
+    "path": "/a/place/a"
+   },
+   "contents": "the-contents"
+  }
+ ],
  "fileMetadata": [
   {
    "location": {
@@ -198,6 +206,14 @@
      "scope": ""
     }
    },
+   "file-contents": {
+    "cataloger": {
+     "enabled": false,
+     "scope": ""
+    },
+    "skip-files-above-size": 0,
+    "globs": null
+   },
    "secrets": {
     "cataloger": {
      "enabled": false,

schema/json/schema-1.0.6.json

@@ -1,935 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "$ref": "#/definitions/Document",
-  "definitions": {
-    "ApkFileRecord": {
-      "required": [
-        "path"
-      ],
-      "properties": {
-        "path": {
-          "type": "string"
-        },
-        "ownerUid": {
-          "type": "string"
-        },
-        "ownerGid": {
-          "type": "string"
-        },
-        "permissions": {
-          "type": "string"
-        },
-        "checksum": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "ApkMetadata": {
-      "required": [
-        "package",
-        "originPackage",
-        "maintainer",
-        "version",
-        "license",
-        "architecture",
-        "url",
-        "description",
-        "size",
-        "installedSize",
-        "pullDependencies",
-        "pullChecksum",
-        "gitCommitOfApkPort",
-        "files"
-      ],
-      "properties": {
-        "package": {
-          "type": "string"
-        },
-        "originPackage": {
-          "type": "string"
-        },
-        "maintainer": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "license": {
-          "type": "string"
-        },
-        "architecture": {
-          "type": "string"
-        },
-        "url": {
-          "type": "string"
-        },
-        "description": {
-          "type": "string"
-        },
-        "size": {
-          "type": "integer"
-        },
-        "installedSize": {
-          "type": "integer"
-        },
-        "pullDependencies": {
-          "type": "string"
-        },
-        "pullChecksum": {
-          "type": "string"
-        },
-        "gitCommitOfApkPort": {
-          "type": "string"
-        },
-        "files": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/ApkFileRecord"
-          },
-          "type": "array"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "CargoPackageMetadata": {
-      "required": [
-        "name",
-        "version",
-        "source",
-        "checksum",
-        "dependencies"
-      ],
-      "properties": {
-        "name": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "source": {
-          "type": "string"
-        },
-        "checksum": {
-          "type": "string"
-        },
-        "dependencies": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "Classification": {
-      "required": [
-        "class",
-        "metadata"
-      ],
-      "properties": {
-        "class": {
-          "type": "string"
-        },
-        "metadata": {
-          "patternProperties": {
-            ".*": {
-              "type": "string"
-            }
-          },
-          "type": "object"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "Descriptor": {
-      "required": [
-        "name",
-        "version"
-      ],
-      "properties": {
-        "name": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "configuration": {
-          "additionalProperties": true
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "Digest": {
-      "required": [
-        "algorithm",
-        "value"
-      ],
-      "properties": {
-        "algorithm": {
-          "type": "string"
-        },
-        "value": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "Distribution": {
-      "required": [
-        "name",
-        "version",
-        "idLike"
-      ],
-      "properties": {
-        "name": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "idLike": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "Document": {
-      "required": [
-        "artifacts",
-        "artifactRelationships",
-        "source",
-        "distro",
-        "descriptor",
-        "schema"
-      ],
-      "properties": {
-        "fileClassifications": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/FileClassifications"
-          },
-          "type": "array"
-        },
-        "fileMetadata": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/FileMetadata"
-          },
-          "type": "array"
-        },
-        "secrets": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/Secrets"
-          },
-          "type": "array"
-        },
-        "artifacts": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/Package"
-          },
-          "type": "array"
-        },
-        "artifactRelationships": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/Relationship"
-          },
-          "type": "array"
-        },
-        "source": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/Source"
-        },
-        "distro": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/Distribution"
-        },
-        "descriptor": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/Descriptor"
-        },
-        "schema": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/Schema"
-        },
-        "artifacts.metadata": {
-          "anyOf": [
-            {
-              "type": "null"
-            },
-            {
-              "$ref": "#/definitions/ApkMetadata"
-            },
-            {
-              "$ref": "#/definitions/CargoPackageMetadata"
-            },
-            {
-              "$ref": "#/definitions/DpkgMetadata"
-            },
-            {
-              "$ref": "#/definitions/GemMetadata"
-            },
-            {
-              "$ref": "#/definitions/JavaMetadata"
-            },
-            {
-              "$ref": "#/definitions/NpmPackageJSONMetadata"
-            },
-            {
-              "$ref": "#/definitions/PythonPackageMetadata"
-            },
-            {
-              "$ref": "#/definitions/RpmdbMetadata"
-            }
-          ]
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "DpkgFileRecord": {
-      "required": [
-        "path",
-        "md5"
-      ],
-      "properties": {
-        "path": {
-          "type": "string"
-        },
-        "md5": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "DpkgMetadata": {
-      "required": [
-        "package",
-        "source",
-        "version",
-        "sourceVersion",
-        "architecture",
-        "maintainer",
-        "installedSize",
-        "files"
-      ],
-      "properties": {
-        "package": {
-          "type": "string"
-        },
-        "source": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "sourceVersion": {
-          "type": "string"
-        },
-        "architecture": {
-          "type": "string"
-        },
-        "maintainer": {
-          "type": "string"
-        },
-        "installedSize": {
-          "type": "integer"
-        },
-        "files": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/DpkgFileRecord"
-          },
-          "type": "array"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "FileClassifications": {
-      "required": [
-        "location",
-        "classification"
-      ],
-      "properties": {
-        "location": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/Location"
-        },
-        "classification": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/Classification"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "FileMetadata": {
-      "required": [
-        "location",
-        "metadata"
-      ],
-      "properties": {
-        "location": {
-          "$ref": "#/definitions/Location"
-        },
-        "metadata": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/FileMetadataEntry"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "FileMetadataEntry": {
-      "required": [
-        "mode",
-        "type",
-        "userID",
-        "groupID"
-      ],
-      "properties": {
-        "mode": {
-          "type": "integer"
-        },
-        "type": {
-          "type": "string"
-        },
-        "linkDestination": {
-          "type": "string"
-        },
-        "userID": {
-          "type": "integer"
-        },
-        "groupID": {
-          "type": "integer"
-        },
-        "digests": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/Digest"
-          },
-          "type": "array"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "GemMetadata": {
-      "required": [
-        "name",
-        "version"
-      ],
-      "properties": {
-        "name": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "files": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
-        },
-        "authors": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
-        },
-        "licenses": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
-        },
-        "homepage": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "JavaManifest": {
-      "properties": {
-        "main": {
-          "patternProperties": {
-            ".*": {
-              "type": "string"
-            }
-          },
-          "type": "object"
-        },
-        "namedSections": {
-          "patternProperties": {
-            ".*": {
-              "patternProperties": {
-                ".*": {
-                  "type": "string"
-                }
-              },
-              "type": "object"
-            }
-          },
-          "type": "object"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "JavaMetadata": {
-      "required": [
-        "virtualPath"
-      ],
-      "properties": {
-        "virtualPath": {
-          "type": "string"
-        },
-        "manifest": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/JavaManifest"
-        },
-        "pomProperties": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/PomProperties"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "Location": {
-      "required": [
-        "path"
-      ],
-      "properties": {
-        "path": {
-          "type": "string"
-        },
-        "layerID": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "NpmPackageJSONMetadata": {
-      "required": [
-        "author",
-        "licenses",
-        "homepage",
-        "description",
-        "url"
-      ],
-      "properties": {
-        "files": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
-        },
-        "author": {
-          "type": "string"
-        },
-        "licenses": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
-        },
-        "homepage": {
-          "type": "string"
-        },
-        "description": {
-          "type": "string"
-        },
-        "url": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "Package": {
-      "required": [
-        "id",
-        "name",
-        "version",
-        "type",
-        "foundBy",
-        "locations",
-        "licenses",
-        "language",
-        "cpes",
-        "purl",
-        "metadataType",
-        "metadata"
-      ],
-      "properties": {
-        "id": {
-          "type": "string"
-        },
-        "name": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "type": {
-          "type": "string"
-        },
-        "foundBy": {
-          "type": "string"
-        },
-        "locations": {
-          "items": {
-            "$ref": "#/definitions/Location"
-          },
-          "type": "array"
-        },
-        "licenses": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
-        },
-        "language": {
-          "type": "string"
-        },
-        "cpes": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
-        },
-        "purl": {
-          "type": "string"
-        },
-        "metadataType": {
-          "type": "string"
-        },
-        "metadata": {
-          "additionalProperties": true
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "PomProperties": {
-      "required": [
-        "path",
-        "name",
-        "groupId",
-        "artifactId",
-        "version",
-        "extraFields"
-      ],
-      "properties": {
-        "path": {
-          "type": "string"
-        },
-        "name": {
-          "type": "string"
-        },
-        "groupId": {
-          "type": "string"
-        },
-        "artifactId": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "extraFields": {
-          "patternProperties": {
-            ".*": {
-              "type": "string"
-            }
-          },
-          "type": "object"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "PythonFileDigest": {
-      "required": [
-        "algorithm",
-        "value"
-      ],
-      "properties": {
-        "algorithm": {
-          "type": "string"
-        },
-        "value": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "PythonFileRecord": {
-      "required": [
-        "path"
-      ],
-      "properties": {
-        "path": {
-          "type": "string"
-        },
-        "digest": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/PythonFileDigest"
-        },
-        "size": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "PythonPackageMetadata": {
-      "required": [
-        "name",
-        "version",
-        "license",
-        "author",
-        "authorEmail",
-        "platform",
-        "sitePackagesRootPath"
-      ],
-      "properties": {
-        "name": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "license": {
-          "type": "string"
-        },
-        "author": {
-          "type": "string"
-        },
-        "authorEmail": {
-          "type": "string"
-        },
-        "platform": {
-          "type": "string"
-        },
-        "files": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/PythonFileRecord"
-          },
-          "type": "array"
-        },
-        "sitePackagesRootPath": {
-          "type": "string"
-        },
-        "topLevelPackages": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "Relationship": {
-      "required": [
-        "parent",
-        "child",
-        "type",
-        "metadata"
-      ],
-      "properties": {
-        "parent": {
-          "type": "string"
-        },
-        "child": {
-          "type": "string"
-        },
-        "type": {
-          "type": "string"
-        },
-        "metadata": {
-          "additionalProperties": true
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "RpmdbFileRecord": {
-      "required": [
-        "path",
-        "mode",
-        "size",
-        "sha256"
-      ],
-      "properties": {
-        "path": {
-          "type": "string"
-        },
-        "mode": {
-          "type": "integer"
-        },
-        "size": {
-          "type": "integer"
-        },
-        "sha256": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "RpmdbMetadata": {
-      "required": [
-        "name",
-        "version",
-        "epoch",
-        "architecture",
-        "release",
-        "sourceRpm",
-        "size",
-        "license",
-        "vendor",
-        "files"
-      ],
-      "properties": {
-        "name": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "epoch": {
-          "type": "integer"
-        },
-        "architecture": {
-          "type": "string"
-        },
-        "release": {
-          "type": "string"
-        },
-        "sourceRpm": {
-          "type": "string"
-        },
-        "size": {
-          "type": "integer"
-        },
-        "license": {
-          "type": "string"
-        },
-        "vendor": {
-          "type": "string"
-        },
-        "files": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/RpmdbFileRecord"
-          },
-          "type": "array"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "Schema": {
-      "required": [
-        "version",
-        "url"
-      ],
-      "properties": {
-        "version": {
-          "type": "string"
-        },
-        "url": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "SearchResult": {
-      "required": [
-        "classification",
-        "lineNumber",
-        "lineOffset",
-        "seekPosition",
-        "length"
-      ],
-      "properties": {
-        "classification": {
-          "type": "string"
-        },
-        "lineNumber": {
-          "type": "integer"
-        },
-        "lineOffset": {
-          "type": "integer"
-        },
-        "seekPosition": {
-          "type": "integer"
-        },
-        "length": {
-          "type": "integer"
-        },
-        "value": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "Secrets": {
-      "required": [
-        "location",
-        "secrets"
-      ],
-      "properties": {
-        "location": {
-          "$ref": "#/definitions/Location"
-        },
-        "secrets": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/SearchResult"
-          },
-          "type": "array"
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    },
-    "Source": {
-      "required": [
-        "type",
-        "target"
-      ],
-      "properties": {
-        "type": {
-          "type": "string"
-        },
-        "target": {
-          "additionalProperties": true
-        }
-      },
-      "additionalProperties": true,
-      "type": "object"
-    }
-  }
-}

schema/json/schema-1.1.0.json

@@ -125,6 +125,27 @@
       "additionalProperties": true,
       "type": "object"
     },
+    "Classification": {
+      "required": [
+        "class",
+        "metadata"
+      ],
+      "properties": {
+        "class": {
+          "type": "string"
+        },
+        "metadata": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        }
+      },
+      "additionalProperties": true,
+      "type": "object"
+    },
     "Descriptor": {
       "required": [
         "name",
@@ -190,6 +211,20 @@
         "schema"
       ],
       "properties": {
+        "fileClassifications": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/FileClassifications"
+          },
+          "type": "array"
+        },
+        "fileContents": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/FileContents"
+          },
+          "type": "array"
+        },
         "fileMetadata": {
           "items": {
             "$schema": "http://json-schema.org/draft-04/schema#",
@@ -302,6 +337,40 @@
       "additionalProperties": true,
       "type": "object"
     },
+    "FileClassifications": {
+      "required": [
+        "location",
+        "classification"
+      ],
+      "properties": {
+        "location": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Location"
+        },
+        "classification": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Classification"
+        }
+      },
+      "additionalProperties": true,
+      "type": "object"
+    },
+    "FileContents": {
+      "required": [
+        "location",
+        "contents"
+      ],
+      "properties": {
+        "location": {
+          "$ref": "#/definitions/Location"
+        },
+        "contents": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": true,
+      "type": "object"
+    },
     "FileMetadata": {
       "required": [
         "location",
@@ -309,7 +378,6 @@
       ],
       "properties": {
         "location": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
           "$ref": "#/definitions/Location"
         },
         "metadata": {

syft/file/contents_cataloger.go

@@ -0,0 +1,68 @@
+package file
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"io"
+
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/source"
+)
+
+type ContentsCataloger struct {
+	globs                     []string
+	skipFilesAboveSizeInBytes int64
+}
+
+func NewContentsCataloger(globs []string, skipFilesAboveSize int64) (*ContentsCataloger, error) {
+	return &ContentsCataloger{
+		globs:                     globs,
+		skipFilesAboveSizeInBytes: skipFilesAboveSize,
+	}, nil
+}
+
+func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Location]string, error) {
+	results := make(map[source.Location]string)
+	var locations []source.Location
+
+	locations, err := resolver.FilesByGlob(i.globs...)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, location := range locations {
+		metadata, err := resolver.FileMetadataByLocation(location)
+		if err != nil {
+			return nil, err
+		}
+
+		if i.skipFilesAboveSizeInBytes > 0 && metadata.Size > i.skipFilesAboveSizeInBytes {
+			continue
+		}
+
+		result, err := i.catalogLocation(resolver, location)
+		if err != nil {
+			return nil, err
+		}
+		results[location] = result
+	}
+	log.Debugf("file contents cataloger processed %d files", len(results))
+
+	return results, nil
+}
+
+func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) (string, error) {
+	contentReader, err := resolver.FileContentsByLocation(location)
+	if err != nil {
+		return "", err
+	}
+	defer contentReader.Close()
+
+	buf := &bytes.Buffer{}
+	if _, err = io.Copy(base64.NewEncoder(base64.StdEncoding, buf), contentReader); err != nil {
+		return "", fmt.Errorf("unable to observe contents of %+v: %w", location.RealPath, err)
+	}
+
+	return buf.String(), nil
+}

syft/file/contents_cataloger_test.go

@@ -0,0 +1,79 @@
+package file
+
+import (
+	"testing"
+
+	"github.com/anchore/syft/syft/source"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestContentsCataloger(t *testing.T) {
+	allFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
+
+	tests := []struct {
+		name     string
+		globs    []string
+		maxSize  int64
+		files    []string
+		expected map[source.Location]string
+	}{
+		{
+			name:  "multi-pattern",
+			globs: []string{"test-fixtures/last/*.txt", "test-fixtures/*.txt"},
+			files: allFiles,
+			expected: map[source.Location]string{
+				source.NewLocation("test-fixtures/last/path.txt"):    "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
+				source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+				source.NewLocation("test-fixtures/a-path.txt"):       "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+			},
+		},
+		{
+			name:     "no-patterns",
+			globs:    []string{},
+			files:    []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"},
+			expected: map[source.Location]string{},
+		},
+		{
+			name:  "all-txt",
+			globs: []string{"**/*.txt"},
+			files: allFiles,
+			expected: map[source.Location]string{
+				source.NewLocation("test-fixtures/last/path.txt"):    "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
+				source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+				source.NewLocation("test-fixtures/a-path.txt"):       "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+			},
+		},
+		{
+			name:  "subpath",
+			globs: []string{"test-fixtures/*.txt"},
+			files: allFiles,
+			expected: map[source.Location]string{
+				source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+				source.NewLocation("test-fixtures/a-path.txt"):       "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+			},
+		},
+		{
+			name:    "size-filter",
+			maxSize: 42,
+			globs:   []string{"**/*.txt"},
+			files:   allFiles,
+			expected: map[source.Location]string{
+				source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
+				source.NewLocation("test-fixtures/a-path.txt"):    "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			c, err := NewContentsCataloger(test.globs, test.maxSize)
+			assert.NoError(t, err)
+
+			resolver := source.NewMockResolverForPaths(test.files...)
+			actual, err := c.Catalog(resolver)
+			assert.NoError(t, err)
+			assert.Equal(t, test.expected, actual, "mismatched contents")
+
+		})
+	}
+}

syft/pkg/cataloger/common/generic_cataloger_test.go

@@ -6,6 +6,8 @@ import (
 	"io/ioutil"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
 )
@@ -25,7 +27,7 @@ func parser(_ string, reader io.Reader) ([]pkg.Package, error) {
 func TestGenericCataloger(t *testing.T) {
 
 	globParsers := map[string]ParserFn{
-		"**a-path.txt": parser,
+		"**/a-path.txt": parser,
 	}
 	pathParsers := map[string]ParserFn{
 		"test-fixtures/another-path.txt": parser,
@@ -46,13 +48,8 @@ func TestGenericCataloger(t *testing.T) {
 	}
 
 	actualPkgs, err := cataloger.Catalog(resolver)
-	if err != nil {
+	assert.NoError(t, err)
-		t.Fatalf("cataloger catalog action failed: %+v", err)
+	assert.Len(t, actualPkgs, len(expectedPkgs))
-	}
-
-	if len(actualPkgs) != len(expectedPkgs) {
-		t.Fatalf("unexpected packages len: %d", len(actualPkgs))
-	}
 
 	for _, p := range actualPkgs {
 		ref := p.Locations[0]

syft/source/mock_resolver.go

@@ -5,7 +5,7 @@ import (
 	"io"
 	"os"
 
-	"github.com/anchore/syft/internal/file"
+	"github.com/bmatcuk/doublestar/v2"
 )
 
 var _ FileResolver = (*MockResolver)(nil)
@@ -84,7 +84,11 @@ func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 	var results []Location
 	for _, pattern := range patterns {
 		for _, location := range r.Locations {
-			if file.GlobMatch(pattern, location.RealPath) {
+			matches, err := doublestar.Match(pattern, location.RealPath)
+			if err != nil {
+				return nil, err
+			}
+			if matches {
 				results = append(results, location)
 			}
 		}

test/cli/power_user_cmd_test.go

@@ -51,6 +51,17 @@ func TestPowerUserCmdFlags(t *testing.T) {
 				assertSuccessfulReturnCode,
 			},
 		},
+		{
+			name: "content-cataloger-wired-up",
+			args: []string{"power-user", "docker-archive:" + getFixtureImage(t, "image-secrets")},
+			env: map[string]string{
+				"SYFT_FILE_CONTENTS_GLOBS": "/api-key.txt",
+			},
+			assertions: []traitAssertion{
+				assertInOutput(`"contents": "c29tZV9BcEkta0V5ID0gIjEyMzQ1QTdhOTAxYjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MCIK"`), // proof of the content cataloger
+				assertSuccessfulReturnCode,
+			},
+		},
 	}
 
 	for _, test := range tests {