diff --git a/README.md b/README.md index 642a1dc4d..af0d17357 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,25 @@ file-classification: # SYFT_FILE_CLASSIFICATION_CATALOGER_SCOPE env var scope: "squashed" +# cataloging file contents is exposed through the power-user subcommand +file-contents: + cataloger: + # enable/disable cataloging of secrets + # SYFT_FILE_CONTENTS_CATALOGER_ENABLED env var + enabled: true + + # the search space to look for secrets (options: all-layers, squashed) + # SYFT_FILE_CONTENTS_CATALOGER_SCOPE env var + scope: "squashed" + + # skip searching a file entirely if it is above the given size (default = 1MB; unit = bytes) + # SYFT_FILE_CONTENTS_SKIP_FILES_ABOVE_SIZE env var + skip-files-above-size: 1048576 + + # file globs for the cataloger to match on + # SYFT_FILE_CONTENTS_GLOBS env var + globs: [] + # cataloging file metadata is exposed through the power-user subcommand file-metadata: cataloger: @@ -149,9 +168,9 @@ secrets: # SYFT_SECRETS_REVEAL_VALUES env var reveal-values: false - # skip searching a file entirely if it is above the given size (default = 10MB; unit = bytes) + # skip searching a file entirely if it is above the given size (default = 1MB; unit = bytes) # SYFT_SECRETS_SKIP_FILES_ABOVE_SIZE env var - skip-files-above-size: 10485760 + skip-files-above-size: 1048576 # name-regex pairs to consider when searching files for secrets. Note: the regex must match single line patterns # but may also have OPTIONAL multiline capture groups. Regexes with a named capture group of "value" will diff --git a/cmd/power_user_tasks.go b/cmd/power_user_tasks.go index feacb7174..df543b9c9 100644 --- a/cmd/power_user_tasks.go +++ b/cmd/power_user_tasks.go @@ -21,6 +21,7 @@ func powerUserTasks() ([]powerUserTask, error) { catalogFileDigestsTask, catalogSecretsTask, catalogFileClassificationsTask, + catalogContentsTask, } for _, generator := range generators { @@ -185,3 +186,30 @@ func catalogFileClassificationsTask() (powerUserTask, error) { return task, nil } + +func catalogContentsTask() (powerUserTask, error) { + if !appConfig.FileContents.Cataloger.Enabled { + return nil, nil + } + + contentsCataloger, err := file.NewContentsCataloger(appConfig.FileContents.Globs, appConfig.FileContents.SkipFilesAboveSize) + if err != nil { + return nil, err + } + + task := func(results *poweruser.JSONDocumentConfig, src source.Source) error { + resolver, err := src.FileResolver(appConfig.FileContents.Cataloger.ScopeOpt) + if err != nil { + return err + } + + result, err := contentsCataloger.Catalog(resolver) + if err != nil { + return err + } + results.FileContents = result + return nil + } + + return task, nil +} diff --git a/internal/config/application.go b/internal/config/application.go index 3c0f2f112..9e0d58b0b 100644 --- a/internal/config/application.go +++ b/internal/config/application.go @@ -38,6 +38,7 @@ type Application struct { Package packages `yaml:"package" json:"package" mapstructure:"package"` FileMetadata FileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"` FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"` + FileContents fileContents `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"` Secrets secrets `yaml:"secrets" json:"secrets" mapstructure:"secrets"` } diff --git a/internal/config/file_contents.go b/internal/config/file_contents.go new file mode 100644 index 000000000..3b3eb4ca9 --- /dev/null +++ b/internal/config/file_contents.go @@ -0,0 +1,24 @@ +package config + +import ( + "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/syft/source" + "github.com/spf13/viper" +) + +type fileContents struct { + Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"` + SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"` + Globs []string `yaml:"globs" json:"globs" mapstructure:"globs"` +} + +func (cfg fileContents) loadDefaultValues(v *viper.Viper) { + v.SetDefault("file-contents.cataloger.enabled", true) + v.SetDefault("file-contents.cataloger.scope", source.SquashedScope) + v.SetDefault("file-contents.skip-files-above-size", 1*file.MB) + v.SetDefault("file-contents.globs", []string{}) +} + +func (cfg *fileContents) parseConfigValues() error { + return cfg.Cataloger.parseConfigValues() +} diff --git a/internal/presenter/poweruser/json_document.go b/internal/presenter/poweruser/json_document.go index b4e930307..d468bfaf5 100644 --- a/internal/presenter/poweruser/json_document.go +++ b/internal/presenter/poweruser/json_document.go @@ -10,6 +10,7 @@ type JSONDocument struct { // require these fields. As an accepted rule in this repo all collections should still be initialized in the // context of being used in a JSON document. FileClassifications []JSONFileClassifications `json:"fileClassifications,omitempty"` // note: must have omitempty + FileContents []JSONFileContents `json:"fileContents,omitempty"` // note: must have omitempty FileMetadata []JSONFileMetadata `json:"fileMetadata,omitempty"` // note: must have omitempty Secrets []JSONSecrets `json:"secrets,omitempty"` // note: must have omitempty packages.JSONDocument @@ -29,6 +30,7 @@ func NewJSONDocument(config JSONDocumentConfig) (JSONDocument, error) { return JSONDocument{ FileClassifications: NewJSONFileClassifications(config.FileClassifications), + FileContents: NewJSONFileContents(config.FileContents), FileMetadata: fileMetadata, Secrets: NewJSONSecrets(config.Secrets), JSONDocument: pkgsDoc, diff --git a/internal/presenter/poweruser/json_document_config.go b/internal/presenter/poweruser/json_document_config.go index 7d9cdffc3..20db5c759 100644 --- a/internal/presenter/poweruser/json_document_config.go +++ b/internal/presenter/poweruser/json_document_config.go @@ -14,6 +14,7 @@ type JSONDocumentConfig struct { FileMetadata map[source.Location]source.FileMetadata FileDigests map[source.Location][]file.Digest FileClassifications map[source.Location][]file.Classification + FileContents map[source.Location]string Secrets map[source.Location][]file.SearchResult Distro *distro.Distro SourceMetadata source.Metadata diff --git a/internal/presenter/poweruser/json_file_contents.go b/internal/presenter/poweruser/json_file_contents.go new file mode 100644 index 000000000..3105a9507 --- /dev/null +++ b/internal/presenter/poweruser/json_file_contents.go @@ -0,0 +1,31 @@ +package poweruser + +import ( + "sort" + + "github.com/anchore/syft/syft/source" +) + +type JSONFileContents struct { + Location source.Location `json:"location"` + Contents string `json:"contents"` +} + +func NewJSONFileContents(data map[source.Location]string) []JSONFileContents { + results := make([]JSONFileContents, 0) + for location, contents := range data { + results = append(results, JSONFileContents{ + Location: location, + Contents: contents, + }) + } + + // sort by real path then virtual path to ensure the result is stable across multiple runs + sort.SliceStable(results, func(i, j int) bool { + if results[i].Location.RealPath == results[j].Location.RealPath { + return results[i].Location.VirtualPath < results[j].Location.VirtualPath + } + return results[i].Location.RealPath < results[j].Location.RealPath + }) + return results +} diff --git a/internal/presenter/poweruser/json_presenter_test.go b/internal/presenter/poweruser/json_presenter_test.go index 97724da69..a8928bd0f 100644 --- a/internal/presenter/poweruser/json_presenter_test.go +++ b/internal/presenter/poweruser/json_presenter_test.go @@ -125,6 +125,9 @@ func TestJSONPresenter(t *testing.T) { }, }, }, + FileContents: map[source.Location]string{ + source.NewLocation("/a/place/a"): "the-contents", + }, Distro: &distro.Distro{ Type: distro.RedHat, RawVersion: "7", diff --git a/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden b/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden index d44b5a9a8..0e92c7967 100644 --- a/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden +++ b/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden @@ -1,4 +1,12 @@ { + "fileContents": [ + { + "location": { + "path": "/a/place/a" + }, + "contents": "the-contents" + } + ], "fileMetadata": [ { "location": { @@ -198,6 +206,14 @@ "scope": "" } }, + "file-contents": { + "cataloger": { + "enabled": false, + "scope": "" + }, + "skip-files-above-size": 0, + "globs": null + }, "secrets": { "cataloger": { "enabled": false, diff --git a/schema/json/schema-1.0.6.json b/schema/json/schema-1.0.6.json deleted file mode 100644 index e269d1e57..000000000 --- a/schema/json/schema-1.0.6.json +++ /dev/null @@ -1,935 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Document", - "definitions": { - "ApkFileRecord": { - "required": [ - "path" - ], - "properties": { - "path": { - "type": "string" - }, - "ownerUid": { - "type": "string" - }, - "ownerGid": { - "type": "string" - }, - "permissions": { - "type": "string" - }, - "checksum": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "ApkMetadata": { - "required": [ - "package", - "originPackage", - "maintainer", - "version", - "license", - "architecture", - "url", - "description", - "size", - "installedSize", - "pullDependencies", - "pullChecksum", - "gitCommitOfApkPort", - "files" - ], - "properties": { - "package": { - "type": "string" - }, - "originPackage": { - "type": "string" - }, - "maintainer": { - "type": "string" - }, - "version": { - "type": "string" - }, - "license": { - "type": "string" - }, - "architecture": { - "type": "string" - }, - "url": { - "type": "string" - }, - "description": { - "type": "string" - }, - "size": { - "type": "integer" - }, - "installedSize": { - "type": "integer" - }, - "pullDependencies": { - "type": "string" - }, - "pullChecksum": { - "type": "string" - }, - "gitCommitOfApkPort": { - "type": "string" - }, - "files": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/ApkFileRecord" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "CargoPackageMetadata": { - "required": [ - "name", - "version", - "source", - "checksum", - "dependencies" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "source": { - "type": "string" - }, - "checksum": { - "type": "string" - }, - "dependencies": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Classification": { - "required": [ - "class", - "metadata" - ], - "properties": { - "class": { - "type": "string" - }, - "metadata": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Descriptor": { - "required": [ - "name", - "version" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "configuration": { - "additionalProperties": true - } - }, - "additionalProperties": true, - "type": "object" - }, - "Digest": { - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Distribution": { - "required": [ - "name", - "version", - "idLike" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "idLike": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Document": { - "required": [ - "artifacts", - "artifactRelationships", - "source", - "distro", - "descriptor", - "schema" - ], - "properties": { - "fileClassifications": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/FileClassifications" - }, - "type": "array" - }, - "fileMetadata": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/FileMetadata" - }, - "type": "array" - }, - "secrets": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Secrets" - }, - "type": "array" - }, - "artifacts": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Package" - }, - "type": "array" - }, - "artifactRelationships": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Relationship" - }, - "type": "array" - }, - "source": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Source" - }, - "distro": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Distribution" - }, - "descriptor": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Descriptor" - }, - "schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Schema" - }, - "artifacts.metadata": { - "anyOf": [ - { - "type": "null" - }, - { - "$ref": "#/definitions/ApkMetadata" - }, - { - "$ref": "#/definitions/CargoPackageMetadata" - }, - { - "$ref": "#/definitions/DpkgMetadata" - }, - { - "$ref": "#/definitions/GemMetadata" - }, - { - "$ref": "#/definitions/JavaMetadata" - }, - { - "$ref": "#/definitions/NpmPackageJSONMetadata" - }, - { - "$ref": "#/definitions/PythonPackageMetadata" - }, - { - "$ref": "#/definitions/RpmdbMetadata" - } - ] - } - }, - "additionalProperties": true, - "type": "object" - }, - "DpkgFileRecord": { - "required": [ - "path", - "md5" - ], - "properties": { - "path": { - "type": "string" - }, - "md5": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "DpkgMetadata": { - "required": [ - "package", - "source", - "version", - "sourceVersion", - "architecture", - "maintainer", - "installedSize", - "files" - ], - "properties": { - "package": { - "type": "string" - }, - "source": { - "type": "string" - }, - "version": { - "type": "string" - }, - "sourceVersion": { - "type": "string" - }, - "architecture": { - "type": "string" - }, - "maintainer": { - "type": "string" - }, - "installedSize": { - "type": "integer" - }, - "files": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/DpkgFileRecord" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "FileClassifications": { - "required": [ - "location", - "classification" - ], - "properties": { - "location": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Location" - }, - "classification": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Classification" - } - }, - "additionalProperties": true, - "type": "object" - }, - "FileMetadata": { - "required": [ - "location", - "metadata" - ], - "properties": { - "location": { - "$ref": "#/definitions/Location" - }, - "metadata": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/FileMetadataEntry" - } - }, - "additionalProperties": true, - "type": "object" - }, - "FileMetadataEntry": { - "required": [ - "mode", - "type", - "userID", - "groupID" - ], - "properties": { - "mode": { - "type": "integer" - }, - "type": { - "type": "string" - }, - "linkDestination": { - "type": "string" - }, - "userID": { - "type": "integer" - }, - "groupID": { - "type": "integer" - }, - "digests": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Digest" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "GemMetadata": { - "required": [ - "name", - "version" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "files": { - "items": { - "type": "string" - }, - "type": "array" - }, - "authors": { - "items": { - "type": "string" - }, - "type": "array" - }, - "licenses": { - "items": { - "type": "string" - }, - "type": "array" - }, - "homepage": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "JavaManifest": { - "properties": { - "main": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object" - }, - "namedSections": { - "patternProperties": { - ".*": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object" - } - }, - "type": "object" - } - }, - "additionalProperties": true, - "type": "object" - }, - "JavaMetadata": { - "required": [ - "virtualPath" - ], - "properties": { - "virtualPath": { - "type": "string" - }, - "manifest": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/JavaManifest" - }, - "pomProperties": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/PomProperties" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Location": { - "required": [ - "path" - ], - "properties": { - "path": { - "type": "string" - }, - "layerID": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "NpmPackageJSONMetadata": { - "required": [ - "author", - "licenses", - "homepage", - "description", - "url" - ], - "properties": { - "files": { - "items": { - "type": "string" - }, - "type": "array" - }, - "author": { - "type": "string" - }, - "licenses": { - "items": { - "type": "string" - }, - "type": "array" - }, - "homepage": { - "type": "string" - }, - "description": { - "type": "string" - }, - "url": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Package": { - "required": [ - "id", - "name", - "version", - "type", - "foundBy", - "locations", - "licenses", - "language", - "cpes", - "purl", - "metadataType", - "metadata" - ], - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "type": { - "type": "string" - }, - "foundBy": { - "type": "string" - }, - "locations": { - "items": { - "$ref": "#/definitions/Location" - }, - "type": "array" - }, - "licenses": { - "items": { - "type": "string" - }, - "type": "array" - }, - "language": { - "type": "string" - }, - "cpes": { - "items": { - "type": "string" - }, - "type": "array" - }, - "purl": { - "type": "string" - }, - "metadataType": { - "type": "string" - }, - "metadata": { - "additionalProperties": true - } - }, - "additionalProperties": true, - "type": "object" - }, - "PomProperties": { - "required": [ - "path", - "name", - "groupId", - "artifactId", - "version", - "extraFields" - ], - "properties": { - "path": { - "type": "string" - }, - "name": { - "type": "string" - }, - "groupId": { - "type": "string" - }, - "artifactId": { - "type": "string" - }, - "version": { - "type": "string" - }, - "extraFields": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object" - } - }, - "additionalProperties": true, - "type": "object" - }, - "PythonFileDigest": { - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "PythonFileRecord": { - "required": [ - "path" - ], - "properties": { - "path": { - "type": "string" - }, - "digest": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/PythonFileDigest" - }, - "size": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "PythonPackageMetadata": { - "required": [ - "name", - "version", - "license", - "author", - "authorEmail", - "platform", - "sitePackagesRootPath" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "license": { - "type": "string" - }, - "author": { - "type": "string" - }, - "authorEmail": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "files": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/PythonFileRecord" - }, - "type": "array" - }, - "sitePackagesRootPath": { - "type": "string" - }, - "topLevelPackages": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Relationship": { - "required": [ - "parent", - "child", - "type", - "metadata" - ], - "properties": { - "parent": { - "type": "string" - }, - "child": { - "type": "string" - }, - "type": { - "type": "string" - }, - "metadata": { - "additionalProperties": true - } - }, - "additionalProperties": true, - "type": "object" - }, - "RpmdbFileRecord": { - "required": [ - "path", - "mode", - "size", - "sha256" - ], - "properties": { - "path": { - "type": "string" - }, - "mode": { - "type": "integer" - }, - "size": { - "type": "integer" - }, - "sha256": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "RpmdbMetadata": { - "required": [ - "name", - "version", - "epoch", - "architecture", - "release", - "sourceRpm", - "size", - "license", - "vendor", - "files" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "epoch": { - "type": "integer" - }, - "architecture": { - "type": "string" - }, - "release": { - "type": "string" - }, - "sourceRpm": { - "type": "string" - }, - "size": { - "type": "integer" - }, - "license": { - "type": "string" - }, - "vendor": { - "type": "string" - }, - "files": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/RpmdbFileRecord" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Schema": { - "required": [ - "version", - "url" - ], - "properties": { - "version": { - "type": "string" - }, - "url": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "SearchResult": { - "required": [ - "classification", - "lineNumber", - "lineOffset", - "seekPosition", - "length" - ], - "properties": { - "classification": { - "type": "string" - }, - "lineNumber": { - "type": "integer" - }, - "lineOffset": { - "type": "integer" - }, - "seekPosition": { - "type": "integer" - }, - "length": { - "type": "integer" - }, - "value": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Secrets": { - "required": [ - "location", - "secrets" - ], - "properties": { - "location": { - "$ref": "#/definitions/Location" - }, - "secrets": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/SearchResult" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Source": { - "required": [ - "type", - "target" - ], - "properties": { - "type": { - "type": "string" - }, - "target": { - "additionalProperties": true - } - }, - "additionalProperties": true, - "type": "object" - } - } -} diff --git a/schema/json/schema-1.1.0.json b/schema/json/schema-1.1.0.json index 29af14cd4..687a81c19 100644 --- a/schema/json/schema-1.1.0.json +++ b/schema/json/schema-1.1.0.json @@ -125,6 +125,27 @@ "additionalProperties": true, "type": "object" }, + "Classification": { + "required": [ + "class", + "metadata" + ], + "properties": { + "class": { + "type": "string" + }, + "metadata": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "additionalProperties": true, + "type": "object" + }, "Descriptor": { "required": [ "name", @@ -190,6 +211,20 @@ "schema" ], "properties": { + "fileClassifications": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/FileClassifications" + }, + "type": "array" + }, + "fileContents": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/FileContents" + }, + "type": "array" + }, "fileMetadata": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", @@ -302,6 +337,40 @@ "additionalProperties": true, "type": "object" }, + "FileClassifications": { + "required": [ + "location", + "classification" + ], + "properties": { + "location": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Location" + }, + "classification": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Classification" + } + }, + "additionalProperties": true, + "type": "object" + }, + "FileContents": { + "required": [ + "location", + "contents" + ], + "properties": { + "location": { + "$ref": "#/definitions/Location" + }, + "contents": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, "FileMetadata": { "required": [ "location", @@ -309,7 +378,6 @@ ], "properties": { "location": { - "$schema": "http://json-schema.org/draft-04/schema#", "$ref": "#/definitions/Location" }, "metadata": { diff --git a/syft/file/contents_cataloger.go b/syft/file/contents_cataloger.go new file mode 100644 index 000000000..97652868c --- /dev/null +++ b/syft/file/contents_cataloger.go @@ -0,0 +1,68 @@ +package file + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/source" +) + +type ContentsCataloger struct { + globs []string + skipFilesAboveSizeInBytes int64 +} + +func NewContentsCataloger(globs []string, skipFilesAboveSize int64) (*ContentsCataloger, error) { + return &ContentsCataloger{ + globs: globs, + skipFilesAboveSizeInBytes: skipFilesAboveSize, + }, nil +} + +func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Location]string, error) { + results := make(map[source.Location]string) + var locations []source.Location + + locations, err := resolver.FilesByGlob(i.globs...) + if err != nil { + return nil, err + } + + for _, location := range locations { + metadata, err := resolver.FileMetadataByLocation(location) + if err != nil { + return nil, err + } + + if i.skipFilesAboveSizeInBytes > 0 && metadata.Size > i.skipFilesAboveSizeInBytes { + continue + } + + result, err := i.catalogLocation(resolver, location) + if err != nil { + return nil, err + } + results[location] = result + } + log.Debugf("file contents cataloger processed %d files", len(results)) + + return results, nil +} + +func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) (string, error) { + contentReader, err := resolver.FileContentsByLocation(location) + if err != nil { + return "", err + } + defer contentReader.Close() + + buf := &bytes.Buffer{} + if _, err = io.Copy(base64.NewEncoder(base64.StdEncoding, buf), contentReader); err != nil { + return "", fmt.Errorf("unable to observe contents of %+v: %w", location.RealPath, err) + } + + return buf.String(), nil +} diff --git a/syft/file/contents_cataloger_test.go b/syft/file/contents_cataloger_test.go new file mode 100644 index 000000000..c6c1b0db9 --- /dev/null +++ b/syft/file/contents_cataloger_test.go @@ -0,0 +1,79 @@ +package file + +import ( + "testing" + + "github.com/anchore/syft/syft/source" + "github.com/stretchr/testify/assert" +) + +func TestContentsCataloger(t *testing.T) { + allFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"} + + tests := []struct { + name string + globs []string + maxSize int64 + files []string + expected map[source.Location]string + }{ + { + name: "multi-pattern", + globs: []string{"test-fixtures/last/*.txt", "test-fixtures/*.txt"}, + files: allFiles, + expected: map[source.Location]string{ + source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + }, + }, + { + name: "no-patterns", + globs: []string{}, + files: []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}, + expected: map[source.Location]string{}, + }, + { + name: "all-txt", + globs: []string{"**/*.txt"}, + files: allFiles, + expected: map[source.Location]string{ + source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + }, + }, + { + name: "subpath", + globs: []string{"test-fixtures/*.txt"}, + files: allFiles, + expected: map[source.Location]string{ + source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + }, + }, + { + name: "size-filter", + maxSize: 42, + globs: []string{"**/*.txt"}, + files: allFiles, + expected: map[source.Location]string{ + source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + c, err := NewContentsCataloger(test.globs, test.maxSize) + assert.NoError(t, err) + + resolver := source.NewMockResolverForPaths(test.files...) + actual, err := c.Catalog(resolver) + assert.NoError(t, err) + assert.Equal(t, test.expected, actual, "mismatched contents") + + }) + } +} diff --git a/syft/pkg/cataloger/common/generic_cataloger_test.go b/syft/pkg/cataloger/common/generic_cataloger_test.go index 8c1f4215c..a30a3d5f7 100644 --- a/syft/pkg/cataloger/common/generic_cataloger_test.go +++ b/syft/pkg/cataloger/common/generic_cataloger_test.go @@ -6,6 +6,8 @@ import ( "io/ioutil" "testing" + "github.com/stretchr/testify/assert" + "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -25,7 +27,7 @@ func parser(_ string, reader io.Reader) ([]pkg.Package, error) { func TestGenericCataloger(t *testing.T) { globParsers := map[string]ParserFn{ - "**a-path.txt": parser, + "**/a-path.txt": parser, } pathParsers := map[string]ParserFn{ "test-fixtures/another-path.txt": parser, @@ -46,13 +48,8 @@ func TestGenericCataloger(t *testing.T) { } actualPkgs, err := cataloger.Catalog(resolver) - if err != nil { - t.Fatalf("cataloger catalog action failed: %+v", err) - } - - if len(actualPkgs) != len(expectedPkgs) { - t.Fatalf("unexpected packages len: %d", len(actualPkgs)) - } + assert.NoError(t, err) + assert.Len(t, actualPkgs, len(expectedPkgs)) for _, p := range actualPkgs { ref := p.Locations[0] diff --git a/syft/source/mock_resolver.go b/syft/source/mock_resolver.go index d97278722..9bd569475 100644 --- a/syft/source/mock_resolver.go +++ b/syft/source/mock_resolver.go @@ -5,7 +5,7 @@ import ( "io" "os" - "github.com/anchore/syft/internal/file" + "github.com/bmatcuk/doublestar/v2" ) var _ FileResolver = (*MockResolver)(nil) @@ -84,7 +84,11 @@ func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) { var results []Location for _, pattern := range patterns { for _, location := range r.Locations { - if file.GlobMatch(pattern, location.RealPath) { + matches, err := doublestar.Match(pattern, location.RealPath) + if err != nil { + return nil, err + } + if matches { results = append(results, location) } } diff --git a/test/cli/power_user_cmd_test.go b/test/cli/power_user_cmd_test.go index 3246c20c4..2f7dcbbe9 100644 --- a/test/cli/power_user_cmd_test.go +++ b/test/cli/power_user_cmd_test.go @@ -51,6 +51,17 @@ func TestPowerUserCmdFlags(t *testing.T) { assertSuccessfulReturnCode, }, }, + { + name: "content-cataloger-wired-up", + args: []string{"power-user", "docker-archive:" + getFixtureImage(t, "image-secrets")}, + env: map[string]string{ + "SYFT_FILE_CONTENTS_GLOBS": "/api-key.txt", + }, + assertions: []traitAssertion{ + assertInOutput(`"contents": "c29tZV9BcEkta0V5ID0gIjEyMzQ1QTdhOTAxYjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MCIK"`), // proof of the content cataloger + assertSuccessfulReturnCode, + }, + }, } for _, test := range tests {