From c1551a03c5d8a778d5b74883ba183a4fc63c7f1f Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 6 Apr 2021 16:41:43 -0400 Subject: [PATCH 1/6] add file contents cataloger Signed-off-by: Alex Goodman --- cmd/power_user_tasks.go | 28 ++++++ internal/config/application.go | 1 + internal/config/contents.go | 24 +++++ internal/presenter/poweruser/json_document.go | 2 + .../poweruser/json_document_config.go | 1 + .../presenter/poweruser/json_file_contents.go | 31 +++++++ syft/file/contents_cataloger.go | 68 ++++++++++++++ syft/file/contents_cataloger_test.go | 89 +++++++++++++++++++ syft/source/mock_resolver.go | 8 +- test/cli/power_user_cmd_test.go | 11 +++ 10 files changed, 261 insertions(+), 2 deletions(-) create mode 100644 internal/config/contents.go create mode 100644 internal/presenter/poweruser/json_file_contents.go create mode 100644 syft/file/contents_cataloger.go create mode 100644 syft/file/contents_cataloger_test.go diff --git a/cmd/power_user_tasks.go b/cmd/power_user_tasks.go index feacb7174..da7cc97ea 100644 --- a/cmd/power_user_tasks.go +++ b/cmd/power_user_tasks.go @@ -21,6 +21,7 @@ func powerUserTasks() ([]powerUserTask, error) { catalogFileDigestsTask, catalogSecretsTask, catalogFileClassificationsTask, + catalogContentsTask, } for _, generator := range generators { @@ -185,3 +186,30 @@ func catalogFileClassificationsTask() (powerUserTask, error) { return task, nil } + +func catalogContentsTask() (powerUserTask, error) { + if !appConfig.Contents.Cataloger.Enabled { + return nil, nil + } + + contentsCataloger, err := file.NewContentsCataloger(appConfig.Contents.Globs, appConfig.Contents.SkipFilesAboveSize) + if err != nil { + return nil, err + } + + task := func(results *poweruser.JSONDocumentConfig, src source.Source) error { + resolver, err := src.FileResolver(appConfig.Contents.Cataloger.ScopeOpt) + if err != nil { + return err + } + + result, err := contentsCataloger.Catalog(resolver) + if err != nil { + return err + } + results.FileContents = result + return nil + } + + return task, nil +} diff --git a/internal/config/application.go b/internal/config/application.go index 3c0f2f112..c78b21abf 100644 --- a/internal/config/application.go +++ b/internal/config/application.go @@ -38,6 +38,7 @@ type Application struct { Package packages `yaml:"package" json:"package" mapstructure:"package"` FileMetadata FileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"` FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"` + Contents contents `yaml:"contents" json:"contents" mapstructure:"contents"` Secrets secrets `yaml:"secrets" json:"secrets" mapstructure:"secrets"` } diff --git a/internal/config/contents.go b/internal/config/contents.go new file mode 100644 index 000000000..5c660013c --- /dev/null +++ b/internal/config/contents.go @@ -0,0 +1,24 @@ +package config + +import ( + "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/syft/source" + "github.com/spf13/viper" +) + +type contents struct { + Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"` + SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"` + Globs []string `yaml:"globs" json:"globs" mapstructure:"globs"` +} + +func (cfg contents) loadDefaultValues(v *viper.Viper) { + v.SetDefault("contents.cataloger.enabled", true) + v.SetDefault("contents.cataloger.scope", source.SquashedScope) + v.SetDefault("contents.skip-files-above-size", 1*file.MB) + v.SetDefault("contents.globs", []string{}) +} + +func (cfg *contents) parseConfigValues() error { + return cfg.Cataloger.parseConfigValues() +} diff --git a/internal/presenter/poweruser/json_document.go b/internal/presenter/poweruser/json_document.go index b4e930307..d468bfaf5 100644 --- a/internal/presenter/poweruser/json_document.go +++ b/internal/presenter/poweruser/json_document.go @@ -10,6 +10,7 @@ type JSONDocument struct { // require these fields. As an accepted rule in this repo all collections should still be initialized in the // context of being used in a JSON document. FileClassifications []JSONFileClassifications `json:"fileClassifications,omitempty"` // note: must have omitempty + FileContents []JSONFileContents `json:"fileContents,omitempty"` // note: must have omitempty FileMetadata []JSONFileMetadata `json:"fileMetadata,omitempty"` // note: must have omitempty Secrets []JSONSecrets `json:"secrets,omitempty"` // note: must have omitempty packages.JSONDocument @@ -29,6 +30,7 @@ func NewJSONDocument(config JSONDocumentConfig) (JSONDocument, error) { return JSONDocument{ FileClassifications: NewJSONFileClassifications(config.FileClassifications), + FileContents: NewJSONFileContents(config.FileContents), FileMetadata: fileMetadata, Secrets: NewJSONSecrets(config.Secrets), JSONDocument: pkgsDoc, diff --git a/internal/presenter/poweruser/json_document_config.go b/internal/presenter/poweruser/json_document_config.go index 7d9cdffc3..20db5c759 100644 --- a/internal/presenter/poweruser/json_document_config.go +++ b/internal/presenter/poweruser/json_document_config.go @@ -14,6 +14,7 @@ type JSONDocumentConfig struct { FileMetadata map[source.Location]source.FileMetadata FileDigests map[source.Location][]file.Digest FileClassifications map[source.Location][]file.Classification + FileContents map[source.Location]string Secrets map[source.Location][]file.SearchResult Distro *distro.Distro SourceMetadata source.Metadata diff --git a/internal/presenter/poweruser/json_file_contents.go b/internal/presenter/poweruser/json_file_contents.go new file mode 100644 index 000000000..3105a9507 --- /dev/null +++ b/internal/presenter/poweruser/json_file_contents.go @@ -0,0 +1,31 @@ +package poweruser + +import ( + "sort" + + "github.com/anchore/syft/syft/source" +) + +type JSONFileContents struct { + Location source.Location `json:"location"` + Contents string `json:"contents"` +} + +func NewJSONFileContents(data map[source.Location]string) []JSONFileContents { + results := make([]JSONFileContents, 0) + for location, contents := range data { + results = append(results, JSONFileContents{ + Location: location, + Contents: contents, + }) + } + + // sort by real path then virtual path to ensure the result is stable across multiple runs + sort.SliceStable(results, func(i, j int) bool { + if results[i].Location.RealPath == results[j].Location.RealPath { + return results[i].Location.VirtualPath < results[j].Location.VirtualPath + } + return results[i].Location.RealPath < results[j].Location.RealPath + }) + return results +} diff --git a/syft/file/contents_cataloger.go b/syft/file/contents_cataloger.go new file mode 100644 index 000000000..2a67b4c2e --- /dev/null +++ b/syft/file/contents_cataloger.go @@ -0,0 +1,68 @@ +package file + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/source" +) + +type ContentsCataloger struct { + globs []string + skipFilesAboveSize int64 +} + +func NewContentsCataloger(globs []string, skipFilesAboveSize int64) (*ContentsCataloger, error) { + return &ContentsCataloger{ + globs: globs, + skipFilesAboveSize: skipFilesAboveSize, + }, nil +} + +func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Location]string, error) { + results := make(map[source.Location]string) + var locations []source.Location + + locations, err := resolver.FilesByGlob(i.globs...) + if err != nil { + return nil, err + } + + for _, location := range locations { + metadata, err := resolver.FileMetadataByLocation(location) + if err != nil { + return nil, err + } + + if i.skipFilesAboveSize > 0 && metadata.Size > i.skipFilesAboveSize { + continue + } + + result, err := i.catalogLocation(resolver, location) + if err != nil { + return nil, err + } + results[location] = result + } + log.Debugf("file contents cataloger processed %d files", len(results)) + + return results, nil +} + +func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) (string, error) { + contentReader, err := resolver.FileContentsByLocation(location) + if err != nil { + return "", err + } + defer contentReader.Close() + + buf := &bytes.Buffer{} + if _, err = io.Copy(base64.NewEncoder(base64.StdEncoding, buf), contentReader); err != nil { + return "", fmt.Errorf("unable to observe contents of %+v: %+v", location.RealPath, err) + } + + return buf.String(), nil +} diff --git a/syft/file/contents_cataloger_test.go b/syft/file/contents_cataloger_test.go new file mode 100644 index 000000000..eb83d21aa --- /dev/null +++ b/syft/file/contents_cataloger_test.go @@ -0,0 +1,89 @@ +package file + +import ( + "testing" + + "github.com/anchore/syft/syft/source" + "github.com/stretchr/testify/assert" +) + +func TestContentsCataloger(t *testing.T) { + allFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"} + + tests := []struct { + name string + globs []string + maxSize int64 + files []string + expected map[source.Location]string + catalogErr bool + }{ + { + name: "multi-pattern", + globs: []string{"test-fixtures/last/*.txt", "test-fixtures/*.txt"}, + files: allFiles, + expected: map[source.Location]string{ + source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + }, + }, + { + name: "no-patterns", + globs: []string{}, + files: []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}, + expected: map[source.Location]string{}, + }, + { + name: "all-txt", + globs: []string{"**/*.txt"}, + files: allFiles, + expected: map[source.Location]string{ + source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + }, + }, + { + name: "subpath", + globs: []string{"test-fixtures/*.txt"}, + files: allFiles, + expected: map[source.Location]string{ + source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + }, + }, + { + name: "size-filter", + maxSize: 42, + globs: []string{"**/*.txt"}, + files: allFiles, + expected: map[source.Location]string{ + source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh", + source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + c, err := NewContentsCataloger(test.globs, test.maxSize) + if err != nil { + t.Fatalf("could not create cataloger: %+v", err) + } + + resolver := source.NewMockResolverForPaths(test.files...) + actual, err := c.Catalog(resolver) + if err != nil && !test.catalogErr { + t.Fatalf("could not catalog (but should have been able to): %+v", err) + } else if err == nil && test.catalogErr { + t.Fatalf("expected catalog error but did not get one") + } else if test.catalogErr && err != nil { + return + } + + assert.Equal(t, test.expected, actual, "mismatched contents") + + }) + } +} diff --git a/syft/source/mock_resolver.go b/syft/source/mock_resolver.go index d97278722..9bd569475 100644 --- a/syft/source/mock_resolver.go +++ b/syft/source/mock_resolver.go @@ -5,7 +5,7 @@ import ( "io" "os" - "github.com/anchore/syft/internal/file" + "github.com/bmatcuk/doublestar/v2" ) var _ FileResolver = (*MockResolver)(nil) @@ -84,7 +84,11 @@ func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) { var results []Location for _, pattern := range patterns { for _, location := range r.Locations { - if file.GlobMatch(pattern, location.RealPath) { + matches, err := doublestar.Match(pattern, location.RealPath) + if err != nil { + return nil, err + } + if matches { results = append(results, location) } } diff --git a/test/cli/power_user_cmd_test.go b/test/cli/power_user_cmd_test.go index 3246c20c4..fb7c8cf48 100644 --- a/test/cli/power_user_cmd_test.go +++ b/test/cli/power_user_cmd_test.go @@ -51,6 +51,17 @@ func TestPowerUserCmdFlags(t *testing.T) { assertSuccessfulReturnCode, }, }, + { + name: "content-cataloger-wired-up", + args: []string{"power-user", "docker-archive:" + getFixtureImage(t, "image-secrets")}, + env: map[string]string{ + "SYFT_CONTENTS_GLOBS": "/api-key.txt", + }, + assertions: []traitAssertion{ + assertInOutput(`"contents": "c29tZV9BcEkta0V5ID0gIjEyMzQ1QTdhOTAxYjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MCIK"`), // proof of the content cataloger + assertSuccessfulReturnCode, + }, + }, } for _, test := range tests { From d451a5ad3092c7679c7b497d2056aab94efd6832 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 6 Apr 2021 16:46:15 -0400 Subject: [PATCH 2/6] update the json schema and tests with file contents section Signed-off-by: Alex Goodman --- .../poweruser/json_presenter_test.go | 3 + .../snapshot/TestJSONPresenter.golden | 16 + schema/json/schema-1.0.6.json | 935 ------------------ schema/json/schema-1.1.0.json | 70 +- .../common/generic_cataloger_test.go | 4 +- 5 files changed, 90 insertions(+), 938 deletions(-) delete mode 100644 schema/json/schema-1.0.6.json diff --git a/internal/presenter/poweruser/json_presenter_test.go b/internal/presenter/poweruser/json_presenter_test.go index 97724da69..a8928bd0f 100644 --- a/internal/presenter/poweruser/json_presenter_test.go +++ b/internal/presenter/poweruser/json_presenter_test.go @@ -125,6 +125,9 @@ func TestJSONPresenter(t *testing.T) { }, }, }, + FileContents: map[source.Location]string{ + source.NewLocation("/a/place/a"): "the-contents", + }, Distro: &distro.Distro{ Type: distro.RedHat, RawVersion: "7", diff --git a/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden b/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden index d44b5a9a8..76c0d521f 100644 --- a/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden +++ b/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden @@ -1,4 +1,12 @@ { + "fileContents": [ + { + "location": { + "path": "/a/place/a" + }, + "contents": "the-contents" + } + ], "fileMetadata": [ { "location": { @@ -198,6 +206,14 @@ "scope": "" } }, + "contents": { + "cataloger": { + "enabled": false, + "scope": "" + }, + "skip-files-above-size": 0, + "globs": null + }, "secrets": { "cataloger": { "enabled": false, diff --git a/schema/json/schema-1.0.6.json b/schema/json/schema-1.0.6.json deleted file mode 100644 index e269d1e57..000000000 --- a/schema/json/schema-1.0.6.json +++ /dev/null @@ -1,935 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Document", - "definitions": { - "ApkFileRecord": { - "required": [ - "path" - ], - "properties": { - "path": { - "type": "string" - }, - "ownerUid": { - "type": "string" - }, - "ownerGid": { - "type": "string" - }, - "permissions": { - "type": "string" - }, - "checksum": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "ApkMetadata": { - "required": [ - "package", - "originPackage", - "maintainer", - "version", - "license", - "architecture", - "url", - "description", - "size", - "installedSize", - "pullDependencies", - "pullChecksum", - "gitCommitOfApkPort", - "files" - ], - "properties": { - "package": { - "type": "string" - }, - "originPackage": { - "type": "string" - }, - "maintainer": { - "type": "string" - }, - "version": { - "type": "string" - }, - "license": { - "type": "string" - }, - "architecture": { - "type": "string" - }, - "url": { - "type": "string" - }, - "description": { - "type": "string" - }, - "size": { - "type": "integer" - }, - "installedSize": { - "type": "integer" - }, - "pullDependencies": { - "type": "string" - }, - "pullChecksum": { - "type": "string" - }, - "gitCommitOfApkPort": { - "type": "string" - }, - "files": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/ApkFileRecord" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "CargoPackageMetadata": { - "required": [ - "name", - "version", - "source", - "checksum", - "dependencies" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "source": { - "type": "string" - }, - "checksum": { - "type": "string" - }, - "dependencies": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Classification": { - "required": [ - "class", - "metadata" - ], - "properties": { - "class": { - "type": "string" - }, - "metadata": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Descriptor": { - "required": [ - "name", - "version" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "configuration": { - "additionalProperties": true - } - }, - "additionalProperties": true, - "type": "object" - }, - "Digest": { - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Distribution": { - "required": [ - "name", - "version", - "idLike" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "idLike": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Document": { - "required": [ - "artifacts", - "artifactRelationships", - "source", - "distro", - "descriptor", - "schema" - ], - "properties": { - "fileClassifications": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/FileClassifications" - }, - "type": "array" - }, - "fileMetadata": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/FileMetadata" - }, - "type": "array" - }, - "secrets": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Secrets" - }, - "type": "array" - }, - "artifacts": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Package" - }, - "type": "array" - }, - "artifactRelationships": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Relationship" - }, - "type": "array" - }, - "source": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Source" - }, - "distro": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Distribution" - }, - "descriptor": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Descriptor" - }, - "schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Schema" - }, - "artifacts.metadata": { - "anyOf": [ - { - "type": "null" - }, - { - "$ref": "#/definitions/ApkMetadata" - }, - { - "$ref": "#/definitions/CargoPackageMetadata" - }, - { - "$ref": "#/definitions/DpkgMetadata" - }, - { - "$ref": "#/definitions/GemMetadata" - }, - { - "$ref": "#/definitions/JavaMetadata" - }, - { - "$ref": "#/definitions/NpmPackageJSONMetadata" - }, - { - "$ref": "#/definitions/PythonPackageMetadata" - }, - { - "$ref": "#/definitions/RpmdbMetadata" - } - ] - } - }, - "additionalProperties": true, - "type": "object" - }, - "DpkgFileRecord": { - "required": [ - "path", - "md5" - ], - "properties": { - "path": { - "type": "string" - }, - "md5": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "DpkgMetadata": { - "required": [ - "package", - "source", - "version", - "sourceVersion", - "architecture", - "maintainer", - "installedSize", - "files" - ], - "properties": { - "package": { - "type": "string" - }, - "source": { - "type": "string" - }, - "version": { - "type": "string" - }, - "sourceVersion": { - "type": "string" - }, - "architecture": { - "type": "string" - }, - "maintainer": { - "type": "string" - }, - "installedSize": { - "type": "integer" - }, - "files": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/DpkgFileRecord" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "FileClassifications": { - "required": [ - "location", - "classification" - ], - "properties": { - "location": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Location" - }, - "classification": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Classification" - } - }, - "additionalProperties": true, - "type": "object" - }, - "FileMetadata": { - "required": [ - "location", - "metadata" - ], - "properties": { - "location": { - "$ref": "#/definitions/Location" - }, - "metadata": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/FileMetadataEntry" - } - }, - "additionalProperties": true, - "type": "object" - }, - "FileMetadataEntry": { - "required": [ - "mode", - "type", - "userID", - "groupID" - ], - "properties": { - "mode": { - "type": "integer" - }, - "type": { - "type": "string" - }, - "linkDestination": { - "type": "string" - }, - "userID": { - "type": "integer" - }, - "groupID": { - "type": "integer" - }, - "digests": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Digest" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "GemMetadata": { - "required": [ - "name", - "version" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "files": { - "items": { - "type": "string" - }, - "type": "array" - }, - "authors": { - "items": { - "type": "string" - }, - "type": "array" - }, - "licenses": { - "items": { - "type": "string" - }, - "type": "array" - }, - "homepage": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "JavaManifest": { - "properties": { - "main": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object" - }, - "namedSections": { - "patternProperties": { - ".*": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object" - } - }, - "type": "object" - } - }, - "additionalProperties": true, - "type": "object" - }, - "JavaMetadata": { - "required": [ - "virtualPath" - ], - "properties": { - "virtualPath": { - "type": "string" - }, - "manifest": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/JavaManifest" - }, - "pomProperties": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/PomProperties" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Location": { - "required": [ - "path" - ], - "properties": { - "path": { - "type": "string" - }, - "layerID": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "NpmPackageJSONMetadata": { - "required": [ - "author", - "licenses", - "homepage", - "description", - "url" - ], - "properties": { - "files": { - "items": { - "type": "string" - }, - "type": "array" - }, - "author": { - "type": "string" - }, - "licenses": { - "items": { - "type": "string" - }, - "type": "array" - }, - "homepage": { - "type": "string" - }, - "description": { - "type": "string" - }, - "url": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Package": { - "required": [ - "id", - "name", - "version", - "type", - "foundBy", - "locations", - "licenses", - "language", - "cpes", - "purl", - "metadataType", - "metadata" - ], - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "type": { - "type": "string" - }, - "foundBy": { - "type": "string" - }, - "locations": { - "items": { - "$ref": "#/definitions/Location" - }, - "type": "array" - }, - "licenses": { - "items": { - "type": "string" - }, - "type": "array" - }, - "language": { - "type": "string" - }, - "cpes": { - "items": { - "type": "string" - }, - "type": "array" - }, - "purl": { - "type": "string" - }, - "metadataType": { - "type": "string" - }, - "metadata": { - "additionalProperties": true - } - }, - "additionalProperties": true, - "type": "object" - }, - "PomProperties": { - "required": [ - "path", - "name", - "groupId", - "artifactId", - "version", - "extraFields" - ], - "properties": { - "path": { - "type": "string" - }, - "name": { - "type": "string" - }, - "groupId": { - "type": "string" - }, - "artifactId": { - "type": "string" - }, - "version": { - "type": "string" - }, - "extraFields": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object" - } - }, - "additionalProperties": true, - "type": "object" - }, - "PythonFileDigest": { - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "PythonFileRecord": { - "required": [ - "path" - ], - "properties": { - "path": { - "type": "string" - }, - "digest": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/PythonFileDigest" - }, - "size": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "PythonPackageMetadata": { - "required": [ - "name", - "version", - "license", - "author", - "authorEmail", - "platform", - "sitePackagesRootPath" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "license": { - "type": "string" - }, - "author": { - "type": "string" - }, - "authorEmail": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "files": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/PythonFileRecord" - }, - "type": "array" - }, - "sitePackagesRootPath": { - "type": "string" - }, - "topLevelPackages": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Relationship": { - "required": [ - "parent", - "child", - "type", - "metadata" - ], - "properties": { - "parent": { - "type": "string" - }, - "child": { - "type": "string" - }, - "type": { - "type": "string" - }, - "metadata": { - "additionalProperties": true - } - }, - "additionalProperties": true, - "type": "object" - }, - "RpmdbFileRecord": { - "required": [ - "path", - "mode", - "size", - "sha256" - ], - "properties": { - "path": { - "type": "string" - }, - "mode": { - "type": "integer" - }, - "size": { - "type": "integer" - }, - "sha256": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "RpmdbMetadata": { - "required": [ - "name", - "version", - "epoch", - "architecture", - "release", - "sourceRpm", - "size", - "license", - "vendor", - "files" - ], - "properties": { - "name": { - "type": "string" - }, - "version": { - "type": "string" - }, - "epoch": { - "type": "integer" - }, - "architecture": { - "type": "string" - }, - "release": { - "type": "string" - }, - "sourceRpm": { - "type": "string" - }, - "size": { - "type": "integer" - }, - "license": { - "type": "string" - }, - "vendor": { - "type": "string" - }, - "files": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/RpmdbFileRecord" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Schema": { - "required": [ - "version", - "url" - ], - "properties": { - "version": { - "type": "string" - }, - "url": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "SearchResult": { - "required": [ - "classification", - "lineNumber", - "lineOffset", - "seekPosition", - "length" - ], - "properties": { - "classification": { - "type": "string" - }, - "lineNumber": { - "type": "integer" - }, - "lineOffset": { - "type": "integer" - }, - "seekPosition": { - "type": "integer" - }, - "length": { - "type": "integer" - }, - "value": { - "type": "string" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Secrets": { - "required": [ - "location", - "secrets" - ], - "properties": { - "location": { - "$ref": "#/definitions/Location" - }, - "secrets": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/SearchResult" - }, - "type": "array" - } - }, - "additionalProperties": true, - "type": "object" - }, - "Source": { - "required": [ - "type", - "target" - ], - "properties": { - "type": { - "type": "string" - }, - "target": { - "additionalProperties": true - } - }, - "additionalProperties": true, - "type": "object" - } - } -} diff --git a/schema/json/schema-1.1.0.json b/schema/json/schema-1.1.0.json index 29af14cd4..687a81c19 100644 --- a/schema/json/schema-1.1.0.json +++ b/schema/json/schema-1.1.0.json @@ -125,6 +125,27 @@ "additionalProperties": true, "type": "object" }, + "Classification": { + "required": [ + "class", + "metadata" + ], + "properties": { + "class": { + "type": "string" + }, + "metadata": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object" + } + }, + "additionalProperties": true, + "type": "object" + }, "Descriptor": { "required": [ "name", @@ -190,6 +211,20 @@ "schema" ], "properties": { + "fileClassifications": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/FileClassifications" + }, + "type": "array" + }, + "fileContents": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/FileContents" + }, + "type": "array" + }, "fileMetadata": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", @@ -302,6 +337,40 @@ "additionalProperties": true, "type": "object" }, + "FileClassifications": { + "required": [ + "location", + "classification" + ], + "properties": { + "location": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Location" + }, + "classification": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Classification" + } + }, + "additionalProperties": true, + "type": "object" + }, + "FileContents": { + "required": [ + "location", + "contents" + ], + "properties": { + "location": { + "$ref": "#/definitions/Location" + }, + "contents": { + "type": "string" + } + }, + "additionalProperties": true, + "type": "object" + }, "FileMetadata": { "required": [ "location", @@ -309,7 +378,6 @@ ], "properties": { "location": { - "$schema": "http://json-schema.org/draft-04/schema#", "$ref": "#/definitions/Location" }, "metadata": { diff --git a/syft/pkg/cataloger/common/generic_cataloger_test.go b/syft/pkg/cataloger/common/generic_cataloger_test.go index 8c1f4215c..594436447 100644 --- a/syft/pkg/cataloger/common/generic_cataloger_test.go +++ b/syft/pkg/cataloger/common/generic_cataloger_test.go @@ -25,7 +25,7 @@ func parser(_ string, reader io.Reader) ([]pkg.Package, error) { func TestGenericCataloger(t *testing.T) { globParsers := map[string]ParserFn{ - "**a-path.txt": parser, + "**/a-path.txt": parser, } pathParsers := map[string]ParserFn{ "test-fixtures/another-path.txt": parser, @@ -51,7 +51,7 @@ func TestGenericCataloger(t *testing.T) { } if len(actualPkgs) != len(expectedPkgs) { - t.Fatalf("unexpected packages len: %d", len(actualPkgs)) + t.Fatalf("unexpected packages len: %d != %d", len(expectedPkgs), len(actualPkgs)) } for _, p := range actualPkgs { From 44915b435f3c7051bd236d92517161aa9489fa17 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Wed, 7 Apr 2021 16:28:45 -0400 Subject: [PATCH 3/6] add documentation for the file contents cataloger config options Signed-off-by: Alex Goodman --- README.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 642a1dc4d..5401f13f7 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,25 @@ file-classification: # SYFT_FILE_CLASSIFICATION_CATALOGER_SCOPE env var scope: "squashed" +# cataloging file contents is exposed through the power-user subcommand +contents: + cataloger: + # enable/disable cataloging of secrets + # SYFT_CONTENTS_CATALOGER_ENABLED env var + enabled: true + + # the search space to look for secrets (options: all-layers, squashed) + # SYFT_CONTENTS_CATALOGER_SCOPE env var + scope: "squashed" + + # skip searching a file entirely if it is above the given size (default = 1MB; unit = bytes) + # SYFT_CONTENTS_SKIP_FILES_ABOVE_SIZE env var + skip-files-above-size: 1048576 + + # file globs for the cataloger to match on + # SYFT_CONTENTS_GLOBS env var + globs: [] + # cataloging file metadata is exposed through the power-user subcommand file-metadata: cataloger: @@ -149,9 +168,9 @@ secrets: # SYFT_SECRETS_REVEAL_VALUES env var reveal-values: false - # skip searching a file entirely if it is above the given size (default = 10MB; unit = bytes) + # skip searching a file entirely if it is above the given size (default = 1MB; unit = bytes) # SYFT_SECRETS_SKIP_FILES_ABOVE_SIZE env var - skip-files-above-size: 10485760 + skip-files-above-size: 1048576 # name-regex pairs to consider when searching files for secrets. Note: the regex must match single line patterns # but may also have OPTIONAL multiline capture groups. Regexes with a named capture group of "value" will From 51b13f8221da35097e91bd1c20e4aea0dd087daa Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 9 Apr 2021 08:56:01 -0400 Subject: [PATCH 4/6] rename Contents to FileContents in app config and documentation Signed-off-by: Alex Goodman --- README.md | 10 +++++----- cmd/power_user_tasks.go | 4 ++-- internal/config/application.go | 2 +- internal/config/{contents.go => file_contents.go} | 14 +++++++------- .../snapshot/TestJSONPresenter.golden | 2 +- test/cli/power_user_cmd_test.go | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) rename internal/config/{contents.go => file_contents.go} (57%) diff --git a/README.md b/README.md index 5401f13f7..af0d17357 100644 --- a/README.md +++ b/README.md @@ -120,22 +120,22 @@ file-classification: scope: "squashed" # cataloging file contents is exposed through the power-user subcommand -contents: +file-contents: cataloger: # enable/disable cataloging of secrets - # SYFT_CONTENTS_CATALOGER_ENABLED env var + # SYFT_FILE_CONTENTS_CATALOGER_ENABLED env var enabled: true # the search space to look for secrets (options: all-layers, squashed) - # SYFT_CONTENTS_CATALOGER_SCOPE env var + # SYFT_FILE_CONTENTS_CATALOGER_SCOPE env var scope: "squashed" # skip searching a file entirely if it is above the given size (default = 1MB; unit = bytes) - # SYFT_CONTENTS_SKIP_FILES_ABOVE_SIZE env var + # SYFT_FILE_CONTENTS_SKIP_FILES_ABOVE_SIZE env var skip-files-above-size: 1048576 # file globs for the cataloger to match on - # SYFT_CONTENTS_GLOBS env var + # SYFT_FILE_CONTENTS_GLOBS env var globs: [] # cataloging file metadata is exposed through the power-user subcommand diff --git a/cmd/power_user_tasks.go b/cmd/power_user_tasks.go index da7cc97ea..37b0ba896 100644 --- a/cmd/power_user_tasks.go +++ b/cmd/power_user_tasks.go @@ -188,11 +188,11 @@ func catalogFileClassificationsTask() (powerUserTask, error) { } func catalogContentsTask() (powerUserTask, error) { - if !appConfig.Contents.Cataloger.Enabled { + if !appConfig.FileContents.Cataloger.Enabled { return nil, nil } - contentsCataloger, err := file.NewContentsCataloger(appConfig.Contents.Globs, appConfig.Contents.SkipFilesAboveSize) + contentsCataloger, err := file.NewContentsCataloger(appConfig.FileContents.Globs, appConfig.FileContents.SkipFilesAboveSize) if err != nil { return nil, err } diff --git a/internal/config/application.go b/internal/config/application.go index c78b21abf..9e0d58b0b 100644 --- a/internal/config/application.go +++ b/internal/config/application.go @@ -38,7 +38,7 @@ type Application struct { Package packages `yaml:"package" json:"package" mapstructure:"package"` FileMetadata FileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"` FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"` - Contents contents `yaml:"contents" json:"contents" mapstructure:"contents"` + FileContents fileContents `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"` Secrets secrets `yaml:"secrets" json:"secrets" mapstructure:"secrets"` } diff --git a/internal/config/contents.go b/internal/config/file_contents.go similarity index 57% rename from internal/config/contents.go rename to internal/config/file_contents.go index 5c660013c..3b3eb4ca9 100644 --- a/internal/config/contents.go +++ b/internal/config/file_contents.go @@ -6,19 +6,19 @@ import ( "github.com/spf13/viper" ) -type contents struct { +type fileContents struct { Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"` SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"` Globs []string `yaml:"globs" json:"globs" mapstructure:"globs"` } -func (cfg contents) loadDefaultValues(v *viper.Viper) { - v.SetDefault("contents.cataloger.enabled", true) - v.SetDefault("contents.cataloger.scope", source.SquashedScope) - v.SetDefault("contents.skip-files-above-size", 1*file.MB) - v.SetDefault("contents.globs", []string{}) +func (cfg fileContents) loadDefaultValues(v *viper.Viper) { + v.SetDefault("file-contents.cataloger.enabled", true) + v.SetDefault("file-contents.cataloger.scope", source.SquashedScope) + v.SetDefault("file-contents.skip-files-above-size", 1*file.MB) + v.SetDefault("file-contents.globs", []string{}) } -func (cfg *contents) parseConfigValues() error { +func (cfg *fileContents) parseConfigValues() error { return cfg.Cataloger.parseConfigValues() } diff --git a/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden b/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden index 76c0d521f..0e92c7967 100644 --- a/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden +++ b/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden @@ -206,7 +206,7 @@ "scope": "" } }, - "contents": { + "file-contents": { "cataloger": { "enabled": false, "scope": "" diff --git a/test/cli/power_user_cmd_test.go b/test/cli/power_user_cmd_test.go index fb7c8cf48..2f7dcbbe9 100644 --- a/test/cli/power_user_cmd_test.go +++ b/test/cli/power_user_cmd_test.go @@ -55,7 +55,7 @@ func TestPowerUserCmdFlags(t *testing.T) { name: "content-cataloger-wired-up", args: []string{"power-user", "docker-archive:" + getFixtureImage(t, "image-secrets")}, env: map[string]string{ - "SYFT_CONTENTS_GLOBS": "/api-key.txt", + "SYFT_FILE_CONTENTS_GLOBS": "/api-key.txt", }, assertions: []traitAssertion{ assertInOutput(`"contents": "c29tZV9BcEkta0V5ID0gIjEyMzQ1QTdhOTAxYjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MCIK"`), // proof of the content cataloger From b5d4b2f7b2f4ca1b23440c3bd840d45791c1889b Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 12 Apr 2021 17:24:08 -0400 Subject: [PATCH 5/6] simplify test assertions + rename file contents cataloger size limiter var Signed-off-by: Alex Goodman --- syft/file/contents_cataloger.go | 12 +++++----- syft/file/contents_cataloger_test.go | 24 ++++++------------- .../common/generic_cataloger_test.go | 11 ++++----- 3 files changed, 17 insertions(+), 30 deletions(-) diff --git a/syft/file/contents_cataloger.go b/syft/file/contents_cataloger.go index 2a67b4c2e..97652868c 100644 --- a/syft/file/contents_cataloger.go +++ b/syft/file/contents_cataloger.go @@ -11,14 +11,14 @@ import ( ) type ContentsCataloger struct { - globs []string - skipFilesAboveSize int64 + globs []string + skipFilesAboveSizeInBytes int64 } func NewContentsCataloger(globs []string, skipFilesAboveSize int64) (*ContentsCataloger, error) { return &ContentsCataloger{ - globs: globs, - skipFilesAboveSize: skipFilesAboveSize, + globs: globs, + skipFilesAboveSizeInBytes: skipFilesAboveSize, }, nil } @@ -37,7 +37,7 @@ func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Lo return nil, err } - if i.skipFilesAboveSize > 0 && metadata.Size > i.skipFilesAboveSize { + if i.skipFilesAboveSizeInBytes > 0 && metadata.Size > i.skipFilesAboveSizeInBytes { continue } @@ -61,7 +61,7 @@ func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, locati buf := &bytes.Buffer{} if _, err = io.Copy(base64.NewEncoder(base64.StdEncoding, buf), contentReader); err != nil { - return "", fmt.Errorf("unable to observe contents of %+v: %+v", location.RealPath, err) + return "", fmt.Errorf("unable to observe contents of %+v: %w", location.RealPath, err) } return buf.String(), nil diff --git a/syft/file/contents_cataloger_test.go b/syft/file/contents_cataloger_test.go index eb83d21aa..c6c1b0db9 100644 --- a/syft/file/contents_cataloger_test.go +++ b/syft/file/contents_cataloger_test.go @@ -11,12 +11,11 @@ func TestContentsCataloger(t *testing.T) { allFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"} tests := []struct { - name string - globs []string - maxSize int64 - files []string - expected map[source.Location]string - catalogErr bool + name string + globs []string + maxSize int64 + files []string + expected map[source.Location]string }{ { name: "multi-pattern", @@ -68,20 +67,11 @@ func TestContentsCataloger(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { c, err := NewContentsCataloger(test.globs, test.maxSize) - if err != nil { - t.Fatalf("could not create cataloger: %+v", err) - } + assert.NoError(t, err) resolver := source.NewMockResolverForPaths(test.files...) actual, err := c.Catalog(resolver) - if err != nil && !test.catalogErr { - t.Fatalf("could not catalog (but should have been able to): %+v", err) - } else if err == nil && test.catalogErr { - t.Fatalf("expected catalog error but did not get one") - } else if test.catalogErr && err != nil { - return - } - + assert.NoError(t, err) assert.Equal(t, test.expected, actual, "mismatched contents") }) diff --git a/syft/pkg/cataloger/common/generic_cataloger_test.go b/syft/pkg/cataloger/common/generic_cataloger_test.go index 594436447..a30a3d5f7 100644 --- a/syft/pkg/cataloger/common/generic_cataloger_test.go +++ b/syft/pkg/cataloger/common/generic_cataloger_test.go @@ -6,6 +6,8 @@ import ( "io/ioutil" "testing" + "github.com/stretchr/testify/assert" + "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -46,13 +48,8 @@ func TestGenericCataloger(t *testing.T) { } actualPkgs, err := cataloger.Catalog(resolver) - if err != nil { - t.Fatalf("cataloger catalog action failed: %+v", err) - } - - if len(actualPkgs) != len(expectedPkgs) { - t.Fatalf("unexpected packages len: %d != %d", len(expectedPkgs), len(actualPkgs)) - } + assert.NoError(t, err) + assert.Len(t, actualPkgs, len(expectedPkgs)) for _, p := range actualPkgs { ref := p.Locations[0] From cd542bab5b3f0482e48aef176ae8e15f66c20381 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 12 Apr 2021 17:37:38 -0400 Subject: [PATCH 6/6] ensure file content cataloger is using the proper scope Signed-off-by: Alex Goodman --- cmd/power_user_tasks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/power_user_tasks.go b/cmd/power_user_tasks.go index 37b0ba896..df543b9c9 100644 --- a/cmd/power_user_tasks.go +++ b/cmd/power_user_tasks.go @@ -198,7 +198,7 @@ func catalogContentsTask() (powerUserTask, error) { } task := func(results *poweruser.JSONDocumentConfig, src source.Source) error { - resolver, err := src.FileResolver(appConfig.Contents.Cataloger.ScopeOpt) + resolver, err := src.FileResolver(appConfig.FileContents.Cataloger.ScopeOpt) if err != nil { return err }