mirror of
https://github.com/anchore/syft.git
synced 2025-11-19 17:33:18 +01:00
Merge pull request #370 from anchore/add-file-contents-cataloger
Add file contents cataloger
This commit is contained in:
Alex Goodman
GitHub
commit
c02ab88d5f
23
README.md
23
README.md
@ -119,6 +119,25 @@ file-classification:
|
||||
# SYFT_FILE_CLASSIFICATION_CATALOGER_SCOPE env var
|
||||
scope: "squashed"
|
||||
|
||||
# cataloging file contents is exposed through the power-user subcommand
|
||||
file-contents:
|
||||
cataloger:
|
||||
# enable/disable cataloging of secrets
|
||||
# SYFT_FILE_CONTENTS_CATALOGER_ENABLED env var
|
||||
enabled: true
|
||||
|
||||
# the search space to look for secrets (options: all-layers, squashed)
|
||||
# SYFT_FILE_CONTENTS_CATALOGER_SCOPE env var
|
||||
scope: "squashed"
|
||||
|
||||
# skip searching a file entirely if it is above the given size (default = 1MB; unit = bytes)
|
||||
# SYFT_FILE_CONTENTS_SKIP_FILES_ABOVE_SIZE env var
|
||||
skip-files-above-size: 1048576
|
||||
|
||||
# file globs for the cataloger to match on
|
||||
# SYFT_FILE_CONTENTS_GLOBS env var
|
||||
globs: []
|
||||
|
||||
# cataloging file metadata is exposed through the power-user subcommand
|
||||
file-metadata:
|
||||
cataloger:
|
||||
@ -149,9 +168,9 @@ secrets:
|
||||
# SYFT_SECRETS_REVEAL_VALUES env var
|
||||
reveal-values: false
|
||||
|
||||
# skip searching a file entirely if it is above the given size (default = 10MB; unit = bytes)
|
||||
# skip searching a file entirely if it is above the given size (default = 1MB; unit = bytes)
|
||||
# SYFT_SECRETS_SKIP_FILES_ABOVE_SIZE env var
|
||||
skip-files-above-size: 10485760
|
||||
skip-files-above-size: 1048576
|
||||
|
||||
# name-regex pairs to consider when searching files for secrets. Note: the regex must match single line patterns
|
||||
# but may also have OPTIONAL multiline capture groups. Regexes with a named capture group of "value" will
|
||||
|
||||
@ -21,6 +21,7 @@ func powerUserTasks() ([]powerUserTask, error) {
|
||||
catalogFileDigestsTask,
|
||||
catalogSecretsTask,
|
||||
catalogFileClassificationsTask,
|
||||
catalogContentsTask,
|
||||
}
|
||||
|
||||
for _, generator := range generators {
|
||||
@ -185,3 +186,30 @@ func catalogFileClassificationsTask() (powerUserTask, error) {
|
||||
|
||||
return task, nil
|
||||
}
|
||||
|
||||
func catalogContentsTask() (powerUserTask, error) {
|
||||
if !appConfig.FileContents.Cataloger.Enabled {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
contentsCataloger, err := file.NewContentsCataloger(appConfig.FileContents.Globs, appConfig.FileContents.SkipFilesAboveSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
task := func(results *poweruser.JSONDocumentConfig, src source.Source) error {
|
||||
resolver, err := src.FileResolver(appConfig.FileContents.Cataloger.ScopeOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result, err := contentsCataloger.Catalog(resolver)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
results.FileContents = result
|
||||
return nil
|
||||
}
|
||||
|
||||
return task, nil
|
||||
}
|
||||
|
||||
@ -38,6 +38,7 @@ type Application struct {
|
||||
Package packages `yaml:"package" json:"package" mapstructure:"package"`
|
||||
FileMetadata FileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"`
|
||||
FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"`
|
||||
FileContents fileContents `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"`
|
||||
Secrets secrets `yaml:"secrets" json:"secrets" mapstructure:"secrets"`
|
||||
}
|
||||
|
||||
|
||||
24
internal/config/file_contents.go
Normal file
24
internal/config/file_contents.go
Normal file
@ -0,0 +1,24 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
type fileContents struct {
|
||||
Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
|
||||
SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"`
|
||||
Globs []string `yaml:"globs" json:"globs" mapstructure:"globs"`
|
||||
}
|
||||
|
||||
func (cfg fileContents) loadDefaultValues(v *viper.Viper) {
|
||||
v.SetDefault("file-contents.cataloger.enabled", true)
|
||||
v.SetDefault("file-contents.cataloger.scope", source.SquashedScope)
|
||||
v.SetDefault("file-contents.skip-files-above-size", 1*file.MB)
|
||||
v.SetDefault("file-contents.globs", []string{})
|
||||
}
|
||||
|
||||
func (cfg *fileContents) parseConfigValues() error {
|
||||
return cfg.Cataloger.parseConfigValues()
|
||||
}
|
||||
@ -10,6 +10,7 @@ type JSONDocument struct {
|
||||
// require these fields. As an accepted rule in this repo all collections should still be initialized in the
|
||||
// context of being used in a JSON document.
|
||||
FileClassifications []JSONFileClassifications `json:"fileClassifications,omitempty"` // note: must have omitempty
|
||||
FileContents []JSONFileContents `json:"fileContents,omitempty"` // note: must have omitempty
|
||||
FileMetadata []JSONFileMetadata `json:"fileMetadata,omitempty"` // note: must have omitempty
|
||||
Secrets []JSONSecrets `json:"secrets,omitempty"` // note: must have omitempty
|
||||
packages.JSONDocument
|
||||
@ -29,6 +30,7 @@ func NewJSONDocument(config JSONDocumentConfig) (JSONDocument, error) {
|
||||
|
||||
return JSONDocument{
|
||||
FileClassifications: NewJSONFileClassifications(config.FileClassifications),
|
||||
FileContents: NewJSONFileContents(config.FileContents),
|
||||
FileMetadata: fileMetadata,
|
||||
Secrets: NewJSONSecrets(config.Secrets),
|
||||
JSONDocument: pkgsDoc,
|
||||
|
||||
@ -14,6 +14,7 @@ type JSONDocumentConfig struct {
|
||||
FileMetadata map[source.Location]source.FileMetadata
|
||||
FileDigests map[source.Location][]file.Digest
|
||||
FileClassifications map[source.Location][]file.Classification
|
||||
FileContents map[source.Location]string
|
||||
Secrets map[source.Location][]file.SearchResult
|
||||
Distro *distro.Distro
|
||||
SourceMetadata source.Metadata
|
||||
|
||||
31
internal/presenter/poweruser/json_file_contents.go
Normal file
31
internal/presenter/poweruser/json_file_contents.go
Normal file
@ -0,0 +1,31 @@
|
||||
package poweruser
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
type JSONFileContents struct {
|
||||
Location source.Location `json:"location"`
|
||||
Contents string `json:"contents"`
|
||||
}
|
||||
|
||||
func NewJSONFileContents(data map[source.Location]string) []JSONFileContents {
|
||||
results := make([]JSONFileContents, 0)
|
||||
for location, contents := range data {
|
||||
results = append(results, JSONFileContents{
|
||||
Location: location,
|
||||
Contents: contents,
|
||||
})
|
||||
}
|
||||
|
||||
// sort by real path then virtual path to ensure the result is stable across multiple runs
|
||||
sort.SliceStable(results, func(i, j int) bool {
|
||||
if results[i].Location.RealPath == results[j].Location.RealPath {
|
||||
return results[i].Location.VirtualPath < results[j].Location.VirtualPath
|
||||
}
|
||||
return results[i].Location.RealPath < results[j].Location.RealPath
|
||||
})
|
||||
return results
|
||||
}
|
||||
@ -125,6 +125,9 @@ func TestJSONPresenter(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
FileContents: map[source.Location]string{
|
||||
source.NewLocation("/a/place/a"): "the-contents",
|
||||
},
|
||||
Distro: &distro.Distro{
|
||||
Type: distro.RedHat,
|
||||
RawVersion: "7",
|
||||
|
||||
@ -1,4 +1,12 @@
|
||||
{
|
||||
"fileContents": [
|
||||
{
|
||||
"location": {
|
||||
"path": "/a/place/a"
|
||||
},
|
||||
"contents": "the-contents"
|
||||
}
|
||||
],
|
||||
"fileMetadata": [
|
||||
{
|
||||
"location": {
|
||||
@ -198,6 +206,14 @@
|
||||
"scope": ""
|
||||
}
|
||||
},
|
||||
"file-contents": {
|
||||
"cataloger": {
|
||||
"enabled": false,
|
||||
"scope": ""
|
||||
},
|
||||
"skip-files-above-size": 0,
|
||||
"globs": null
|
||||
},
|
||||
"secrets": {
|
||||
"cataloger": {
|
||||
"enabled": false,
|
||||
|
||||
@ -1,935 +0,0 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Document",
|
||||
"definitions": {
|
||||
"ApkFileRecord": {
|
||||
"required": [
|
||||
"path"
|
||||
],
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string"
|
||||
},
|
||||
"ownerUid": {
|
||||
"type": "string"
|
||||
},
|
||||
"ownerGid": {
|
||||
"type": "string"
|
||||
},
|
||||
"permissions": {
|
||||
"type": "string"
|
||||
},
|
||||
"checksum": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"ApkMetadata": {
|
||||
"required": [
|
||||
"package",
|
||||
"originPackage",
|
||||
"maintainer",
|
||||
"version",
|
||||
"license",
|
||||
"architecture",
|
||||
"url",
|
||||
"description",
|
||||
"size",
|
||||
"installedSize",
|
||||
"pullDependencies",
|
||||
"pullChecksum",
|
||||
"gitCommitOfApkPort",
|
||||
"files"
|
||||
],
|
||||
"properties": {
|
||||
"package": {
|
||||
"type": "string"
|
||||
},
|
||||
"originPackage": {
|
||||
"type": "string"
|
||||
},
|
||||
"maintainer": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"license": {
|
||||
"type": "string"
|
||||
},
|
||||
"architecture": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"size": {
|
||||
"type": "integer"
|
||||
},
|
||||
"installedSize": {
|
||||
"type": "integer"
|
||||
},
|
||||
"pullDependencies": {
|
||||
"type": "string"
|
||||
},
|
||||
"pullChecksum": {
|
||||
"type": "string"
|
||||
},
|
||||
"gitCommitOfApkPort": {
|
||||
"type": "string"
|
||||
},
|
||||
"files": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/ApkFileRecord"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"CargoPackageMetadata": {
|
||||
"required": [
|
||||
"name",
|
||||
"version",
|
||||
"source",
|
||||
"checksum",
|
||||
"dependencies"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"source": {
|
||||
"type": "string"
|
||||
},
|
||||
"checksum": {
|
||||
"type": "string"
|
||||
},
|
||||
"dependencies": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Classification": {
|
||||
"required": [
|
||||
"class",
|
||||
"metadata"
|
||||
],
|
||||
"properties": {
|
||||
"class": {
|
||||
"type": "string"
|
||||
},
|
||||
"metadata": {
|
||||
"patternProperties": {
|
||||
".*": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Descriptor": {
|
||||
"required": [
|
||||
"name",
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"configuration": {
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Digest": {
|
||||
"required": [
|
||||
"algorithm",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"algorithm": {
|
||||
"type": "string"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Distribution": {
|
||||
"required": [
|
||||
"name",
|
||||
"version",
|
||||
"idLike"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"idLike": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Document": {
|
||||
"required": [
|
||||
"artifacts",
|
||||
"artifactRelationships",
|
||||
"source",
|
||||
"distro",
|
||||
"descriptor",
|
||||
"schema"
|
||||
],
|
||||
"properties": {
|
||||
"fileClassifications": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/FileClassifications"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"fileMetadata": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/FileMetadata"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"secrets": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Secrets"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"artifacts": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Package"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"artifactRelationships": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Relationship"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"source": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Source"
|
||||
},
|
||||
"distro": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Distribution"
|
||||
},
|
||||
"descriptor": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Descriptor"
|
||||
},
|
||||
"schema": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Schema"
|
||||
},
|
||||
"artifacts.metadata": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/ApkMetadata"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/CargoPackageMetadata"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/DpkgMetadata"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/GemMetadata"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/JavaMetadata"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/NpmPackageJSONMetadata"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/PythonPackageMetadata"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/RpmdbMetadata"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"DpkgFileRecord": {
|
||||
"required": [
|
||||
"path",
|
||||
"md5"
|
||||
],
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string"
|
||||
},
|
||||
"md5": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"DpkgMetadata": {
|
||||
"required": [
|
||||
"package",
|
||||
"source",
|
||||
"version",
|
||||
"sourceVersion",
|
||||
"architecture",
|
||||
"maintainer",
|
||||
"installedSize",
|
||||
"files"
|
||||
],
|
||||
"properties": {
|
||||
"package": {
|
||||
"type": "string"
|
||||
},
|
||||
"source": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"sourceVersion": {
|
||||
"type": "string"
|
||||
},
|
||||
"architecture": {
|
||||
"type": "string"
|
||||
},
|
||||
"maintainer": {
|
||||
"type": "string"
|
||||
},
|
||||
"installedSize": {
|
||||
"type": "integer"
|
||||
},
|
||||
"files": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/DpkgFileRecord"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"FileClassifications": {
|
||||
"required": [
|
||||
"location",
|
||||
"classification"
|
||||
],
|
||||
"properties": {
|
||||
"location": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Location"
|
||||
},
|
||||
"classification": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Classification"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"FileMetadata": {
|
||||
"required": [
|
||||
"location",
|
||||
"metadata"
|
||||
],
|
||||
"properties": {
|
||||
"location": {
|
||||
"$ref": "#/definitions/Location"
|
||||
},
|
||||
"metadata": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/FileMetadataEntry"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"FileMetadataEntry": {
|
||||
"required": [
|
||||
"mode",
|
||||
"type",
|
||||
"userID",
|
||||
"groupID"
|
||||
],
|
||||
"properties": {
|
||||
"mode": {
|
||||
"type": "integer"
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"linkDestination": {
|
||||
"type": "string"
|
||||
},
|
||||
"userID": {
|
||||
"type": "integer"
|
||||
},
|
||||
"groupID": {
|
||||
"type": "integer"
|
||||
},
|
||||
"digests": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Digest"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"GemMetadata": {
|
||||
"required": [
|
||||
"name",
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"files": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"authors": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"licenses": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"homepage": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"JavaManifest": {
|
||||
"properties": {
|
||||
"main": {
|
||||
"patternProperties": {
|
||||
".*": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"namedSections": {
|
||||
"patternProperties": {
|
||||
".*": {
|
||||
"patternProperties": {
|
||||
".*": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"JavaMetadata": {
|
||||
"required": [
|
||||
"virtualPath"
|
||||
],
|
||||
"properties": {
|
||||
"virtualPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"manifest": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/JavaManifest"
|
||||
},
|
||||
"pomProperties": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/PomProperties"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Location": {
|
||||
"required": [
|
||||
"path"
|
||||
],
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string"
|
||||
},
|
||||
"layerID": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"NpmPackageJSONMetadata": {
|
||||
"required": [
|
||||
"author",
|
||||
"licenses",
|
||||
"homepage",
|
||||
"description",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"files": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"author": {
|
||||
"type": "string"
|
||||
},
|
||||
"licenses": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"homepage": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Package": {
|
||||
"required": [
|
||||
"id",
|
||||
"name",
|
||||
"version",
|
||||
"type",
|
||||
"foundBy",
|
||||
"locations",
|
||||
"licenses",
|
||||
"language",
|
||||
"cpes",
|
||||
"purl",
|
||||
"metadataType",
|
||||
"metadata"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"foundBy": {
|
||||
"type": "string"
|
||||
},
|
||||
"locations": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/Location"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"licenses": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"language": {
|
||||
"type": "string"
|
||||
},
|
||||
"cpes": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"purl": {
|
||||
"type": "string"
|
||||
},
|
||||
"metadataType": {
|
||||
"type": "string"
|
||||
},
|
||||
"metadata": {
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"PomProperties": {
|
||||
"required": [
|
||||
"path",
|
||||
"name",
|
||||
"groupId",
|
||||
"artifactId",
|
||||
"version",
|
||||
"extraFields"
|
||||
],
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"groupId": {
|
||||
"type": "string"
|
||||
},
|
||||
"artifactId": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"extraFields": {
|
||||
"patternProperties": {
|
||||
".*": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"PythonFileDigest": {
|
||||
"required": [
|
||||
"algorithm",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"algorithm": {
|
||||
"type": "string"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"PythonFileRecord": {
|
||||
"required": [
|
||||
"path"
|
||||
],
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string"
|
||||
},
|
||||
"digest": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/PythonFileDigest"
|
||||
},
|
||||
"size": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"PythonPackageMetadata": {
|
||||
"required": [
|
||||
"name",
|
||||
"version",
|
||||
"license",
|
||||
"author",
|
||||
"authorEmail",
|
||||
"platform",
|
||||
"sitePackagesRootPath"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"license": {
|
||||
"type": "string"
|
||||
},
|
||||
"author": {
|
||||
"type": "string"
|
||||
},
|
||||
"authorEmail": {
|
||||
"type": "string"
|
||||
},
|
||||
"platform": {
|
||||
"type": "string"
|
||||
},
|
||||
"files": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/PythonFileRecord"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"sitePackagesRootPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"topLevelPackages": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Relationship": {
|
||||
"required": [
|
||||
"parent",
|
||||
"child",
|
||||
"type",
|
||||
"metadata"
|
||||
],
|
||||
"properties": {
|
||||
"parent": {
|
||||
"type": "string"
|
||||
},
|
||||
"child": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"metadata": {
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"RpmdbFileRecord": {
|
||||
"required": [
|
||||
"path",
|
||||
"mode",
|
||||
"size",
|
||||
"sha256"
|
||||
],
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string"
|
||||
},
|
||||
"mode": {
|
||||
"type": "integer"
|
||||
},
|
||||
"size": {
|
||||
"type": "integer"
|
||||
},
|
||||
"sha256": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"RpmdbMetadata": {
|
||||
"required": [
|
||||
"name",
|
||||
"version",
|
||||
"epoch",
|
||||
"architecture",
|
||||
"release",
|
||||
"sourceRpm",
|
||||
"size",
|
||||
"license",
|
||||
"vendor",
|
||||
"files"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"epoch": {
|
||||
"type": "integer"
|
||||
},
|
||||
"architecture": {
|
||||
"type": "string"
|
||||
},
|
||||
"release": {
|
||||
"type": "string"
|
||||
},
|
||||
"sourceRpm": {
|
||||
"type": "string"
|
||||
},
|
||||
"size": {
|
||||
"type": "integer"
|
||||
},
|
||||
"license": {
|
||||
"type": "string"
|
||||
},
|
||||
"vendor": {
|
||||
"type": "string"
|
||||
},
|
||||
"files": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/RpmdbFileRecord"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Schema": {
|
||||
"required": [
|
||||
"version",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"SearchResult": {
|
||||
"required": [
|
||||
"classification",
|
||||
"lineNumber",
|
||||
"lineOffset",
|
||||
"seekPosition",
|
||||
"length"
|
||||
],
|
||||
"properties": {
|
||||
"classification": {
|
||||
"type": "string"
|
||||
},
|
||||
"lineNumber": {
|
||||
"type": "integer"
|
||||
},
|
||||
"lineOffset": {
|
||||
"type": "integer"
|
||||
},
|
||||
"seekPosition": {
|
||||
"type": "integer"
|
||||
},
|
||||
"length": {
|
||||
"type": "integer"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Secrets": {
|
||||
"required": [
|
||||
"location",
|
||||
"secrets"
|
||||
],
|
||||
"properties": {
|
||||
"location": {
|
||||
"$ref": "#/definitions/Location"
|
||||
},
|
||||
"secrets": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/SearchResult"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Source": {
|
||||
"required": [
|
||||
"type",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -125,6 +125,27 @@
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Classification": {
|
||||
"required": [
|
||||
"class",
|
||||
"metadata"
|
||||
],
|
||||
"properties": {
|
||||
"class": {
|
||||
"type": "string"
|
||||
},
|
||||
"metadata": {
|
||||
"patternProperties": {
|
||||
".*": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"Descriptor": {
|
||||
"required": [
|
||||
"name",
|
||||
@ -190,6 +211,20 @@
|
||||
"schema"
|
||||
],
|
||||
"properties": {
|
||||
"fileClassifications": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/FileClassifications"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"fileContents": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/FileContents"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"fileMetadata": {
|
||||
"items": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
@ -302,6 +337,40 @@
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"FileClassifications": {
|
||||
"required": [
|
||||
"location",
|
||||
"classification"
|
||||
],
|
||||
"properties": {
|
||||
"location": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Location"
|
||||
},
|
||||
"classification": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Classification"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"FileContents": {
|
||||
"required": [
|
||||
"location",
|
||||
"contents"
|
||||
],
|
||||
"properties": {
|
||||
"location": {
|
||||
"$ref": "#/definitions/Location"
|
||||
},
|
||||
"contents": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
"FileMetadata": {
|
||||
"required": [
|
||||
"location",
|
||||
@ -309,7 +378,6 @@
|
||||
],
|
||||
"properties": {
|
||||
"location": {
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
"$ref": "#/definitions/Location"
|
||||
},
|
||||
"metadata": {
|
||||
|
||||
68
syft/file/contents_cataloger.go
Normal file
68
syft/file/contents_cataloger.go
Normal file
@ -0,0 +1,68 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
type ContentsCataloger struct {
|
||||
globs []string
|
||||
skipFilesAboveSizeInBytes int64
|
||||
}
|
||||
|
||||
func NewContentsCataloger(globs []string, skipFilesAboveSize int64) (*ContentsCataloger, error) {
|
||||
return &ContentsCataloger{
|
||||
globs: globs,
|
||||
skipFilesAboveSizeInBytes: skipFilesAboveSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Location]string, error) {
|
||||
results := make(map[source.Location]string)
|
||||
var locations []source.Location
|
||||
|
||||
locations, err := resolver.FilesByGlob(i.globs...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, location := range locations {
|
||||
metadata, err := resolver.FileMetadataByLocation(location)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if i.skipFilesAboveSizeInBytes > 0 && metadata.Size > i.skipFilesAboveSizeInBytes {
|
||||
continue
|
||||
}
|
||||
|
||||
result, err := i.catalogLocation(resolver, location)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results[location] = result
|
||||
}
|
||||
log.Debugf("file contents cataloger processed %d files", len(results))
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) (string, error) {
|
||||
contentReader, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer contentReader.Close()
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
if _, err = io.Copy(base64.NewEncoder(base64.StdEncoding, buf), contentReader); err != nil {
|
||||
return "", fmt.Errorf("unable to observe contents of %+v: %w", location.RealPath, err)
|
||||
}
|
||||
|
||||
return buf.String(), nil
|
||||
}
|
||||
79
syft/file/contents_cataloger_test.go
Normal file
79
syft/file/contents_cataloger_test.go
Normal file
@ -0,0 +1,79 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestContentsCataloger(t *testing.T) {
|
||||
allFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
globs []string
|
||||
maxSize int64
|
||||
files []string
|
||||
expected map[source.Location]string
|
||||
}{
|
||||
{
|
||||
name: "multi-pattern",
|
||||
globs: []string{"test-fixtures/last/*.txt", "test-fixtures/*.txt"},
|
||||
files: allFiles,
|
||||
expected: map[source.Location]string{
|
||||
source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
|
||||
source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
|
||||
source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "no-patterns",
|
||||
globs: []string{},
|
||||
files: []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"},
|
||||
expected: map[source.Location]string{},
|
||||
},
|
||||
{
|
||||
name: "all-txt",
|
||||
globs: []string{"**/*.txt"},
|
||||
files: allFiles,
|
||||
expected: map[source.Location]string{
|
||||
source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
|
||||
source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
|
||||
source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "subpath",
|
||||
globs: []string{"test-fixtures/*.txt"},
|
||||
files: allFiles,
|
||||
expected: map[source.Location]string{
|
||||
source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
|
||||
source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "size-filter",
|
||||
maxSize: 42,
|
||||
globs: []string{"**/*.txt"},
|
||||
files: allFiles,
|
||||
expected: map[source.Location]string{
|
||||
source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
|
||||
source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
c, err := NewContentsCataloger(test.globs, test.maxSize)
|
||||
assert.NoError(t, err)
|
||||
|
||||
resolver := source.NewMockResolverForPaths(test.files...)
|
||||
actual, err := c.Catalog(resolver)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, test.expected, actual, "mismatched contents")
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -6,6 +6,8 @@ import (
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
@ -25,7 +27,7 @@ func parser(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func TestGenericCataloger(t *testing.T) {
|
||||
|
||||
globParsers := map[string]ParserFn{
|
||||
"**a-path.txt": parser,
|
||||
"**/a-path.txt": parser,
|
||||
}
|
||||
pathParsers := map[string]ParserFn{
|
||||
"test-fixtures/another-path.txt": parser,
|
||||
@ -46,13 +48,8 @@ func TestGenericCataloger(t *testing.T) {
|
||||
}
|
||||
|
||||
actualPkgs, err := cataloger.Catalog(resolver)
|
||||
if err != nil {
|
||||
t.Fatalf("cataloger catalog action failed: %+v", err)
|
||||
}
|
||||
|
||||
if len(actualPkgs) != len(expectedPkgs) {
|
||||
t.Fatalf("unexpected packages len: %d", len(actualPkgs))
|
||||
}
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, actualPkgs, len(expectedPkgs))
|
||||
|
||||
for _, p := range actualPkgs {
|
||||
ref := p.Locations[0]
|
||||
|
||||
@ -5,7 +5,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/anchore/syft/internal/file"
|
||||
"github.com/bmatcuk/doublestar/v2"
|
||||
)
|
||||
|
||||
var _ FileResolver = (*MockResolver)(nil)
|
||||
@ -84,7 +84,11 @@ func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) {
|
||||
var results []Location
|
||||
for _, pattern := range patterns {
|
||||
for _, location := range r.Locations {
|
||||
if file.GlobMatch(pattern, location.RealPath) {
|
||||
matches, err := doublestar.Match(pattern, location.RealPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if matches {
|
||||
results = append(results, location)
|
||||
}
|
||||
}
|
||||
|
||||
@ -51,6 +51,17 @@ func TestPowerUserCmdFlags(t *testing.T) {
|
||||
assertSuccessfulReturnCode,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "content-cataloger-wired-up",
|
||||
args: []string{"power-user", "docker-archive:" + getFixtureImage(t, "image-secrets")},
|
||||
env: map[string]string{
|
||||
"SYFT_FILE_CONTENTS_GLOBS": "/api-key.txt",
|
||||
},
|
||||
assertions: []traitAssertion{
|
||||
assertInOutput(`"contents": "c29tZV9BcEkta0V5ID0gIjEyMzQ1QTdhOTAxYjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MCIK"`), // proof of the content cataloger
|
||||
assertSuccessfulReturnCode,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user