diff --git a/internal/file/copy.go b/internal/archive/safe_copy.go similarity index 83% rename from internal/file/copy.go rename to internal/archive/safe_copy.go index d569dcc4c..0867f7a5a 100644 --- a/internal/file/copy.go +++ b/internal/archive/safe_copy.go @@ -1,12 +1,14 @@ -package file +package archive import ( "errors" "fmt" "io" + + "github.com/anchore/syft/syft/file" ) -const perFileReadLimit = 2 * GB +const perFileReadLimit = 2 * file.GB // safeCopy limits the copy from the reader. This is useful when extracting files from archives to // protect against decompression bomb attacks. diff --git a/internal/file/tar_file_traversal.go b/internal/archive/tar_file_traversal.go similarity index 75% rename from internal/file/tar_file_traversal.go rename to internal/archive/tar_file_traversal.go index 4280935c6..712a1c41b 100644 --- a/internal/file/tar_file_traversal.go +++ b/internal/archive/tar_file_traversal.go @@ -1,38 +1,39 @@ -package file +package archive import ( "fmt" "io/ioutil" "path/filepath" + "github.com/anchore/syft/syft/file" "github.com/bmatcuk/doublestar/v4" "github.com/mholt/archiver/v3" ) // ExtractGlobsFromTarToUniqueTempFile extracts paths matching the given globs within the given archive to a temporary directory, returning file openers for each file extracted. -func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...string) (map[string]Opener, error) { - results := make(map[string]Opener) +func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...string) (map[string]file.Opener, error) { + results := make(map[string]file.Opener) // don't allow for full traversal, only select traversal from given paths if len(globs) == 0 { return results, nil } - visitor := func(file archiver.File) error { - defer file.Close() + visitor := func(f archiver.File) error { + defer f.Close() // ignore directories - if file.FileInfo.IsDir() { + if f.FileInfo.IsDir() { return nil } // ignore any filename that doesn't match the given globs... - if !matchesAnyGlob(file.Name(), globs...) { + if !matchesAnyGlob(f.Name(), globs...) { return nil } // we have a file we want to extract.... - tempfilePrefix := filepath.Base(filepath.Clean(file.Name())) + "-" + tempfilePrefix := filepath.Base(filepath.Clean(f.Name())) + "-" tempFile, err := ioutil.TempFile(dir, tempfilePrefix) if err != nil { return fmt.Errorf("unable to create temp file: %w", err) @@ -42,11 +43,11 @@ func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...strin // provides a ReadCloser. It is up to the caller to handle closing the file explicitly. defer tempFile.Close() - if err := safeCopy(tempFile, file.ReadCloser); err != nil { - return fmt.Errorf("unable to copy source=%q for tar=%q: %w", file.Name(), archivePath, err) + if err := safeCopy(tempFile, f.ReadCloser); err != nil { + return fmt.Errorf("unable to copy source=%q for tar=%q: %w", f.Name(), archivePath, err) } - results[file.Name()] = Opener{path: tempFile.Name()} + results[f.Name()] = file.NewOpener(tempFile.Name()) return nil } diff --git a/internal/file/test-fixtures/generate-zip-fixture-from-source-dir.sh b/internal/archive/test-fixtures/generate-zip-fixture-from-source-dir.sh similarity index 100% rename from internal/file/test-fixtures/generate-zip-fixture-from-source-dir.sh rename to internal/archive/test-fixtures/generate-zip-fixture-from-source-dir.sh diff --git a/internal/file/test-fixtures/zip-source/b-file.txt b/internal/archive/test-fixtures/zip-source/b-file.txt similarity index 100% rename from internal/file/test-fixtures/zip-source/b-file.txt rename to internal/archive/test-fixtures/zip-source/b-file.txt diff --git a/internal/file/test-fixtures/zip-source/some-dir/a-file.txt b/internal/archive/test-fixtures/zip-source/some-dir/a-file.txt similarity index 100% rename from internal/file/test-fixtures/zip-source/some-dir/a-file.txt rename to internal/archive/test-fixtures/zip-source/some-dir/a-file.txt diff --git a/internal/file/zip_file_helpers_test.go b/internal/archive/zip_file_helpers_test.go similarity index 99% rename from internal/file/zip_file_helpers_test.go rename to internal/archive/zip_file_helpers_test.go index 7bf89a08c..6d97be802 100644 --- a/internal/file/zip_file_helpers_test.go +++ b/internal/archive/zip_file_helpers_test.go @@ -1,4 +1,4 @@ -package file +package archive import ( "io/ioutil" diff --git a/internal/file/zip_file_manifest.go b/internal/archive/zip_file_manifest.go similarity index 94% rename from internal/file/zip_file_manifest.go rename to internal/archive/zip_file_manifest.go index 700c1472b..e22d60b80 100644 --- a/internal/file/zip_file_manifest.go +++ b/internal/archive/zip_file_manifest.go @@ -1,7 +1,8 @@ -package file +package archive import ( "fmt" + "github.com/anchore/syft/syft/file" "os" "sort" "strings" @@ -48,7 +49,7 @@ func (z ZipFileManifest) GlobMatch(patterns ...string) []string { // so that glob logic is consistent inside and outside of ZIP archives normalizedEntry := normalizeZipEntryName(entry) - if GlobMatch(pattern, normalizedEntry) { + if file.GlobMatch(pattern, normalizedEntry) { uniqueMatches.Add(entry) } } diff --git a/internal/file/zip_file_manifest_test.go b/internal/archive/zip_file_manifest_test.go similarity index 99% rename from internal/file/zip_file_manifest_test.go rename to internal/archive/zip_file_manifest_test.go index f89e88ca9..2b0a849df 100644 --- a/internal/file/zip_file_manifest_test.go +++ b/internal/archive/zip_file_manifest_test.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package file +package archive import ( "encoding/json" diff --git a/internal/file/zip_file_traversal.go b/internal/archive/zip_file_traversal.go similarity index 92% rename from internal/file/zip_file_traversal.go rename to internal/archive/zip_file_traversal.go index e71502e6e..2a4eac716 100644 --- a/internal/file/zip_file_traversal.go +++ b/internal/archive/zip_file_traversal.go @@ -1,9 +1,10 @@ -package file +package archive import ( "archive/zip" "bytes" "fmt" + "github.com/anchore/syft/syft/file" "io/ioutil" "os" "path/filepath" @@ -12,14 +13,6 @@ import ( "github.com/anchore/syft/internal/log" ) -const ( - // represents the order of bytes - _ = iota - KB = 1 << (10 * iota) - MB - GB -) - type errZipSlipDetected struct { Prefix string JoinArgs []string @@ -71,16 +64,16 @@ func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths } // ExtractFromZipToUniqueTempFile extracts select paths for the given archive to a temporary directory, returning file openers for each file extracted. -func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]Opener, error) { - results := make(map[string]Opener) +func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]file.Opener, error) { + results := make(map[string]file.Opener) // don't allow for full traversal, only select traversal from given paths if len(paths) == 0 { return results, nil } - visitor := func(file *zip.File) error { - tempfilePrefix := filepath.Base(filepath.Clean(file.Name)) + "-" + visitor := func(f *zip.File) error { + tempfilePrefix := filepath.Base(filepath.Clean(f.Name)) + "-" tempFile, err := ioutil.TempFile(dir, tempfilePrefix) if err != nil { @@ -91,26 +84,26 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m // provides a ReadCloser. It is up to the caller to handle closing the file explicitly. defer tempFile.Close() - zippedFile, err := file.Open() + zippedFile, err := f.Open() if err != nil { - return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err) + return fmt.Errorf("unable to read file=%q from zip=%q: %w", f.Name, archivePath, err) } defer func() { err := zippedFile.Close() if err != nil { - log.Errorf("unable to close source file=%q from zip=%q: %+v", file.Name, archivePath, err) + log.Errorf("unable to close source file=%q from zip=%q: %+v", f.Name, archivePath, err) } }() - if file.FileInfo().IsDir() { - return fmt.Errorf("unable to extract directories, only files: %s", file.Name) + if f.FileInfo().IsDir() { + return fmt.Errorf("unable to extract directories, only files: %s", f.Name) } if err := safeCopy(tempFile, zippedFile); err != nil { - return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err) + return fmt.Errorf("unable to copy source=%q for zip=%q: %w", f.Name, archivePath, err) } - results[file.Name] = Opener{path: tempFile.Name()} + results[f.Name] = file.NewOpener(tempFile.Name()) return nil } diff --git a/internal/file/zip_file_traversal_test.go b/internal/archive/zip_file_traversal_test.go similarity index 99% rename from internal/file/zip_file_traversal_test.go rename to internal/archive/zip_file_traversal_test.go index 6c7a9ea46..41a13552d 100644 --- a/internal/file/zip_file_traversal_test.go +++ b/internal/archive/zip_file_traversal_test.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package file +package archive import ( "crypto/sha256" diff --git a/internal/file/zip_read_closer.go b/internal/archive/zip_read_closer.go similarity index 99% rename from internal/file/zip_read_closer.go rename to internal/archive/zip_read_closer.go index 83edb6b08..fb485a6c4 100644 --- a/internal/file/zip_read_closer.go +++ b/internal/archive/zip_read_closer.go @@ -1,4 +1,4 @@ -package file +package archive import ( "archive/zip" diff --git a/internal/file/zip_read_closer_test.go b/internal/archive/zip_read_closer_test.go similarity index 98% rename from internal/file/zip_read_closer_test.go rename to internal/archive/zip_read_closer_test.go index 349bfcc9b..0229315d7 100644 --- a/internal/file/zip_read_closer_test.go +++ b/internal/archive/zip_read_closer_test.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package file +package archive import ( "os" diff --git a/internal/config/application.go b/internal/config/application.go index 5296d51e6..253b179e6 100644 --- a/internal/config/application.go +++ b/internal/config/application.go @@ -45,7 +45,7 @@ type Application struct { FileMetadata FileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"` FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"` FileContents fileContents `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"` - Secrets secrets `yaml:"secrets" json:"secrets" mapstructure:"secrets"` + Secrets secretsCfg `yaml:"secrets" json:"secrets" mapstructure:"secrets"` Registry registry `yaml:"registry" json:"registry" mapstructure:"registry"` Exclusions []string `yaml:"exclude" json:"exclude" mapstructure:"exclude"` Attest attest `yaml:"attest" json:"attest" mapstructure:"attest"` diff --git a/internal/config/file_contents.go b/internal/config/file_contents.go index 2a00a9d04..870e5eb6e 100644 --- a/internal/config/file_contents.go +++ b/internal/config/file_contents.go @@ -1,8 +1,8 @@ package config import ( - internalFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/file/cataloger/filecontents" "github.com/anchore/syft/syft/source" "github.com/spf13/viper" ) @@ -16,7 +16,7 @@ type fileContents struct { func (cfg fileContents) loadDefaultValues(v *viper.Viper) { v.SetDefault("file-contents.cataloger.enabled", catalogerEnabledDefault) v.SetDefault("file-contents.cataloger.scope", source.SquashedScope) - v.SetDefault("file-contents.skip-files-above-size", 1*internalFile.MB) + v.SetDefault("file-contents.skip-files-above-size", 1*file.MB) v.SetDefault("file-contents.globs", []string{}) } @@ -24,8 +24,8 @@ func (cfg *fileContents) parseConfigValues() error { return cfg.Cataloger.parseConfigValues() } -func (cfg fileContents) ToConfig() file.ContentsCatalogerConfig { - return file.ContentsCatalogerConfig{ +func (cfg fileContents) ToConfig() filecontents.CatalogerConfig { + return filecontents.CatalogerConfig{ Globs: cfg.Globs, SkipFilesAboveSizeInBytes: cfg.SkipFilesAboveSize, } diff --git a/internal/config/secrets.go b/internal/config/secrets.go index 5cbf6ae99..7fb112bd8 100644 --- a/internal/config/secrets.go +++ b/internal/config/secrets.go @@ -2,13 +2,13 @@ package config import ( "fmt" - internalFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/file/cataloger/secrets" "github.com/anchore/syft/syft/source" "github.com/spf13/viper" ) -type secrets struct { +type secretsCfg struct { Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"` AdditionalPatterns map[string]string `yaml:"additional-patterns" json:"additional-patterns" mapstructure:"additional-patterns"` ExcludePatternNames []string `yaml:"exclude-pattern-names" json:"exclude-pattern-names" mapstructure:"exclude-pattern-names"` @@ -16,25 +16,25 @@ type secrets struct { SkipFilesAboveSize int64 `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"` } -func (cfg secrets) loadDefaultValues(v *viper.Viper) { +func (cfg secretsCfg) loadDefaultValues(v *viper.Viper) { v.SetDefault("secrets.cataloger.enabled", catalogerEnabledDefault) v.SetDefault("secrets.cataloger.scope", source.AllLayersScope) v.SetDefault("secrets.reveal-values", false) - v.SetDefault("secrets.skip-files-above-size", 1*internalFile.MB) + v.SetDefault("secrets.skip-files-above-size", 1*file.MB) v.SetDefault("secrets.additional-patterns", map[string]string{}) v.SetDefault("secrets.exclude-pattern-names", []string{}) } -func (cfg *secrets) parseConfigValues() error { +func (cfg *secretsCfg) parseConfigValues() error { return cfg.Cataloger.parseConfigValues() } -func (cfg secrets) ToConfig() (*file.SecretsCatalogerConfig, error) { - patterns, err := file.GenerateSearchPatterns(file.DefaultSecretsPatterns, cfg.AdditionalPatterns, cfg.ExcludePatternNames) +func (cfg secretsCfg) ToConfig() (*secrets.CatalogerConfig, error) { + patterns, err := file.GenerateSearchPatterns(secrets.DefaultSecretsPatterns, cfg.AdditionalPatterns, cfg.ExcludePatternNames) if err != nil { return nil, fmt.Errorf("unable to process secrets config patterns: %w", err) } - return &file.SecretsCatalogerConfig{ + return &secrets.CatalogerConfig{ Patterns: patterns, RevealValues: cfg.RevealValues, MaxFileSize: cfg.SkipFilesAboveSize, diff --git a/syft/catalog.go b/syft/catalog.go index 03a2d5416..6d7c63aca 100644 --- a/syft/catalog.go +++ b/syft/catalog.go @@ -19,12 +19,12 @@ func Catalog(src *source.Source, options ...CatalogingOption) (*sbom.SBOM, error var tasks []task generators := []taskGenerator{ - generateCatalogPackagesTask, - generateCatalogFileMetadataTask, - generateCatalogFileDigestsTask, - generateCatalogSecretsTask, - generateCatalogFileClassificationsTask, - generateCatalogContentsTask, + generatePackagesCatalogingTask, + generateFileMetadataCatalogingTask, + generateFileDigestsCatalogingTask, + generateSecretsCatalogingTask, + generateFileClassifierTask, + generateContentsCatalogingTask, } for _, generator := range generators { diff --git a/syft/cataloging_config.go b/syft/cataloging_config.go index d9b73dbb0..ee04b0183 100644 --- a/syft/cataloging_config.go +++ b/syft/cataloging_config.go @@ -2,6 +2,8 @@ package syft import ( "crypto" + "github.com/anchore/syft/syft/file/cataloger/filecontents" + "github.com/anchore/syft/syft/file/cataloger/secrets" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/version" @@ -25,13 +27,13 @@ type CatalogingConfig struct { DigestHashes []crypto.Hash // secrets CaptureSecrets bool - SecretsConfig file.SecretsCatalogerConfig + SecretsConfig secrets.CatalogerConfig SecretsScope source.Scope // file classification ClassifyFiles bool FileClassifiers []file.Classifier // file contents - ContentsConfig file.ContentsCatalogerConfig + ContentsConfig filecontents.CatalogerConfig } func DefaultCatalogingConfig() CatalogingConfig { @@ -40,8 +42,8 @@ func DefaultCatalogingConfig() CatalogingConfig { ToolName: internal.ApplicationName, ToolVersion: version.Guess(), SecretsScope: source.AllLayersScope, - SecretsConfig: file.DefaultSecretsCatalogerConfig(), + SecretsConfig: secrets.DefaultCatalogerConfig(), FileClassifiers: file.DefaultClassifiers(), - ContentsConfig: file.DefaultContentsCatalogerConfig(), + ContentsConfig: filecontents.DefaultCatalogerConfig(), } } diff --git a/syft/cataloging_option.go b/syft/cataloging_option.go index b2e698568..913668730 100644 --- a/syft/cataloging_option.go +++ b/syft/cataloging_option.go @@ -3,6 +3,7 @@ package syft import ( "crypto" "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/file/cataloger/secrets" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/packages" "github.com/anchore/syft/syft/source" @@ -82,7 +83,7 @@ func WithFileDigests(hashes ...crypto.Hash) CatalogingOption { } } -func WithSecrets(secretConfig *file.SecretsCatalogerConfig) CatalogingOption { +func WithSecrets(secretConfig *secrets.CatalogerConfig) CatalogingOption { return func(_ *source.Source, config *CatalogingConfig) error { config.CaptureSecrets = true if secretConfig != nil { diff --git a/syft/file/byte_sizes.go b/syft/file/byte_sizes.go new file mode 100644 index 000000000..7aea46756 --- /dev/null +++ b/syft/file/byte_sizes.go @@ -0,0 +1,9 @@ +package file + +const ( + // represents the order of bytes + _ = iota + KB = 1 << (10 * iota) + MB + GB +) diff --git a/syft/file/classification_cataloger.go b/syft/file/cataloger/fileclassifier/cataloger.go similarity index 58% rename from syft/file/classification_cataloger.go rename to syft/file/cataloger/fileclassifier/cataloger.go index 01a0685ef..f44f4bea6 100644 --- a/syft/file/classification_cataloger.go +++ b/syft/file/cataloger/fileclassifier/cataloger.go @@ -1,25 +1,26 @@ -package file +package fileclassifier import ( "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/source" ) -type ClassificationCataloger struct { - classifiers []Classifier +type Cataloger struct { + classifiers []file.Classifier } -func NewClassificationCataloger(classifiers []Classifier) (*ClassificationCataloger, error) { - return &ClassificationCataloger{ +func NewCataloger(classifiers []file.Classifier) (*Cataloger, error) { + return &Cataloger{ classifiers: classifiers, }, nil } -func (i *ClassificationCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]Classification, error) { - results := make(map[source.Coordinates][]Classification) +func (i *Cataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]file.Classification, error) { + results := make(map[source.Coordinates][]file.Classification) numResults := 0 - for _, location := range allRegularFiles(resolver) { + for _, location := range source.AllRegularFiles(resolver) { for _, classifier := range i.classifiers { result, err := classifier.Classify(resolver, location) if err != nil { diff --git a/syft/file/classification_cataloger_test.go b/syft/file/cataloger/fileclassifier/cataloger_test.go similarity index 88% rename from syft/file/classification_cataloger_test.go rename to syft/file/cataloger/fileclassifier/cataloger_test.go index da6fb37cb..3cdd5ab4e 100644 --- a/syft/file/classification_cataloger_test.go +++ b/syft/file/cataloger/fileclassifier/cataloger_test.go @@ -1,9 +1,10 @@ -package file +package fileclassifier import ( - "github.com/anchore/stereoscope/pkg/imagetest" "testing" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/source" "github.com/stretchr/testify/assert" ) @@ -13,14 +14,14 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) { name string fixtureDir string location string - expected []Classification + expected []file.Classification expectedErr func(assert.TestingT, error, ...interface{}) bool }{ { name: "positive-libpython3.7.so", fixtureDir: "test-fixtures/classifiers/positive", location: "libpython3.7.so", - expected: []Classification{ + expected: []file.Classification{ { Class: "python-binary", Metadata: map[string]string{ @@ -34,7 +35,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) { name: "positive-python3.6", fixtureDir: "test-fixtures/classifiers/positive", location: "python3.6", - expected: []Classification{ + expected: []file.Classification{ { Class: "python-binary", Metadata: map[string]string{ @@ -48,7 +49,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) { name: "positive-patchlevel.h", fixtureDir: "test-fixtures/classifiers/positive", location: "patchlevel.h", - expected: []Classification{ + expected: []file.Classification{ { Class: "cpython-source", Metadata: map[string]string{ @@ -62,7 +63,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) { name: "positive-go", fixtureDir: "test-fixtures/classifiers/positive", location: "go", - expected: []Classification{ + expected: []file.Classification{ { Class: "go-binary", Metadata: map[string]string{ @@ -76,7 +77,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) { name: "positive-go-hint", fixtureDir: "test-fixtures/classifiers/positive", location: "VERSION", - expected: []Classification{ + expected: []file.Classification{ { Class: "go-binary-hint", Metadata: map[string]string{ @@ -90,7 +91,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) { name: "positive-busybox", fixtureDir: "test-fixtures/classifiers/positive", location: "[", // note: busybox is a link to [ - expected: []Classification{ + expected: []file.Classification{ { Class: "busybox-binary", Metadata: map[string]string{ @@ -105,7 +106,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - c, err := NewClassificationCataloger(DefaultClassifiers) + c, err := NewCataloger(file.DefaultClassifiers()) test.expectedErr(t, err) src, err := source.NewFromDirectory(test.fixtureDir) @@ -138,14 +139,14 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T name string fixtureImage string location string - expected []Classification + expected []file.Classification expectedErr func(assert.TestingT, error, ...interface{}) bool }{ { name: "busybox-regression", fixtureImage: "image-busybox", location: "/bin/[", - expected: []Classification{ + expected: []file.Classification{ { Class: "busybox-binary", Metadata: map[string]string{ @@ -160,7 +161,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T for _, test := range tests { t.Run(test.name, func(t *testing.T) { - c, err := NewClassificationCataloger(DefaultClassifiers) + c, err := NewCataloger(file.DefaultClassifiers()) test.expectedErr(t, err) img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureImage) @@ -191,7 +192,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T func TestClassifierCataloger_DefaultClassifiers_NegativeCases(t *testing.T) { - c, err := NewClassificationCataloger(DefaultClassifiers) + c, err := NewCataloger(file.DefaultClassifiers()) assert.NoError(t, err) src, err := source.NewFromDirectory("test-fixtures/classifiers/negative") diff --git a/syft/file/test-fixtures/classifiers/negative/.gitignore b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/negative/.gitignore similarity index 100% rename from syft/file/test-fixtures/classifiers/negative/.gitignore rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/negative/.gitignore diff --git a/syft/file/test-fixtures/classifiers/negative/busybox b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/negative/busybox similarity index 100% rename from syft/file/test-fixtures/classifiers/negative/busybox rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/negative/busybox diff --git a/syft/file/test-fixtures/classifiers/negative/go b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/negative/go similarity index 100% rename from syft/file/test-fixtures/classifiers/negative/go rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/negative/go diff --git a/syft/file/test-fixtures/classifiers/negative/libpython2.7.so b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/negative/libpython2.7.so similarity index 100% rename from syft/file/test-fixtures/classifiers/negative/libpython2.7.so rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/negative/libpython2.7.so diff --git a/syft/file/test-fixtures/classifiers/negative/python2.6 b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/negative/python2.6 similarity index 100% rename from syft/file/test-fixtures/classifiers/negative/python2.6 rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/negative/python2.6 diff --git a/syft/file/test-fixtures/classifiers/positive/.gitignore b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/.gitignore similarity index 100% rename from syft/file/test-fixtures/classifiers/positive/.gitignore rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/.gitignore diff --git a/syft/file/test-fixtures/classifiers/positive/VERSION b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/VERSION similarity index 100% rename from syft/file/test-fixtures/classifiers/positive/VERSION rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/VERSION diff --git a/syft/file/test-fixtures/classifiers/positive/[ b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/[ similarity index 100% rename from syft/file/test-fixtures/classifiers/positive/[ rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/[ diff --git a/syft/file/test-fixtures/classifiers/positive/busybox b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/busybox similarity index 100% rename from syft/file/test-fixtures/classifiers/positive/busybox rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/busybox diff --git a/syft/file/test-fixtures/classifiers/positive/go b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/go similarity index 100% rename from syft/file/test-fixtures/classifiers/positive/go rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/go diff --git a/syft/file/test-fixtures/classifiers/positive/libpython3.7.so b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/libpython3.7.so similarity index 100% rename from syft/file/test-fixtures/classifiers/positive/libpython3.7.so rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/libpython3.7.so diff --git a/syft/file/test-fixtures/classifiers/positive/patchlevel.h b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/patchlevel.h similarity index 100% rename from syft/file/test-fixtures/classifiers/positive/patchlevel.h rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/patchlevel.h diff --git a/syft/file/test-fixtures/classifiers/positive/python3.6 b/syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/python3.6 similarity index 100% rename from syft/file/test-fixtures/classifiers/positive/python3.6 rename to syft/file/cataloger/fileclassifier/test-fixtures/classifiers/positive/python3.6 diff --git a/syft/file/test-fixtures/image-busybox/Dockerfile b/syft/file/cataloger/fileclassifier/test-fixtures/image-busybox/Dockerfile similarity index 100% rename from syft/file/test-fixtures/image-busybox/Dockerfile rename to syft/file/cataloger/fileclassifier/test-fixtures/image-busybox/Dockerfile diff --git a/syft/file/contents_cataloger.go b/syft/file/cataloger/filecontents/cataloger.go similarity index 75% rename from syft/file/contents_cataloger.go rename to syft/file/cataloger/filecontents/cataloger.go index cdc5607a8..85ce3d097 100644 --- a/syft/file/contents_cataloger.go +++ b/syft/file/cataloger/filecontents/cataloger.go @@ -1,41 +1,40 @@ -package file +package filecontents import ( "bytes" "encoding/base64" "fmt" - "github.com/anchore/syft/internal/file" "io" "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/source" ) -type ContentsCatalogerConfig struct { +type CatalogerConfig struct { Globs []string SkipFilesAboveSizeInBytes int64 } -type ContentsCataloger struct { - config ContentsCatalogerConfig +type Cataloger struct { + config CatalogerConfig } -func DefaultContentsCatalogerConfig() ContentsCatalogerConfig { - return ContentsCatalogerConfig{ +func DefaultCatalogerConfig() CatalogerConfig { + return CatalogerConfig{ Globs: nil, SkipFilesAboveSizeInBytes: 1 * file.MB, } } -func NewContentsCataloger(config ContentsCatalogerConfig) (*ContentsCataloger, error) { - return &ContentsCataloger{ +func NewCataloger(config CatalogerConfig) (*Cataloger, error) { + return &Cataloger{ config: config, }, nil } -func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates]string, error) { +func (i *Cataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates]string, error) { results := make(map[source.Coordinates]string) var locations []source.Location @@ -68,7 +67,7 @@ func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Co return results, nil } -func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) (string, error) { +func (i *Cataloger) catalogLocation(resolver source.FileResolver, location source.Location) (string, error) { contentReader, err := resolver.FileContentsByLocation(location) if err != nil { return "", err diff --git a/syft/file/contents_cataloger_test.go b/syft/file/cataloger/filecontents/cataloger_test.go similarity index 94% rename from syft/file/contents_cataloger_test.go rename to syft/file/cataloger/filecontents/cataloger_test.go index d13025c4c..f7286278f 100644 --- a/syft/file/contents_cataloger_test.go +++ b/syft/file/cataloger/filecontents/cataloger_test.go @@ -1,4 +1,4 @@ -package file +package filecontents import ( "testing" @@ -66,7 +66,10 @@ func TestContentsCataloger(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - c, err := NewContentsCataloger(test.globs, test.maxSize) + c, err := NewCataloger(CatalogerConfig{ + Globs: test.globs, + SkipFilesAboveSizeInBytes: test.maxSize, + }) assert.NoError(t, err) resolver := source.NewMockResolverForPaths(test.files...) diff --git a/syft/file/test-fixtures/a-path.txt b/syft/file/cataloger/filecontents/test-fixtures/a-path.txt similarity index 100% rename from syft/file/test-fixtures/a-path.txt rename to syft/file/cataloger/filecontents/test-fixtures/a-path.txt diff --git a/syft/file/test-fixtures/another-path.txt b/syft/file/cataloger/filecontents/test-fixtures/another-path.txt similarity index 100% rename from syft/file/test-fixtures/another-path.txt rename to syft/file/cataloger/filecontents/test-fixtures/another-path.txt diff --git a/syft/file/test-fixtures/last/empty/empty b/syft/file/cataloger/filecontents/test-fixtures/last/empty/empty similarity index 100% rename from syft/file/test-fixtures/last/empty/empty rename to syft/file/cataloger/filecontents/test-fixtures/last/empty/empty diff --git a/syft/file/test-fixtures/last/path.txt b/syft/file/cataloger/filecontents/test-fixtures/last/path.txt similarity index 100% rename from syft/file/test-fixtures/last/path.txt rename to syft/file/cataloger/filecontents/test-fixtures/last/path.txt diff --git a/syft/file/digest_cataloger.go b/syft/file/cataloger/filedigests/cataloger.go similarity index 80% rename from syft/file/digest_cataloger.go rename to syft/file/cataloger/filedigests/cataloger.go index 8d6c7c52c..7529912d4 100644 --- a/syft/file/digest_cataloger.go +++ b/syft/file/cataloger/filedigests/cataloger.go @@ -1,37 +1,37 @@ -package file +package filedigests import ( "crypto" "errors" "fmt" - "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/bus" - "github.com/anchore/syft/syft/event" - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" "hash" "io" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" - + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/source" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" ) var errUndigestableFile = errors.New("undigestable file") -type DigestsCataloger struct { +type Cataloger struct { hashes []crypto.Hash } -func NewDigestsCataloger(hashes []crypto.Hash) (*DigestsCataloger, error) { - return &DigestsCataloger{ +func NewCataloger(hashes []crypto.Hash) (*Cataloger, error) { + return &Cataloger{ hashes: hashes, }, nil } -func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]Digest, error) { - results := make(map[source.Coordinates][]Digest) - locations := allRegularFiles(resolver) +func (i *Cataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]file.Digest, error) { + results := make(map[source.Coordinates][]file.Digest) + locations := source.AllRegularFiles(resolver) stage, prog := digestsCatalogingProgress(int64(len(locations))) for _, location := range locations { stage.Current = location.RealPath @@ -57,7 +57,7 @@ func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Coo return results, nil } -func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]Digest, error) { +func (i *Cataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]file.Digest, error) { meta, err := resolver.FileMetadataByLocation(location) if err != nil { return nil, err @@ -88,16 +88,16 @@ func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, locatio } if size == 0 { - return make([]Digest, 0), nil + return make([]file.Digest, 0), nil } - result := make([]Digest, len(i.hashes)) + result := make([]file.Digest, len(i.hashes)) // only capture digests when there is content. It is important to do this based on SIZE and not // FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only // file type but a body is still allowed. for idx, hasher := range hashers { - result[idx] = Digest{ - Algorithm: DigestAlgorithmName(i.hashes[idx]), + result[idx] = file.Digest{ + Algorithm: file.DigestAlgorithmName(i.hashes[idx]), Value: fmt.Sprintf("%+x", hasher.Sum(nil)), } } diff --git a/syft/file/digest_cataloger_test.go b/syft/file/cataloger/filedigests/cataloger_test.go similarity index 82% rename from syft/file/digest_cataloger_test.go rename to syft/file/cataloger/filedigests/cataloger_test.go index 277921508..521fea712 100644 --- a/syft/file/digest_cataloger_test.go +++ b/syft/file/cataloger/filedigests/cataloger_test.go @@ -1,25 +1,23 @@ -package file +package filedigests import ( "crypto" "fmt" - "github.com/stretchr/testify/require" "io/ioutil" "os" "path/filepath" "testing" - "github.com/anchore/stereoscope/pkg/file" - + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" - - "github.com/stretchr/testify/assert" - + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/source" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func testDigests(t testing.TB, root string, files []string, hashes ...crypto.Hash) map[source.Coordinates][]Digest { - digests := make(map[source.Coordinates][]Digest) +func testDigests(t testing.TB, root string, files []string, hashes ...crypto.Hash) map[source.Coordinates][]file.Digest { + digests := make(map[source.Coordinates][]file.Digest) for _, f := range files { fh, err := os.Open(filepath.Join(root, f)) @@ -33,15 +31,15 @@ func testDigests(t testing.TB, root string, files []string, hashes ...crypto.Has if len(b) == 0 { // we don't keep digests for empty files - digests[source.NewLocation(f).Coordinates] = []Digest{} + digests[source.NewLocation(f).Coordinates] = []file.Digest{} continue } for _, hash := range hashes { h := hash.New() h.Write(b) - digests[source.NewLocation(f).Coordinates] = append(digests[source.NewLocation(f).Coordinates], Digest{ - Algorithm: CleanDigestAlgorithmName(hash.String()), + digests[source.NewLocation(f).Coordinates] = append(digests[source.NewLocation(f).Coordinates], file.Digest{ + Algorithm: file.CleanDigestAlgorithmName(hash.String()), Value: fmt.Sprintf("%x", h.Sum(nil)), }) } @@ -56,7 +54,7 @@ func TestDigestsCataloger(t *testing.T) { name string digests []crypto.Hash files []string - expected map[source.Coordinates][]Digest + expected map[source.Coordinates][]file.Digest }{ { name: "md5", @@ -74,7 +72,7 @@ func TestDigestsCataloger(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - c, err := NewDigestsCataloger(test.digests) + c, err := NewCataloger(test.digests) require.NoError(t, err) src, err := source.NewFromDirectory("test-fixtures/last/") @@ -92,14 +90,7 @@ func TestDigestsCataloger(t *testing.T) { } func TestDigestsCataloger_MixFileTypes(t *testing.T) { - testImage := "image-file-type-mix" - - if *updateImageGoldenFiles { - imagetest.UpdateGoldenFixtureImage(t, testImage) - } - - img := imagetest.GetGoldenFixtureImage(t, testImage) - + img := imagetest.GetFixtureImage(t, "docker-archive", "image-file-type-mix") src, err := source.NewFromImage(img, "---") if err != nil { t.Fatalf("could not create source: %+v", err) @@ -140,7 +131,7 @@ func TestDigestsCataloger_MixFileTypes(t *testing.T) { for _, test := range tests { t.Run(test.path, func(t *testing.T) { - c, err := NewDigestsCataloger([]crypto.Hash{crypto.MD5}) + c, err := NewCataloger([]crypto.Hash{crypto.MD5}) if err != nil { t.Fatalf("unable to get cataloger: %+v", err) } @@ -150,7 +141,7 @@ func TestDigestsCataloger_MixFileTypes(t *testing.T) { t.Fatalf("could not catalog: %+v", err) } - _, ref, err := img.SquashedTree().File(file.Path(test.path)) + _, ref, err := img.SquashedTree().File(stereoscopeFile.Path(test.path)) if err != nil { t.Fatalf("unable to get file=%q : %+v", test.path, err) } diff --git a/syft/file/test-fixtures/image-file-type-mix/Dockerfile b/syft/file/cataloger/filedigests/test-fixtures/image-file-type-mix/Dockerfile similarity index 81% rename from syft/file/test-fixtures/image-file-type-mix/Dockerfile rename to syft/file/cataloger/filedigests/test-fixtures/image-file-type-mix/Dockerfile index c2d61ef4d..f1caa0453 100644 --- a/syft/file/test-fixtures/image-file-type-mix/Dockerfile +++ b/syft/file/cataloger/filedigests/test-fixtures/image-file-type-mix/Dockerfile @@ -4,7 +4,6 @@ ADD file-1.txt . RUN chmod 644 file-1.txt RUN chown 1:2 file-1.txt RUN ln -s file-1.txt symlink-1 -# note: hard links may behave inconsistently, this should be a golden image RUN ln file-1.txt hardlink-1 RUN mknod char-device-1 c 89 1 RUN mknod block-device-1 b 0 1 diff --git a/syft/file/test-fixtures/image-file-type-mix/file-1.txt b/syft/file/cataloger/filedigests/test-fixtures/image-file-type-mix/file-1.txt similarity index 100% rename from syft/file/test-fixtures/image-file-type-mix/file-1.txt rename to syft/file/cataloger/filedigests/test-fixtures/image-file-type-mix/file-1.txt diff --git a/syft/file/cataloger/filedigests/test-fixtures/last/empty/empty b/syft/file/cataloger/filedigests/test-fixtures/last/empty/empty new file mode 100644 index 000000000..e69de29bb diff --git a/syft/file/cataloger/filedigests/test-fixtures/last/path.txt b/syft/file/cataloger/filedigests/test-fixtures/last/path.txt new file mode 100644 index 000000000..3d4a165ab --- /dev/null +++ b/syft/file/cataloger/filedigests/test-fixtures/last/path.txt @@ -0,0 +1 @@ +test-fixtures/last/path.txt file contents! \ No newline at end of file diff --git a/syft/file/metadata_cataloger.go b/syft/file/cataloger/filemetadata/cataloger.go similarity index 83% rename from syft/file/metadata_cataloger.go rename to syft/file/cataloger/filemetadata/cataloger.go index e9cf28b90..634363cf9 100644 --- a/syft/file/metadata_cataloger.go +++ b/syft/file/cataloger/filemetadata/cataloger.go @@ -1,4 +1,4 @@ -package file +package filemetadata import ( "github.com/anchore/syft/internal/bus" @@ -9,14 +9,14 @@ import ( "github.com/wagoodman/go-progress" ) -type MetadataCataloger struct { +type Cataloger struct { } -func NewMetadataCataloger() *MetadataCataloger { - return &MetadataCataloger{} +func NewCataloger() *Cataloger { + return &Cataloger{} } -func (i *MetadataCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates]source.FileMetadata, error) { +func (i *Cataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates]source.FileMetadata, error) { results := make(map[source.Coordinates]source.FileMetadata) var locations []source.Location for location := range resolver.AllLocations() { diff --git a/syft/file/metadata_cataloger_test.go b/syft/file/cataloger/filemetadata/cataloger_test.go similarity index 88% rename from syft/file/metadata_cataloger_test.go rename to syft/file/cataloger/filemetadata/cataloger_test.go index 9ca27a6d0..243195c10 100644 --- a/syft/file/metadata_cataloger_test.go +++ b/syft/file/cataloger/filemetadata/cataloger_test.go @@ -1,7 +1,6 @@ -package file +package filemetadata import ( - "flag" "os" "testing" @@ -11,18 +10,10 @@ import ( "github.com/stretchr/testify/assert" ) -var updateImageGoldenFiles = flag.Bool("update-image", false, "update the golden fixture images used for testing") - func TestFileMetadataCataloger(t *testing.T) { - testImage := "image-file-type-mix" + img := imagetest.GetFixtureImage(t, "docker-archive", "image-file-type-mix") - if *updateImageGoldenFiles { - imagetest.UpdateGoldenFixtureImage(t, testImage) - } - - img := imagetest.GetGoldenFixtureImage(t, testImage) - - c := NewMetadataCataloger() + c := NewCataloger() src, err := source.NewFromImage(img, "---") if err != nil { diff --git a/syft/file/cataloger/filemetadata/test-fixtures/image-file-type-mix/Dockerfile b/syft/file/cataloger/filemetadata/test-fixtures/image-file-type-mix/Dockerfile new file mode 100644 index 000000000..f1caa0453 --- /dev/null +++ b/syft/file/cataloger/filemetadata/test-fixtures/image-file-type-mix/Dockerfile @@ -0,0 +1,12 @@ +FROM busybox:latest + +ADD file-1.txt . +RUN chmod 644 file-1.txt +RUN chown 1:2 file-1.txt +RUN ln -s file-1.txt symlink-1 +RUN ln file-1.txt hardlink-1 +RUN mknod char-device-1 c 89 1 +RUN mknod block-device-1 b 0 1 +RUN mknod fifo-1 p +RUN mkdir /dir +RUN rm -rf home etc/group etc/localtime etc/mtab etc/network etc/passwd etc/shadow var usr bin/* \ No newline at end of file diff --git a/syft/file/cataloger/filemetadata/test-fixtures/image-file-type-mix/file-1.txt b/syft/file/cataloger/filemetadata/test-fixtures/image-file-type-mix/file-1.txt new file mode 100644 index 000000000..d86db8155 --- /dev/null +++ b/syft/file/cataloger/filemetadata/test-fixtures/image-file-type-mix/file-1.txt @@ -0,0 +1 @@ +file 1! \ No newline at end of file diff --git a/syft/file/secrets_cataloger.go b/syft/file/cataloger/secrets/cataloger.go similarity index 84% rename from syft/file/secrets_cataloger.go rename to syft/file/cataloger/secrets/cataloger.go index a1c5147e3..0d6031b03 100644 --- a/syft/file/secrets_cataloger.go +++ b/syft/file/cataloger/secrets/cataloger.go @@ -1,23 +1,22 @@ -package file +package secrets import ( "bytes" "fmt" - "github.com/anchore/syft/internal/bus" - "github.com/anchore/syft/internal/file" - "github.com/anchore/syft/syft/event" - "github.com/anchore/syft/syft/event/monitor" - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" "io" "io/ioutil" "regexp" "sort" "github.com/anchore/syft/internal" - + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/source" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" ) var DefaultSecretsPatterns = map[string]string{ @@ -28,38 +27,38 @@ var DefaultSecretsPatterns = map[string]string{ "generic-api-key": `(?i)api(-|_)?key["'=:\s]*?(?P[A-Z0-9]{20,60})["']?(\s|$)`, } -type SecretsCatalogerConfig struct { +type CatalogerConfig struct { Patterns map[string]*regexp.Regexp RevealValues bool MaxFileSize int64 } -type SecretsCataloger struct { - config SecretsCatalogerConfig +type Cataloger struct { + config CatalogerConfig } -func DefaultSecretsCatalogerConfig() SecretsCatalogerConfig { - patterns, err := GenerateSearchPatterns(DefaultSecretsPatterns, nil, nil) +func DefaultCatalogerConfig() CatalogerConfig { + patterns, err := file.GenerateSearchPatterns(DefaultSecretsPatterns, nil, nil) if err != nil { patterns = make(map[string]*regexp.Regexp) log.Errorf("unable to create default secrets config: %w", err) } - return SecretsCatalogerConfig{ + return CatalogerConfig{ Patterns: patterns, RevealValues: false, MaxFileSize: 1 * file.MB, } } -func NewSecretsCataloger(config SecretsCatalogerConfig) (*SecretsCataloger, error) { - return &SecretsCataloger{ +func NewCataloger(config CatalogerConfig) (*Cataloger, error) { + return &Cataloger{ config: config, }, nil } -func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]SearchResult, error) { - results := make(map[source.Coordinates][]SearchResult) - locations := allRegularFiles(resolver) +func (i *Cataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]file.SearchResult, error) { + results := make(map[source.Coordinates][]file.SearchResult) + locations := source.AllRegularFiles(resolver) stage, prog, secretsDiscovered := newSecretsCatalogerMonitor(int64(len(locations))) for _, location := range locations { stage.Current = location.RealPath @@ -83,7 +82,7 @@ func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Coo return results, nil } -func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]SearchResult, error) { +func (i *Cataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]file.SearchResult, error) { metadata, err := resolver.FileMetadataByLocation(location) if err != nil { return nil, err diff --git a/syft/file/secrets_cataloger_test.go b/syft/file/cataloger/secrets/cataloger_test.go similarity index 92% rename from syft/file/secrets_cataloger_test.go rename to syft/file/cataloger/secrets/cataloger_test.go index 696f3865c..36fb25df9 100644 --- a/syft/file/secrets_cataloger_test.go +++ b/syft/file/cataloger/secrets/cataloger_test.go @@ -1,13 +1,11 @@ -package file +package secrets import ( "regexp" "testing" - "github.com/anchore/syft/internal/file" - + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/source" - "github.com/stretchr/testify/assert" ) @@ -18,7 +16,7 @@ func TestSecretsCataloger(t *testing.T) { reveal bool maxSize int64 patterns map[string]string - expected []SearchResult + expected []file.SearchResult constructorErr bool catalogErr bool }{ @@ -29,7 +27,7 @@ func TestSecretsCataloger(t *testing.T) { patterns: map[string]string{ "simple-secret-key": `^secret_key=.*`, }, - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "simple-secret-key", LineNumber: 2, @@ -47,7 +45,7 @@ func TestSecretsCataloger(t *testing.T) { patterns: map[string]string{ "simple-secret-key": `^secret_key=.*`, }, - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "simple-secret-key", LineNumber: 2, @@ -65,7 +63,7 @@ func TestSecretsCataloger(t *testing.T) { patterns: map[string]string{ "simple-secret-key": `^secret_key=(?P.*)`, }, - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "simple-secret-key", LineNumber: 2, @@ -83,7 +81,7 @@ func TestSecretsCataloger(t *testing.T) { patterns: map[string]string{ "simple-secret-key": `secret_key=.*`, }, - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "simple-secret-key", LineNumber: 1, @@ -126,7 +124,7 @@ func TestSecretsCataloger(t *testing.T) { patterns: map[string]string{ "simple-secret-key": `secret_key=(?P.*)`, }, - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "simple-secret-key", LineNumber: 1, @@ -177,7 +175,11 @@ func TestSecretsCataloger(t *testing.T) { regexObjs[name] = obj } - c, err := NewSecretsCataloger(regexObjs, test.reveal, test.maxSize) + c, err := NewCataloger(CatalogerConfig{ + Patterns: regexObjs, + RevealValues: test.reveal, + MaxFileSize: test.maxSize, + }) if err != nil && !test.constructorErr { t.Fatalf("could not create cataloger (but should have been able to): %+v", err) } else if err == nil && test.constructorErr { @@ -208,18 +210,18 @@ func TestSecretsCataloger(t *testing.T) { } func TestSecretsCataloger_DefaultSecrets(t *testing.T) { - regexObjs, err := GenerateSearchPatterns(DefaultSecretsPatterns, nil, nil) + regexObjs, err := file.GenerateSearchPatterns(DefaultSecretsPatterns, nil, nil) if err != nil { t.Fatalf("unable to get patterns: %+v", err) } tests := []struct { fixture string - expected []SearchResult + expected []file.SearchResult }{ { fixture: "test-fixtures/secrets/default/aws.env", - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "aws-access-key", LineNumber: 2, @@ -240,7 +242,7 @@ func TestSecretsCataloger_DefaultSecrets(t *testing.T) { }, { fixture: "test-fixtures/secrets/default/aws.ini", - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "aws-access-key", LineNumber: 3, @@ -261,7 +263,7 @@ func TestSecretsCataloger_DefaultSecrets(t *testing.T) { }, { fixture: "test-fixtures/secrets/default/private-key.pem", - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "pem-private-key", LineNumber: 2, @@ -281,7 +283,7 @@ z3P668YfhUbKdRF6S42Cg6zn }, { fixture: "test-fixtures/secrets/default/private-key-openssl.pem", - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "pem-private-key", LineNumber: 2, @@ -303,7 +305,7 @@ z3P668YfhUbKdRF6S42Cg6zn // note: this test proves that the PEM regex matches the smallest possible match // since the test catches two adjacent secrets fixture: "test-fixtures/secrets/default/private-keys.pem", - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "pem-private-key", LineNumber: 1, @@ -346,7 +348,7 @@ j4f668YfhUbKdRF6S6734856 // 2. a named capture group with the correct line number and line offset case // 3. the named capture group is in a different line than the match start, and both the match start and the capture group have different line offsets fixture: "test-fixtures/secrets/default/docker-config.json", - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "docker-config-auth", LineNumber: 5, @@ -363,7 +365,7 @@ j4f668YfhUbKdRF6S6734856 }, { fixture: "test-fixtures/secrets/default/api-key.txt", - expected: []SearchResult{ + expected: []file.SearchResult{ { Classification: "generic-api-key", LineNumber: 2, @@ -419,7 +421,11 @@ j4f668YfhUbKdRF6S6734856 for _, test := range tests { t.Run(test.fixture, func(t *testing.T) { - c, err := NewSecretsCataloger(regexObjs, true, 10*file.MB) + c, err := NewCataloger(CatalogerConfig{ + Patterns: regexObjs, + RevealValues: true, + MaxFileSize: 10 * file.MB, + }) if err != nil { t.Fatalf("could not create cataloger: %+v", err) } diff --git a/syft/file/newline_counter.go b/syft/file/cataloger/secrets/newline_counter.go similarity index 97% rename from syft/file/newline_counter.go rename to syft/file/cataloger/secrets/newline_counter.go index fec905523..d3c8ef894 100644 --- a/syft/file/newline_counter.go +++ b/syft/file/cataloger/secrets/newline_counter.go @@ -1,4 +1,4 @@ -package file +package secrets import "io" diff --git a/syft/file/newline_counter_test.go b/syft/file/cataloger/secrets/newline_counter_test.go similarity index 98% rename from syft/file/newline_counter_test.go rename to syft/file/cataloger/secrets/newline_counter_test.go index 24282bceb..0760e892c 100644 --- a/syft/file/newline_counter_test.go +++ b/syft/file/cataloger/secrets/newline_counter_test.go @@ -1,4 +1,4 @@ -package file +package secrets import ( "bufio" diff --git a/syft/file/secrets_search_by_line_strategy.go b/syft/file/cataloger/secrets/secrets_search_by_line_strategy.go similarity index 93% rename from syft/file/secrets_search_by_line_strategy.go rename to syft/file/cataloger/secrets/secrets_search_by_line_strategy.go index 0fd87ff63..c6cc01667 100644 --- a/syft/file/secrets_search_by_line_strategy.go +++ b/syft/file/cataloger/secrets/secrets_search_by_line_strategy.go @@ -1,9 +1,10 @@ -package file +package secrets import ( "bufio" "errors" "fmt" + "github.com/anchore/syft/syft/file" "io" "io/ioutil" "regexp" @@ -13,7 +14,7 @@ import ( "github.com/anchore/syft/syft/source" ) -func catalogLocationByLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp) ([]SearchResult, error) { +func catalogLocationByLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp) ([]file.SearchResult, error) { readCloser, err := resolver.FileContentsByLocation(location) if err != nil { return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) @@ -22,7 +23,7 @@ func catalogLocationByLine(resolver source.FileResolver, location source.Locatio var scanner = bufio.NewReader(readCloser) var position int64 - var allSecrets []SearchResult + var allSecrets []file.SearchResult var lineNo int64 var readErr error for !errors.Is(readErr, io.EOF) { @@ -45,8 +46,8 @@ func catalogLocationByLine(resolver source.FileResolver, location source.Locatio return allSecrets, nil } -func searchForSecretsWithinLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp, line []byte, lineNo int64, position int64) ([]SearchResult, error) { - var secrets []SearchResult +func searchForSecretsWithinLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp, line []byte, lineNo int64, position int64) ([]file.SearchResult, error) { + var secrets []file.SearchResult for name, pattern := range patterns { matches := pattern.FindAllIndex(line, -1) for i, match := range matches { @@ -91,7 +92,7 @@ func readerAtPosition(resolver source.FileResolver, location source.Location, se return readCloser, nil } -func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *regexp.Regexp, lineNo, lineOffset, seekPosition int64) *SearchResult { +func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *regexp.Regexp, lineNo, lineOffset, seekPosition int64) *file.SearchResult { reader := &newlineCounter{RuneReader: bufio.NewReader(readCloser)} positions := pattern.FindReaderSubmatchIndex(reader) if len(positions) == 0 { @@ -127,7 +128,7 @@ func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *r lineOffsetOfSecret += lineOffset } - return &SearchResult{ + return &file.SearchResult{ Classification: name, SeekPosition: start + seekPosition, Length: stop - start, diff --git a/syft/file/test-fixtures/secrets/default/api-key.txt b/syft/file/cataloger/secrets/test-fixtures/secrets/default/api-key.txt similarity index 100% rename from syft/file/test-fixtures/secrets/default/api-key.txt rename to syft/file/cataloger/secrets/test-fixtures/secrets/default/api-key.txt diff --git a/syft/file/test-fixtures/secrets/default/aws.env b/syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.env similarity index 100% rename from syft/file/test-fixtures/secrets/default/aws.env rename to syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.env diff --git a/syft/file/test-fixtures/secrets/default/aws.ini b/syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.ini similarity index 100% rename from syft/file/test-fixtures/secrets/default/aws.ini rename to syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.ini diff --git a/syft/file/test-fixtures/secrets/default/docker-config.json b/syft/file/cataloger/secrets/test-fixtures/secrets/default/docker-config.json similarity index 100% rename from syft/file/test-fixtures/secrets/default/docker-config.json rename to syft/file/cataloger/secrets/test-fixtures/secrets/default/docker-config.json diff --git a/syft/file/test-fixtures/secrets/default/not-docker-config.json b/syft/file/cataloger/secrets/test-fixtures/secrets/default/not-docker-config.json similarity index 100% rename from syft/file/test-fixtures/secrets/default/not-docker-config.json rename to syft/file/cataloger/secrets/test-fixtures/secrets/default/not-docker-config.json diff --git a/syft/file/test-fixtures/secrets/default/private-key-false-positive.pem b/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-false-positive.pem similarity index 100% rename from syft/file/test-fixtures/secrets/default/private-key-false-positive.pem rename to syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-false-positive.pem diff --git a/syft/file/test-fixtures/secrets/default/private-key-openssl.pem b/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-openssl.pem similarity index 100% rename from syft/file/test-fixtures/secrets/default/private-key-openssl.pem rename to syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-openssl.pem diff --git a/syft/file/test-fixtures/secrets/default/private-key.pem b/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key.pem similarity index 100% rename from syft/file/test-fixtures/secrets/default/private-key.pem rename to syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key.pem diff --git a/syft/file/test-fixtures/secrets/default/private-keys.pem b/syft/file/cataloger/secrets/test-fixtures/secrets/default/private-keys.pem similarity index 100% rename from syft/file/test-fixtures/secrets/default/private-keys.pem rename to syft/file/cataloger/secrets/test-fixtures/secrets/default/private-keys.pem diff --git a/syft/file/test-fixtures/secrets/multiple.txt b/syft/file/cataloger/secrets/test-fixtures/secrets/multiple.txt similarity index 100% rename from syft/file/test-fixtures/secrets/multiple.txt rename to syft/file/cataloger/secrets/test-fixtures/secrets/multiple.txt diff --git a/syft/file/test-fixtures/secrets/simple.txt b/syft/file/cataloger/secrets/test-fixtures/secrets/simple.txt similarity index 100% rename from syft/file/test-fixtures/secrets/simple.txt rename to syft/file/cataloger/secrets/test-fixtures/secrets/simple.txt diff --git a/syft/file/classification.go b/syft/file/classification.go new file mode 100644 index 000000000..48e638a6e --- /dev/null +++ b/syft/file/classification.go @@ -0,0 +1,6 @@ +package file + +type Classification struct { + Class string `json:"class"` + Metadata map[string]string `json:"metadata"` +} diff --git a/syft/file/classifier.go b/syft/file/classifier.go index d447dbd70..7efb15a5b 100644 --- a/syft/file/classifier.go +++ b/syft/file/classifier.go @@ -11,6 +11,12 @@ import ( "github.com/anchore/syft/syft/source" ) +type Classifier struct { + Class string + FilepathPatterns []*regexp.Regexp + EvidencePatternTemplates []string +} + func DefaultClassifiers() []Classifier { return []Classifier{ { @@ -62,17 +68,6 @@ func DefaultClassifiers() []Classifier { } } -type Classifier struct { - Class string - FilepathPatterns []*regexp.Regexp - EvidencePatternTemplates []string -} - -type Classification struct { - Class string `json:"class"` - Metadata map[string]string `json:"metadata"` -} - func (c Classifier) Classify(resolver source.FileResolver, location source.Location) (*Classification, error) { doesFilepathMatch, filepathNamedGroupValues := filepathMatches(c.FilepathPatterns, location) if !doesFilepathMatch { diff --git a/internal/file/glob_match.go b/syft/file/glob_match.go similarity index 100% rename from internal/file/glob_match.go rename to syft/file/glob_match.go diff --git a/internal/file/glob_match_test.go b/syft/file/glob_match_test.go similarity index 100% rename from internal/file/glob_match_test.go rename to syft/file/glob_match_test.go diff --git a/internal/file/opener.go b/syft/file/opener.go similarity index 78% rename from internal/file/opener.go rename to syft/file/opener.go index 969930f11..46ca3150d 100644 --- a/internal/file/opener.go +++ b/syft/file/opener.go @@ -10,6 +10,12 @@ type Opener struct { path string } +func NewOpener(path string) Opener { + return Opener{ + path: path, + } +} + // Open the stored path as a io.ReadCloser. func (o Opener) Open() (io.ReadCloser, error) { return os.Open(o.path) diff --git a/syft/file/test-fixtures/snapshot/stereoscope-fixture-image-file-type-mix.golden b/syft/file/test-fixtures/snapshot/stereoscope-fixture-image-file-type-mix.golden deleted file mode 100644 index e85036214..000000000 Binary files a/syft/file/test-fixtures/snapshot/stereoscope-fixture-image-file-type-mix.golden and /dev/null differ diff --git a/syft/file/test-fixtures/symlinked-root/nested/link-root b/syft/file/test-fixtures/symlinked-root/nested/link-root deleted file mode 120000 index 24659224a..000000000 --- a/syft/file/test-fixtures/symlinked-root/nested/link-root +++ /dev/null @@ -1 +0,0 @@ -../real-root \ No newline at end of file diff --git a/syft/file/test-fixtures/symlinked-root/real-root/file1.txt b/syft/file/test-fixtures/symlinked-root/real-root/file1.txt deleted file mode 100644 index 5452844a2..000000000 --- a/syft/file/test-fixtures/symlinked-root/real-root/file1.txt +++ /dev/null @@ -1 +0,0 @@ -contents! diff --git a/syft/file/test-fixtures/symlinked-root/real-root/nested/file2.txt b/syft/file/test-fixtures/symlinked-root/real-root/nested/file2.txt deleted file mode 100644 index 5f7e2f213..000000000 --- a/syft/file/test-fixtures/symlinked-root/real-root/nested/file2.txt +++ /dev/null @@ -1 +0,0 @@ -more contents! diff --git a/syft/file/test-fixtures/symlinked-root/real-root/nested/linked-file1.txt b/syft/file/test-fixtures/symlinked-root/real-root/nested/linked-file1.txt deleted file mode 120000 index 4e7feb2d8..000000000 --- a/syft/file/test-fixtures/symlinked-root/real-root/nested/linked-file1.txt +++ /dev/null @@ -1 +0,0 @@ -../file1.txt \ No newline at end of file diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 4ff2369bc..584bc9df4 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -2,15 +2,16 @@ package java import ( "fmt" - "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/internal/archive" "io" "path" "strings" - "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" ) // integrity check @@ -35,7 +36,7 @@ var archiveFormatGlobs = []string{ } type archiveParser struct { - fileManifest file.ZipFileManifest + fileManifest archive.ZipFileManifest virtualPath string archivePath string contentPath string @@ -74,7 +75,7 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err) } - fileManifest, err := file.NewZipFileManifest(archivePath) + fileManifest, err := archive.NewZipFileManifest(archivePath) if err != nil { return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err) } @@ -144,7 +145,7 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) { } // fetch the manifest file - contents, err := file.ContentsFromZip(j.archivePath, manifestMatches...) + contents, err := archive.ContentsFromZip(j.archivePath, manifestMatches...) if err != nil { return nil, fmt.Errorf("unable to extract java manifests (%s): %w", j.virtualPath, err) } @@ -213,9 +214,9 @@ func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ( // discoverPkgsFromZip finds Java archives within Java archives, returning all listed Java packages found and // associating each discovered package to the given parent package. -func discoverPkgsFromZip(virtualPath, archivePath, contentPath string, fileManifest file.ZipFileManifest, parentPkg *pkg.Package) ([]*pkg.Package, []artifact.Relationship, error) { +func discoverPkgsFromZip(virtualPath, archivePath, contentPath string, fileManifest archive.ZipFileManifest, parentPkg *pkg.Package) ([]*pkg.Package, []artifact.Relationship, error) { // search and parse pom.properties files & fetch the contents - openers, err := file.ExtractFromZipToUniqueTempFile(archivePath, contentPath, fileManifest.GlobMatch(archiveFormatGlobs...)...) + openers, err := archive.ExtractFromZipToUniqueTempFile(archivePath, contentPath, fileManifest.GlobMatch(archiveFormatGlobs...)...) if err != nil { return nil, nil, fmt.Errorf("unable to extract files from zip: %w", err) } @@ -274,7 +275,7 @@ func discoverPkgsFromOpener(virtualPath, pathWithinArchive string, archiveOpener } func pomPropertiesByParentPath(archivePath string, extractPaths []string, virtualPath string) (map[string]pkg.PomProperties, error) { - contentsOfMavenPropertiesFiles, err := file.ContentsFromZip(archivePath, extractPaths...) + contentsOfMavenPropertiesFiles, err := archive.ContentsFromZip(archivePath, extractPaths...) if err != nil { return nil, fmt.Errorf("unable to extract maven files: %w", err) } @@ -302,7 +303,7 @@ func pomPropertiesByParentPath(archivePath string, extractPaths []string, virtua } func pomProjectByParentPath(archivePath string, extractPaths []string, virtualPath string) (map[string]pkg.PomProject, error) { - contentsOfMavenProjectFiles, err := file.ContentsFromZip(archivePath, extractPaths...) + contentsOfMavenProjectFiles, err := archive.ContentsFromZip(archivePath, extractPaths...) if err != nil { return nil, fmt.Errorf("unable to extract maven files: %w", err) } diff --git a/syft/pkg/cataloger/java/tar_wrapped_archive_parser.go b/syft/pkg/cataloger/java/tar_wrapped_archive_parser.go index 5e1bd62f0..03e5a845e 100644 --- a/syft/pkg/cataloger/java/tar_wrapped_archive_parser.go +++ b/syft/pkg/cataloger/java/tar_wrapped_archive_parser.go @@ -2,11 +2,10 @@ package java import ( "fmt" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/syft/pkg/cataloger/generic" "io" - "github.com/anchore/syft/internal/file" - "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" ) @@ -59,7 +58,7 @@ func parseTarWrappedJavaArchive(virtualPath string, reader io.Reader) ([]*pkg.Pa } func discoverPkgsFromTar(virtualPath, archivePath, contentPath string) ([]*pkg.Package, []artifact.Relationship, error) { - openers, err := file.ExtractGlobsFromTarToUniqueTempFile(archivePath, contentPath, archiveFormatGlobs...) + openers, err := archive.ExtractGlobsFromTarToUniqueTempFile(archivePath, contentPath, archiveFormatGlobs...) if err != nil { return nil, nil, fmt.Errorf("unable to extract files from tar: %w", err) } diff --git a/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go b/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go index c3b5a8103..9346f83f2 100644 --- a/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go +++ b/syft/pkg/cataloger/java/zip_wrapped_archive_parser.go @@ -2,11 +2,10 @@ package java import ( "fmt" + "github.com/anchore/syft/internal/archive" "github.com/anchore/syft/syft/pkg/cataloger/generic" "io" - "github.com/anchore/syft/internal/file" - "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" ) @@ -33,7 +32,7 @@ func parseZipWrappedJavaArchive(virtualPath string, reader io.Reader) ([]*pkg.Pa // functions support zips with shell scripts prepended to the file. Specifically, the helpers use the central // header at the end of the file to determine where the beginning of the zip payload is (unlike the standard lib // or archiver). - fileManifest, err := file.NewZipFileManifest(archivePath) + fileManifest, err := archive.NewZipFileManifest(archivePath) if err != nil { return nil, nil, fmt.Errorf("unable to read files from java archive: %w", err) } diff --git a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go index 4c6a880f1..b189e6cc3 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go @@ -3,11 +3,11 @@ package python import ( "bufio" "fmt" + "github.com/anchore/syft/syft/file" "io" "path/filepath" "strings" - "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" "github.com/mitchellh/mapstructure" diff --git a/syft/file/all_regular_files.go b/syft/source/all_regular_files.go similarity index 76% rename from syft/file/all_regular_files.go rename to syft/source/all_regular_files.go index e7612c8b2..18d225549 100644 --- a/syft/file/all_regular_files.go +++ b/syft/source/all_regular_files.go @@ -1,11 +1,10 @@ -package file +package source import ( "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/source" ) -func allRegularFiles(resolver source.FileResolver) (locations []source.Location) { +func AllRegularFiles(resolver FileResolver) (locations []Location) { for location := range resolver.AllLocations() { resolvedLocations, err := resolver.FilesByPath(location.RealPath) if err != nil { @@ -20,7 +19,7 @@ func allRegularFiles(resolver source.FileResolver) (locations []source.Location) continue } - if metadata.Type != source.RegularFile { + if metadata.Type != RegularFile { continue } locations = append(locations, resolvedLocation) diff --git a/syft/file/all_regular_files_test.go b/syft/source/all_regular_files_test.go similarity index 68% rename from syft/file/all_regular_files_test.go rename to syft/source/all_regular_files_test.go index aad33b348..24c378948 100644 --- a/syft/file/all_regular_files_test.go +++ b/syft/source/all_regular_files_test.go @@ -1,8 +1,7 @@ -package file +package source import ( "github.com/anchore/stereoscope/pkg/imagetest" - "github.com/anchore/syft/syft/source" "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -16,25 +15,19 @@ func Test_allRegularFiles(t *testing.T) { } tests := []struct { name string - setup func() source.FileResolver + setup func() FileResolver wantRealPaths *strset.Set wantVirtualPaths *strset.Set }{ { name: "image", - setup: func() source.FileResolver { - testImage := "image-file-type-mix" + setup: func() FileResolver { + img := imagetest.GetFixtureImage(t, "docker-archive", "image-file-type-mix") - if *updateImageGoldenFiles { - imagetest.UpdateGoldenFixtureImage(t, testImage) - } - - img := imagetest.GetGoldenFixtureImage(t, testImage) - - s, err := source.NewFromImage(img, "---") + s, err := NewFromImage(img, "---") require.NoError(t, err) - r, err := s.FileResolver(source.SquashedScope) + r, err := s.FileResolver(SquashedScope) require.NoError(t, err) return r @@ -44,10 +37,10 @@ func Test_allRegularFiles(t *testing.T) { }, { name: "directory", - setup: func() source.FileResolver { - s, err := source.NewFromDirectory("test-fixtures/symlinked-root/nested/link-root") + setup: func() FileResolver { + s, err := NewFromDirectory("test-fixtures/symlinked-root/nested/link-root") require.NoError(t, err) - r, err := s.FileResolver(source.SquashedScope) + r, err := s.FileResolver(SquashedScope) require.NoError(t, err) return r }, @@ -58,7 +51,7 @@ func Test_allRegularFiles(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { resolver := tt.setup() - locations := allRegularFiles(resolver) + locations := AllRegularFiles(resolver) realLocations := strset.New() virtualLocations := strset.New() for _, l := range locations { diff --git a/syft/source/test-fixtures/image-file-type-mix/Dockerfile b/syft/source/test-fixtures/image-file-type-mix/Dockerfile new file mode 100644 index 000000000..f1caa0453 --- /dev/null +++ b/syft/source/test-fixtures/image-file-type-mix/Dockerfile @@ -0,0 +1,12 @@ +FROM busybox:latest + +ADD file-1.txt . +RUN chmod 644 file-1.txt +RUN chown 1:2 file-1.txt +RUN ln -s file-1.txt symlink-1 +RUN ln file-1.txt hardlink-1 +RUN mknod char-device-1 c 89 1 +RUN mknod block-device-1 b 0 1 +RUN mknod fifo-1 p +RUN mkdir /dir +RUN rm -rf home etc/group etc/localtime etc/mtab etc/network etc/passwd etc/shadow var usr bin/* \ No newline at end of file diff --git a/syft/source/test-fixtures/image-file-type-mix/file-1.txt b/syft/source/test-fixtures/image-file-type-mix/file-1.txt new file mode 100644 index 000000000..d86db8155 --- /dev/null +++ b/syft/source/test-fixtures/image-file-type-mix/file-1.txt @@ -0,0 +1 @@ +file 1! \ No newline at end of file diff --git a/syft/tasks.go b/syft/tasks.go index d549916f1..20fac37bc 100644 --- a/syft/tasks.go +++ b/syft/tasks.go @@ -3,7 +3,11 @@ package syft import ( "fmt" "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/file/cataloger/fileclassifier" + "github.com/anchore/syft/syft/file/cataloger/filecontents" + "github.com/anchore/syft/syft/file/cataloger/filedigests" + "github.com/anchore/syft/syft/file/cataloger/filemetadata" + "github.com/anchore/syft/syft/file/cataloger/secrets" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg/cataloger/packages" "github.com/anchore/syft/syft/sbom" @@ -13,7 +17,7 @@ import ( type task func(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error) type taskGenerator func(CatalogingConfig) (task, error) -func generateCatalogPackagesTask(config CatalogingConfig) (task, error) { +func generatePackagesCatalogingTask(config CatalogingConfig) (task, error) { if len(config.PackageCatalogers) == 0 { return nil, nil } @@ -38,12 +42,12 @@ func generateCatalogPackagesTask(config CatalogingConfig) (task, error) { }, nil } -func generateCatalogFileMetadataTask(config CatalogingConfig) (task, error) { +func generateFileMetadataCatalogingTask(config CatalogingConfig) (task, error) { if !config.CaptureFileMetadata { return nil, nil } - metadataCataloger := file.NewMetadataCataloger() + cataloger := filemetadata.NewCataloger() return func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(config.Scope) @@ -51,7 +55,7 @@ func generateCatalogFileMetadataTask(config CatalogingConfig) (task, error) { return nil, err } - result, err := metadataCataloger.Catalog(resolver) + result, err := cataloger.Catalog(resolver) if err != nil { return nil, err } @@ -61,12 +65,12 @@ func generateCatalogFileMetadataTask(config CatalogingConfig) (task, error) { } -func generateCatalogFileDigestsTask(config CatalogingConfig) (task, error) { +func generateFileDigestsCatalogingTask(config CatalogingConfig) (task, error) { if len(config.DigestHashes) == 0 { return nil, nil } - digestsCataloger, err := file.NewDigestsCataloger(config.DigestHashes) + cataloger, err := filedigests.NewCataloger(config.DigestHashes) if err != nil { return nil, err } @@ -77,7 +81,7 @@ func generateCatalogFileDigestsTask(config CatalogingConfig) (task, error) { return nil, err } - result, err := digestsCataloger.Catalog(resolver) + result, err := cataloger.Catalog(resolver) if err != nil { return nil, err } @@ -87,12 +91,12 @@ func generateCatalogFileDigestsTask(config CatalogingConfig) (task, error) { } -func generateCatalogContentsTask(config CatalogingConfig) (task, error) { - if len(config.ContentsConfig.Globs) > 0 { +func generateContentsCatalogingTask(config CatalogingConfig) (task, error) { + if len(config.ContentsConfig.Globs) == 0 { return nil, nil } - contentsCataloger, err := file.NewContentsCataloger(config.ContentsConfig) + cataloger, err := filecontents.NewCataloger(config.ContentsConfig) if err != nil { return nil, err } @@ -103,7 +107,7 @@ func generateCatalogContentsTask(config CatalogingConfig) (task, error) { return nil, err } - result, err := contentsCataloger.Catalog(resolver) + result, err := cataloger.Catalog(resolver) if err != nil { return nil, err } @@ -112,12 +116,12 @@ func generateCatalogContentsTask(config CatalogingConfig) (task, error) { }, nil } -func generateCatalogSecretsTask(config CatalogingConfig) (task, error) { +func generateSecretsCatalogingTask(config CatalogingConfig) (task, error) { if !config.CaptureSecrets { return nil, nil } - secretsCataloger, err := file.NewSecretsCataloger(config.SecretsConfig) + cataloger, err := secrets.NewCataloger(config.SecretsConfig) if err != nil { return nil, err } @@ -128,7 +132,7 @@ func generateCatalogSecretsTask(config CatalogingConfig) (task, error) { return nil, err } - result, err := secretsCataloger.Catalog(resolver) + result, err := cataloger.Catalog(resolver) if err != nil { return nil, err } @@ -138,12 +142,12 @@ func generateCatalogSecretsTask(config CatalogingConfig) (task, error) { } -func generateCatalogFileClassificationsTask(config CatalogingConfig) (task, error) { +func generateFileClassifierTask(config CatalogingConfig) (task, error) { if !config.ClassifyFiles { return nil, nil } - classifierCataloger, err := file.NewClassificationCataloger(config.FileClassifiers) + cataloger, err := fileclassifier.NewCataloger(config.FileClassifiers) if err != nil { return nil, err } @@ -154,7 +158,7 @@ func generateCatalogFileClassificationsTask(config CatalogingConfig) (task, erro return nil, err } - result, err := classifierCataloger.Catalog(resolver) + result, err := cataloger.Catalog(resolver) if err != nil { return nil, err }