keep file catalogers separate from file-related definitions

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

internal/archive/safe_copy.go

@@ -1,12 +1,14 @@
-package file
+package archive
 
 import (
 	"errors"
 	"fmt"
 	"io"
+
+	"github.com/anchore/syft/syft/file"
 )
 
-const perFileReadLimit = 2 * GB
+const perFileReadLimit = 2 * file.GB
 
 // safeCopy limits the copy from the reader. This is useful when extracting files from archives to
 // protect against decompression bomb attacks.

internal/archive/tar_file_traversal.go

@@ -1,38 +1,39 @@
-package file
+package archive
 
 import (
 	"fmt"
 	"io/ioutil"
 	"path/filepath"
 
+	"github.com/anchore/syft/syft/file"
 	"github.com/bmatcuk/doublestar/v4"
 	"github.com/mholt/archiver/v3"
 )
 
 // ExtractGlobsFromTarToUniqueTempFile extracts paths matching the given globs within the given archive to a temporary directory, returning file openers for each file extracted.
-func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...string) (map[string]Opener, error) {
-	results := make(map[string]Opener)
+func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...string) (map[string]file.Opener, error) {
+	results := make(map[string]file.Opener)
 
 	// don't allow for full traversal, only select traversal from given paths
 	if len(globs) == 0 {
 		return results, nil
 	}
 
-	visitor := func(file archiver.File) error {
-		defer file.Close()
+	visitor := func(f archiver.File) error {
+		defer f.Close()
 
 		// ignore directories
-		if file.FileInfo.IsDir() {
+		if f.FileInfo.IsDir() {
 			return nil
 		}
 
 		// ignore any filename that doesn't match the given globs...
-		if !matchesAnyGlob(file.Name(), globs...) {
+		if !matchesAnyGlob(f.Name(), globs...) {
 			return nil
 		}
 
 		// we have a file we want to extract....
-		tempfilePrefix := filepath.Base(filepath.Clean(file.Name())) + "-"
+		tempfilePrefix := filepath.Base(filepath.Clean(f.Name())) + "-"
 		tempFile, err := ioutil.TempFile(dir, tempfilePrefix)
 		if err != nil {
 			return fmt.Errorf("unable to create temp file: %w", err)
@@ -42,11 +43,11 @@ func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...strin
 		// provides a ReadCloser. It is up to the caller to handle closing the file explicitly.
 		defer tempFile.Close()
 
-		if err := safeCopy(tempFile, file.ReadCloser); err != nil {
-			return fmt.Errorf("unable to copy source=%q for tar=%q: %w", file.Name(), archivePath, err)
+		if err := safeCopy(tempFile, f.ReadCloser); err != nil {
+			return fmt.Errorf("unable to copy source=%q for tar=%q: %w", f.Name(), archivePath, err)
 		}
 
-		results[file.Name()] = Opener{path: tempFile.Name()}
+		results[f.Name()] = file.NewOpener(tempFile.Name())
 
 		return nil
 	}

internal/archive/zip_file_helpers_test.go

@@ -1,4 +1,4 @@
-package file
+package archive
 
 import (
 	"io/ioutil"

internal/archive/zip_file_manifest.go

@@ -1,7 +1,8 @@
-package file
+package archive
 
 import (
 	"fmt"
+	"github.com/anchore/syft/syft/file"
 	"os"
 	"sort"
 	"strings"
@@ -48,7 +49,7 @@ func (z ZipFileManifest) GlobMatch(patterns ...string) []string {
 			// so that glob logic is consistent inside and outside of ZIP archives
 			normalizedEntry := normalizeZipEntryName(entry)
 
-			if GlobMatch(pattern, normalizedEntry) {
+			if file.GlobMatch(pattern, normalizedEntry) {
 				uniqueMatches.Add(entry)
 			}
 		}

internal/archive/zip_file_manifest_test.go

@@ -1,7 +1,7 @@
 //go:build !windows
 // +build !windows
 
-package file
+package archive
 
 import (
 	"encoding/json"

internal/archive/zip_file_traversal.go

@@ -1,9 +1,10 @@
-package file
+package archive
 
 import (
 	"archive/zip"
 	"bytes"
 	"fmt"
+	"github.com/anchore/syft/syft/file"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -12,14 +13,6 @@ import (
 	"github.com/anchore/syft/internal/log"
 )
 
-const (
-	// represents the order of bytes
-	_  = iota
-	KB = 1 << (10 * iota)
-	MB
-	GB
-)
-
 type errZipSlipDetected struct {
 	Prefix   string
 	JoinArgs []string
@@ -71,16 +64,16 @@ func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths
 }
 
 // ExtractFromZipToUniqueTempFile extracts select paths for the given archive to a temporary directory, returning file openers for each file extracted.
-func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]Opener, error) {
-	results := make(map[string]Opener)
+func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]file.Opener, error) {
+	results := make(map[string]file.Opener)
 
 	// don't allow for full traversal, only select traversal from given paths
 	if len(paths) == 0 {
 		return results, nil
 	}
 
-	visitor := func(file *zip.File) error {
-		tempfilePrefix := filepath.Base(filepath.Clean(file.Name)) + "-"
+	visitor := func(f *zip.File) error {
+		tempfilePrefix := filepath.Base(filepath.Clean(f.Name)) + "-"
 
 		tempFile, err := ioutil.TempFile(dir, tempfilePrefix)
 		if err != nil {
@@ -91,26 +84,26 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
 		// provides a ReadCloser. It is up to the caller to handle closing the file explicitly.
 		defer tempFile.Close()
 
-		zippedFile, err := file.Open()
+		zippedFile, err := f.Open()
 		if err != nil {
-			return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
+			return fmt.Errorf("unable to read file=%q from zip=%q: %w", f.Name, archivePath, err)
 		}
 		defer func() {
 			err := zippedFile.Close()
 			if err != nil {
-				log.Errorf("unable to close source file=%q from zip=%q: %+v", file.Name, archivePath, err)
+				log.Errorf("unable to close source file=%q from zip=%q: %+v", f.Name, archivePath, err)
 			}
 		}()
 
-		if file.FileInfo().IsDir() {
-			return fmt.Errorf("unable to extract directories, only files: %s", file.Name)
+		if f.FileInfo().IsDir() {
+			return fmt.Errorf("unable to extract directories, only files: %s", f.Name)
 		}
 
 		if err := safeCopy(tempFile, zippedFile); err != nil {
-			return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err)
+			return fmt.Errorf("unable to copy source=%q for zip=%q: %w", f.Name, archivePath, err)
 		}
 
-		results[file.Name] = Opener{path: tempFile.Name()}
+		results[f.Name] = file.NewOpener(tempFile.Name())
 
 		return nil
 	}

internal/archive/zip_file_traversal_test.go

@@ -1,7 +1,7 @@
 //go:build !windows
 // +build !windows
 
-package file
+package archive
 
 import (
 	"crypto/sha256"

internal/archive/zip_read_closer.go

@@ -1,4 +1,4 @@
-package file
+package archive
 
 import (
 	"archive/zip"

internal/archive/zip_read_closer_test.go

@@ -1,7 +1,7 @@
 //go:build !windows
 // +build !windows
 
-package file
+package archive
 
 import (
 	"os"

internal/config/application.go

@@ -45,7 +45,7 @@ type Application struct {
 	FileMetadata       FileMetadata       `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"`
 	FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"`
 	FileContents       fileContents       `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"`
-	Secrets            secrets            `yaml:"secrets" json:"secrets" mapstructure:"secrets"`
+	Secrets            secretsCfg         `yaml:"secrets" json:"secrets" mapstructure:"secrets"`
 	Registry           registry           `yaml:"registry" json:"registry" mapstructure:"registry"`
 	Exclusions         []string           `yaml:"exclude" json:"exclude" mapstructure:"exclude"`
 	Attest             attest             `yaml:"attest" json:"attest" mapstructure:"attest"`

internal/config/file_contents.go

@@ -1,8 +1,8 @@
 package config
 
 import (
-	internalFile "github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/file/cataloger/filecontents"
 	"github.com/anchore/syft/syft/source"
 	"github.com/spf13/viper"
 )
@@ -16,7 +16,7 @@ type fileContents struct {
 func (cfg fileContents) loadDefaultValues(v *viper.Viper) {
 	v.SetDefault("file-contents.cataloger.enabled", catalogerEnabledDefault)
 	v.SetDefault("file-contents.cataloger.scope", source.SquashedScope)
-	v.SetDefault("file-contents.skip-files-above-size", 1*internalFile.MB)
+	v.SetDefault("file-contents.skip-files-above-size", 1*file.MB)
 	v.SetDefault("file-contents.globs", []string{})
 }
 
@@ -24,8 +24,8 @@ func (cfg *fileContents) parseConfigValues() error {
 	return cfg.Cataloger.parseConfigValues()
 }
 
-func (cfg fileContents) ToConfig() file.ContentsCatalogerConfig {
-	return file.ContentsCatalogerConfig{
+func (cfg fileContents) ToConfig() filecontents.CatalogerConfig {
+	return filecontents.CatalogerConfig{
 		Globs:                     cfg.Globs,
 		SkipFilesAboveSizeInBytes: cfg.SkipFilesAboveSize,
 	}

internal/config/secrets.go

@@ -2,13 +2,13 @@ package config
 
 import (
 	"fmt"
-	internalFile "github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/file/cataloger/secrets"
 	"github.com/anchore/syft/syft/source"
 	"github.com/spf13/viper"
 )
 
-type secrets struct {
+type secretsCfg struct {
 	Cataloger           catalogerOptions  `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
 	AdditionalPatterns  map[string]string `yaml:"additional-patterns" json:"additional-patterns" mapstructure:"additional-patterns"`
 	ExcludePatternNames []string          `yaml:"exclude-pattern-names" json:"exclude-pattern-names" mapstructure:"exclude-pattern-names"`
@@ -16,25 +16,25 @@ type secrets struct {
 	SkipFilesAboveSize  int64             `yaml:"skip-files-above-size" json:"skip-files-above-size" mapstructure:"skip-files-above-size"`
 }
 
-func (cfg secrets) loadDefaultValues(v *viper.Viper) {
+func (cfg secretsCfg) loadDefaultValues(v *viper.Viper) {
 	v.SetDefault("secrets.cataloger.enabled", catalogerEnabledDefault)
 	v.SetDefault("secrets.cataloger.scope", source.AllLayersScope)
 	v.SetDefault("secrets.reveal-values", false)
-	v.SetDefault("secrets.skip-files-above-size", 1*internalFile.MB)
+	v.SetDefault("secrets.skip-files-above-size", 1*file.MB)
 	v.SetDefault("secrets.additional-patterns", map[string]string{})
 	v.SetDefault("secrets.exclude-pattern-names", []string{})
 }
 
-func (cfg *secrets) parseConfigValues() error {
+func (cfg *secretsCfg) parseConfigValues() error {
 	return cfg.Cataloger.parseConfigValues()
 }
 
-func (cfg secrets) ToConfig() (*file.SecretsCatalogerConfig, error) {
-	patterns, err := file.GenerateSearchPatterns(file.DefaultSecretsPatterns, cfg.AdditionalPatterns, cfg.ExcludePatternNames)
+func (cfg secretsCfg) ToConfig() (*secrets.CatalogerConfig, error) {
+	patterns, err := file.GenerateSearchPatterns(secrets.DefaultSecretsPatterns, cfg.AdditionalPatterns, cfg.ExcludePatternNames)
 	if err != nil {
 		return nil, fmt.Errorf("unable to process secrets config patterns: %w", err)
 	}
-	return &file.SecretsCatalogerConfig{
+	return &secrets.CatalogerConfig{
 		Patterns:     patterns,
 		RevealValues: cfg.RevealValues,
 		MaxFileSize:  cfg.SkipFilesAboveSize,

internal/file/glob_match.go

@@ -1,46 +0,0 @@
-package file
-
-// GlobMatch evaluates the given glob pattern against the given "name" string, indicating if there is a match or not.
-// Source: https://research.swtch.com/glob.go
-func GlobMatch(pattern, name string) bool {
-	px := 0
-	nx := 0
-	nextPx := 0
-	nextNx := 0
-	for px < len(pattern) || nx < len(name) {
-		if px < len(pattern) {
-			c := pattern[px]
-			switch c {
-			default: // ordinary character
-				if nx < len(name) && name[nx] == c {
-					px++
-					nx++
-					continue
-				}
-			case '?': // single-character wildcard
-				if nx < len(name) {
-					px++
-					nx++
-					continue
-				}
-			case '*': // zero-or-more-character wildcard
-				// Try to match at nx.
-				// If that doesn't work out,
-				// restart at nx+1 next.
-				nextPx = px
-				nextNx = nx + 1
-				px++
-				continue
-			}
-		}
-		// Mismatch. Maybe restart.
-		if 0 < nextNx && nextNx <= len(name) {
-			px = nextPx
-			nx = nextNx
-			continue
-		}
-		return false
-	}
-	// Matched all of pattern to all of name. Success.
-	return true
-}

internal/file/glob_match_test.go

@@ -1,39 +0,0 @@
-package file
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestGlobMatch(t *testing.T) {
-	var tests = []struct {
-		pattern string
-		data    string
-		ok      bool
-	}{
-		{"", "", true},
-		{"x", "", false},
-		{"", "x", false},
-		{"abc", "abc", true},
-		{"*", "abc", true},
-		{"*c", "abc", true},
-		{"*b", "abc", false},
-		{"a*", "abc", true},
-		{"b*", "abc", false},
-		{"a*", "a", true},
-		{"*a", "a", true},
-		{"a*b*c*d*e*", "axbxcxdxe", true},
-		{"a*b*c*d*e*", "axbxcxdxexxx", true},
-		{"a*b?c*x", "abxbbxdbxebxczzx", true},
-		{"a*b?c*x", "abxbbxdbxebxczzy", false},
-		{"a*a*a*a*b", strings.Repeat("a", 100), false},
-		{"*x", "xxx", true},
-		{"/home/place/**", "/home/place/a/thing", true},
-	}
-
-	for _, test := range tests {
-		if GlobMatch(test.pattern, test.data) != test.ok {
-			t.Errorf("failed glob='%s' data='%s'", test.pattern, test.data)
-		}
-	}
-}

internal/file/opener.go

@@ -1,16 +0,0 @@
-package file
-
-import (
-	"io"
-	"os"
-)
-
-// Opener is an object that stores a path to later be opened as a file.
-type Opener struct {
-	path string
-}
-
-// Open the stored path as a io.ReadCloser.
-func (o Opener) Open() (io.ReadCloser, error) {
-	return os.Open(o.path)
-}