Migrate location-related structs to the file package (#1751)

* migrate location structs to file package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* replace source.Location refs with file package call

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove hardlink test for file based catalogers

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove hardlink test for all-regular-files testing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* migrate file resolver implementations to separate package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* [wip] migrate resolvers to internal

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* migrate resolvers to syft/internal

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

---------

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
Signed-off-by: <>

syft/file/cataloger/filecontent/cataloger.go

@@ -0,0 +1,79 @@
+package filecontent
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"io"
+
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/file"
+)
+
+// Deprecated: will be removed in syft v1.0.0
+type Cataloger struct {
+	globs                     []string
+	skipFilesAboveSizeInBytes int64
+}
+
+// Deprecated: will be removed in syft v1.0.0
+func NewCataloger(globs []string, skipFilesAboveSize int64) (*Cataloger, error) {
+	return &Cataloger{
+		globs:                     globs,
+		skipFilesAboveSizeInBytes: skipFilesAboveSize,
+	}, nil
+}
+
+func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string, error) {
+	results := make(map[file.Coordinates]string)
+	var locations []file.Location
+
+	locations, err := resolver.FilesByGlob(i.globs...)
+	if err != nil {
+		return nil, err
+	}
+	for _, location := range locations {
+		metadata, err := resolver.FileMetadataByLocation(location)
+		if err != nil {
+			return nil, err
+		}
+
+		if i.skipFilesAboveSizeInBytes > 0 && metadata.Size() > i.skipFilesAboveSizeInBytes {
+			continue
+		}
+
+		result, err := i.catalogLocation(resolver, location)
+		if internal.IsErrPathPermission(err) {
+			log.Debugf("file contents cataloger skipping - %+v", err)
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		results[location.Coordinates] = result
+	}
+	log.Debugf("file contents cataloger processed %d files", len(results))
+
+	return results, nil
+}
+
+func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) (string, error) {
+	contentReader, err := resolver.FileContentsByLocation(location)
+	if err != nil {
+		return "", err
+	}
+	defer internal.CloseAndLogError(contentReader, location.VirtualPath)
+
+	buf := &bytes.Buffer{}
+	encoder := base64.NewEncoder(base64.StdEncoding, buf)
+	if _, err = io.Copy(encoder, contentReader); err != nil {
+		return "", internal.ErrPath{Context: "contents-cataloger", Path: location.RealPath, Err: err}
+	}
+	// note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes
+	if err := encoder.Close(); err != nil {
+		return "", fmt.Errorf("unable to close base64 encoder: %w", err)
+	}
+
+	return buf.String(), nil
+}

syft/file/cataloger/filecontent/cataloger_test.go

@@ -0,0 +1,80 @@
+package filecontent
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anchore/syft/syft/file"
+)
+
+func TestContentsCataloger(t *testing.T) {
+	allFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
+
+	tests := []struct {
+		name     string
+		globs    []string
+		maxSize  int64
+		files    []string
+		expected map[file.Coordinates]string
+	}{
+		{
+			name:  "multi-pattern",
+			globs: []string{"test-fixtures/last/*.txt", "test-fixtures/*.txt"},
+			files: allFiles,
+			expected: map[file.Coordinates]string{
+				file.NewLocation("test-fixtures/last/path.txt").Coordinates:    "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
+				file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+				file.NewLocation("test-fixtures/a-path.txt").Coordinates:       "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+			},
+		},
+		{
+			name:     "no-patterns",
+			globs:    []string{},
+			files:    []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"},
+			expected: map[file.Coordinates]string{},
+		},
+		{
+			name:  "all-txt",
+			globs: []string{"**/*.txt"},
+			files: allFiles,
+			expected: map[file.Coordinates]string{
+				file.NewLocation("test-fixtures/last/path.txt").Coordinates:    "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
+				file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+				file.NewLocation("test-fixtures/a-path.txt").Coordinates:       "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+			},
+		},
+		{
+			name:  "subpath",
+			globs: []string{"test-fixtures/*.txt"},
+			files: allFiles,
+			expected: map[file.Coordinates]string{
+				file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+				file.NewLocation("test-fixtures/a-path.txt").Coordinates:       "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+			},
+		},
+		{
+			name:    "size-filter",
+			maxSize: 42,
+			globs:   []string{"**/*.txt"},
+			files:   allFiles,
+			expected: map[file.Coordinates]string{
+				file.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
+				file.NewLocation("test-fixtures/a-path.txt").Coordinates:    "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			c, err := NewCataloger(test.globs, test.maxSize)
+			assert.NoError(t, err)
+
+			resolver := file.NewMockResolverForPaths(test.files...)
+			actual, err := c.Catalog(resolver)
+			assert.NoError(t, err)
+			assert.Equal(t, test.expected, actual, "mismatched contents")
+
+		})
+	}
+}

syft/file/cataloger/filecontent/test-fixtures/a-path.txt

@@ -0,0 +1 @@
+test-fixtures/a-path.txt file contents!

syft/file/cataloger/filecontent/test-fixtures/another-path.txt

@@ -0,0 +1 @@
+test-fixtures/another-path.txt file contents!

syft/file/cataloger/filecontent/test-fixtures/last/path.txt

@@ -0,0 +1 @@
+test-fixtures/last/path.txt file contents!

syft/file/cataloger/filedigest/cataloger.go

@@ -0,0 +1,109 @@
+package filedigest
+
+import (
+	"crypto"
+	"errors"
+
+	"github.com/wagoodman/go-partybus"
+	"github.com/wagoodman/go-progress"
+
+	stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/bus"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/event"
+	"github.com/anchore/syft/syft/file"
+	internal2 "github.com/anchore/syft/syft/file/cataloger/internal"
+)
+
+var ErrUndigestableFile = errors.New("undigestable file")
+
+type Cataloger struct {
+	hashes []crypto.Hash
+}
+
+func NewCataloger(hashes []crypto.Hash) *Cataloger {
+	return &Cataloger{
+		hashes: hashes,
+	}
+}
+
+func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates][]file.Digest, error) {
+	results := make(map[file.Coordinates][]file.Digest)
+	var locations []file.Location
+
+	if len(coordinates) == 0 {
+		locations = internal2.AllRegularFiles(resolver)
+	} else {
+		for _, c := range coordinates {
+			locations = append(locations, file.NewLocationFromCoordinates(c))
+		}
+	}
+
+	stage, prog := digestsCatalogingProgress(int64(len(locations)))
+	for _, location := range locations {
+		stage.Current = location.RealPath
+		result, err := i.catalogLocation(resolver, location)
+
+		if errors.Is(err, ErrUndigestableFile) {
+			continue
+		}
+
+		if internal.IsErrPathPermission(err) {
+			log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
+			continue
+		}
+
+		if err != nil {
+			return nil, err
+		}
+		prog.Increment()
+		results[location.Coordinates] = result
+	}
+	log.Debugf("file digests cataloger processed %d files", prog.Current())
+	prog.SetCompleted()
+	return results, nil
+}
+
+func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) ([]file.Digest, error) {
+	meta, err := resolver.FileMetadataByLocation(location)
+	if err != nil {
+		return nil, err
+	}
+
+	// we should only attempt to report digests for files that are regular files (don't attempt to resolve links)
+	if meta.Type != stereoscopeFile.TypeRegular {
+		return nil, ErrUndigestableFile
+	}
+
+	contentReader, err := resolver.FileContentsByLocation(location)
+	if err != nil {
+		return nil, err
+	}
+	defer internal.CloseAndLogError(contentReader, location.VirtualPath)
+
+	digests, err := file.NewDigestsFromFile(contentReader, i.hashes)
+	if err != nil {
+		return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
+	}
+
+	return digests, nil
+}
+
+func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
+	stage := &progress.Stage{}
+	prog := progress.NewManual(locations)
+
+	bus.Publish(partybus.Event{
+		Type: event.FileDigestsCatalogerStarted,
+		Value: struct {
+			progress.Stager
+			progress.Progressable
+		}{
+			Stager:       progress.Stager(stage),
+			Progressable: prog,
+		},
+	})
+
+	return stage, prog
+}

syft/file/cataloger/filedigest/cataloger_test.go

@@ -0,0 +1,161 @@
+package filedigest
+
+import (
+	"crypto"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/source"
+)
+
+func testDigests(t testing.TB, root string, files []string, hashes ...crypto.Hash) map[file.Coordinates][]file.Digest {
+	digests := make(map[file.Coordinates][]file.Digest)
+
+	for _, f := range files {
+		fh, err := os.Open(filepath.Join(root, f))
+		if err != nil {
+			t.Fatalf("could not open %q : %+v", f, err)
+		}
+		b, err := io.ReadAll(fh)
+		if err != nil {
+			t.Fatalf("could not read %q : %+v", f, err)
+		}
+
+		if len(b) == 0 {
+			// we don't keep digests for empty files
+			digests[file.NewLocation(f).Coordinates] = []file.Digest{}
+			continue
+		}
+
+		for _, hash := range hashes {
+			h := hash.New()
+			h.Write(b)
+			digests[file.NewLocation(f).Coordinates] = append(digests[file.NewLocation(f).Coordinates], file.Digest{
+				Algorithm: file.CleanDigestAlgorithmName(hash.String()),
+				Value:     fmt.Sprintf("%x", h.Sum(nil)),
+			})
+		}
+	}
+
+	return digests
+}
+
+func TestDigestsCataloger(t *testing.T) {
+
+	tests := []struct {
+		name     string
+		digests  []crypto.Hash
+		files    []string
+		expected map[file.Coordinates][]file.Digest
+	}{
+		{
+			name:     "md5",
+			digests:  []crypto.Hash{crypto.MD5},
+			files:    []string{"test-fixtures/last/empty/empty", "test-fixtures/last/path.txt"},
+			expected: testDigests(t, "test-fixtures/last", []string{"empty/empty", "path.txt"}, crypto.MD5),
+		},
+		{
+			name:     "md5-sha1-sha256",
+			digests:  []crypto.Hash{crypto.MD5, crypto.SHA1, crypto.SHA256},
+			files:    []string{"test-fixtures/last/empty/empty", "test-fixtures/last/path.txt"},
+			expected: testDigests(t, "test-fixtures/last", []string{"empty/empty", "path.txt"}, crypto.MD5, crypto.SHA1, crypto.SHA256),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			c := NewCataloger(test.digests)
+
+			src, err := source.NewFromDirectory("test-fixtures/last/")
+			require.NoError(t, err)
+
+			resolver, err := src.FileResolver(source.SquashedScope)
+			require.NoError(t, err)
+
+			actual, err := c.Catalog(resolver)
+			require.NoError(t, err)
+
+			assert.Equal(t, test.expected, actual, "mismatched digests")
+		})
+	}
+}
+
+func TestDigestsCataloger_MixFileTypes(t *testing.T) {
+	testImage := "image-file-type-mix"
+
+	img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
+
+	src, err := source.NewFromImage(img, "---")
+	if err != nil {
+		t.Fatalf("could not create source: %+v", err)
+	}
+
+	resolver, err := src.FileResolver(source.SquashedScope)
+	if err != nil {
+		t.Fatalf("could not create resolver: %+v", err)
+	}
+
+	tests := []struct {
+		path     string
+		expected string
+	}{
+		{
+			path:     "/file-1.txt",
+			expected: "888c139e550867814eb7c33b84d76e4d",
+		},
+		// this is difficult to reproduce in a cross-platform way
+		//{
+		//	path: "/hardlink-1",
+		//},
+		{
+			path: "/symlink-1",
+		},
+		{
+			path: "/char-device-1",
+		},
+		{
+			path: "/block-device-1",
+		},
+		{
+			path: "/fifo-1",
+		},
+		{
+			path: "/bin",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.path, func(t *testing.T) {
+			c := NewCataloger([]crypto.Hash{crypto.MD5})
+
+			actual, err := c.Catalog(resolver)
+			if err != nil {
+				t.Fatalf("could not catalog: %+v", err)
+			}
+
+			_, ref, err := img.SquashedTree().File(stereoscopeFile.Path(test.path))
+			if err != nil {
+				t.Fatalf("unable to get file=%q : %+v", test.path, err)
+			}
+			l := file.NewLocationFromImage(test.path, *ref.Reference, img)
+
+			if len(actual[l.Coordinates]) == 0 {
+				if test.expected != "" {
+					t.Fatalf("no digest found, but expected one")
+				}
+
+			} else {
+				assert.Equal(t, actual[l.Coordinates][0].Value, test.expected, "mismatched digests")
+			}
+		})
+	}
+}

syft/file/cataloger/filedigest/test-fixtures/image-file-type-mix/Dockerfile

@@ -0,0 +1,13 @@
+FROM busybox:1.28.1@sha256:c7b0a24019b0e6eda714ec0fa137ad42bc44a754d9cea17d14fba3a80ccc1ee4
+
+ADD file-1.txt .
+RUN chmod 644 file-1.txt
+RUN chown 1:2 file-1.txt
+RUN ln -s file-1.txt symlink-1
+# note: hard links may behave inconsistently, this should be a golden image
+RUN ln file-1.txt hardlink-1
+RUN mknod char-device-1 c 89 1
+RUN mknod block-device-1 b 0 1
+RUN mknod fifo-1 p
+RUN mkdir /dir
+RUN rm -rf home etc/group etc/localtime etc/mtab etc/network etc/passwd etc/shadow var usr bin/*

syft/file/cataloger/filedigest/test-fixtures/image-file-type-mix/file-1.txt

@@ -0,0 +1 @@
+file 1!

syft/file/cataloger/filedigest/test-fixtures/last/path.txt

@@ -0,0 +1 @@
+test-fixtures/last/path.txt file contents!

syft/file/cataloger/filemetadata/cataloger.go

@@ -0,0 +1,71 @@
+package filemetadata
+
+import (
+	"github.com/wagoodman/go-partybus"
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/syft/internal/bus"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/event"
+	"github.com/anchore/syft/syft/file"
+)
+
+type Cataloger struct {
+}
+
+func NewCataloger() *Cataloger {
+	return &Cataloger{}
+}
+
+func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates]file.Metadata, error) {
+	results := make(map[file.Coordinates]file.Metadata)
+	var locations <-chan file.Location
+
+	if len(coordinates) == 0 {
+		locations = resolver.AllLocations()
+	} else {
+		locations = func() <-chan file.Location {
+			ch := make(chan file.Location)
+			go func() {
+				close(ch)
+				for _, c := range coordinates {
+					ch <- file.NewLocationFromCoordinates(c)
+				}
+			}()
+			return ch
+		}()
+	}
+
+	stage, prog := metadataCatalogingProgress(int64(len(locations)))
+	for location := range locations {
+		stage.Current = location.RealPath
+		metadata, err := resolver.FileMetadataByLocation(location)
+		if err != nil {
+			return nil, err
+		}
+
+		results[location.Coordinates] = metadata
+		prog.Increment()
+	}
+	log.Debugf("file metadata cataloger processed %d files", prog.Current())
+	prog.SetCompleted()
+	return results, nil
+}
+
+func metadataCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
+	stage := &progress.Stage{}
+	prog := progress.NewManual(locations)
+
+	bus.Publish(partybus.Event{
+		Type: event.FileMetadataCatalogerStarted,
+		Value: struct {
+			progress.Stager
+			progress.Progressable
+		}{
+			Stager:       progress.Stager(stage),
+			Progressable: prog,
+		},
+	})
+
+	return stage, prog
+}

syft/file/cataloger/filemetadata/cataloger_test.go

@@ -0,0 +1,158 @@
+package filemetadata
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/source"
+)
+
+func TestFileMetadataCataloger(t *testing.T) {
+	testImage := "image-file-type-mix"
+
+	img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
+
+	c := NewCataloger()
+
+	src, err := source.NewFromImage(img, "---")
+	if err != nil {
+		t.Fatalf("could not create source: %+v", err)
+	}
+
+	resolver, err := src.FileResolver(source.SquashedScope)
+	if err != nil {
+		t.Fatalf("could not create resolver: %+v", err)
+	}
+
+	actual, err := c.Catalog(resolver)
+	if err != nil {
+		t.Fatalf("could not catalog: %+v", err)
+	}
+
+	tests := []struct {
+		path     string
+		exists   bool
+		expected file.Metadata
+		err      bool
+	}{
+		// note: it is difficult to add a hardlink-based test in a cross-platform way and is already covered well in stereoscope
+		{
+			path:   "/file-1.txt",
+			exists: true,
+			expected: file.Metadata{
+				FileInfo: stereoscopeFile.ManualInfo{
+					NameValue: "file-1.txt",
+					ModeValue: 0644,
+					SizeValue: 7,
+				},
+				Path:     "/file-1.txt",
+				Type:     stereoscopeFile.TypeRegular,
+				UserID:   1,
+				GroupID:  2,
+				MIMEType: "text/plain",
+			},
+		},
+		{
+			path:   "/symlink-1",
+			exists: true,
+			expected: file.Metadata{
+				Path: "/symlink-1",
+				FileInfo: stereoscopeFile.ManualInfo{
+					NameValue: "symlink-1",
+					ModeValue: 0777 | os.ModeSymlink,
+				},
+				Type:            stereoscopeFile.TypeSymLink,
+				LinkDestination: "file-1.txt",
+				UserID:          0,
+				GroupID:         0,
+				MIMEType:        "",
+			},
+		},
+		{
+			path:   "/char-device-1",
+			exists: true,
+			expected: file.Metadata{
+				Path: "/char-device-1",
+				FileInfo: stereoscopeFile.ManualInfo{
+					NameValue: "char-device-1",
+					ModeValue: 0644 | os.ModeDevice | os.ModeCharDevice,
+				},
+				Type:     stereoscopeFile.TypeCharacterDevice,
+				UserID:   0,
+				GroupID:  0,
+				MIMEType: "",
+			},
+		},
+		{
+			path:   "/block-device-1",
+			exists: true,
+			expected: file.Metadata{
+				Path: "/block-device-1",
+				FileInfo: stereoscopeFile.ManualInfo{
+					NameValue: "block-device-1",
+					ModeValue: 0644 | os.ModeDevice,
+				},
+				Type:     stereoscopeFile.TypeBlockDevice,
+				UserID:   0,
+				GroupID:  0,
+				MIMEType: "",
+			},
+		},
+		{
+			path:   "/fifo-1",
+			exists: true,
+			expected: file.Metadata{
+				Path: "/fifo-1",
+				FileInfo: stereoscopeFile.ManualInfo{
+					NameValue: "fifo-1",
+					ModeValue: 0644 | os.ModeNamedPipe,
+				},
+				Type:     stereoscopeFile.TypeFIFO,
+				UserID:   0,
+				GroupID:  0,
+				MIMEType: "",
+			},
+		},
+		{
+			path:   "/bin",
+			exists: true,
+			expected: file.Metadata{
+				Path: "/bin",
+				FileInfo: stereoscopeFile.ManualInfo{
+					NameValue: "bin",
+					ModeValue: 0755 | os.ModeDir,
+				},
+				Type:     stereoscopeFile.TypeDirectory,
+				UserID:   0,
+				GroupID:  0,
+				MIMEType: "",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.path, func(t *testing.T) {
+			_, ref, err := img.SquashedTree().File(stereoscopeFile.Path(test.path))
+			require.NoError(t, err)
+
+			l := file.NewLocationFromImage(test.path, *ref.Reference, img)
+
+			if _, ok := actual[l.Coordinates]; ok {
+				// we're not interested in keeping the test fixtures up to date with the latest file modification times
+				// thus ModTime is not under test
+				fi := test.expected.FileInfo.(stereoscopeFile.ManualInfo)
+				fi.ModTimeValue = actual[l.Coordinates].ModTime()
+				test.expected.FileInfo = fi
+			}
+
+			assert.True(t, test.expected.Equal(actual[l.Coordinates]))
+		})
+	}
+
+}

syft/file/cataloger/filemetadata/test-fixtures/image-file-type-mix/Dockerfile

@@ -0,0 +1,13 @@
+FROM busybox:1.28.1@sha256:c7b0a24019b0e6eda714ec0fa137ad42bc44a754d9cea17d14fba3a80ccc1ee4
+
+ADD file-1.txt .
+RUN chmod 644 file-1.txt
+RUN chown 1:2 file-1.txt
+RUN ln -s file-1.txt symlink-1
+# note: hard links may behave inconsistently, this should be a golden image
+RUN ln file-1.txt hardlink-1
+RUN mknod char-device-1 c 89 1
+RUN mknod block-device-1 b 0 1
+RUN mknod fifo-1 p
+RUN mkdir /dir
+RUN rm -rf home etc/group etc/localtime etc/mtab etc/network etc/passwd etc/shadow var usr bin/*

syft/file/cataloger/filemetadata/test-fixtures/image-file-type-mix/file-1.txt

@@ -0,0 +1 @@
+file 1!

syft/file/cataloger/internal/all_regular_files.go

@@ -0,0 +1,31 @@
+package internal
+
+import (
+	stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/file"
+)
+
+func AllRegularFiles(resolver file.Resolver) (locations []file.Location) {
+	for location := range resolver.AllLocations() {
+		resolvedLocations, err := resolver.FilesByPath(location.RealPath)
+		if err != nil {
+			log.Warnf("unable to resolve %+v: %+v", location, err)
+			continue
+		}
+
+		for _, resolvedLocation := range resolvedLocations {
+			metadata, err := resolver.FileMetadataByLocation(resolvedLocation)
+			if err != nil {
+				log.Warnf("unable to get metadata for %+v: %+v", location, err)
+				continue
+			}
+
+			if metadata.Type != stereoscopeFile.TypeRegular {
+				continue
+			}
+			locations = append(locations, resolvedLocation)
+		}
+	}
+	return locations
+}

syft/file/cataloger/internal/all_regular_files_test.go

@@ -0,0 +1,77 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/scylladb/go-set/strset"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/source"
+)
+
+func Test_allRegularFiles(t *testing.T) {
+	tests := []struct {
+		name             string
+		setup            func() file.Resolver
+		wantRealPaths    *strset.Set
+		wantVirtualPaths *strset.Set
+	}{
+		{
+			name: "image",
+			setup: func() file.Resolver {
+				testImage := "image-file-type-mix"
+
+				img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
+
+				s, err := source.NewFromImage(img, "---")
+				require.NoError(t, err)
+
+				r, err := s.FileResolver(source.SquashedScope)
+				require.NoError(t, err)
+
+				return r
+			},
+			wantRealPaths:    strset.New("/file-1.txt"),
+			wantVirtualPaths: strset.New("/file-1.txt", "/symlink-1", "/hardlink-1"),
+		},
+		{
+			name: "directory",
+			setup: func() file.Resolver {
+				s, err := source.NewFromDirectory("test-fixtures/symlinked-root/nested/link-root")
+				require.NoError(t, err)
+				r, err := s.FileResolver(source.SquashedScope)
+				require.NoError(t, err)
+				return r
+			},
+			wantRealPaths:    strset.New("file1.txt", "nested/file2.txt"),
+			wantVirtualPaths: strset.New("nested/linked-file1.txt"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			resolver := tt.setup()
+			locations := AllRegularFiles(resolver)
+			realLocations := strset.New()
+			virtualLocations := strset.New()
+			for _, l := range locations {
+				realLocations.Add(l.RealPath)
+				if l.VirtualPath != "" {
+					virtualLocations.Add(l.VirtualPath)
+				}
+			}
+
+			// this is difficult to reproduce in a cross-platform way
+			realLocations.Remove("/hardlink-1")
+			virtualLocations.Remove("/hardlink-1")
+			tt.wantRealPaths.Remove("/hardlink-1")
+			tt.wantVirtualPaths.Remove("/hardlink-1")
+
+			assert.ElementsMatch(t, tt.wantRealPaths.List(), realLocations.List(), "real paths differ: "+cmp.Diff(tt.wantRealPaths.List(), realLocations.List()))
+			assert.ElementsMatch(t, tt.wantVirtualPaths.List(), virtualLocations.List(), "virtual paths differ: "+cmp.Diff(tt.wantVirtualPaths.List(), virtualLocations.List()))
+		})
+	}
+}

syft/file/cataloger/internal/test-fixtures/image-file-type-mix/Dockerfile

@@ -0,0 +1,13 @@
+FROM busybox:1.28.1@sha256:c7b0a24019b0e6eda714ec0fa137ad42bc44a754d9cea17d14fba3a80ccc1ee4
+
+ADD file-1.txt .
+RUN chmod 644 file-1.txt
+RUN chown 1:2 file-1.txt
+RUN ln -s file-1.txt symlink-1
+# note: hard links may behave inconsistently, this should be a golden image
+RUN ln file-1.txt hardlink-1
+RUN mknod char-device-1 c 89 1
+RUN mknod block-device-1 b 0 1
+RUN mknod fifo-1 p
+RUN mkdir /dir
+RUN rm -rf home etc/group etc/localtime etc/mtab etc/network etc/passwd etc/shadow var usr bin/*

syft/file/cataloger/internal/test-fixtures/image-file-type-mix/file-1.txt

@@ -0,0 +1 @@
+file 1!

syft/file/cataloger/internal/test-fixtures/symlinked-root/nested/link-root

@@ -0,0 +1 @@
+../real-root

syft/file/cataloger/internal/test-fixtures/symlinked-root/real-root/file1.txt

@@ -0,0 +1 @@
+contents!

syft/file/cataloger/internal/test-fixtures/symlinked-root/real-root/nested/file2.txt

@@ -0,0 +1 @@
+more contents!

syft/file/cataloger/internal/test-fixtures/symlinked-root/real-root/nested/linked-file1.txt

@@ -0,0 +1 @@
+../file1.txt

syft/file/cataloger/secrets/cataloger.go

@@ -0,0 +1,158 @@
+package secrets
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"regexp"
+	"sort"
+
+	"github.com/wagoodman/go-partybus"
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/bus"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/event"
+	"github.com/anchore/syft/syft/file"
+	internal2 "github.com/anchore/syft/syft/file/cataloger/internal"
+)
+
+var DefaultSecretsPatterns = map[string]string{
+	"aws-access-key":     `(?i)aws_access_key_id["'=:\s]*?(?P<value>(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})`,
+	"aws-secret-key":     `(?i)aws_secret_access_key["'=:\s]*?(?P<value>[0-9a-zA-Z/+]{40})`,
+	"pem-private-key":    `-----BEGIN (\S+ )?PRIVATE KEY(\sBLOCK)?-----((?P<value>(\n.*?)+)-----END (\S+ )?PRIVATE KEY(\sBLOCK)?-----)?`,
+	"docker-config-auth": `"auths"((.*\n)*.*?"auth"\s*:\s*"(?P<value>[^"]+)")?`,
+	"generic-api-key":    `(?i)api(-|_)?key["'=:\s]*?(?P<value>[A-Z0-9]{20,60})["']?(\s|$)`,
+}
+
+// Deprecated: will be removed in syft v1.0.0
+type Cataloger struct {
+	patterns           map[string]*regexp.Regexp
+	revealValues       bool
+	skipFilesAboveSize int64
+}
+
+// Deprecated: will be removed in syft v1.0.0
+func NewCataloger(patterns map[string]*regexp.Regexp, revealValues bool, maxFileSize int64) (*Cataloger, error) {
+	return &Cataloger{
+		patterns:           patterns,
+		revealValues:       revealValues,
+		skipFilesAboveSize: maxFileSize,
+	}, nil
+}
+
+func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates][]file.SearchResult, error) {
+	results := make(map[file.Coordinates][]file.SearchResult)
+	locations := internal2.AllRegularFiles(resolver)
+	stage, prog, secretsDiscovered := secretsCatalogingProgress(int64(len(locations)))
+	for _, location := range locations {
+		stage.Current = location.RealPath
+		result, err := i.catalogLocation(resolver, location)
+		if internal.IsErrPathPermission(err) {
+			log.Debugf("secrets cataloger skipping - %+v", err)
+			continue
+		}
+
+		if err != nil {
+			return nil, err
+		}
+		if len(result) > 0 {
+			secretsDiscovered.Add(int64(len(result)))
+			results[location.Coordinates] = result
+		}
+		prog.Increment()
+	}
+	log.Debugf("secrets cataloger discovered %d secrets", secretsDiscovered.Current())
+	prog.SetCompleted()
+	return results, nil
+}
+
+func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) ([]file.SearchResult, error) {
+	metadata, err := resolver.FileMetadataByLocation(location)
+	if err != nil {
+		return nil, err
+	}
+
+	if metadata.Size() == 0 {
+		return nil, nil
+	}
+
+	if i.skipFilesAboveSize > 0 && metadata.Size() > i.skipFilesAboveSize {
+		return nil, nil
+	}
+
+	// TODO: in the future we can swap out search strategies here
+	secrets, err := catalogLocationByLine(resolver, location, i.patterns)
+	if err != nil {
+		return nil, internal.ErrPath{Context: "secrets-cataloger", Path: location.RealPath, Err: err}
+	}
+
+	if i.revealValues {
+		for idx, secret := range secrets {
+			value, err := extractValue(resolver, location, secret.SeekPosition, secret.Length)
+			if err != nil {
+				return nil, err
+			}
+			secrets[idx].Value = value
+		}
+	}
+
+	// sort by the start location of each secret as it appears in the location
+	sort.SliceStable(secrets, func(i, j int) bool {
+		return secrets[i].SeekPosition < secrets[j].SeekPosition
+	})
+
+	return secrets, nil
+}
+
+func extractValue(resolver file.Resolver, location file.Location, start, length int64) (string, error) {
+	readCloser, err := resolver.FileContentsByLocation(location)
+	if err != nil {
+		return "", fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
+	}
+	defer internal.CloseAndLogError(readCloser, location.VirtualPath)
+
+	n, err := io.CopyN(io.Discard, readCloser, start)
+	if err != nil {
+		return "", fmt.Errorf("unable to read contents for location=%q : %w", location, err)
+	}
+	if n != start {
+		return "", fmt.Errorf("unexpected seek location for location=%q : %d != %d", location, n, start)
+	}
+
+	var buf bytes.Buffer
+	n, err = io.CopyN(&buf, readCloser, length)
+	if err != nil {
+		return "", fmt.Errorf("unable to read secret value for location=%q : %w", location, err)
+	}
+	if n != length {
+		return "", fmt.Errorf("unexpected secret length for location=%q : %d != %d", location, n, length)
+	}
+
+	return buf.String(), nil
+}
+
+type Monitor struct {
+	progress.Stager
+	SecretsDiscovered progress.Monitorable
+	progress.Progressable
+}
+
+func secretsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual, *progress.Manual) {
+	stage := &progress.Stage{}
+	secretsDiscovered := &progress.Manual{}
+	prog := progress.NewManual(locations)
+
+	bus.Publish(partybus.Event{
+		Type:   event.SecretsCatalogerStarted,
+		Source: secretsDiscovered,
+		Value: Monitor{
+			Stager:            progress.Stager(stage),
+			SecretsDiscovered: secretsDiscovered,
+			Progressable:      prog,
+		},
+	})
+
+	return stage, prog, secretsDiscovered
+}

syft/file/cataloger/secrets/cataloger_test.go

@@ -0,0 +1,443 @@
+package secrets
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	intFile "github.com/anchore/syft/internal/file"
+	"github.com/anchore/syft/syft/file"
+)
+
+func TestSecretsCataloger(t *testing.T) {
+	tests := []struct {
+		name           string
+		fixture        string
+		reveal         bool
+		maxSize        int64
+		patterns       map[string]string
+		expected       []file.SearchResult
+		constructorErr bool
+		catalogErr     bool
+	}{
+		{
+			name:    "go-case-find-and-reveal",
+			fixture: "test-fixtures/secrets/simple.txt",
+			reveal:  true,
+			patterns: map[string]string{
+				"simple-secret-key": `^secret_key=.*`,
+			},
+			expected: []file.SearchResult{
+				{
+					Classification: "simple-secret-key",
+					LineNumber:     2,
+					LineOffset:     0,
+					SeekPosition:   34,
+					Length:         21,
+					Value:          "secret_key=clear_text",
+				},
+			},
+		},
+		{
+			name:    "dont-reveal-secret-value",
+			fixture: "test-fixtures/secrets/simple.txt",
+			reveal:  false,
+			patterns: map[string]string{
+				"simple-secret-key": `^secret_key=.*`,
+			},
+			expected: []file.SearchResult{
+				{
+					Classification: "simple-secret-key",
+					LineNumber:     2,
+					LineOffset:     0,
+					SeekPosition:   34,
+					Length:         21,
+					Value:          "",
+				},
+			},
+		},
+		{
+			name:    "reveal-named-capture-group",
+			fixture: "test-fixtures/secrets/simple.txt",
+			reveal:  true,
+			patterns: map[string]string{
+				"simple-secret-key": `^secret_key=(?P<value>.*)`,
+			},
+			expected: []file.SearchResult{
+				{
+					Classification: "simple-secret-key",
+					LineNumber:     2,
+					LineOffset:     11,
+					SeekPosition:   45,
+					Length:         10,
+					Value:          "clear_text",
+				},
+			},
+		},
+		{
+			name:    "multiple-secret-instances",
+			fixture: "test-fixtures/secrets/multiple.txt",
+			reveal:  true,
+			patterns: map[string]string{
+				"simple-secret-key": `secret_key=.*`,
+			},
+			expected: []file.SearchResult{
+				{
+					Classification: "simple-secret-key",
+					LineNumber:     1,
+					LineOffset:     0,
+					SeekPosition:   0,
+					Length:         22,
+					Value:          "secret_key=clear_text1",
+				},
+				{
+					Classification: "simple-secret-key",
+					LineNumber:     3,
+					LineOffset:     0,
+					SeekPosition:   57,
+					Length:         22,
+					Value:          "secret_key=clear_text2",
+				},
+				{
+					Classification: "simple-secret-key",
+					LineNumber:     4,
+					// note: this test captures a line offset case
+					LineOffset:   1,
+					SeekPosition: 81,
+					Length:       22,
+					Value:        "secret_key=clear_text3",
+				},
+				{
+					Classification: "simple-secret-key",
+					LineNumber:     6,
+					LineOffset:     0,
+					SeekPosition:   139,
+					Length:         22,
+					Value:          "secret_key=clear_text4",
+				},
+			},
+		},
+		{
+			name:    "multiple-secret-instances-with-capture-group",
+			fixture: "test-fixtures/secrets/multiple.txt",
+			reveal:  true,
+			patterns: map[string]string{
+				"simple-secret-key": `secret_key=(?P<value>.*)`,
+			},
+			expected: []file.SearchResult{
+				{
+					Classification: "simple-secret-key",
+					LineNumber:     1,
+					// note: value capture group location
+					LineOffset:   11,
+					SeekPosition: 11,
+					Length:       11,
+					Value:        "clear_text1",
+				},
+				{
+					Classification: "simple-secret-key",
+					LineNumber:     3,
+					LineOffset:     11,
+					SeekPosition:   68,
+					Length:         11,
+					Value:          "clear_text2",
+				},
+				{
+					Classification: "simple-secret-key",
+					LineNumber:     4,
+					// note: value capture group location + offset
+					LineOffset:   12,
+					SeekPosition: 92,
+					Length:       11,
+					Value:        "clear_text3",
+				},
+				{
+					Classification: "simple-secret-key",
+					LineNumber:     6,
+					LineOffset:     11,
+					SeekPosition:   150,
+					Length:         11,
+					Value:          "clear_text4",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			regexObjs := make(map[string]*regexp.Regexp)
+			for name, pattern := range test.patterns {
+				// always assume given patterns should be multiline
+				obj, err := regexp.Compile(`` + pattern)
+				if err != nil {
+					t.Fatalf("unable to parse regex: %+v", err)
+				}
+				regexObjs[name] = obj
+			}
+
+			c, err := NewCataloger(regexObjs, test.reveal, test.maxSize)
+			if err != nil && !test.constructorErr {
+				t.Fatalf("could not create cataloger (but should have been able to): %+v", err)
+			} else if err == nil && test.constructorErr {
+				t.Fatalf("expected constructor error but did not get one")
+			} else if test.constructorErr && err != nil {
+				return
+			}
+
+			resolver := file.NewMockResolverForPaths(test.fixture)
+
+			actualResults, err := c.Catalog(resolver)
+			if err != nil && !test.catalogErr {
+				t.Fatalf("could not catalog (but should have been able to): %+v", err)
+			} else if err == nil && test.catalogErr {
+				t.Fatalf("expected catalog error but did not get one")
+			} else if test.catalogErr && err != nil {
+				return
+			}
+
+			loc := file.NewLocation(test.fixture)
+			if _, exists := actualResults[loc.Coordinates]; !exists {
+				t.Fatalf("could not find location=%q in results", loc)
+			}
+
+			assert.Equal(t, test.expected, actualResults[loc.Coordinates], "mismatched secrets")
+		})
+	}
+}
+
+func TestSecretsCataloger_DefaultSecrets(t *testing.T) {
+	regexObjs, err := GenerateSearchPatterns(DefaultSecretsPatterns, nil, nil)
+	if err != nil {
+		t.Fatalf("unable to get patterns: %+v", err)
+	}
+
+	tests := []struct {
+		fixture  string
+		expected []file.SearchResult
+	}{
+		{
+			fixture: "test-fixtures/secrets/default/aws.env",
+			expected: []file.SearchResult{
+				{
+					Classification: "aws-access-key",
+					LineNumber:     2,
+					LineOffset:     25,
+					SeekPosition:   64,
+					Length:         20,
+					Value:          "AKIAIOSFODNN7EXAMPLE",
+				},
+				{
+					Classification: "aws-secret-key",
+					LineNumber:     3,
+					LineOffset:     29,
+					SeekPosition:   114,
+					Length:         40,
+					Value:          "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+				},
+			},
+		},
+		{
+			fixture: "test-fixtures/secrets/default/aws.ini",
+			expected: []file.SearchResult{
+				{
+					Classification: "aws-access-key",
+					LineNumber:     3,
+					LineOffset:     18,
+					SeekPosition:   67,
+					Length:         20,
+					Value:          "AKIAIOSFODNN7EXAMPLE",
+				},
+				{
+					Classification: "aws-secret-key",
+					LineNumber:     4,
+					LineOffset:     22,
+					SeekPosition:   110,
+					Length:         40,
+					Value:          "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+				},
+			},
+		},
+		{
+			fixture: "test-fixtures/secrets/default/private-key.pem",
+			expected: []file.SearchResult{
+				{
+					Classification: "pem-private-key",
+					LineNumber:     2,
+					LineOffset:     27,
+					SeekPosition:   66,
+					Length:         351,
+					Value: `
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
+cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
+VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
+bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
+BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
+z3P668YfhUbKdRF6S42Cg6zn
+`,
+				},
+			},
+		},
+		{
+			fixture: "test-fixtures/secrets/default/private-key-openssl.pem",
+			expected: []file.SearchResult{
+				{
+					Classification: "pem-private-key",
+					LineNumber:     2,
+					LineOffset:     35,
+					SeekPosition:   74,
+					Length:         351,
+					Value: `
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
+cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
+VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
+bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
+BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
+z3P668YfhUbKdRF6S42Cg6zn
+`,
+				},
+			},
+		},
+		{
+			// note: this test proves that the PEM regex matches the smallest possible match
+			// since the test catches two adjacent secrets
+			fixture: "test-fixtures/secrets/default/private-keys.pem",
+			expected: []file.SearchResult{
+				{
+					Classification: "pem-private-key",
+					LineNumber:     1,
+					LineOffset:     35,
+					SeekPosition:   35,
+					Length:         351,
+					Value: `
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
+cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
+VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
+bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
+BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
+z3P668YfhUbKdRF6S42Cg6zn
+`,
+				},
+				{
+					Classification: "pem-private-key",
+					LineNumber:     9,
+					LineOffset:     35,
+					SeekPosition:   455,
+					Length:         351,
+					Value: `
+MIIEvgTHISISNOTAREALKEYoIBAQDBj08DBj08DBj08DBj08DBj08DBsp5++4an3
+cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgY5
+VQQDDBcqLmF3cy10ZXN0SISNOTAREALKEYoIBAQDBj08DfffKoZIhvcNAQEBBQA7
+bml6SISNOTAREALKEYoIBAQDBj08DdssBggrBgEFBQcBAQSBkzCBkDBNBggrBgE8
+BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmd1
+j4f668YfhUbKdRF6S6734856
+`,
+				},
+			},
+		},
+		{
+			fixture:  "test-fixtures/secrets/default/private-key-false-positive.pem",
+			expected: nil,
+		},
+		{
+			// this test represents:
+			// 1. a docker config
+			// 2. a named capture group with the correct line number and line offset case
+			// 3. the named capture group is in a different line than the match start, and both the match start and the capture group have different line offsets
+			fixture: "test-fixtures/secrets/default/docker-config.json",
+			expected: []file.SearchResult{
+				{
+					Classification: "docker-config-auth",
+					LineNumber:     5,
+					LineOffset:     15,
+					SeekPosition:   100,
+					Length:         10,
+					Value:          "tOpsyKreTz",
+				},
+			},
+		},
+		{
+			fixture:  "test-fixtures/secrets/default/not-docker-config.json",
+			expected: nil,
+		},
+		{
+			fixture: "test-fixtures/secrets/default/api-key.txt",
+			expected: []file.SearchResult{
+				{
+					Classification: "generic-api-key",
+					LineNumber:     2,
+					LineOffset:     7,
+					SeekPosition:   33,
+					Length:         20,
+					Value:          "12345A7a901b34567890",
+				},
+				{
+					Classification: "generic-api-key",
+					LineNumber:     3,
+					LineOffset:     9,
+					SeekPosition:   63,
+					Length:         30,
+					Value:          "12345A7a901b345678901234567890",
+				},
+				{
+					Classification: "generic-api-key",
+					LineNumber:     4,
+					LineOffset:     10,
+					SeekPosition:   104,
+					Length:         40,
+					Value:          "12345A7a901b3456789012345678901234567890",
+				},
+				{
+					Classification: "generic-api-key",
+					LineNumber:     5,
+					LineOffset:     10,
+					SeekPosition:   156,
+					Length:         50,
+					Value:          "12345A7a901b34567890123456789012345678901234567890",
+				},
+				{
+					Classification: "generic-api-key",
+					LineNumber:     6,
+					LineOffset:     16,
+					SeekPosition:   224,
+					Length:         60,
+					Value:          "12345A7a901b345678901234567890123456789012345678901234567890",
+				},
+				{
+					Classification: "generic-api-key",
+					LineNumber:     14,
+					LineOffset:     8,
+					SeekPosition:   502,
+					Length:         20,
+					Value:          "11111111111111111111",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.fixture, func(t *testing.T) {
+
+			c, err := NewCataloger(regexObjs, true, 10*intFile.MB)
+			if err != nil {
+				t.Fatalf("could not create cataloger: %+v", err)
+			}
+
+			resolver := file.NewMockResolverForPaths(test.fixture)
+
+			actualResults, err := c.Catalog(resolver)
+			if err != nil {
+				t.Fatalf("could not catalog: %+v", err)
+			}
+
+			loc := file.NewLocation(test.fixture)
+			if _, exists := actualResults[loc.Coordinates]; !exists && test.expected != nil {
+				t.Fatalf("could not find location=%q in results", loc)
+			} else if !exists && test.expected == nil {
+				return
+			}
+
+			assert.Equal(t, test.expected, actualResults[loc.Coordinates], "mismatched secrets")
+		})
+	}
+}

syft/file/cataloger/secrets/generate_search_patterns.go

@@ -0,0 +1,56 @@
+package secrets
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/bmatcuk/doublestar/v4"
+	"github.com/hashicorp/go-multierror"
+)
+
+// GenerateSearchPatterns takes a set of named base patterns, a set of additional named patterns and an name exclusion list and generates a final
+// set of regular expressions (indexed by name). The sets are aggregated roughly as such: (base - excluded) + additional.
+func GenerateSearchPatterns(basePatterns map[string]string, additionalPatterns map[string]string, excludePatternNames []string) (map[string]*regexp.Regexp, error) {
+	var regexObjs = make(map[string]*regexp.Regexp)
+	var errs error
+
+	addFn := func(name, pattern string) {
+		// always enable multiline search option for extracting secrets with multiline values
+		obj, err := regexp.Compile(`(?m)` + pattern)
+		if err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("unable to parse %q regular expression: %w", name, err))
+		}
+		regexObjs[name] = obj
+	}
+
+	// add all base cases... unless that base case was asked to be excluded
+	for name, pattern := range basePatterns {
+		if !matchesExclusion(excludePatternNames, name) {
+			addFn(name, pattern)
+		}
+	}
+
+	// add all additional cases
+	for name, pattern := range additionalPatterns {
+		addFn(name, pattern)
+	}
+
+	if errs != nil {
+		return nil, errs
+	}
+
+	return regexObjs, nil
+}
+
+func matchesExclusion(excludePatternNames []string, name string) bool {
+	for _, exclude := range excludePatternNames {
+		matches, err := doublestar.Match(exclude, name)
+		if err != nil {
+			return false
+		}
+		if matches {
+			return true
+		}
+	}
+	return false
+}

syft/file/cataloger/secrets/generate_search_patterns_test.go

@@ -0,0 +1,125 @@
+package secrets
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGenerateSearchPatterns(t *testing.T) {
+	tests := []struct {
+		name       string
+		base       map[string]string
+		additional map[string]string
+		exclude    []string
+		expected   map[string]string
+	}{
+		{
+			name: "use-base-set",
+			base: map[string]string{
+				"in-default": `^secret_key=.*`,
+			},
+			expected: map[string]string{
+				"in-default": `(?m)^secret_key=.*`,
+			},
+		},
+		{
+			name: "exclude-from-base-set",
+			base: map[string]string{
+				"in-default":      `^secret_key=.*`,
+				"also-in-default": `^also-in-default=.*`,
+			},
+			exclude: []string{"also-in-default"},
+			expected: map[string]string{
+				"in-default": `(?m)^secret_key=.*`,
+			},
+		},
+		{
+			name: "exclude-multiple-from-base-set",
+			base: map[string]string{
+				"in-default":             `^secret_key=.*`,
+				"also-in-default":        `^also-in-default=.*`,
+				"furthermore-in-default": `^furthermore-in-default=.*`,
+			},
+			exclude: []string{"also-in-default", "furthermore-in-default"},
+			expected: map[string]string{
+				"in-default": `(?m)^secret_key=.*`,
+			},
+		},
+		{
+			name: "exclude-all",
+			base: map[string]string{
+				"in-default":      `^secret_key=.*`,
+				"also-in-default": `^also-in-default=.*`,
+			},
+			exclude:  []string{"*"},
+			expected: map[string]string{},
+		},
+		{
+			name: "exclude-some",
+			base: map[string]string{
+				"real":            `^real=.*`,
+				"in-default":      `^secret_key=.*`,
+				"also-in-default": `^also-in-default=.*`,
+			},
+			exclude: []string{"*-default"},
+			expected: map[string]string{
+				"real": `(?m)^real=.*`,
+			},
+		},
+		{
+			name: "additional-pattern-unison",
+			base: map[string]string{
+				"in-default": `^secret_key=.*`,
+			},
+			additional: map[string]string{
+				"additional": `^additional=.*`,
+			},
+			expected: map[string]string{
+				"in-default": `(?m)^secret_key=.*`,
+				"additional": `(?m)^additional=.*`,
+			},
+		},
+		{
+			name: "override",
+			base: map[string]string{
+				"in-default": `^secret_key=.*`,
+			},
+			additional: map[string]string{
+				"in-default": `^additional=.*`,
+			},
+			expected: map[string]string{
+				"in-default": `(?m)^additional=.*`,
+			},
+		},
+		{
+			name: "exclude-and-override",
+			base: map[string]string{
+				"in-default": `^secret_key=.*`,
+			},
+			exclude: []string{"in-default"},
+			additional: map[string]string{
+				"in-default": `^additional=.*`,
+			},
+			expected: map[string]string{
+				"in-default": `(?m)^additional=.*`,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			actualObj, err := GenerateSearchPatterns(test.base, test.additional, test.exclude)
+			if err != nil {
+				t.Fatalf("unable to combine: %+v", err)
+			}
+
+			actual := make(map[string]string)
+			for n, v := range actualObj {
+				actual[n] = v.String()
+			}
+
+			assert.Equal(t, test.expected, actual, "mismatched combination")
+		})
+	}
+}

syft/file/cataloger/secrets/newline_counter.go

@@ -0,0 +1,39 @@
+package secrets
+
+import "io"
+
+type newlineCounter struct {
+	io.RuneReader
+	numBytes int64
+	newLines []int64
+}
+
+func (c *newlineCounter) ReadRune() (r rune, size int, err error) {
+	r, size, err = c.RuneReader.ReadRune()
+	c.numBytes += int64(size)
+	if r == '\n' {
+		c.newLines = append(c.newLines, c.numBytes)
+	}
+	return
+}
+
+func (c *newlineCounter) newlinesBefore(pos int64) int {
+	var result int
+	for _, nlPos := range c.newLines {
+		if nlPos <= pos {
+			result++
+		}
+	}
+	return result
+}
+
+func (c *newlineCounter) newlinePositionBefore(pos int64) int64 {
+	var last int64
+	for _, nlPos := range c.newLines {
+		if nlPos > pos {
+			break
+		}
+		last = nlPos
+	}
+	return last
+}

syft/file/cataloger/secrets/newline_counter_test.go

@@ -0,0 +1,35 @@
+package secrets
+
+import (
+	"bufio"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLineCounter_ReadRune(t *testing.T) {
+	counter := &newlineCounter{RuneReader: bufio.NewReader(strings.NewReader("hi\nwhat's the weather like today?\ndunno...\n"))}
+	var err error
+	for err == nil {
+		_, _, err = counter.ReadRune()
+	}
+	if err != io.EOF {
+		t.Fatalf("should have gotten an eof, got %+v", err)
+	}
+	assert.Equal(t, 3, len(counter.newLines), "bad line count")
+	assert.Equal(t, []int64{3, 34, 43}, counter.newLines, "bad line positions")
+}
+
+func TestLineCounter_newlinesBefore(t *testing.T) {
+	counter := &newlineCounter{RuneReader: bufio.NewReader(strings.NewReader("hi\nwhat's the weather like today?\ndunno...\n"))}
+	var err error
+	for err == nil {
+		_, _, err = counter.ReadRune()
+	}
+	if err != io.EOF {
+		t.Fatalf("should have gotten an eof, got %+v", err)
+	}
+	assert.Equal(t, 1, counter.newlinesBefore(10), "bad line count")
+}

syft/file/cataloger/secrets/secrets_search_by_line_strategy.go

@@ -0,0 +1,135 @@
+package secrets
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"regexp"
+
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/syft/file"
+)
+
+func catalogLocationByLine(resolver file.Resolver, location file.Location, patterns map[string]*regexp.Regexp) ([]file.SearchResult, error) {
+	readCloser, err := resolver.FileContentsByLocation(location)
+	if err != nil {
+		return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
+	}
+	defer internal.CloseAndLogError(readCloser, location.VirtualPath)
+
+	var scanner = bufio.NewReader(readCloser)
+	var position int64
+	var allSecrets []file.SearchResult
+	var lineNo int64
+	var readErr error
+	for !errors.Is(readErr, io.EOF) {
+		lineNo++
+		var line []byte
+		// TODO: we're at risk of large memory usage for very long lines
+		line, readErr = scanner.ReadBytes('\n')
+		if readErr != nil && readErr != io.EOF {
+			return nil, readErr
+		}
+
+		lineSecrets, err := searchForSecretsWithinLine(resolver, location, patterns, line, lineNo, position)
+		if err != nil {
+			return nil, err
+		}
+		position += int64(len(line))
+		allSecrets = append(allSecrets, lineSecrets...)
+	}
+
+	return allSecrets, nil
+}
+
+func searchForSecretsWithinLine(resolver file.Resolver, location file.Location, patterns map[string]*regexp.Regexp, line []byte, lineNo int64, position int64) ([]file.SearchResult, error) {
+	var secrets []file.SearchResult
+	for name, pattern := range patterns {
+		matches := pattern.FindAllIndex(line, -1)
+		for i, match := range matches {
+			if i%2 == 1 {
+				// FindAllIndex returns pairs of numbers for each match, we are only interested in the starting (first)
+				// position in each pair.
+				continue
+			}
+
+			lineOffset := int64(match[0])
+			seekLocation := position + lineOffset
+			reader, err := readerAtPosition(resolver, location, seekLocation)
+			if err != nil {
+				return nil, err
+			}
+
+			secret := extractSecretFromPosition(reader, name, pattern, lineNo, lineOffset, seekLocation)
+			if secret != nil {
+				secrets = append(secrets, *secret)
+			}
+			internal.CloseAndLogError(reader, location.VirtualPath)
+		}
+	}
+
+	return secrets, nil
+}
+
+func readerAtPosition(resolver file.Resolver, location file.Location, seekPosition int64) (io.ReadCloser, error) {
+	readCloser, err := resolver.FileContentsByLocation(location)
+	if err != nil {
+		return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
+	}
+	if seekPosition > 0 {
+		n, err := io.CopyN(io.Discard, readCloser, seekPosition)
+		if err != nil {
+			return nil, fmt.Errorf("unable to read contents for location=%q while searching for secrets: %w", location, err)
+		}
+		if n != seekPosition {
+			return nil, fmt.Errorf("unexpected seek location for location=%q while searching for secrets: %d != %d", location, n, seekPosition)
+		}
+	}
+	return readCloser, nil
+}
+
+func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *regexp.Regexp, lineNo, lineOffset, seekPosition int64) *file.SearchResult {
+	reader := &newlineCounter{RuneReader: bufio.NewReader(readCloser)}
+	positions := pattern.FindReaderSubmatchIndex(reader)
+	if len(positions) == 0 {
+		// no matches found
+		return nil
+	}
+
+	index := pattern.SubexpIndex("value")
+	var indexOffset int
+	if index != -1 {
+		// there is a capture group, use the capture group selection as the secret value. To do this we want to
+		// use the position at the discovered offset. Note: all positions come in pairs, so you will need to adjust
+		// the offset accordingly (multiply by 2).
+		indexOffset = index * 2
+	}
+	// get the start and stop of the secret value. Note: this covers both when there is a capture group
+	// and when there is not a capture group (full value match)
+	start, stop := int64(positions[indexOffset]), int64(positions[indexOffset+1])
+
+	if start < 0 || stop < 0 {
+		// no match location found. This can happen when there is a value capture group specified by the user
+		// and there was a match on the overall regex, but not for the capture group (which is possible if the capture
+		// group is optional).
+		return nil
+	}
+
+	// lineNoOfSecret are the number of lines which occur before the start of the secret value
+	var lineNoOfSecret = lineNo + int64(reader.newlinesBefore(start))
+	// lineOffsetOfSecret are the number of bytes that occur after the last newline but before the secret value.
+	var lineOffsetOfSecret = start - reader.newlinePositionBefore(start)
+	if lineNoOfSecret == lineNo {
+		// the secret value starts in the same line as the overall match, so we must consider that line offset
+		lineOffsetOfSecret += lineOffset
+	}
+
+	return &file.SearchResult{
+		Classification: name,
+		SeekPosition:   start + seekPosition,
+		Length:         stop - start,
+		LineNumber:     lineNoOfSecret,
+		LineOffset:     lineOffsetOfSecret,
+	}
+}

syft/file/cataloger/secrets/test-fixtures/secrets/default/api-key.txt

@@ -0,0 +1,14 @@
+# these should be matches
+apikey=12345A7a901b34567890
+api_key =12345A7a901b345678901234567890
+API-KEY= '12345A7a901b3456789012345678901234567890'
+API-key: "12345A7a901b34567890123456789012345678901234567890"
+some_ApI-kEy = "12345A7a901b345678901234567890123456789012345678901234567890"
+
+# these should be non matches
+api_key = "toolong12345A7a901b345678901234567890123456789012345678901234567890"
+api_key = "tooshort"
+not_api_k3y = "badkeyname12345A7a901b34567890"
+
+# value at EOF should match
+api_key=11111111111111111111

syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.env

@@ -0,0 +1,3 @@
+# note: these are NOT real credentials
+export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
+export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY

syft/file/cataloger/secrets/test-fixtures/secrets/default/aws.ini

@@ -0,0 +1,4 @@
+# note: these are NOT real credentials
+[default]
+aws_access_key_id=AKIAIOSFODNN7EXAMPLE
+aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY

syft/file/cataloger/secrets/test-fixtures/secrets/default/docker-config.json

@@ -0,0 +1,10 @@
+{
+  "experimental" : "disabled",
+  "auths" : {
+    "https://index.docker.io/v1/" : {
+      "auth": "tOpsyKreTz"
+    }
+  },
+  "stackOrchestrator" : "swarm",
+  "credsStore" : "desktop"
+}

syft/file/cataloger/secrets/test-fixtures/secrets/default/not-docker-config.json

@@ -0,0 +1,4 @@
+{
+  "endpoint" : "http://somewhere",
+  "auth" : "basic"
+}

syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-false-positive.pem

@@ -0,0 +1 @@
+-----BEGIN OPENSSL PRIVATE KEY-----

syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key-openssl.pem

@@ -0,0 +1,9 @@
+# note: this is NOT a real private key
+-----BEGIN OPENSSL PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
+cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
+VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
+bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
+BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
+z3P668YfhUbKdRF6S42Cg6zn
+-----END OPENSSL PRIVATE KEY-----

syft/file/cataloger/secrets/test-fixtures/secrets/default/private-key.pem

@@ -0,0 +1,10 @@
+# note: this is NOT a real private key
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
+cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
+VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
+bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
+BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
+z3P668YfhUbKdRF6S42Cg6zn
+-----END PRIVATE KEY-----
+other embedded text

syft/file/cataloger/secrets/test-fixtures/secrets/default/private-keys.pem

@@ -0,0 +1,16 @@
+-----BEGIN OPENSSL PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDBj08sp5++4anG
+cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgYD
+VQQDDBcqLmF3cy10ZXN0LnByb2dyZXNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD
+bml6YXRpb252YWxzaGEyZzIuY3JsMIGgBggrBgEFBQcBAQSBkzCBkDBNBggrBgEF
+BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmdh
+z3P668YfhUbKdRF6S42Cg6zn
+-----END OPENSSL PRIVATE KEY-----
+-----BEGIN OPENSSL PRIVATE KEY-----
+MIIEvgTHISISNOTAREALKEYoIBAQDBj08DBj08DBj08DBj08DBj08DBsp5++4an3
+cmQxJjAkBgNVBAoTHVByb2dyZXNzIFNvZnR3YXJlIENvcnBvcmF0aW9uMSAwHgY5
+VQQDDBcqLmF3cy10ZXN0SISNOTAREALKEYoIBAQDBj08DfffKoZIhvcNAQEBBQA7
+bml6SISNOTAREALKEYoIBAQDBj08DdssBggrBgEFBQcBAQSBkzCBkDBNBggrBgE8
+BQcwAoZBaHR0cDovL3NlY3VyZS5nbG9iYWxzaWduLmNvbS9jYWNlcnQvZ3Nvcmd1
+j4f668YfhUbKdRF6S6734856
+-----END OPENSSL PRIVATE KEY-----

syft/file/cataloger/secrets/test-fixtures/secrets/multiple.txt

@@ -0,0 +1,6 @@
+secret_key=clear_text1
+other text that should be ignored
+secret_key=clear_text2
+ secret_key=clear_text3
+also things that should be ignored
+secret_key=clear_text4

syft/file/cataloger/secrets/test-fixtures/secrets/simple.txt

@@ -0,0 +1,4 @@
+other text that should be ignored
+secret_key=clear_text
+---secret_key=clear_text
+also things that should be ignored