feat: parallelize catalogers per-file and hash contents in parallel (#3636)

Signed-off-by: Keith Zantow <kzantow@gmail.com>

syft/file/cataloger/executable/cataloger.go

@@ -2,6 +2,7 @@ package executable
 
 import (
 	"bytes"
+	"context"
 	"debug/elf"
 	"debug/macho"
 	"encoding/binary"
@@ -11,11 +12,13 @@ import (
 	"github.com/bmatcuk/doublestar/v4"
 	"github.com/dustin/go-humanize"
 
+	"github.com/anchore/go-sync"
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/mimetype"
 	"github.com/anchore/syft/internal/unknown"
+	"github.com/anchore/syft/syft/cataloging"
 	"github.com/anchore/syft/syft/event/monitor"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/internal/unionreader"
@@ -46,8 +49,10 @@ func NewCataloger(cfg Config) *Cataloger {
 }
 
 func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]file.Executable, error) {
-	var errs error
+	return i.CatalogCtx(context.Background(), resolver)
+}
 
+func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map[file.Coordinates]file.Executable, error) {
 	locs, err := resolver.FilesByMIMEType(i.config.MIMETypes...)
 	if err != nil {
 		return nil, fmt.Errorf("unable to get file locations for binaries: %w", err)
@@ -61,19 +66,20 @@ func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]file.E
 	prog := catalogingProgress(int64(len(locs)))
 
 	results := make(map[file.Coordinates]file.Executable)
-	for _, loc := range locs {
+	errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(locs), func(loc file.Location) (*file.Executable, error) {
 		prog.AtomicStage.Set(loc.Path())
 
 		exec, err := processExecutableLocation(loc, resolver)
 		if err != nil {
-			errs = unknown.Append(errs, loc, err)
+			err = unknown.New(loc, err)
 		}
-
+		return exec, err
+	}, func(loc file.Location, exec *file.Executable) {
 		if exec != nil {
 			prog.Increment()
 			results[loc.Coordinates] = *exec
 		}
-	}
+	})
 
 	log.Debugf("executable cataloger processed %d files", len(results))
 

syft/file/cataloger/filedigest/cataloger.go

@@ -8,12 +8,14 @@ import (
 
 	"github.com/dustin/go-humanize"
 
+	"github.com/anchore/go-sync"
 	stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/bus"
 	intFile "github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/unknown"
+	"github.com/anchore/syft/syft/cataloging"
 	"github.com/anchore/syft/syft/event/monitor"
 	"github.com/anchore/syft/syft/file"
 	intCataloger "github.com/anchore/syft/syft/file/cataloger/internal"
@@ -34,7 +36,6 @@ func NewCataloger(hashes []crypto.Hash) *Cataloger {
 func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates][]file.Digest, error) {
 	results := make(map[file.Coordinates][]file.Digest)
 	var locations []file.Location
-	var errs error
 
 	if len(coordinates) == 0 {
 		locations = intCataloger.AllRegularFiles(ctx, resolver)
@@ -49,41 +50,44 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
 	}
 
 	prog := catalogingProgress(int64(len(locations)))
-	for _, location := range locations {
-		result, err := i.catalogLocation(resolver, location)
+
+	err := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(locations), func(location file.Location) ([]file.Digest, error) {
+		result, err := i.catalogLocation(ctx, resolver, location)
 
 		if errors.Is(err, ErrUndigestableFile) {
-			continue
+			return nil, nil
 		}
 
 		prog.AtomicStage.Set(location.Path())
 
 		if internal.IsErrPathPermission(err) {
 			log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
-			errs = unknown.Append(errs, location, err)
-			continue
+			return nil, unknown.New(location, err)
 		}
 
 		if err != nil {
 			prog.SetError(err)
-			errs = unknown.Append(errs, location, err)
-			continue
+			return nil, unknown.New(location, err)
 		}
 
 		prog.Increment()
 
-		results[location.Coordinates] = result
-	}
+		return result, nil
+	}, func(location file.Location, digests []file.Digest) {
+		if len(digests) > 0 {
+			results[location.Coordinates] = digests
+		}
+	})
 
 	log.Debugf("file digests cataloger processed %d files", prog.Current())
 
 	prog.AtomicStage.Set(fmt.Sprintf("%s files", humanize.Comma(prog.Current())))
 	prog.SetCompleted()
 
-	return results, errs
+	return results, err
 }
 
-func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) ([]file.Digest, error) {
+func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, location file.Location) ([]file.Digest, error) {
 	meta, err := resolver.FileMetadataByLocation(location)
 	if err != nil {
 		return nil, err
@@ -100,7 +104,7 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
 	}
 	defer internal.CloseAndLogError(contentReader, location.AccessPath)
 
-	digests, err := intFile.NewDigestsFromFile(contentReader, i.hashes)
+	digests, err := intFile.NewDigestsFromFile(ctx, contentReader, i.hashes)
 	if err != nil {
 		return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
 	}

syft/file/cataloger/filedigest/cataloger_test.go

@@ -65,13 +65,13 @@ func TestDigestsCataloger(t *testing.T) {
 			name:     "md5",
 			digests:  []crypto.Hash{crypto.MD5},
 			files:    []string{"test-fixtures/last/empty/empty", "test-fixtures/last/path.txt"},
-			expected: testDigests(t, "test-fixtures/last", []string{"empty/empty", "path.txt"}, crypto.MD5),
+			expected: testDigests(t, "test-fixtures/last", []string{"path.txt"}, crypto.MD5),
 		},
 		{
 			name:     "md5-sha1-sha256",
 			digests:  []crypto.Hash{crypto.MD5, crypto.SHA1, crypto.SHA256},
 			files:    []string{"test-fixtures/last/empty/empty", "test-fixtures/last/path.txt"},
-			expected: testDigests(t, "test-fixtures/last", []string{"empty/empty", "path.txt"}, crypto.MD5, crypto.SHA1, crypto.SHA256),
+			expected: testDigests(t, "test-fixtures/last", []string{"path.txt"}, crypto.MD5, crypto.SHA1, crypto.SHA256),
 		},
 	}