feat: parallelize catalogers per-file and hash contents in parallel (#3636)

Signed-off-by: Keith Zantow <kzantow@gmail.com>
This commit is contained in:
Keith Zantow
2025-03-26 11:10:08 -04:00
committed by GitHub
parent dbe29ed4ab
commit 4a9437808e
20 changed files with 161 additions and 118 deletions
+5 -2
View File
@@ -1,12 +1,15 @@
package file
import (
"context"
"crypto"
"fmt"
"hash"
"io"
"strings"
"github.com/anchore/go-sync"
"github.com/anchore/syft/syft/cataloging"
"github.com/anchore/syft/syft/file"
)
@@ -21,7 +24,7 @@ func supportedHashAlgorithms() []crypto.Hash {
}
}
func NewDigestsFromFile(closer io.ReadCloser, hashes []crypto.Hash) ([]file.Digest, error) {
func NewDigestsFromFile(ctx context.Context, closer io.ReadCloser, hashes []crypto.Hash) ([]file.Digest, error) {
hashes = NormalizeHashes(hashes)
// create a set of hasher objects tied together with a single writer to feed content into
hashers := make([]hash.Hash, len(hashes))
@@ -31,7 +34,7 @@ func NewDigestsFromFile(closer io.ReadCloser, hashes []crypto.Hash) ([]file.Dige
writers[idx] = hashers[idx]
}
size, err := io.Copy(io.MultiWriter(writers...), closer)
size, err := io.Copy(sync.ParallelWriter(ctx, cataloging.ExecutorCPU, writers...), closer)
if err != nil {
return nil, err
}
+2 -1
View File
@@ -1,6 +1,7 @@
package file
import (
"context"
"crypto"
"os"
"testing"
@@ -81,7 +82,7 @@ func TestNewDigestsFromFile(t *testing.T) {
fh, err := os.Open(tt.fixture)
require.NoError(t, err)
got, err := NewDigestsFromFile(fh, tt.hashes)
got, err := NewDigestsFromFile(context.TODO(), fh, tt.hashes)
tt.wantErr(t, err)
if err != nil {
return