Deduplicate digests from user configuration (#2522)

* deduplicate digests from user configuration

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* backout pointer reciever change on imageSource

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

cmd/syft/cli/options/file.go

@@ -2,6 +2,9 @@ package options
 
 import (
 	"fmt"
+	"sort"
+
+	"github.com/scylladb/go-set/strset"
 
 	intFile "github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/syft/file"
@@ -35,6 +38,10 @@ func defaultFileConfig() fileConfig {
 }
 
 func (c *fileConfig) PostLoad() error {
+	digests := strset.New(c.Metadata.Digests...).List()
+	sort.Strings(digests)
+	c.Metadata.Digests = digests
+
 	switch c.Metadata.Selection {
 	case file.NoFilesSelection, file.FilesOwnedByPackageSelection, file.AllFilesSelection:
 		return nil

cmd/syft/cli/options/file_test.go

@@ -0,0 +1,56 @@
+package options
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/anchore/syft/syft/file"
+)
+
+func Test_fileConfig_PostLoad(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     fileConfig
+		assert  func(t *testing.T, cfg fileConfig)
+		wantErr assert.ErrorAssertionFunc
+	}{
+		{
+			name: "deduplicate digests",
+			cfg: fileConfig{
+				Metadata: fileMetadata{
+					Selection: file.NoFilesSelection,
+					Digests:   []string{"sha1", "sha1"},
+				},
+			},
+			assert: func(t *testing.T, cfg fileConfig) {
+				assert.Equal(t, []string{"sha1"}, cfg.Metadata.Digests)
+			},
+		},
+		{
+			name: "error on invalid selection",
+			cfg: fileConfig{
+				Metadata: fileMetadata{
+					Selection: file.Selection("invalid"),
+				},
+			},
+			wantErr: assert.Error,
+		},
+		{
+			name:    "error on empty selection",
+			cfg:     fileConfig{},
+			wantErr: assert.Error,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.wantErr == nil {
+				tt.wantErr = assert.NoError
+			}
+			tt.wantErr(t, tt.cfg.PostLoad())
+			if tt.assert != nil {
+				tt.assert(t, tt.cfg)
+			}
+		})
+	}
+}

cmd/syft/cli/options/source.go

@@ -2,6 +2,7 @@ package options
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/scylladb/go-set/strset"
@@ -34,6 +35,13 @@ func defaultSourceConfig() sourceConfig {
 	}
 }
 
+func (c *fileSource) PostLoad() error {
+	digests := strset.New(c.Digests...).List()
+	sort.Strings(digests)
+	c.Digests = digests
+	return nil
+}
+
 func (c imageSource) PostLoad() error {
 	return checkDefaultSourceValues(c.DefaultPullSource)
 }

cmd/syft/cli/options/source_test.go

@@ -0,0 +1,37 @@
+package options
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_fileSource_PostLoad(t *testing.T) {
+	tests := []struct {
+		name    string
+		cfg     fileSource
+		assert  func(t *testing.T, cfg fileSource)
+		wantErr assert.ErrorAssertionFunc
+	}{
+		{
+			name: "deduplicate digests",
+			cfg: fileSource{
+				Digests: []string{"sha1", "sha1"},
+			},
+			assert: func(t *testing.T, cfg fileSource) {
+				assert.Equal(t, []string{"sha1"}, cfg.Digests)
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.wantErr == nil {
+				tt.wantErr = assert.NoError
+			}
+			tt.wantErr(t, tt.cfg.PostLoad())
+			if tt.assert != nil {
+				tt.assert(t, tt.cfg)
+			}
+		})
+	}
+}

internal/file/digest.go

@@ -22,6 +22,7 @@ func supportedHashAlgorithms() []crypto.Hash {
 }
 
 func NewDigestsFromFile(closer io.ReadCloser, hashes []crypto.Hash) ([]file.Digest, error) {
+	hashes = NormalizeHashes(hashes)
 	// create a set of hasher objects tied together with a single writer to feed content into
 	hashers := make([]hash.Hash, len(hashes))
 	writers := make([]io.Writer, len(hashes))
@@ -67,7 +68,7 @@ func Hashers(names ...string) ([]crypto.Hash, error) {
 		}
 		hashers = append(hashers, hashObj)
 	}
-	return hashers, nil
+	return NormalizeHashes(hashers), nil
 }
 
 func CleanDigestAlgorithmName(name string) string {

internal/file/normalize_hashes.go

@@ -0,0 +1,24 @@
+package file
+
+import (
+	"crypto"
+	"sort"
+
+	"github.com/scylladb/go-set/uset"
+)
+
+func NormalizeHashes(hashes []crypto.Hash) []crypto.Hash {
+	set := uset.New()
+	for _, h := range hashes {
+		set.Add(uint(h))
+	}
+	list := set.List()
+	sort.Slice(list, func(i, j int) bool {
+		return list[i] < list[j]
+	})
+	result := make([]crypto.Hash, len(list))
+	for i, v := range list {
+		result[i] = crypto.Hash(v)
+	}
+	return result
+}

internal/file/normalize_hashes_test.go

@@ -0,0 +1,44 @@
+package file
+
+import (
+	"crypto"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNormalizeHashes(t *testing.T) {
+
+	tests := []struct {
+		name  string
+		input []crypto.Hash
+		want  []crypto.Hash
+	}{
+		{
+			name: "deduplicate hashes",
+			input: []crypto.Hash{
+				crypto.SHA1,
+				crypto.SHA1,
+			},
+			want: []crypto.Hash{
+				crypto.SHA1,
+			},
+		},
+		{
+			name: "sort hashes",
+			input: []crypto.Hash{
+				crypto.SHA512,
+				crypto.SHA1,
+			},
+			want: []crypto.Hash{
+				crypto.SHA1,
+				crypto.SHA512,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, NormalizeHashes(tt.input))
+		})
+	}
+}

syft/cataloging/filecataloging/config.go

@@ -73,6 +73,6 @@ func (cfg Config) WithSelection(selection file.Selection) Config {
 }
 
 func (cfg Config) WithHashers(hashers []crypto.Hash) Config {
-	cfg.Hashers = hashers
+	cfg.Hashers = intFile.NormalizeHashes(hashers)
 	return cfg
 }

syft/file/cataloger/filedigest/cataloger.go

@@ -25,7 +25,7 @@ type Cataloger struct {
 
 func NewCataloger(hashes []crypto.Hash) *Cataloger {
 	return &Cataloger{
-		hashes: hashes,
+		hashes: intFile.NormalizeHashes(hashes),
 	}
 }