Deduplicate packages across multiple container image layers (#930)

syft/source/coordinate_set.go

@@ -0,0 +1,86 @@
+package source
+
+import (
+	"sort"
+
+	"github.com/mitchellh/hashstructure/v2"
+	"github.com/scylladb/go-set/strset"
+)
+
+type CoordinateSet struct {
+	set map[Coordinates]struct{}
+}
+
+func NewCoordinateSet(coordinates ...Coordinates) (s CoordinateSet) {
+	for _, l := range coordinates {
+		s.Add(l)
+	}
+
+	return s
+}
+
+func (s *CoordinateSet) Add(coordinates ...Coordinates) {
+	if s.set == nil {
+		s.set = make(map[Coordinates]struct{})
+	}
+	for _, l := range coordinates {
+		s.set[l] = struct{}{}
+	}
+}
+
+func (s CoordinateSet) Remove(coordinates ...Coordinates) {
+	if s.set == nil {
+		return
+	}
+	for _, l := range coordinates {
+		delete(s.set, l)
+	}
+}
+
+func (s CoordinateSet) Contains(l Coordinates) bool {
+	if s.set == nil {
+		return false
+	}
+	_, ok := s.set[l]
+	return ok
+}
+
+func (s CoordinateSet) Paths() []string {
+	if s.set == nil {
+		return nil
+	}
+
+	paths := strset.New()
+	for _, c := range s.ToSlice() {
+		paths.Add(c.RealPath)
+	}
+	pathSlice := paths.List()
+	sort.Strings(pathSlice)
+	return pathSlice
+}
+
+func (s CoordinateSet) ToSlice() []Coordinates {
+	if s.set == nil {
+		return nil
+	}
+	coordinates := make([]Coordinates, len(s.set))
+	idx := 0
+	for v := range s.set {
+		coordinates[idx] = v
+		idx++
+	}
+	sort.SliceStable(coordinates, func(i, j int) bool {
+		if coordinates[i].RealPath == coordinates[j].RealPath {
+			return coordinates[i].FileSystemID < coordinates[j].FileSystemID
+		}
+		return coordinates[i].RealPath < coordinates[j].RealPath
+	})
+	return coordinates
+}
+
+func (s CoordinateSet) Hash() (uint64, error) {
+	return hashstructure.Hash(s.ToSlice(), hashstructure.FormatV2, &hashstructure.HashOptions{
+		ZeroNil:      true,
+		SlicesAsSets: true,
+	})
+}

syft/source/coordinate_set_test.go

@@ -0,0 +1,117 @@
+package source
+
+import (
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/stretchr/testify/require"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCoordinatesSet(t *testing.T) {
+
+	binA := Coordinates{
+		RealPath:     "/bin",
+		FileSystemID: "a",
+	}
+
+	binB := Coordinates{
+		RealPath:     "/bin",
+		FileSystemID: "b",
+	}
+
+	tests := []struct {
+		name     string
+		input    []Coordinates
+		expected []Coordinates
+	}{
+		{
+			name: "de-dup same location",
+			input: []Coordinates{
+				binA, binA, binA,
+			},
+			expected: []Coordinates{
+				binA,
+			},
+		},
+		{
+			name: "dont de-dup different filesystem",
+			input: []Coordinates{
+				binB, binA,
+			},
+			expected: []Coordinates{
+				binA, binB,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			set := NewCoordinateSet(test.input...)
+			assert.Equal(t, test.expected, set.ToSlice())
+		})
+	}
+}
+
+func TestCoordinateSet_Hash(t *testing.T) {
+	etcA := Coordinates{
+		RealPath:     "/etc",
+		FileSystemID: "a",
+	}
+
+	etcB := Coordinates{
+		RealPath:     "/etc",
+		FileSystemID: "b",
+	}
+
+	binA := Coordinates{
+		RealPath:     "/bin",
+		FileSystemID: "a",
+	}
+
+	binB := Coordinates{
+		RealPath:     "/bin",
+		FileSystemID: "b",
+	}
+
+	tests := []struct {
+		name string
+		setA CoordinateSet
+		setB CoordinateSet
+		want assert.ComparisonAssertionFunc
+	}{
+		{
+			name: "empty sets have the same hash",
+			setA: NewCoordinateSet(),
+			setB: NewCoordinateSet(),
+			want: assert.Equal,
+		},
+		{
+			name: "sets with same elements have the same hash",
+			setA: NewCoordinateSet(binA, etcA),
+			setB: NewCoordinateSet(etcA, binA),
+			want: assert.Equal,
+		},
+		{
+			name: "sets with different elements have different hashes",
+			setA: NewCoordinateSet(binA, etcA),
+			setB: NewCoordinateSet(binA),
+			want: assert.NotEqual,
+		},
+		{
+			name: "sets with same paths but different FS IDs have different hashes",
+			setA: NewCoordinateSet(etcA, binA),
+			setB: NewCoordinateSet(etcB, binB),
+			want: assert.NotEqual,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotA, err := artifact.IDByHash(tt.setA)
+			require.NoError(t, err)
+			gotB, err := artifact.IDByHash(tt.setB)
+			require.NoError(t, err)
+			tt.want(t, gotA, gotB)
+		})
+	}
+}

syft/source/coordinates.go

@@ -2,7 +2,6 @@ package source
 
 import (
 	"fmt"
-	"sort"
 
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
@@ -14,18 +13,6 @@ type Coordinates struct {
 	FileSystemID string `json:"layerID,omitempty" cyclonedx:"layerID"` // An ID representing the filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank.
 }
 
-// CoordinateSet represents a set of string types.
-type CoordinateSet map[Coordinates]struct{}
-
-// NewCoordinateSet creates a CoordinateSet populated with values from the given slice.
-func NewCoordinateSet(start ...Coordinates) CoordinateSet {
-	ret := make(CoordinateSet)
-	for _, s := range start {
-		ret.Add(s)
-	}
-	return ret
-}
-
 func (c Coordinates) ID() artifact.ID {
 	f, err := artifact.IDByHash(c)
 	if err != nil {
@@ -45,37 +32,3 @@ func (c Coordinates) String() string {
 	}
 	return fmt.Sprintf("Location<%s>", str)
 }
-
-// Add a string to the set.
-func (s CoordinateSet) Add(i Coordinates) {
-	s[i] = struct{}{}
-}
-
-// Remove a string from the set.
-func (s CoordinateSet) Remove(i Coordinates) {
-	delete(s, i)
-}
-
-// Contains indicates if the given string is contained within the set.
-func (s CoordinateSet) Contains(i Coordinates) bool {
-	_, ok := s[i]
-	return ok
-}
-
-// ToSlice returns a sorted slice of Locations that are contained within the set.
-func (s CoordinateSet) ToSlice() []Coordinates {
-	ret := make([]Coordinates, len(s))
-	idx := 0
-	for v := range s {
-		ret[idx] = v
-		idx++
-	}
-
-	sort.SliceStable(ret, func(i, j int) bool {
-		if ret[i].RealPath == ret[j].RealPath {
-			return ret[i].FileSystemID < ret[j].FileSystemID
-		}
-		return ret[i].RealPath < ret[j].RealPath
-	})
-	return ret
-}

syft/source/coordinates_test.go

@@ -1,51 +0,0 @@
-package source
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestCoordinateSet(t *testing.T) {
-
-	binA := Coordinates{
-		RealPath:     "/bin",
-		FileSystemID: "a",
-	}
-
-	binB := Coordinates{
-		RealPath:     "/bin",
-		FileSystemID: "b",
-	}
-
-	tests := []struct {
-		name     string
-		input    []Coordinates
-		expected []Coordinates
-	}{
-		{
-			name: "de-dup same location",
-			input: []Coordinates{
-				binA, binA, binA,
-			},
-			expected: []Coordinates{
-				binA,
-			},
-		},
-		{
-			name: "dont de-dup different filesystem",
-			input: []Coordinates{
-				binB, binA,
-			},
-			expected: []Coordinates{
-				binA, binB,
-			},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			assert.Equal(t, test.expected, NewCoordinateSet(test.input...).ToSlice())
-		})
-	}
-}

syft/source/location.go

@@ -109,3 +109,9 @@ func (l Location) String() string {
 	}
 	return fmt.Sprintf("Location<%s>", str)
 }
+
+func (l Location) Equals(other Location) bool {
+	return l.RealPath == other.RealPath &&
+		l.VirtualPath == other.VirtualPath &&
+		l.FileSystemID == other.FileSystemID
+}

syft/source/location_set.go

@@ -0,0 +1,78 @@
+package source
+
+import (
+	"sort"
+
+	"github.com/mitchellh/hashstructure/v2"
+)
+
+type LocationSet struct {
+	set map[Location]struct{}
+}
+
+func NewLocationSet(locations ...Location) (s LocationSet) {
+	for _, l := range locations {
+		s.Add(l)
+	}
+
+	return s
+}
+
+func (s *LocationSet) Add(locations ...Location) {
+	if s.set == nil {
+		s.set = make(map[Location]struct{})
+	}
+	for _, l := range locations {
+		s.set[l] = struct{}{}
+	}
+}
+
+func (s LocationSet) Remove(locations ...Location) {
+	if s.set == nil {
+		return
+	}
+	for _, l := range locations {
+		delete(s.set, l)
+	}
+}
+
+func (s LocationSet) Contains(l Location) bool {
+	if s.set == nil {
+		return false
+	}
+	_, ok := s.set[l]
+	return ok
+}
+
+func (s LocationSet) ToSlice() []Location {
+	if s.set == nil {
+		return nil
+	}
+	locations := make([]Location, len(s.set))
+	idx := 0
+	for v := range s.set {
+		locations[idx] = v
+		idx++
+	}
+	sort.Sort(Locations(locations))
+	return locations
+}
+
+func (s *LocationSet) CoordinateSet() CoordinateSet {
+	if s.set == nil {
+		return NewCoordinateSet()
+	}
+	set := NewCoordinateSet()
+	for l := range s.set {
+		set.Add(l.Coordinates)
+	}
+	return set
+}
+
+func (s LocationSet) Hash() (uint64, error) {
+	// access paths and filesystem IDs are not considered when hashing a location set, only the real paths
+	return hashstructure.Hash(s.CoordinateSet().Paths(), hashstructure.FormatV2, &hashstructure.HashOptions{
+		ZeroNil:      true,
+		SlicesAsSets: true,
+	})
+}

syft/source/location_set_test.go

@@ -0,0 +1,178 @@
+package source
+
+import (
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/stretchr/testify/require"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLocationSet(t *testing.T) {
+
+	etcHostsLinkVar := Location{
+		Coordinates: Coordinates{
+			RealPath:     "/etc/hosts",
+			FileSystemID: "a",
+		},
+		VirtualPath: "/var/etc/hosts",
+	}
+
+	etcHostsLinkHome := Location{
+		Coordinates: Coordinates{
+			RealPath:     "/etc/hosts",
+			FileSystemID: "a",
+		},
+		VirtualPath: "/home/wagoodman/hosts",
+	}
+
+	binA := Location{
+		Coordinates: Coordinates{
+			RealPath:     "/bin",
+			FileSystemID: "a",
+		},
+		VirtualPath: "/usr/bin",
+	}
+
+	binB := Location{
+		Coordinates: Coordinates{
+			RealPath:     "/bin",
+			FileSystemID: "b",
+		},
+		VirtualPath: "/usr/bin",
+	}
+
+	tests := []struct {
+		name     string
+		input    []Location
+		expected []Location
+	}{
+		{
+			name: "de-dup same location",
+			input: []Location{
+				binA, binA, binA,
+			},
+			expected: []Location{
+				binA,
+			},
+		},
+		{
+			name: "dont de-dup different filesystem",
+			input: []Location{
+				binB, binA,
+			},
+			expected: []Location{
+				binA, binB,
+			},
+		},
+		{
+			name: "dont de-dup different virtual paths",
+			input: []Location{
+				etcHostsLinkVar, etcHostsLinkHome,
+			},
+			expected: []Location{
+				etcHostsLinkHome, etcHostsLinkVar,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			set := NewLocationSet(test.input...)
+			assert.Equal(t, test.expected, set.ToSlice())
+		})
+	}
+}
+
+func TestLocationSet_Hash(t *testing.T) {
+	etcAlink := Location{
+		Coordinates: Coordinates{
+			RealPath:     "/etc/hosts",
+			FileSystemID: "a",
+		},
+		VirtualPath: "/var/etc/hosts",
+	}
+
+	etcA := Location{
+		Coordinates: Coordinates{
+			RealPath:     "/etc/hosts",
+			FileSystemID: "a",
+		},
+	}
+
+	etcB := Location{
+		Coordinates: Coordinates{
+			RealPath:     "/etc/hosts",
+			FileSystemID: "b",
+		},
+	}
+
+	binA := Location{
+		Coordinates: Coordinates{
+			RealPath:     "/bin",
+			FileSystemID: "a",
+		},
+		VirtualPath: "/usr/bin",
+	}
+
+	binB := Location{
+		Coordinates: Coordinates{
+			RealPath:     "/bin",
+			FileSystemID: "b",
+		},
+		VirtualPath: "/usr/bin",
+	}
+
+	tests := []struct {
+		name string
+		setA LocationSet
+		setB LocationSet
+		want assert.ComparisonAssertionFunc
+	}{
+		{
+			name: "empty sets have the same hash",
+			setA: NewLocationSet(),
+			setB: NewLocationSet(),
+			want: assert.Equal,
+		},
+		{
+			name: "sets with same elements accessed through different paths have the same hash",
+			setA: NewLocationSet(binA, etcA),
+			setB: NewLocationSet(etcAlink, binA),
+			want: assert.Equal,
+		},
+		{
+			name: "sets with same elements have the same hash",
+			setA: NewLocationSet(binA, etcA),
+			setB: NewLocationSet(etcA, binA),
+			want: assert.Equal,
+		},
+		{
+			name: "sets with different element counts have different hashes",
+			setA: NewLocationSet(binA, etcA),
+			setB: NewLocationSet(binA),
+			want: assert.NotEqual,
+		},
+		{
+			name: "sets with same path but different FS IDs have the same hash",
+			setA: NewLocationSet(binA),
+			setB: NewLocationSet(binB),
+			want: assert.Equal,
+		},
+		{
+			name: "sets with same paths but different FS IDs have the same hash",
+			setA: NewLocationSet(etcA, binA),
+			setB: NewLocationSet(binB, etcB),
+			want: assert.Equal,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotA, err := artifact.IDByHash(tt.setA)
+			require.NoError(t, err)
+			gotB, err := artifact.IDByHash(tt.setB)
+			require.NoError(t, err)
+			tt.want(t, gotA, gotB)
+		})
+	}
+}

syft/source/locations.go

@@ -0,0 +1,21 @@
+package source
+
+type Locations []Location
+
+func (l Locations) Len() int {
+	return len(l)
+}
+
+func (l Locations) Less(i, j int) bool {
+	if l[i].RealPath == l[j].RealPath {
+		if l[i].VirtualPath == l[j].VirtualPath {
+			return l[i].FileSystemID < l[j].FileSystemID
+		}
+		return l[i].VirtualPath < l[j].VirtualPath
+	}
+	return l[i].RealPath < l[j].RealPath
+}
+
+func (l Locations) Swap(i, j int) {
+	l[i], l[j] = l[j], l[i]
+}