order locations by container layer order (#3858)

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2025-05-13 00:02:07 -04:00
committed by GitHub
parent e3e69596bd
commit 59b880f26a
18 changed files with 1589 additions and 83 deletions
+34 -21
View File
@@ -10,6 +10,7 @@ import (
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
formatInternal "github.com/anchore/syft/syft/format/internal"
"github.com/anchore/syft/syft/format/syftjson/model"
"github.com/anchore/syft/syft/internal/packagemetadata"
"github.com/anchore/syft/syft/internal/sourcemetadata"
@@ -33,10 +34,12 @@ func metadataType(metadata interface{}, legacy bool) string {
// ToFormatModel transforms the sbom import a format-specific model.
func ToFormatModel(s sbom.SBOM, cfg EncoderConfig) model.Document {
locationSorter, coordinateSorter := formatInternal.GetLocationSorters(s)
return model.Document{
Artifacts: toPackageModels(s.Artifacts.Packages, cfg),
Artifacts: toPackageModels(s.Artifacts.Packages, locationSorter, cfg),
ArtifactRelationships: toRelationshipModel(s.Relationships),
Files: toFile(s),
Files: toFile(s, coordinateSorter),
Source: toSourceModel(s.Source),
Distro: toLinuxReleaser(s.Artifacts.LinuxDistribution),
Descriptor: toDescriptor(s.Descriptor),
@@ -81,7 +84,7 @@ func toDescriptor(d sbom.Descriptor) model.Descriptor {
}
}
func toFile(s sbom.SBOM) []model.File {
func toFile(s sbom.SBOM, coordinateSorter func(a, b file.Coordinates) int) []model.File {
results := make([]model.File, 0)
artifacts := s.Artifacts
@@ -141,13 +144,21 @@ func toFile(s sbom.SBOM) []model.File {
})
}
// sort by real path then virtual path to ensure the result is stable across multiple runs
sort.SliceStable(results, func(i, j int) bool {
return results[i].Location.RealPath < results[j].Location.RealPath
})
// sort to ensure we're stable across multiple runs
// should order by the layer order from the container image then by real path
sortFiles(results, coordinateSorter)
return results
}
func sortFiles(files []model.File, coordinateSorter func(a, b file.Coordinates) int) {
fileSorter := func(a, b model.File) int {
return coordinateSorter(a.Location, b.Location)
}
sort.SliceStable(files, func(i, j int) bool {
return fileSorter(files[i], files[j]) < 0
})
}
func toFileMetadataEntry(coordinates file.Coordinates, metadata *file.Metadata) *model.FileMetadataEntry {
if metadata == nil {
return nil
@@ -155,7 +166,7 @@ func toFileMetadataEntry(coordinates file.Coordinates, metadata *file.Metadata)
var mode int
var size int64
if metadata != nil && metadata.FileInfo != nil {
if metadata.FileInfo != nil {
var err error
mode, err = strconv.Atoi(fmt.Sprintf("%o", metadata.Mode()))
@@ -203,25 +214,19 @@ func toFileType(ty stereoscopeFile.Type) string {
}
}
func toPackageModels(catalog *pkg.Collection, cfg EncoderConfig) []model.Package {
func toPackageModels(catalog *pkg.Collection, locationSorter func(a, b file.Location) int, cfg EncoderConfig) []model.Package {
artifacts := make([]model.Package, 0)
if catalog == nil {
return artifacts
}
for _, p := range catalog.Sorted() {
artifacts = append(artifacts, toPackageModel(p, cfg))
artifacts = append(artifacts, toPackageModel(p, locationSorter, cfg))
}
return artifacts
}
func toLicenseModel(pkgLicenses []pkg.License) (modelLicenses []model.License) {
func toLicenseModel(pkgLicenses []pkg.License, locationSorter func(a, b file.Location) int) (modelLicenses []model.License) {
for _, l := range pkgLicenses {
// guarantee collection
locations := make([]file.Location, 0)
if v := l.Locations.ToSlice(); v != nil {
locations = v
}
// format model must have allocated collections
urls := l.URLs
if urls == nil {
@@ -234,14 +239,14 @@ func toLicenseModel(pkgLicenses []pkg.License) (modelLicenses []model.License) {
Contents: l.Contents,
Type: l.Type,
URLs: urls,
Locations: locations,
Locations: toLocationsModel(l.Locations, locationSorter),
})
}
return
}
// toPackageModel crates a new Package from the given pkg.Package.
func toPackageModel(p pkg.Package, cfg EncoderConfig) model.Package {
func toPackageModel(p pkg.Package, locationSorter func(a, b file.Location) int, cfg EncoderConfig) model.Package {
var cpes = make([]model.CPE, len(p.CPEs))
for i, c := range p.CPEs {
convertedCPE := model.CPE{
@@ -255,7 +260,7 @@ func toPackageModel(p pkg.Package, cfg EncoderConfig) model.Package {
// initializing the array; this is a good choke point for this check
var licenses = make([]model.License, 0)
if !p.Licenses.Empty() {
licenses = toLicenseModel(p.Licenses.ToSlice())
licenses = toLicenseModel(p.Licenses.ToSlice(), locationSorter)
}
return model.Package{
@@ -265,7 +270,7 @@ func toPackageModel(p pkg.Package, cfg EncoderConfig) model.Package {
Version: p.Version,
Type: p.Type,
FoundBy: p.FoundBy,
Locations: p.Locations.ToSlice(),
Locations: toLocationsModel(p.Locations, locationSorter),
Licenses: licenses,
Language: p.Language,
CPEs: cpes,
@@ -278,6 +283,14 @@ func toPackageModel(p pkg.Package, cfg EncoderConfig) model.Package {
}
}
func toLocationsModel(locations file.LocationSet, locationSorter func(a, b file.Location) int) []file.Location {
if locations.Empty() {
return []file.Location{}
}
return locations.ToSlice(locationSorter)
}
func toRelationshipModel(relationships []artifact.Relationship) []model.Relationship {
result := make([]model.Relationship, len(relationships))
for i, r := range relationships {
+699 -1
View File
@@ -345,9 +345,707 @@ func Test_toPackageModel_metadataType(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if d := cmp.Diff(tt.want, toPackageModel(tt.p, tt.cfg), cmpopts.EquateEmpty()); d != "" {
if d := cmp.Diff(tt.want, toPackageModel(tt.p, file.LocationSorter(nil), tt.cfg), cmpopts.EquateEmpty()); d != "" {
t.Errorf("unexpected package (-want +got):\n%s", d)
}
})
}
}
func Test_toPackageModel_layerOrdering(t *testing.T) {
tests := []struct {
name string
p pkg.Package
layerOrder []string
cfg EncoderConfig
want model.Package
}{
{
name: "with layer ordering",
p: pkg.Package{
Name: "pkg-1",
Licenses: pkg.NewLicenseSet(pkg.License{
Value: "MIT",
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-3",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/lic-b",
FileSystemID: "fsid-0",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-2",
}),
),
}),
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-0",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
}),
),
},
layerOrder: []string{
"fsid-0",
"fsid-1",
"fsid-2",
"fsid-3",
},
want: model.Package{
PackageBasicData: model.PackageBasicData{
Name: "pkg-1",
Licenses: []model.License{
{
Value: "MIT",
Locations: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/lic-b",
FileSystemID: "fsid-0", // important!
},
AccessPath: "/lic-b",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-1", // important!
},
AccessPath: "/lic-a",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-2", // important!
},
AccessPath: "/lic-a",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/lic-a",
FileSystemID: "fsid-3", // important!
},
AccessPath: "/lic-a",
},
},
},
},
},
Locations: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-0", // important!
},
AccessPath: "/b",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1", // important!
},
AccessPath: "/a",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2", // important!
},
AccessPath: "/a",
},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3", // important!
},
AccessPath: "/a",
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if d := cmp.Diff(tt.want, toPackageModel(tt.p, file.LocationSorter(tt.layerOrder), tt.cfg), cmpopts.EquateEmpty(), cmpopts.IgnoreUnexported(file.LocationData{})); d != "" {
t.Errorf("unexpected package (-want +got):\n%s", d)
}
})
}
}
func Test_toLocationModel(t *testing.T) {
tests := []struct {
name string
locations file.LocationSet
layers []string
want []file.Location
}{
{
name: "empty location set",
locations: file.NewLocationSet(),
layers: []string{"fsid-1"},
want: []file.Location{},
},
{
name: "nil layer order map",
locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-2",
}),
),
layers: nil, // please don't panic!
want: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
AccessPath: "/a",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-2",
},
AccessPath: "/b",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
},
},
{
name: "go case",
locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
}),
),
layers: []string{
"fsid-1",
"fsid-2",
"fsid-3",
},
want: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
AccessPath: "/b",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
},
AccessPath: "/c",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3",
},
AccessPath: "/a",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
},
},
{
name: "same layer different paths", // prove we can sort by path irrespective of layer
locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
}),
),
layers: []string{
"fsid-1",
},
want: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
AccessPath: "/a",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
AccessPath: "/b",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-1",
},
AccessPath: "/c",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
},
},
{
name: "mixed layers and paths",
locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/z",
FileSystemID: "fsid-3",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
}),
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
}),
),
layers: []string{
"fsid-1",
"fsid-2",
"fsid-3",
},
want: []file.Location{
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
AccessPath: "/b",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
},
AccessPath: "/a",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
},
AccessPath: "/c",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
{
LocationData: file.LocationData{
Coordinates: file.Coordinates{
RealPath: "/z",
FileSystemID: "fsid-3",
},
AccessPath: "/z",
},
LocationMetadata: file.LocationMetadata{Annotations: map[string]string{}},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := toLocationsModel(tt.locations, file.LocationSorter(tt.layers))
if d := cmp.Diff(tt.want, got, cmpopts.IgnoreUnexported(file.LocationData{})); d != "" {
t.Errorf("toLocationsModel() mismatch (-want +got):\n%s", d)
}
})
}
}
func Test_sortFiles(t *testing.T) {
tests := []struct {
name string
files []model.File
layers []string
want []model.File
}{
{
name: "empty files slice",
files: []model.File{},
layers: []string{"fsid-1"},
want: []model.File{},
},
{
name: "nil layer order map",
files: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-2",
},
},
},
layers: nil,
want: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-2",
},
},
},
},
{
name: "layer ordering",
files: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3",
},
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
},
},
},
layers: []string{
"fsid-1",
"fsid-2",
"fsid-3",
},
want: []model.File{
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-2",
},
},
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-3",
},
},
},
},
{
name: "same layer different paths",
files: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-1",
},
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
},
},
layers: []string{
"fsid-1",
},
want: []model.File{
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
},
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/c",
FileSystemID: "fsid-1",
},
},
},
},
{
name: "stability test - preserve original order for equivalent items",
files: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
},
},
layers: []string{
"fsid-1",
},
want: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
},
{
ID: "file-3",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-1",
},
},
},
},
{
name: "complex file metadata doesn't affect sorting",
files: []model.File{
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
},
Metadata: &model.FileMetadataEntry{
Mode: 0644,
Type: "file",
UserID: 1000,
GroupID: 1000,
MIMEType: "text/plain",
Size: 100,
},
Contents: "content1",
Digests: []file.Digest{
{
Algorithm: "sha256",
Value: "abc123",
},
},
},
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
Metadata: &model.FileMetadataEntry{
Mode: 0755,
Type: "directory",
UserID: 0,
GroupID: 0,
MIMEType: "application/directory",
Size: 4096,
},
},
},
layers: []string{
"fsid-1",
"fsid-2",
},
want: []model.File{
{
ID: "file-2",
Location: file.Coordinates{
RealPath: "/b",
FileSystemID: "fsid-1",
},
Metadata: &model.FileMetadataEntry{
Mode: 0755,
Type: "directory",
UserID: 0,
GroupID: 0,
MIMEType: "application/directory",
Size: 4096,
},
},
{
ID: "file-1",
Location: file.Coordinates{
RealPath: "/a",
FileSystemID: "fsid-2",
},
Metadata: &model.FileMetadataEntry{
Mode: 0644,
Type: "file",
UserID: 1000,
GroupID: 1000,
MIMEType: "text/plain",
Size: 100,
},
Contents: "content1",
Digests: []file.Digest{
{
Algorithm: "sha256",
Value: "abc123",
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
files := make([]model.File, len(tt.files))
copy(files, tt.files)
sortFiles(files, file.CoordinatesSorter(tt.layers))
if d := cmp.Diff(tt.want, files); d != "" {
t.Errorf("sortFiles() mismatch (-want +got):\n%s", d)
}
})
}
}