Fix: unmarshal key values in Java, Go, and Conan metadata (#2603)

Previously, Syft represented several metadata fields as map[string]string,
however this representation erased ordering, so Syft now represents these values
as []KeyValue. Add custom unmarshaling so that JSON that was written by
older versions of Syft using the map[string]string representation can be parsed
into the new []KeyValue representation.

Signed-off-by: Will Murphy <will.murphy@anchore.com>

syft/file/cataloger/executable/cataloger.go

@@ -6,6 +6,7 @@ import (
 	"debug/macho"
 	"encoding/binary"
 	"fmt"
+	"sort"
 
 	"github.com/bmatcuk/doublestar/v4"
 	"github.com/dustin/go-humanize"
@@ -28,8 +29,10 @@ type Cataloger struct {
 }
 
 func DefaultConfig() Config {
+	m := mimetype.ExecutableMIMETypeSet.List()
+	sort.Strings(m)
 	return Config{
-		MIMETypes: mimetype.ExecutableMIMETypeSet.List(),
+		MIMETypes: m,
 		Globs:     nil,
 	}
 }

syft/format/syftjson/model/package_test.go

@@ -386,6 +386,143 @@ func Test_UnmarshalJSON(t *testing.T) {
 				assert.Equal(t, reflect.TypeOf(pkg.RustBinaryAuditEntry{}).Name(), reflect.TypeOf(p.Metadata).Name())
 			},
 		},
+		{
+			name: "map-based java metadata",
+			packageData: []byte(`{
+  "id": "e6f845bdaa69ddb2",
+  "name": "SparseBitSet",
+  "version": "1.2",
+  "type": "java-archive",
+  "foundBy": "java-archive-cataloger",
+  "locations": [],
+  "licenses": [],
+  "language": "java",
+  "cpes": [],
+  "purl": "pkg:maven/com.zaxxer/SparseBitSet@1.2",
+  "metadataType": "java-archive",
+  "metadata": {
+    "virtualPath": "/opt/solr-9.4.1/modules/extraction/lib/SparseBitSet-1.2.jar",
+    "manifest": {
+      "main": {
+        "Archiver-Version": "Plexus Archiver",
+        "Build-Jdk": "1.8.0_73",
+        "Built-By": "lbayer",
+        "Created-By": "Apache Maven 3.5.0",
+        "Manifest-Version": "1.0"
+      },
+      "namedSections": {
+        "META-INF/mailcap": {
+          "SHA-256-Digest": "kXN4VupOQOJhduMGwxumj4ijmD/YAlz97a9Mp7CVXtk="
+        },
+        "META-INF/versions/9/module-info.class": {
+          "SHA-256-Digest": "cMeIRa5l8DWPgrVWavr/6TKVBUGixVKGcu6yOTZMlKk="
+        }
+      }
+    }
+  }
+}`),
+			assert: func(p *Package) {
+				meta := p.Metadata.(pkg.JavaArchive)
+				manifest := meta.Manifest
+				assert.Equal(t, "1.8.0_73", manifest.Main.MustGet("Build-Jdk"))
+				require.Equal(t, 2, len(manifest.Sections))
+				assert.Equal(t, "META-INF/mailcap", manifest.Sections[0].MustGet("Name"))
+				assert.Equal(t, "kXN4VupOQOJhduMGwxumj4ijmD/YAlz97a9Mp7CVXtk=", manifest.Sections[0].MustGet("SHA-256-Digest"))
+				assert.Equal(t, "META-INF/versions/9/module-info.class", manifest.Sections[1].MustGet("Name"))
+				assert.Equal(t, "cMeIRa5l8DWPgrVWavr/6TKVBUGixVKGcu6yOTZMlKk=", manifest.Sections[1].MustGet("SHA-256-Digest"))
+			},
+		},
+		{
+			name: "pre key-value golang metadata",
+			packageData: []byte(`{
+  "id": "e348ed25484a94c9",
+  "name": "github.com/anchore/syft",
+  "version": "v0.101.1-SNAPSHOT-4c777834",
+  "type": "go-module",
+  "foundBy": "go-module-binary-cataloger",
+  "locations": [
+    {
+      "path": "/syft",
+      "layerID": "sha256:2274947a5f3527e48d8725a96646aefdcce3d99340c1eefb1e7c894043863c92",
+      "accessPath": "/syft",
+      "annotations": {
+        "evidence": "primary"
+      }
+    }
+  ],
+  "licenses": [],
+  "language": "go",
+  "cpes": [
+    "cpe:2.3:a:anchore:syft:v0.101.1-SNAPSHOT-4c777834:*:*:*:*:*:*:*"
+  ],
+  "purl": "pkg:golang/github.com/anchore/syft@v0.101.1-SNAPSHOT-4c777834",
+  "metadataType": "go-module-buildinfo-entry",
+  "metadata": {
+    "goBuildSettings": {
+      "-buildmode": "exe",
+      "-compiler": "gc",
+      "-ldflags": "-w -s -extldflags '-static' -X main.version=0.101.1-SNAPSHOT-4c777834 -X main.gitCommit=4c777834618b2ad8ad94cd200a45d6670bc1c013 -X main.buildDate=2024-01-22T16:43:49Z -X main.gitDescription=v0.101.1-4-g4c777834 ",
+      "CGO_ENABLED": "0",
+      "GOAMD64": "v1",
+      "GOARCH": "amd64",
+      "GOOS": "linux",
+      "vcs": "git",
+      "vcs.modified": "false",
+      "vcs.revision": "4c777834618b2ad8ad94cd200a45d6670bc1c013",
+      "vcs.time": "2024-01-22T16:31:41Z"
+    },
+    "goCompiledVersion": "go1.21.2",
+    "architecture": "amd64",
+    "mainModule": "github.com/anchore/syft"
+  }
+}
+`),
+			assert: func(p *Package) {
+				buildInfo := p.Metadata.(pkg.GolangBinaryBuildinfoEntry)
+				assert.Equal(t, "exe", buildInfo.BuildSettings.MustGet("-buildmode"))
+			},
+		},
+		{
+			name: "conan lock with legacy options",
+			packageData: []byte(`{
+  "id": "75eb35307226c921",
+  "name": "boost",
+  "version": "1.75.0",
+  "type": "conan",
+  "foundBy": "conan-cataloger",
+  "locations": [
+    {
+      "path": "/conan.lock",
+      "accessPath": "/conan.lock",
+      "annotations": {
+        "evidence": "primary"
+      }
+    }
+  ],
+  "licenses": [],
+  "language": "c++",
+  "cpes": [
+    "cpe:2.3:a:boost:boost:1.75.0:*:*:*:*:*:*:*"
+  ],
+  "purl": "pkg:conan/boost@1.75.0",
+  "metadataType": "c-conan-lock-entry",
+  "metadata": {
+    "ref": "boost/1.75.0#a9c318f067216f900900e044e7af4ab1",
+    "package_id": "dc8aedd23a0f0a773a5fcdcfe1ae3e89c4205978",
+    "prev": "b9d7912e6131dfa453c725593b36c808",
+    "options": {
+      "addr2line_location": "/usr/bin/addr2line",
+      "asio_no_deprecated": "False",
+      "zstd": "False"
+    },
+    "context": "host"
+  }
+}`),
+			assert: func(p *Package) {
+				metadata := p.Metadata.(pkg.ConanV1LockEntry)
+				assert.Equal(t, "False", metadata.Options.MustGet("asio_no_deprecated"))
+			},
+		},
 	}
 
 	for _, test := range tests {

syft/pkg/java.go

@@ -1,6 +1,9 @@
 package pkg
 
 import (
+	"encoding/json"
+	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/anchore/syft/internal"
@@ -70,6 +73,59 @@ type JavaManifest struct {
 	Sections []KeyValues `json:"sections,omitempty"`
 }
 
+type unmarshalJavaManifest JavaManifest
+
+type legacyJavaManifest struct {
+	Main          map[string]string            `json:"main"`
+	NamedSections map[string]map[string]string `json:"namedSections"`
+}
+
+func (m *JavaManifest) UnmarshalJSON(b []byte) error {
+	var either map[string]any
+	err := json.Unmarshal(b, &either)
+	if err != nil {
+		return fmt.Errorf("could not unmarshal java manifest: %w", err)
+	}
+	if _, ok := either["namedSections"]; ok {
+		var lm legacyJavaManifest
+		if err = json.Unmarshal(b, &lm); err != nil {
+			return fmt.Errorf("could not unmarshal java manifest: %w", err)
+		}
+		*m = lm.toNewManifest()
+		return nil
+	}
+	var jm unmarshalJavaManifest
+	err = json.Unmarshal(b, &jm)
+	if err != nil {
+		return fmt.Errorf("could not unmarshal java manifest: %w", err)
+	}
+	*m = JavaManifest(jm)
+	return nil
+}
+
+func (lm legacyJavaManifest) toNewManifest() JavaManifest {
+	var result JavaManifest
+	result.Main = keyValuesFromMap(lm.Main)
+	var sectionNames []string
+	for k := range lm.NamedSections {
+		sectionNames = append(sectionNames, k)
+	}
+	sort.Strings(sectionNames)
+	var sections []KeyValues
+	for _, name := range sectionNames {
+		section := KeyValues{
+			KeyValue{
+				Key:   "Name",
+				Value: name,
+			},
+		}
+		section = append(section, keyValuesFromMap(lm.NamedSections[name])...)
+		sections = append(sections, section)
+	}
+	result.Sections = sections
+	return result
+}
+
 func (m JavaManifest) Section(name string) KeyValues {
 	for _, section := range m.Sections {
 		if sectionName, ok := section.Get("Name"); ok && sectionName == name {

syft/pkg/java_test.go

@@ -3,6 +3,7 @@ package pkg
 import (
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -109,3 +110,103 @@ func TestPomProperties_PkgTypeIndicated(t *testing.T) {
 		})
 	}
 }
+
+func Test_legacyJavaManifest_toNewManifest(t *testing.T) {
+	tests := []struct {
+		name string
+		lm   legacyJavaManifest
+		want JavaManifest
+	}{
+		{
+			name: "empty",
+			lm:   legacyJavaManifest{},
+			want: JavaManifest{},
+		},
+		{
+			name: "main sections are sorted",
+			lm: legacyJavaManifest{
+				Main: map[string]string{
+					"a key": "a value",
+					"b key": "b value",
+					"c key": "c value",
+				},
+			},
+			want: JavaManifest{Main: KeyValues{
+				{
+					Key:   "a key",
+					Value: "a value",
+				},
+				{
+					Key:   "b key",
+					Value: "b value",
+				},
+				{
+					Key:   "c key",
+					Value: "c value",
+				},
+			}},
+		},
+		{
+			name: "named sections have their name in the result",
+			lm: legacyJavaManifest{
+				NamedSections: map[string]map[string]string{
+					"a section": {
+						"a key": "a value",
+						"b key": "b value",
+						"c key": "c value",
+					},
+					"b section": {
+						"d key": "d value",
+						"e key": "e value",
+						"f key": "f value",
+					},
+				},
+			},
+			want: JavaManifest{Sections: []KeyValues{
+				{
+					{
+						Key:   "Name",
+						Value: "a section",
+					},
+					{
+						Key:   "a key",
+						Value: "a value",
+					},
+					{
+						Key:   "b key",
+						Value: "b value",
+					},
+					{
+						Key:   "c key",
+						Value: "c value",
+					},
+				},
+				{
+					{
+						Key:   "Name",
+						Value: "b section",
+					},
+					{
+						Key:   "d key",
+						Value: "d value",
+					},
+					{
+						Key:   "e key",
+						Value: "e value",
+					},
+					{
+						Key:   "f key",
+						Value: "f value",
+					},
+				},
+			}},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if diff := cmp.Diff(tt.want, tt.lm.toNewManifest()); diff != "" {
+				t.Errorf("unexpected diff in manifest (-want +got):\n%s", diff)
+			}
+		})
+	}
+}

syft/pkg/key_value.go

@@ -1,5 +1,11 @@
 package pkg
 
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+)
+
 type KeyValue struct {
 	Key   string `json:"key"`
 	Value string `json:"value"`
@@ -26,3 +32,39 @@ func (k KeyValues) MustGet(key string) string {
 
 	return ""
 }
+
+func keyValuesFromMap(m map[string]string) KeyValues {
+	var result KeyValues
+	var mapKeys []string
+	for k := range m {
+		mapKeys = append(mapKeys, k)
+	}
+	sort.Strings(mapKeys)
+	for _, k := range mapKeys {
+		result = append(result, KeyValue{
+			Key:   k,
+			Value: m[k],
+		})
+	}
+	return result
+}
+
+func (k *KeyValues) UnmarshalJSON(b []byte) error {
+	var kvs []KeyValue
+	if err := json.Unmarshal(b, &kvs); err != nil {
+		var legacyMap map[string]string
+		if err := json.Unmarshal(b, &legacyMap); err != nil {
+			return fmt.Errorf("unable to unmarshal KeyValues: %w", err)
+		}
+		var keys []string
+		for k := range legacyMap {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		for _, k := range keys {
+			kvs = append(kvs, KeyValue{Key: k, Value: legacyMap[k]})
+		}
+	}
+	*k = kvs
+	return nil
+}