Add cataloging of macho multi-architecture binaries (#657)

* add cataloging within universal binaries

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update json test fixtures

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add comments + correct 32 bit multi arch magic check

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

internal/constants.go

@@ -6,5 +6,5 @@ const (
 
 	// JSONSchemaVersion is the current schema version output by the JSON presenter
 	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
-	JSONSchemaVersion = "2.0.0"
+	JSONSchemaVersion = "2.0.1"
 )

internal/formats/common/testutils/utils.go

@@ -60,8 +60,6 @@ func AssertPresenterAgainstGoldenImageSnapshot(t *testing.T, pres presenter.Pres
 	if !bytes.Equal(expected, actual) {
 		dmp := diffmatchpatch.New()
 		diffs := dmp.DiffMain(string(expected), string(actual), true)
-		t.Logf("len: %d\nexpected: %v", len(expected), expected)
-		t.Logf("len: %d\nactual: %v", len(actual), actual)
 		t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
 	}
 }

internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryPresenter.golden

@@ -83,7 +83,7 @@
   }
  },
  "schema": {
-  "version": "2.0.0",
+  "version": "2.0.1",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.0.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.1.json"
  }
 }

internal/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden

@@ -179,7 +179,7 @@
   }
  },
  "schema": {
-  "version": "2.0.0",
+  "version": "2.0.1",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.0.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.1.json"
  }
 }

internal/formats/syftjson/test-fixtures/snapshot/TestImagePresenter.golden

@@ -104,7 +104,7 @@
   }
  },
  "schema": {
-  "version": "2.0.0",
+  "version": "2.0.1",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.0.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.1.json"
  }
 }

schema/json/generate.go

@@ -35,6 +35,7 @@ type artifactMetadataContainer struct {
 	Python pkg.PythonPackageMetadata
 	Rpm    pkg.RpmdbMetadata
 	Cargo  pkg.CargoPackageMetadata
+	Go     pkg.GolangBinMetadata
 }
 
 func main() {

syft/pkg/cataloger/golang/exe.go

@@ -14,6 +14,7 @@ import (
 	"debug/pe"
 	"fmt"
 	"io"
+	"strings"
 )
 
 // An exe is a generic interface to an OS executable (ELF, Mach-O, PE, XCOFF).
@@ -24,6 +25,9 @@ type exe interface {
 	// ReadData reads and returns up to size byte starting at virtual address addr.
 	ReadData(addr, size uint64) ([]byte, error)
 
+	// ArchName returns a string that represents the CPU architecture of the executable.
+	ArchName() string
+
 	// DataStart returns the writable data segment start address.
 	DataStart() uint64
 }
@@ -32,7 +36,7 @@ type exe interface {
 // we changed this signature from accpeting a string
 // to a ReadCloser so we could adapt the code to the
 // stereoscope api. We removed the file open methods.
-func openExe(file io.ReadCloser) (exe, error) {
+func openExe(file io.ReadCloser) ([]exe, error) {
 	/*
 		f, err := os.Open(file)
 		if err != nil {
@@ -57,7 +61,7 @@ func openExe(file io.ReadCloser) (exe, error) {
 			return nil, err
 		}
 
-		return &elfExe{file, e}, nil
+		return []exe{&elfExe{file, e}}, nil
 	}
 
 	if bytes.HasPrefix(data, []byte("MZ")) {
@@ -65,7 +69,7 @@ func openExe(file io.ReadCloser) (exe, error) {
 		if err != nil {
 			return nil, err
 		}
-		return &peExe{file, e}, nil
+		return []exe{&peExe{file, e}}, nil
 	}
 
 	if bytes.HasPrefix(data, []byte("\xFE\xED\xFA")) || bytes.HasPrefix(data[1:], []byte("\xFA\xED\xFE")) {
@@ -73,7 +77,20 @@ func openExe(file io.ReadCloser) (exe, error) {
 		if err != nil {
 			return nil, err
 		}
-		return &machoExe{file, e}, nil
+		return []exe{&machoExe{file, e}}, nil
+	}
+
+	// adding macho multi-architecture support (both for 64bit and 32 bit)... this case is not in the stdlib yet
+	if bytes.HasPrefix(data, []byte("\xCA\xFE\xBA\xBE")) || bytes.HasPrefix(data, []byte("\xCA\xFE\xBA\xBF")) {
+		fatExe, err := macho.NewFatFile(f)
+		if err != nil {
+			return nil, err
+		}
+		var exes []exe
+		for _, arch := range fatExe.Arches {
+			exes = append(exes, &machoExe{file, arch.File})
+		}
+		return exes, nil
 	}
 
 	return nil, fmt.Errorf("unrecognized executable format")
@@ -90,6 +107,14 @@ func (x *elfExe) Close() error {
 	return x.os.Close()
 }
 
+func (x *elfExe) ArchName() string {
+	return cleanElfArch(x.f.Machine)
+}
+
+func cleanElfArch(machine elf.Machine) string {
+	return strings.TrimPrefix(strings.ToLower(machine.String()), "em_")
+}
+
 func (x *elfExe) ReadData(addr, size uint64) ([]byte, error) {
 	for _, prog := range x.f.Progs {
 		if prog.Vaddr <= addr && addr <= prog.Vaddr+prog.Filesz-1 {
@@ -132,6 +157,56 @@ func (x *peExe) Close() error {
 	return x.os.Close()
 }
 
+func (x *peExe) ArchName() string {
+	// from: debug/pe/pe.go
+	switch x.f.Machine {
+	case pe.IMAGE_FILE_MACHINE_AM33:
+		return "amd33"
+	case pe.IMAGE_FILE_MACHINE_AMD64:
+		return "amd64"
+	case pe.IMAGE_FILE_MACHINE_ARM:
+		return "arm"
+	case pe.IMAGE_FILE_MACHINE_ARMNT:
+		return "armnt"
+	case pe.IMAGE_FILE_MACHINE_ARM64:
+		return "arm64"
+	case pe.IMAGE_FILE_MACHINE_EBC:
+		return "ebc"
+	case pe.IMAGE_FILE_MACHINE_I386:
+		return "i386"
+	case pe.IMAGE_FILE_MACHINE_IA64:
+		return "ia64"
+	case pe.IMAGE_FILE_MACHINE_M32R:
+		return "m32r"
+	case pe.IMAGE_FILE_MACHINE_MIPS16:
+		return "mips16"
+	case pe.IMAGE_FILE_MACHINE_MIPSFPU:
+		return "mipsfpu"
+	case pe.IMAGE_FILE_MACHINE_MIPSFPU16:
+		return "mipsfpu16"
+	case pe.IMAGE_FILE_MACHINE_POWERPC:
+		return "ppc"
+	case pe.IMAGE_FILE_MACHINE_POWERPCFP:
+		return "ppcfp"
+	case pe.IMAGE_FILE_MACHINE_R4000:
+		return "r4000"
+	case pe.IMAGE_FILE_MACHINE_SH3:
+		return "sh3"
+	case pe.IMAGE_FILE_MACHINE_SH3DSP:
+		return "sh3dsp"
+	case pe.IMAGE_FILE_MACHINE_SH4:
+		return "sh4"
+	case pe.IMAGE_FILE_MACHINE_SH5:
+		return "sh5"
+	case pe.IMAGE_FILE_MACHINE_THUMB:
+		return "thumb"
+	case pe.IMAGE_FILE_MACHINE_WCEMIPSV2:
+		return "wcemipsv2"
+	default:
+		return fmt.Sprintf("unknown-pe-machine-%d", x.f.Machine)
+	}
+}
+
 func (x *peExe) imageBase() uint64 {
 	switch oh := x.f.OptionalHeader.(type) {
 	case *pe.OptionalHeader32:
@@ -193,6 +268,14 @@ func (x *machoExe) Close() error {
 	return x.os.Close()
 }
 
+func (x *machoExe) ArchName() string {
+	return cleanMachoArch(x.f.Cpu)
+}
+
+func cleanMachoArch(cpu macho.Cpu) string {
+	return strings.TrimPrefix(strings.ToLower(cpu.String()), "cpu")
+}
+
 func (x *machoExe) ReadData(addr, size uint64) ([]byte, error) {
 	for _, load := range x.f.Loads {
 		seg, ok := load.(*macho.Segment)

syft/pkg/cataloger/golang/exe_test.go

@@ -0,0 +1,43 @@
+package golang
+
+import (
+	"debug/elf"
+	"debug/macho"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_cleanElfArch(t *testing.T) {
+	tests := []struct {
+		machine elf.Machine
+		want    string
+	}{
+		{
+			machine: elf.EM_X86_64,
+			want:    "x86_64",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.machine.String(), func(t *testing.T) {
+			assert.Equalf(t, test.want, cleanElfArch(test.machine), "cleanElfArch(%v)", test.machine)
+		})
+	}
+}
+
+func Test_cleanMachoArch(t *testing.T) {
+	tests := []struct {
+		cpu  macho.Cpu
+		want string
+	}{
+		{
+			cpu:  macho.CpuAmd64,
+			want: "amd64",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.cpu.String(), func(t *testing.T) {
+			assert.Equalf(t, test.want, cleanMachoArch(test.cpu), "cleanMachoArch(%v)", test.cpu)
+		})
+	}
+}

syft/pkg/cataloger/golang/parse_go_bin.go

@@ -16,17 +16,20 @@ const (
 
 func parseGoBin(location source.Location, reader io.ReadCloser) ([]pkg.Package, error) {
 	// Identify if bin was compiled by go
-	x, err := openExe(reader)
+	exes, err := openExe(reader)
 	if err != nil {
 		return nil, err
 	}
 
+	var pkgs []pkg.Package
+	for _, x := range exes {
 		goVersion, mod := findVers(x)
-
+		pkgs = append(pkgs, buildGoPkgInfo(location, mod, goVersion, x.ArchName())...)
-	return buildGoPkgInfo(location, mod, goVersion), nil
+	}
+	return pkgs, nil
 }
 
-func buildGoPkgInfo(location source.Location, mod, goVersion string) []pkg.Package {
+func buildGoPkgInfo(location source.Location, mod, goVersion, arch string) []pkg.Package {
 	pkgsSlice := make([]pkg.Package, 0)
 	scanner := bufio.NewScanner(strings.NewReader(mod))
 
@@ -52,6 +55,7 @@ func buildGoPkgInfo(location source.Location, mod, goVersion string) []pkg.Packa
 				Metadata: pkg.GolangBinMetadata{
 					GoCompiledVersion: goVersion,
 					H1Digest:          fields[3],
+					Architecture:      arch,
 				},
 			})
 		}

syft/pkg/cataloger/golang/parse_go_bin_test.go

@@ -8,9 +8,11 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-const goCompiledVersion = "1.17"
-
 func TestBuildGoPkgInfo(t *testing.T) {
+	const (
+		goCompiledVersion = "1.17"
+		archDetails       = "amd64"
+	)
 	tests := []struct {
 		name     string
 		mod      string
@@ -43,6 +45,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 					MetadataType: pkg.GolangBinMetadataType,
 					Metadata: pkg.GolangBinMetadata{
 						GoCompiledVersion: goCompiledVersion,
+						Architecture:      archDetails,
 						H1Digest:          "h1:VSVdnH7cQ7V+B33qSJHTCRlNgra1607Q8PzEmnvb2Ic=",
 					},
 				},
@@ -62,6 +65,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 					MetadataType: pkg.GolangBinMetadataType,
 					Metadata: pkg.GolangBinMetadata{
 						GoCompiledVersion: goCompiledVersion,
+						Architecture:      archDetails,
 						H1Digest:          "h1:DYssiUV1pBmKqzKsm4mqXx8artqC0Q8HgZsVI3lMsAg=",
 					},
 				},
@@ -92,6 +96,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 					MetadataType: pkg.GolangBinMetadataType,
 					Metadata: pkg.GolangBinMetadata{
 						GoCompiledVersion: goCompiledVersion,
+						Architecture:      archDetails,
 						H1Digest:          "h1:KlOXYy8wQWTUJYFgkUI40Lzr06ofg5IRXUK5C7qZt1k=",
 					},
 				},
@@ -111,6 +116,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 					MetadataType: pkg.GolangBinMetadataType,
 					Metadata: pkg.GolangBinMetadata{
 						GoCompiledVersion: goCompiledVersion,
+						Architecture:      archDetails,
 						H1Digest:          "h1:PjhxBct4MZii8FFR8+oeS7QOvxKOTZXgk63EU2XpfJE=",
 					},
 				},
@@ -130,6 +136,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 					MetadataType: pkg.GolangBinMetadataType,
 					Metadata: pkg.GolangBinMetadata{
 						GoCompiledVersion: goCompiledVersion,
+						Architecture:      archDetails,
 						H1Digest:          "h1:Ihq/mm/suC88gF8WFcVwk+OV6Tq+wyA1O0E5UEvDglI=",
 					},
 				},
@@ -146,7 +153,7 @@ func TestBuildGoPkgInfo(t *testing.T) {
 					FileSystemID: "layer-id",
 				},
 			}
-			pkgs := buildGoPkgInfo(location, tt.mod, goCompiledVersion)
+			pkgs := buildGoPkgInfo(location, tt.mod, goCompiledVersion, archDetails)
 			assert.Equal(t, tt.expected, pkgs)
 		})
 	}

syft/pkg/golang_bin_metadata.go

@@ -2,6 +2,7 @@ package pkg
 
 // GolangBinMetadata represents all captured data for a Golang Binary
 type GolangBinMetadata struct {
-	GoCompiledVersion string
+	GoCompiledVersion string `json:"goCompiledVersion"`
-	H1Digest          string
+	Architecture      string `json:"architecture"`
+	H1Digest          string `json:"h1Digest"`
 }