serialize notarization (#806)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

.github/scripts/apple-signing/notarize.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-set -ue
 
 set +xu
 if [ -z "$AC_USERNAME" ]; then
@@ -11,58 +10,33 @@ if [ -z "$AC_PASSWORD" ]; then
 fi
 set -u
 
-# repackage [archive-path]
-#
-# returns an archive compatible for Apple's notarization process, repackaging the input archive as needed
-#
-repackage() {
-  archive=$1
-
-  case "$archive" in
-    *.tar.gz)
-      new_archive=${archive%.tar.gz}.zip
-      (
-        tmp_dir=$(mktemp -d)
-        cd "$tmp_dir"
-        # redirect stdout to stderr to preserve the return value
-        tar xzf "$archive" && zip "$new_archive" ./* 1>&2
-        rm -rf "$tmp_dir"
-      )
-      echo "$new_archive"
-      ;;
-    *.zip)
-      echo "$archive"
-      ;;
-    *) return 1
-      ;;
-  esac
-}
 
 # notarize [archive-path]
 #
 notarize() {
-  archive_path=$1
+  binary_path=$1
+  archive_path=${binary_path}-archive-for-notarization.zip
 
-  title "notarizing binaries found in the release archive"
+  title "archiving release binary into ${archive_path}"
 
-  payload_archive_path=$(repackage "$archive_path")
-  if [ "$?" != "0" ]; then
-    exit_with_error "cannot prepare payload for notarization: $archive_path"
-  fi
+  parent=$(dirname "$binary_path")
+  (
+    cd "${parent}" && zip "${archive_path}" "$(basename ${binary_path})"
+  )
 
-  if [ ! -f "$payload_archive_path" ]; then
-    exit_with_error "cannot find payload for notarization: $payload_archive_path"
+  if [ ! -f "$archive_path" ]; then
+    exit_with_error "cannot find payload for notarization: $archive_path"
   fi
 
   # install gon
   which gon || (brew tap mitchellh/gon && brew install mitchellh/gon/gon)
 
   # create config (note: json via stdin with gon is broken, can only use HCL from file)
-  tmp_file=$(mktemp).hcl
+  hcl_file=$(mktemp).hcl
 
-  cat <<EOF > "$tmp_file"
+  cat <<EOF > "$hcl_file"
 notarize {
-  path = "$payload_archive_path"
+  path = "$archive_path"
   bundle_id = "com.anchore.toolbox.syft"
 }
 
@@ -72,14 +46,8 @@ apple_id {
 }
 EOF
 
-  gon -log-level info "$tmp_file"
+  gon -log-level info "$hcl_file"
 
-  result="$?"
-
-  rm "$tmp_file"
-
-  if [ "$result" -ne "0" ]; then
-      exit_with_error "notarization failed"
-  fi
+  rm "${hcl_file}" "${archive_path}"
 }
 

.github/scripts/apple-signing/setup.sh

@@ -6,6 +6,7 @@ IS_SNAPSHOT="$1"
 ## grab utilities
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 . "$SCRIPT_DIR"/utils.sh
+mkdir -p "$SCRIPT_DIR/log"
 
 main() {
 
@@ -33,17 +34,13 @@ main() {
   echo -n "$MAC_SIGNING_IDENTITY" > "$SCRIPT_DIR/$SIGNING_IDENTITY_FILENAME"
 }
 
-set +u
-if [ -z "$SCRIPT" ]
-then
+# capture all output from a subshell to log output additionally to a file (as well as the terminal)
+( (
+  set +u
+  if [ -n "$SKIP_SIGNING" ]; then
+      commentary "skipping signing setup..."
+  else
     set -u
-    # log all output
-    mkdir -p "$SCRIPT_DIR/log"
-    /usr/bin/script "$SCRIPT_DIR/log/setup.txt" /bin/bash -c "$0 $*"
-    exit $?
-elif [ -n "$SKIP_SIGNING" ]; then
-    commentary "skipping signing setup..."
-else
-  set -u
-  main
-fi
+    main
+  fi
+) 2>&1) | tee "$SCRIPT_DIR/log/setup.txt"

.github/scripts/apple-signing/sign.sh

@@ -1,12 +1,14 @@
 #!/usr/bin/env bash
-set -eu
+set -eu -o pipefail
 
-ARCHIVE_PATH="$1"
+BINARY_PATH="$1"
 IS_SNAPSHOT="$2"
+TARGET_NAME="$3"
 
 ## grab utilities
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 . "$SCRIPT_DIR"/utils.sh
+mkdir -p "$SCRIPT_DIR/log"
 
 
 # sign_binary [binary-path] [signing-identity]
@@ -38,58 +40,14 @@ sign_binary() {
   fi
 
   codesign --verify "$exe_path"  --verbose=4
-
-  if [ $? -ne 0 ]; then
-      exit_with_error "signing verification failed"
-  fi
-}
-
-# sign_binaries_in_archive [archive-abs-path] [signing-identity]
-#
-# signs all binaries within an archive (there must be at least one)
-#
-sign_binaries_in_archive() {
-  archive_abs_path=$1
-  identity=$2
-
-  scratch_path=$(mktemp -d)
-  trap "rm -rf -- $scratch_path" EXIT
-
-  title "getting contents from the release archive: $archive_abs_path"
-  tar -C "$scratch_path" -xvf "$archive_abs_path"
-
-  # invalidate the current archive, we only want an asset with signed binaries from this point forward
-  rm "$archive_abs_path"
-
-  title "signing binaries found in the release archive"
-
-  discovered_binaries=0
-  tmp_pipe=$(mktemp -ut pipe.XXX)
-  mkfifo "$tmp_pipe"
-
-  find "$scratch_path" -perm +111 -type f > "$tmp_pipe" &
-
-  while IFS= read -r binary; do
-    sign_binary "$binary" "$identity"
-    ((discovered_binaries++))
-  done < "$tmp_pipe"
-
-  rm "$tmp_pipe"
-
-  if [ "$discovered_binaries" = "0" ]; then
-      exit_with_error "found no binaries to sign"
-  fi
-
-  title "recreating the release archive: $archive_abs_path"
-  (cd "$scratch_path" && tar -czvf "$archive_abs_path" .)
 }
 
 
 main() {
-  archive_abs_path=$(realpath "$ARCHIVE_PATH")
+  binary_abs_path=$(realpath "$BINARY_PATH")
 
-  if [ ! -f "$archive_abs_path" ]; then
-      echo "archive does not exist: $archive_abs_path"
+  if [ ! -f "$binary_abs_path" ]; then
+      echo "archive does not exist: $binary_abs_path"
   fi
 
   case "$IS_SNAPSHOT" in
@@ -114,27 +72,23 @@ main() {
   MAC_SIGNING_IDENTITY=$(cat "$SCRIPT_DIR/$SIGNING_IDENTITY_FILENAME")
 
   # sign all of the binaries in the archive and recreate the input archive with the signed binaries
-  sign_binaries_in_archive "$archive_abs_path" "$MAC_SIGNING_IDENTITY"
+  sign_binary "$binary_abs_path" "$MAC_SIGNING_IDENTITY"
 
   # send all of the binaries off to apple to bless
   if $perform_notarization ; then
-    notarize "$archive_abs_path"
+    notarize "$binary_abs_path"
   else
     commentary "skipping notarization..."
   fi
 }
 
-set +u
-if [ -z "$SCRIPT" ]
-then
+# capture all output from a subshell to log output additionally to a file (as well as the terminal)
+( (
+  set +u
+  if [ -n "$SKIP_SIGNING" ]; then
+      commentary "skipping signing..."
+  else
     set -u
-    # log all output
-    mkdir -p "$SCRIPT_DIR/log"
-    /usr/bin/script "$SCRIPT_DIR/log/signing-$(basename $ARCHIVE_PATH).txt" /bin/bash -c "$0 $*"
-    exit $?
-elif [ -n "$SKIP_SIGNING" ]; then
-    commentary "skipping signing..."
-else
-  set -u
-  main
-fi
+    main
+  fi
+) 2>&1) | tee "$SCRIPT_DIR/log/signing-$(basename $BINARY_PATH)-$TARGET_NAME.txt"

.goreleaser.yaml

@@ -41,6 +41,11 @@ builds:
     mod_timestamp: *build-timestamp
     env: *build-env
     ldflags: *build-ldflags
+    hooks:
+      post:
+        # we must have signing as a build hook instead of the signs section. The signs section must register a new asset, where we want to replace an existing asset.
+        # a post-build hook has the advantage of not needing to unpackage and repackage a tar.gz with a signed binary
+        - ./.github/scripts/apple-signing/sign.sh "{{ .Path }}" "{{ .IsSnapshot }}" "{{ .Target }}"
 
   - id: windows-build
     binary: syft
@@ -67,16 +72,6 @@ archives:
     builds:
       - windows-build
 
-signs:
-  - artifacts: archive
-    ids:
-      - darwin-archives
-    signature: "${artifact}"
-    cmd: ./.github/scripts/apple-signing/sign.sh
-    args:
-      - "${artifact}"
-      - "{{ .IsSnapshot }}"
-
 nfpms:
   - license: "Apache 2.0"
     maintainer: "Anchore, Inc"

Makefile

@@ -104,7 +104,7 @@ bootstrap-tools: $(TEMPDIR)
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ v1.42.1
 	curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ v0.3.0
 	curl -sSfL https://raw.githubusercontent.com/anchore/chronicle/main/install.sh | sh -s -- -b $(TEMPDIR)/ v0.3.0
-	.github/scripts/goreleaser-install.sh -d -b $(TEMPDIR)/ v1.3.1
+	.github/scripts/goreleaser-install.sh -d -b $(TEMPDIR)/ v1.4.1
 	GOBIN="$(shell realpath $(TEMPDIR))" go install github.com/neilpa/yajsv@v1.4.0
 
 .PHONY: bootstrap-go
@@ -316,9 +316,11 @@ release: clean-dist CHANGELOG.md  ## Build and publish final binaries and packag
 
 	rm -f .github/scripts/apple-signing/log/*.txt
 
+	# note: notarization cannot be done in parallel, thus --parallelism 1
 	bash -c "\
 		$(RELEASE_CMD) \
 			--config $(TEMPDIR)/goreleaser.yaml \
+			--parallelism 1 \
 			--release-notes <(cat CHANGELOG.md)\
 				 || cat .github/scripts/apple-signing/log/*.txt && false"
 

internal/spdxlicense/license_list.go

@@ -1,9 +1,9 @@
 // Code generated by go generate; DO NOT EDIT.
-// This file was generated by robots at 2021-11-15 14:32:22.081911 -0500 EST m=+0.159302107
+// This file was generated by robots at 2022-02-07 09:34:56.622482 -0500 EST m=+0.351558414
 // using data from https://spdx.org/licenses/licenses.json
 package spdxlicense
 
-const Version = "3.15"
+const Version = "3.16"
 
 var licenseIDs = map[string]string{
 	"0bsd":                                   "0BSD",
@@ -64,6 +64,7 @@ var licenseIDs = map[string]string{
 	"apl-1":                                  "APL-1.0",
 	"apl-1.0":                                "APL-1.0",
 	"apl-1.0.0":                              "APL-1.0",
+	"app-s2p":                                "App-s2p",
 	"apsl-1":                                 "APSL-1.0",
 	"apsl-1.0":                               "APSL-1.0",
 	"apsl-1.0.0":                             "APSL-1.0",
@@ -399,6 +400,9 @@ var licenseIDs = map[string]string{
 	"d-fsl-1.0":                                "D-FSL-1.0",
 	"d-fsl-1.0.0":                              "D-FSL-1.0",
 	"diffmark":                                 "diffmark",
+	"dl-de-by-2":                               "DL-DE-BY-2.0",
+	"dl-de-by-2.0":                             "DL-DE-BY-2.0",
+	"dl-de-by-2.0.0":                           "DL-DE-BY-2.0",
 	"doc":                                      "DOC",
 	"dotseqn":                                  "Dotseqn",
 	"drl-1":                                    "DRL-1.0",
@@ -422,6 +426,9 @@ var licenseIDs = map[string]string{
 	"efl-2.0":                                  "EFL-2.0",
 	"efl-2.0.0":                                "EFL-2.0",
 	"egenix":                                   "eGenix",
+	"elastic-2":                                "Elastic-2.0",
+	"elastic-2.0":                              "Elastic-2.0",
+	"elastic-2.0.0":                            "Elastic-2.0",
 	"entessa":                                  "Entessa",
 	"epics":                                    "EPICS",
 	"epl-1":                                    "EPL-1.0",
@@ -596,6 +603,7 @@ var licenseIDs = map[string]string{
 	"ipl-1.0":                                  "IPL-1.0",
 	"ipl-1.0.0":                                "IPL-1.0",
 	"isc":                                      "ISC",
+	"jam":                                      "Jam",
 	"jasper-2":                                 "JasPer-2.0",
 	"jasper-2.0":                               "JasPer-2.0",
 	"jasper-2.0.0":                             "JasPer-2.0",
@@ -909,6 +917,7 @@ var licenseIDs = map[string]string{
 	"sax-pd":                                   "SAX-PD",
 	"saxpath":                                  "Saxpath",
 	"scea":                                     "SCEA",
+	"schemereport":                             "SchemeReport",
 	"sendmail":                                 "Sendmail",
 	"sendmail-8":                               "Sendmail-8.23",
 	"sendmail-8.23":                            "Sendmail-8.23",
@@ -1007,6 +1016,7 @@ var licenseIDs = map[string]string{
 	"wtfpl":                                    "WTFPL",
 	"wxwindows":                                "wxWindows",
 	"x11":                                      "X11",
+	"x11-distribute-modifications-variant":     "X11-distribute-modifications-variant",
 	"xerox":                                    "Xerox",
 	"xfree86-1":                                "XFree86-1.1",
 	"xfree86-1.1":                              "XFree86-1.1",

test/cli/packages_cmd_test.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 )
@@ -201,6 +202,10 @@ func TestPackagesCmdFlags(t *testing.T) {
 }
 
 func TestRegistryAuth(t *testing.T) {
+	host := "localhost:17"
+	image := fmt.Sprintf("%s/something:latest", host)
+	args := []string{"packages", "-vv", fmt.Sprintf("registry:%s", image)}
+
 	tests := []struct {
 		name       string
 		args       []string
@@ -209,55 +214,55 @@ func TestRegistryAuth(t *testing.T) {
 	}{
 		{
 			name: "fallback to keychain",
-			args: []string{"packages", "-vv", "registry:localhost:5000/something:latest"},
+			args: args,
 			assertions: []traitAssertion{
 				assertInOutput("source=OciRegistry"),
-				assertInOutput("localhost:5000/something:latest"),
+				assertInOutput(image),
 				assertInOutput("no registry credentials configured, using the default keychain"),
 			},
 		},
 		{
 			name: "use creds",
-			args: []string{"packages", "-vv", "registry:localhost:5000/something:latest"},
+			args: args,
 			env: map[string]string{
-				"SYFT_REGISTRY_AUTH_AUTHORITY": "localhost:5000",
+				"SYFT_REGISTRY_AUTH_AUTHORITY": host,
 				"SYFT_REGISTRY_AUTH_USERNAME":  "username",
 				"SYFT_REGISTRY_AUTH_PASSWORD":  "password",
 			},
 			assertions: []traitAssertion{
 				assertInOutput("source=OciRegistry"),
-				assertInOutput("localhost:5000/something:latest"),
-				assertInOutput(`using basic auth for registry "localhost:5000"`),
+				assertInOutput(image),
+				assertInOutput(fmt.Sprintf(`using basic auth for registry "%s"`, host)),
 			},
 		},
 		{
 			name: "use token",
-			args: []string{"packages", "-vv", "registry:localhost:5000/something:latest"},
+			args: args,
 			env: map[string]string{
-				"SYFT_REGISTRY_AUTH_AUTHORITY": "localhost:5000",
+				"SYFT_REGISTRY_AUTH_AUTHORITY": host,
 				"SYFT_REGISTRY_AUTH_TOKEN":     "token",
 			},
 			assertions: []traitAssertion{
 				assertInOutput("source=OciRegistry"),
-				assertInOutput("localhost:5000/something:latest"),
-				assertInOutput(`using token for registry "localhost:5000"`),
+				assertInOutput(image),
+				assertInOutput(fmt.Sprintf(`using token for registry "%s"`, host)),
 			},
 		},
 		{
 			name: "not enough info fallsback to keychain",
-			args: []string{"packages", "-vv", "registry:localhost:5000/something:latest"},
+			args: args,
 			env: map[string]string{
-				"SYFT_REGISTRY_AUTH_AUTHORITY": "localhost:5000",
+				"SYFT_REGISTRY_AUTH_AUTHORITY": host,
 			},
 			assertions: []traitAssertion{
 				assertInOutput("source=OciRegistry"),
-				assertInOutput("localhost:5000/something:latest"),
+				assertInOutput(image),
 				assertInOutput(`no registry credentials configured, using the default keychain`),
 			},
 		},
 		{
 			name: "allows insecure http flag",
-			args: []string{"packages", "-vv", "registry:localhost:5000/something:latest"},
+			args: args,
 			env: map[string]string{
 				"SYFT_REGISTRY_INSECURE_USE_HTTP": "true",
 			},

test/install/test_harness.sh

@@ -94,7 +94,7 @@ setup_snapshot_server() {
   worker_pid=$!
 
   # it takes some time for the server to be ready...
-  sleep 0.5
+  sleep 3
 
   echo "$worker_pid"
 }