chore: continue building snapshots for acceptance tests

Signed-off-by: Keith Zantow <kzantow@gmail.com>

.github/workflows/validations.yaml

@@ -92,6 +92,27 @@ jobs:
         run: make integration
 
 
+  Cli-Linux:
+    # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
+    name: "CLI tests (Linux)"
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 #v4.1.0
+
+      - name: Bootstrap environment
+        uses: ./.github/actions/bootstrap
+
+      - name: Restore CLI test-fixture cache
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2
+        with:
+          path: ${{ github.workspace }}/test/cli/test-fixtures/cache
+          key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }}
+
+      - name: Run CLI Tests (Linux)
+        run: make cli
+
+
+
   Build-Snapshot-Artifacts:
     name: "Build snapshot artifacts"
     runs-on: ubuntu-20.04
@@ -183,30 +204,3 @@ jobs:
 
       - name: Run install.sh tests (Mac)
         run: make install-test-ci-mac
-
-
-  Cli-Linux:
-    # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
-    name: "CLI tests (Linux)"
-    needs: [Build-Snapshot-Artifacts]
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 #v4.1.0
-
-      - name: Bootstrap environment
-        uses: ./.github/actions/bootstrap
-
-      - name: Restore CLI test-fixture cache
-        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2
-        with:
-          path: ${{ github.workspace }}/test/cli/test-fixtures/cache
-          key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }}
-
-      - name: Download snapshot build
-        uses: actions/cache/restore@704facf57e6136b1bc63b828d79edcd491f0ee84 #v3.3.2
-        with:
-          path: snapshot
-          key: snapshot-build-${{ github.run_id }}
-
-      - name: Run CLI Tests (Linux)
-        run: make cli

Makefile

@@ -41,7 +41,6 @@ DIST_DIR := ./dist
 SNAPSHOT_DIR := ./snapshot
 CHANGELOG := CHANGELOG.md
 OS := $(shell uname | tr '[:upper:]' '[:lower:]')
-SNAPSHOT_BIN := $(realpath $(shell pwd)/$(SNAPSHOT_DIR)/$(OS)-build_$(OS)_amd64_v1/$(BIN))
 
 ifndef VERSION
 	$(error VERSION is not set)
@@ -161,11 +160,8 @@ validate-cyclonedx-schema:
 	cd schema/cyclonedx && make
 
 .PHONY: cli
-cli: $(SNAPSHOT_DIR)  ## Run CLI tests
-	chmod 755 "$(SNAPSHOT_BIN)"
-	$(SNAPSHOT_BIN) version
-	SYFT_BINARY_LOCATION='$(SNAPSHOT_BIN)' \
-		go test -count=1 -timeout=15m -v ./test/cli
+cli:  ## Run CLI tests
+	go test -count=1 -timeout=15m -v ./test/cli
 
 
 ## Benchmark test targets #################################

test/cli/dir_root_scan_regression_test.go

@@ -12,6 +12,9 @@ func TestDirectoryScanCompletesWithinTimeout(t *testing.T) {
 	// we want to pull the image ahead of the test as to not affect the timeout value
 	pullDockerImage(t, image)
 
+	// run once in case we need to rebuild syft binary, so it doesn't affect the timeout value
+	runSyftInDocker(t, nil, image, "--help")
+
 	var cmd *exec.Cmd
 	var stdout, stderr string
 	done := make(chan struct{})

test/cli/utils_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
-	"math"
 	"os"
 	"os/exec"
 	"path"
@@ -13,8 +12,11 @@ import (
 	"strings"
 	"syscall"
 	"testing"
+	"text/template"
 	"time"
 
+	"gopkg.in/yaml.v3"
+
 	"github.com/anchore/stereoscope/pkg/imagetest"
 )
 
@@ -28,39 +30,6 @@ func logOutputOnFailure(t testing.TB, cmd *exec.Cmd, stdout, stderr string) {
 	}
 }
 
-func setupPKI(t *testing.T, pw string) func() {
-	err := os.Setenv("COSIGN_PASSWORD", pw)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	cosignPath := filepath.Join(repoRoot(t), ".tmp/cosign")
-	cmd := exec.Command(cosignPath, "generate-key-pair")
-	stdout, stderr, _ := runCommand(cmd, nil)
-	if cmd.ProcessState.ExitCode() != 0 {
-		t.Log("STDOUT", stdout)
-		t.Log("STDERR", stderr)
-		t.Fatalf("could not generate keypair")
-	}
-
-	return func() {
-		err := os.Unsetenv("COSIGN_PASSWORD")
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		err = os.Remove("cosign.key")
-		if err != nil {
-			t.Fatalf("could not cleanup cosign.key")
-		}
-
-		err = os.Remove("cosign.pub")
-		if err != nil {
-			t.Fatalf("could not cleanup cosign.key")
-		}
-	}
-}
-
 func getFixtureImage(t testing.TB, fixtureImageName string) string {
 	t.Logf("obtaining fixture image for %s", fixtureImageName)
 	imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName)
@@ -78,7 +47,7 @@ func pullDockerImage(t testing.TB, image string) {
 }
 
 // docker run -v $(pwd)/sbom:/sbom cyclonedx/cyclonedx-cli:latest validate --input-format json --input-version v1_4 --input-file /sbom
-func runCycloneDXInDocker(t testing.TB, env map[string]string, image string, f *os.File, args ...string) (*exec.Cmd, string, string) {
+func runCycloneDXInDocker(_ testing.TB, env map[string]string, image string, f *os.File, args ...string) (*exec.Cmd, string, string) {
 	allArgs := append(
 		[]string{
 			"run",
@@ -102,7 +71,7 @@ func runSyftInDocker(t testing.TB, env map[string]string, image string, args ...
 			"-e",
 			"SYFT_CHECK_FOR_APP_UPDATE=false",
 			"-v",
-			fmt.Sprintf("%s:/syft", getSyftBinaryLocationByOS(t, "linux")),
+			fmt.Sprintf("%s:/syft", getSyftBinaryLocationByOS(t, "linux", runtime.GOARCH)),
 			image,
 			"/syft",
 		},
@@ -219,25 +188,6 @@ func runCommandObj(t testing.TB, cmd *exec.Cmd, env map[string]string, expectErr
 	return stdout, stderr
 }
 
-func runCosign(t testing.TB, env map[string]string, args ...string) (*exec.Cmd, string, string) {
-	cmd := getCommand(t, ".tmp/cosign", args...)
-	if env == nil {
-		env = make(map[string]string)
-	}
-
-	stdout, stderr, err := runCommand(cmd, env)
-
-	if err != nil {
-		t.Errorf("error running cosign: %+v", err)
-	}
-
-	return cmd, stdout, stderr
-}
-
-func getCommand(t testing.TB, location string, args ...string) *exec.Cmd {
-	return exec.Command(filepath.Join(repoRoot(t), location), args...)
-}
-
 func runCommand(cmd *exec.Cmd, env map[string]string) (string, string, error) {
 	if env != nil {
 		cmd.Env = append(os.Environ(), envMapToSlice(env)...)
@@ -267,28 +217,118 @@ func getSyftCommand(t testing.TB, args ...string) *exec.Cmd {
 }
 
 func getSyftBinaryLocation(t testing.TB) string {
-	if os.Getenv("SYFT_BINARY_LOCATION") != "" {
-		// SYFT_BINARY_LOCATION is the absolute path to the snapshot binary
-		return os.Getenv("SYFT_BINARY_LOCATION")
-	}
-	return getSyftBinaryLocationByOS(t, runtime.GOOS)
+	return getSyftBinaryLocationByOS(t, runtime.GOOS, runtime.GOARCH)
 }
 
-func getSyftBinaryLocationByOS(t testing.TB, goOS string) string {
+func getSyftBinaryLocationByOS(t testing.TB, goOS, goArch string) string {
 	// note: for amd64 we need to update the snapshot location with the v1 suffix
 	// see : https://goreleaser.com/customization/build/#why-is-there-a-_v1-suffix-on-amd64-builds
-	archPath := runtime.GOARCH
-	if runtime.GOARCH == "amd64" {
+	archPath := goArch
+	if goArch == "amd64" {
 		archPath = fmt.Sprintf("%s_v1", archPath)
 	}
+
+	bin := ""
 	// note: there is a subtle - vs _ difference between these versions
 	switch goOS {
-	case "darwin", "linux":
-		return path.Join(repoRoot(t), fmt.Sprintf("snapshot/%s-build_%s_%s/syft", goOS, goOS, archPath))
+	case "windows", "darwin", "linux":
+		bin = path.Join(repoRoot(t), fmt.Sprintf("snapshot/%s-build_%s_%s/syft", goOS, goOS, archPath))
 	default:
-		t.Fatalf("unsupported OS: %s", runtime.GOOS)
+		t.Fatalf("unsupported OS: %s", goOS)
+		return ""
 	}
-	return ""
+
+	envName := strings.ToUpper(fmt.Sprintf("SYFT_BINARY_LOCATION_%s_%s", goOS, goArch))
+	if os.Getenv(envName) != bin {
+		buildSyft(t, bin, goOS, goArch)
+		// regardless if we have a successful build, don't attempt to keep building
+		_ = os.Setenv(envName, bin)
+	}
+
+	return bin
+}
+
+func buildSyft(t testing.TB, outfile, goOS, goArch string) {
+	dir := repoRoot(t)
+
+	start := time.Now()
+
+	stdout, stderr, err := buildSyftWithGo(dir, outfile, goOS, goArch)
+
+	took := time.Now().Sub(start).Round(time.Millisecond)
+	if err == nil {
+		if len(stderr) == 0 {
+			t.Logf("binary is up to date: %s in %v", outfile, took)
+		} else {
+			t.Logf("built binary: %s in %v\naffected paths:\n%s", outfile, took, stderr)
+		}
+	} else {
+		t.Fatalf("unable to build binary: %s -- %v\nSTDOUT:\n%s\nSTDERR:\n%s", outfile, err, stdout, stderr)
+	}
+}
+
+func buildSyftWithGo(dir, outfile, goOS, goArch string) (string, string, error) {
+	d := yaml.NewDecoder(strings.NewReader(goreleaserYamlContents(dir)))
+	type releaser struct {
+		Builds []struct {
+			ID      string `yaml:"id"`
+			LDFlags string `yaml:"ldflags"`
+		} `yaml:"builds"`
+	}
+	r := releaser{}
+	_ = d.Decode(&r)
+	ldflags := ""
+	for _, b := range r.Builds {
+		if b.ID == "linux-build" {
+			ldflags = executeTemplate(b.LDFlags, struct {
+				Version string
+				Commit  string
+				Date    string
+				Summary string
+			}{
+				Version: "SNAPSHOT", // should contain "SNAPSHOT" so update checks are skipped
+				Commit:  "COMMIT",
+				Date:    "DATE",
+				Summary: "SUMMARY",
+			})
+			break
+		}
+	}
+
+	cmd := exec.Command("go",
+		"build",
+		"-v",
+		"-o", outfile,
+		"-trimpath",
+		"-ldflags", ldflags,
+		"./cmd/syft",
+	)
+
+	cmd.Dir = dir
+	stdout, stderr, err := runCommand(cmd, map[string]string{
+		"CGO_ENABLED": "0",
+		"GOOS":        goOS,
+		"GOARCH":      goArch,
+	})
+	return stdout, stderr, err
+}
+
+func goreleaserYamlContents(dir string) string {
+	b, _ := os.ReadFile(path.Join(dir, ".goreleaser.yaml"))
+	return string(b)
+}
+
+func executeTemplate(tpl string, data any) string {
+	t, err := template.New("tpl").Parse(tpl)
+	if err != nil {
+		return fmt.Sprintf("ERROR: %v", err)
+	}
+	out := &bytes.Buffer{}
+	err = t.Execute(out, data)
+	if err != nil {
+		return fmt.Sprintf("ERROR: %v", err)
+	}
+	return out.String()
 }
 
 func repoRoot(t testing.TB) string {
@@ -303,40 +343,3 @@ func repoRoot(t testing.TB) string {
 	}
 	return absRepoRoot
 }
-
-func testRetryIntervals(done <-chan struct{}) <-chan time.Duration {
-	return exponentialBackoffDurations(250*time.Millisecond, 4*time.Second, 2, done)
-}
-
-func exponentialBackoffDurations(minDuration, maxDuration time.Duration, step float64, done <-chan struct{}) <-chan time.Duration {
-	sleepDurations := make(chan time.Duration)
-	go func() {
-		defer close(sleepDurations)
-	retryLoop:
-		for attempt := 0; ; attempt++ {
-			duration := exponentialBackoffDuration(minDuration, maxDuration, step, attempt)
-
-			select {
-			case sleepDurations <- duration:
-				break
-			case <-done:
-				break retryLoop
-			}
-
-			if duration == maxDuration {
-				break
-			}
-		}
-	}()
-	return sleepDurations
-}
-
-func exponentialBackoffDuration(minDuration, maxDuration time.Duration, step float64, attempt int) time.Duration {
-	duration := time.Duration(float64(minDuration) * math.Pow(step, float64(attempt)))
-	if duration < minDuration {
-		return minDuration
-	} else if duration > maxDuration {
-		return maxDuration
-	}
-	return duration
-}