fix(make): explicit integration test timeout + single race switch (#5151)

with a cold fixture cache the integration suite builds and saves 18 docker
images across 36 sequential tests, which walks past `go test`'s default 10m
timeout and takes the fixture cache rebuild down with it. that suite now runs
`go test` directly with `-timeout=30m` (gotest.Tasks() has no timeout option),
plus `-count=1` since the built fixtures are the side effect we're actually
after and a test cache hit would skip producing them.

also adds `RACE` as one switch for the race detector across every suite:

- `RACE=false make test` drops `-race` from unit + integration and skips the
  race smoke, worth doing on a cache rebuild where the wall clock is all
  docker builds anyway
- `RACE=true` forces it on locally
- unset behaves as before: on in CI, off locally and on windows

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

.make/main.go

@@ -3,8 +3,10 @@ package main
 import (
 	"path/filepath"
 	"runtime"
+	"strconv"
 
 	. "github.com/anchore/go-make"
+	"github.com/anchore/go-make/config"
 	"github.com/anchore/go-make/file"
 	"github.com/anchore/go-make/lang"
 	"github.com/anchore/go-make/run"
@@ -27,22 +29,43 @@ func main() {
 			gotest.Name("unit"),
 			gotest.ExcludeGlob("**/test/**"),
 			gotest.CoverageThreshold(62),
+			race(),
 		),
 
-		// integration tests: native go-make Task. The race-detector smoke against a
+		// integration tests: run `go test` directly instead of via gotest.Tasks(), which
-		// real image stays bundled here (RunsOn integration) so `make integration`
+		// has no way to set a timeout. The suite is a single package of ~36 sequential
-		// behaves like the Taskfile version did.
+		// tests over 18 docker fixture images, so with a cold fixture cache (every image
-		gotest.Tasks(
+		// built + saved inline, under -race) it runs well past `go test`'s default 10m
-			gotest.Name("integration"),
+		// timeout -- which is exactly the case when regenerating the fixture cache from
-			gotest.IncludeGlob("./cmd/syft/internal/test/integration/..."),
+		// scratch. -count=1 keeps the go test cache from short-circuiting a run whose
-			gotest.Verbose(),
+		// side effect (the built fixtures) is the thing we're after.
-			gotest.NoCoverage(),
+		//
-		),
+		// The race-detector smoke against a real image stays bundled here (RunsOn
+		// integration) so `make integration` behaves like the Taskfile version did.
+		Task{
+			Name:        "integration",
+			Description: "run integration tests",
+			RunsOn:      lang.List("test"),
+			Run: func() {
+				raceFlag := ""
+				if raceEnabled() {
+					raceFlag = " -race"
+				}
+				Run(
+					"go test -count=1 -timeout=30m -v"+raceFlag+" ./cmd/syft/internal/test/integration/...",
+					run.Env("GODEBUG", "dontfreezetheworld=1"),
+				)
+			},
+		},
 		Task{
 			Name:        "integration:race-smoke",
 			Description: "exercise the CLI with the race detector",
 			RunsOn:      lang.List("integration"),
 			Run: func() {
+				if !raceEnabled() {
+					Log("race detector disabled (RACE=false); skipping race smoke")
+					return
+				}
 				Run("go run -race cmd/syft/main.go anchore/test_images:grype-quality-dotnet-69f15d2")
 			},
 		},
@@ -132,6 +155,26 @@ func main() {
 	)
 }
 
+// raceEnabled is the single switch for the race detector across every test suite.
+// Unset it and we keep go-make's behavior (on in CI, off locally and on windows);
+// set RACE=false to turn it off everywhere -- worth doing when rebuilding the test
+// fixture cache from scratch, where every suite is dominated by building docker
+// fixtures and the race detector only adds wall clock. RACE=true forces it on.
+func raceEnabled() bool {
+	if enabled, err := strconv.ParseBool(config.Env("RACE", "")); err == nil {
+		return enabled
+	}
+	return config.CI && !config.Windows
+}
+
+// race applies raceEnabled() to a gotest suite. gotest exposes no functional option
+// for the race detector, but gotest.Config.Race is exported.
+func race() gotest.Option {
+	return func(c *gotest.Config) {
+		c.Race = raceEnabled()
+	}
+}
+
 // snapshotBinPath replicates the SNAPSHOT_BIN computation from the prior Taskfile:
 // <repoRoot>/snapshot/<os>-build_<os>_<arch>/syft, where arch maps amd64->amd64_v1
 // and arm64->arm64_v8.0 to match goreleaser's per-target output directory naming.

.make/main_test.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/anchore/go-make/config"
+	"github.com/anchore/go-make/tasks/gotest"
+)
+
+func TestRaceEnabled(t *testing.T) {
+	tests := []struct {
+		name string
+		env  string // "" means RACE is unset
+		ci   bool
+		want bool
+	}{
+		{name: "default in CI", ci: true, want: true},
+		{name: "default locally", ci: false, want: false},
+		{name: "explicitly off in CI", env: "false", ci: true, want: false},
+		{name: "explicitly on locally", env: "true", ci: false, want: true},
+		{name: "unparseable falls back to default", env: "yes-please", ci: true, want: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.env != "" {
+				t.Setenv("RACE", tt.env)
+			}
+			orig := config.CI
+			config.CI = tt.ci
+			t.Cleanup(func() { config.CI = orig })
+
+			if got := raceEnabled(); got != tt.want {
+				t.Errorf("raceEnabled() = %v, want %v", got, tt.want)
+			}
+
+			// the gotest suites get the same answer through the functional option
+			var cfg gotest.Config
+			race()(&cfg)
+			if cfg.Race != tt.want {
+				t.Errorf("race() set Race = %v, want %v", cfg.Race, tt.want)
+			}
+		})
+	}
+}

Taskfile.yaml

@@ -125,6 +125,7 @@ tasks:
     desc: Run tests for pipeline utils
     cmds:
       - cmd: .github/scripts/labeler_test.py
+      - cmd: go test -C .make ./...
 
   snapshot-smoke-test:
     desc: Run a smoke test on the snapshot builds + docker images