diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index 9ddb8c965..861d7634f 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -90,48 +90,68 @@ jobs: run: make integration - Build-Snapshot-Artifacts: - name: "Build snapshot artifacts" + Cli-Linux: + # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline + name: "CLI tests (Linux)" runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - name: Bootstrap environment uses: ./.github/actions/bootstrap - with: - # why have another build cache key? We don't want unit/integration/etc test build caches to replace - # the snapshot build cache, which includes builds for all OSs and architectures. As long as this key is - # unique from the build-cache-key-prefix in other CI jobs, we should be fine. - # - # note: ideally this value should match what is used in release (just to help with build times). - build-cache-key-prefix: "snapshot" - bootstrap-apt-packages: "" - - name: Build snapshot artifacts - run: make snapshot - - # why not use actions/upload-artifact? It is very slow (3 minutes to upload ~600MB of data, vs 10 seconds with this approach). - # see https://github.com/actions/upload-artifact/issues/199 for more info - - name: Upload snapshot artifacts - uses: actions/cache/save@v3 + - name: Restore CLI test-fixture cache + uses: actions/cache@v3 with: - path: snapshot - key: snapshot-build-${{ github.run_id }} + path: ${{ github.workspace }}/test/cli/test-fixtures/cache + key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }} + + - name: Run CLI Tests (Linux) + run: make cli + + +# Build-Snapshot-Artifacts: +# name: "Build snapshot artifacts" +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@v3 +# +# - name: Bootstrap environment +# uses: ./.github/actions/bootstrap +# with: +# # why have another build cache key? We don't want unit/integration/etc test build caches to replace +# # the snapshot build cache, which includes builds for all OSs and architectures. As long as this key is +# # unique from the build-cache-key-prefix in other CI jobs, we should be fine. +# # +# # note: ideally this value should match what is used in release (just to help with build times). +# build-cache-key-prefix: "snapshot" +# bootstrap-apt-packages: "" +# +# - name: Build snapshot artifacts +# run: make snapshot +# +# # why not use actions/upload-artifact? It is very slow (3 minutes to upload ~600MB of data, vs 10 seconds with this approach). +# # see https://github.com/actions/upload-artifact/issues/199 for more info +# - name: Upload snapshot artifacts +# uses: actions/cache/save@v3 +# with: +# path: snapshot +# key: snapshot-build-${{ github.run_id }} Acceptance-Linux: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "Acceptance tests (Linux)" - needs: [Build-Snapshot-Artifacts] +# needs: [Build-Snapshot-Artifacts] runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - - name: Download snapshot build - uses: actions/cache/restore@v3 - with: - path: snapshot - key: snapshot-build-${{ github.run_id }} +# - name: Download snapshot build +# uses: actions/cache/restore@v3 +# with: +# path: snapshot +# key: snapshot-build-${{ github.run_id }} - name: Run comparison tests (Linux) run: make compare-linux @@ -158,16 +178,16 @@ jobs: Acceptance-Mac: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "Acceptance tests (Mac)" - needs: [Build-Snapshot-Artifacts] +# needs: [Build-Snapshot-Artifacts] runs-on: macos-latest steps: - uses: actions/checkout@v3 - - name: Download snapshot build - uses: actions/cache/restore@v3 - with: - path: snapshot - key: snapshot-build-${{ github.run_id }} +# - name: Download snapshot build +# uses: actions/cache/restore@v3 +# with: +# path: snapshot +# key: snapshot-build-${{ github.run_id }} - name: Restore docker image cache for compare testing id: mac-compare-testing-cache @@ -181,30 +201,3 @@ jobs: - name: Run install.sh tests (Mac) run: make install-test-ci-mac - - - Cli-Linux: - # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline - name: "CLI tests (Linux)" - needs: [Build-Snapshot-Artifacts] - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - - name: Bootstrap environment - uses: ./.github/actions/bootstrap - - - name: Restore CLI test-fixture cache - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/test/cli/test-fixtures/cache - key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }} - - - name: Download snapshot build - uses: actions/cache/restore@v3 - with: - path: snapshot - key: snapshot-build-${{ github.run_id }} - - - name: Run CLI Tests (Linux) - run: make cli diff --git a/Makefile b/Makefile index fd7debaa0..46b2c9b1d 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,6 @@ DIST_DIR := ./dist SNAPSHOT_DIR := ./snapshot CHANGELOG := CHANGELOG.md OS := $(shell uname | tr '[:upper:]' '[:lower:]') -SNAPSHOT_BIN := $(realpath $(shell pwd)/$(SNAPSHOT_DIR)/$(OS)-build_$(OS)_amd64_v1/$(BIN)) ifndef VERSION $(error VERSION is not set) @@ -160,11 +159,8 @@ validate-cyclonedx-schema: cd schema/cyclonedx && make .PHONY: cli -cli: $(SNAPSHOT_DIR) ## Run CLI tests - chmod 755 "$(SNAPSHOT_BIN)" - $(SNAPSHOT_BIN) version - SYFT_BINARY_LOCATION='$(SNAPSHOT_BIN)' \ - go test -count=1 -timeout=15m -v ./test/cli +cli: ## Run CLI tests + go test -count=1 -timeout=15m -v ./test/cli ## Benchmark test targets ################################# @@ -321,15 +317,17 @@ generate-cpe-dictionary-index: ## Build the CPE index based off of the latest a build: CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $@ ./cmd/syft -$(SNAPSHOT_DIR): ## Build snapshot release binaries and packages - $(call title,Building snapshot artifacts) +.PHONY: $(SNAPSHOT_DIR) +$(SNAPSHOT_DIR): ## Build snapshot release for the current platform + $(call title,Building current platform snapshot artifact) # create a config with the dist dir overridden echo "dist: $(SNAPSHOT_DIR)" > $(TEMP_DIR)/goreleaser.yaml cat .goreleaser.yaml >> $(TEMP_DIR)/goreleaser.yaml - # build release snapshots - $(SNAPSHOT_CMD) --config $(TEMP_DIR)/goreleaser.yaml + # build release snapshot + $(SNAPSHOT_CMD) --single-target --config $(TEMP_DIR)/goreleaser.yaml + .PHONY: changelog changelog: clean-changelog ## Generate and show the changelog for the current unreleased version diff --git a/test/cli/dir_root_scan_regression_test.go b/test/cli/dir_root_scan_regression_test.go index 6fbea309a..212e19a92 100644 --- a/test/cli/dir_root_scan_regression_test.go +++ b/test/cli/dir_root_scan_regression_test.go @@ -12,6 +12,9 @@ func TestDirectoryScanCompletesWithinTimeout(t *testing.T) { // we want to pull the image ahead of the test as to not affect the timeout value pullDockerImage(t, image) + // run once in case we need to rebuild syft binary, so it doesn't affect the timeout value + runSyftInDocker(t, nil, image, "--help") + var cmd *exec.Cmd var stdout, stderr string done := make(chan struct{}) diff --git a/test/cli/utils_test.go b/test/cli/utils_test.go index ed560bd88..961a503f4 100644 --- a/test/cli/utils_test.go +++ b/test/cli/utils_test.go @@ -105,7 +105,7 @@ func runSyftInDocker(t testing.TB, env map[string]string, image string, args ... "-e", "SYFT_CHECK_FOR_APP_UPDATE=false", "-v", - fmt.Sprintf("%s:/syft", getSyftBinaryLocationByOS(t, "linux")), + fmt.Sprintf("%s:/syft", getSyftBinaryLocationByOS(t, "linux", runtime.GOARCH)), image, "/syft", }, @@ -270,37 +270,43 @@ func getSyftCommand(t testing.TB, args ...string) *exec.Cmd { } func getSyftBinaryLocation(t testing.TB) string { - if os.Getenv("SYFT_BINARY_LOCATION") != "" { - // SYFT_BINARY_LOCATION is the absolute path to the snapshot binary - return os.Getenv("SYFT_BINARY_LOCATION") - } - bin := getSyftBinaryLocationByOS(t, runtime.GOOS) + return getSyftBinaryLocationByOS(t, runtime.GOOS, runtime.GOARCH) +} - // only run on valid bin target, when not running in CI - if bin != "" && os.Getenv("CI") == "" { - buildSyft(t, bin) +func getSyftBinaryLocationByOS(t testing.TB, goOS, goArch string) string { + // note: for amd64 we need to update the snapshot location with the v1 suffix + // see : https://goreleaser.com/customization/build/#why-is-there-a-_v1-suffix-on-amd64-builds + archPath := goArch + if goArch == "amd64" { + archPath = fmt.Sprintf("%s_v1", archPath) + } + + bin := "" + // note: there is a subtle - vs _ difference between these versions + switch goOS { + case "windows", "darwin", "linux": + bin = path.Join(repoRoot(t), fmt.Sprintf("snapshot/%s-build_%s_%s/syft", goOS, goOS, archPath)) + default: + t.Fatalf("unsupported OS: %s", goOS) + return "" + } + + envName := strings.ToUpper(fmt.Sprintf("SYFT_BINARY_LOCATION_%s_%s", goOS, goArch)) + if os.Getenv(envName) != bin { + buildSyft(t, bin, goOS, goArch) // regardless if we have a successful build, don't attempt to keep building - _ = os.Setenv("SYFT_BINARY_LOCATION", bin) + _ = os.Setenv(envName, bin) } return bin } -func buildSyft(t testing.TB, outfile string) { +func buildSyft(t testing.TB, outfile, goOS, goArch string) { dir := repoRoot(t) start := time.Now() - var stdout, stderr string - var err error - switch "go" { - case "go": - stdout, stderr, err = buildSyftWithGo(dir, outfile) - case "goreleaser": - stdout, stderr, err = buildSyftWithGoreleaser(dir) - case "make": - stdout, stderr, err = buildSyftWithMake(dir) - } + stdout, stderr, err := buildSyftWithGo(dir, outfile, goOS, goArch) took := time.Now().Sub(start).Round(time.Millisecond) if err == nil { @@ -310,16 +316,11 @@ func buildSyft(t testing.TB, outfile string) { t.Logf("built binary: %s in %v\naffected paths:\n%s", outfile, took, stderr) } } else { - t.Logf("unable to build binary: %s %v\nSTDOUT:\n%s\nSTDERR:\n%s", outfile, err, stdout, stderr) + t.Fatalf("unable to build binary: %s -- %v\nSTDOUT:\n%s\nSTDERR:\n%s", outfile, err, stdout, stderr) } } -func goreleaserYamlContents(dir string) string { - b, _ := os.ReadFile(path.Join(dir, ".goreleaser.yaml")) - return string(b) -} - -func buildSyftWithGo(dir string, outfile string) (string, string, error) { +func buildSyftWithGo(dir, outfile, goOS, goArch string) (string, string, error) { d := yaml.NewDecoder(strings.NewReader(goreleaserYamlContents(dir))) type releaser struct { Builds []struct { @@ -359,39 +360,15 @@ func buildSyftWithGo(dir string, outfile string) (string, string, error) { cmd.Dir = dir stdout, stderr, err := runCommand(cmd, map[string]string{ "CGO_ENABLED": "0", + "GOOS": goOS, + "GOARCH": goArch, }) return stdout, stderr, err } -func buildSyftWithMake(dir string) (string, string, error) { - cmd := exec.Command("make", "snapshot") - cmd.Dir = dir - stdout, stderr, err := runCommand(cmd, map[string]string{}) - if strings.Contains(stderr, "error=docker build failed") { - err = nil - } - return stdout, stderr, err -} - -func buildSyftWithGoreleaser(dir string) (string, string, error) { - tmpDir := path.Join(dir, ".tmp") - - goreleaserYaml := goreleaserYamlContents(dir) - - // # create a config with the dist dir overridden - tmpGoreleaserYamlFile := path.Join(tmpDir, "goreleaser.yaml") - _ = os.WriteFile(tmpGoreleaserYamlFile, []byte("dist: snapshot\n"+goreleaserYaml), os.ModePerm) - - cmd := exec.Command(path.Join(tmpDir, "goreleaser"), - "build", - "--snapshot", - "--single-target", - "--clean", - "--config", tmpGoreleaserYamlFile, - ) - cmd.Dir = dir - stdout, stderr, err := runCommand(cmd, map[string]string{}) - return stdout, stderr, err +func goreleaserYamlContents(dir string) string { + b, _ := os.ReadFile(path.Join(dir, ".goreleaser.yaml")) + return string(b) } func executeTemplate(tpl string, data any) string { @@ -407,23 +384,6 @@ func executeTemplate(tpl string, data any) string { return out.String() } -func getSyftBinaryLocationByOS(t testing.TB, goOS string) string { - // note: for amd64 we need to update the snapshot location with the v1 suffix - // see : https://goreleaser.com/customization/build/#why-is-there-a-_v1-suffix-on-amd64-builds - archPath := runtime.GOARCH - if runtime.GOARCH == "amd64" { - archPath = fmt.Sprintf("%s_v1", archPath) - } - // note: there is a subtle - vs _ difference between these versions - switch goOS { - case "darwin", "linux": - return path.Join(repoRoot(t), fmt.Sprintf("snapshot/%s-build_%s_%s/syft", goOS, goOS, archPath)) - default: - t.Fatalf("unsupported OS: %s", runtime.GOOS) - } - return "" -} - func repoRoot(t testing.TB) string { t.Helper() root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()