diff --git a/test/cli/json_schema_test.go b/test/cli/json_schema_test.go new file mode 100644 index 000000000..669123639 --- /dev/null +++ b/test/cli/json_schema_test.go @@ -0,0 +1,91 @@ +package cli + +import ( + "fmt" + "path" + "strings" + "testing" + + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/internal" + "github.com/xeipuuv/gojsonschema" +) + +// this is the path to the json schema directory relative to the root of the repo +const jsonSchemaPath = "schema/json" + +func TestJSONSchema(t *testing.T) { + + imageFixture := func(t *testing.T) string { + fixtureImageName := "image-pkg-coverage" + imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) + tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) + return "docker-archive:" + tarPath + } + + tests := []struct { + name string + subcommand string + args []string + fixture func(*testing.T) string + }{ + { + name: "packages:image:docker-archive:pkg-coverage", + subcommand: "packages", + args: []string{"-o", "json"}, + fixture: imageFixture, + }, + { + name: "power-user:image:docker-archive:pkg-coverage", + subcommand: "power-user", + fixture: imageFixture, + }, + { + name: "packages:dir:pkg-coverage", + subcommand: "packages", + args: []string{"-o", "json"}, + fixture: func(t *testing.T) string { + return "dir:test-fixtures/image-pkg-coverage" + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixtureRef := test.fixture(t) + args := []string{ + test.subcommand, fixtureRef, "-q", + } + for _, a := range test.args { + args = append(args, a) + } + + _, stdout, _ := runSyftCommand(t, nil, args...) + + if len(strings.Trim(stdout, "\n ")) < 100 { + t.Fatalf("bad syft output: %q", stdout) + } + + validateAgainstV1Schema(t, stdout) + }) + } +} + +func validateAgainstV1Schema(t testing.TB, json string) { + fullSchemaPath := path.Join(repoRoot(t), jsonSchemaPath, fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion)) + schemaLoader := gojsonschema.NewReferenceLoader(fmt.Sprintf("file://%s", fullSchemaPath)) + documentLoader := gojsonschema.NewStringLoader(json) + + result, err := gojsonschema.Validate(schemaLoader, documentLoader) + if err != nil { + t.Fatal("unable to validate json schema:", err.Error()) + } + + if !result.Valid() { + t.Errorf("failed json schema validation:") + t.Errorf("JSON:\n%s\n", json) + for _, desc := range result.Errors() { + t.Errorf(" - %s\n", desc) + } + } +} diff --git a/test/cli/packages_cmd_test.go b/test/cli/packages_cmd_test.go new file mode 100644 index 000000000..f5732330e --- /dev/null +++ b/test/cli/packages_cmd_test.go @@ -0,0 +1,139 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/anchore/syft/syft/source" +) + +func TestPackagesCmdFlags(t *testing.T) { + request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage") + + tests := []struct { + name string + args []string + env map[string]string + assertions []traitAssertion + }{ + { + name: "json-output-flag", + args: []string{"packages", "-o", "json", request}, + assertions: []traitAssertion{ + assertJsonReport, + assertSource(source.SquashedScope), + assertSuccessfulReturnCode, + }, + }, + { + name: "output-env-binding", + env: map[string]string{ + "SYFT_OUTPUT": "json", + }, + args: []string{"packages", request}, + assertions: []traitAssertion{ + assertJsonReport, + assertSuccessfulReturnCode, + }, + }, + { + name: "table-output-flag", + args: []string{"packages", "-o", "table", request}, + assertions: []traitAssertion{ + assertTableReport, + assertSuccessfulReturnCode, + }, + }, + { + name: "default-output-flag", + args: []string{"packages", request}, + assertions: []traitAssertion{ + assertTableReport, + assertSuccessfulReturnCode, + }, + }, + { + name: "squashed-scope-flag", + args: []string{"packages", "-o", "json", "-s", "squashed", request}, + assertions: []traitAssertion{ + assertSource(source.SquashedScope), + assertSuccessfulReturnCode, + }, + }, + { + name: "all-layers-scope-flag", + args: []string{"packages", "-o", "json", "-s", "all-layers", request}, + assertions: []traitAssertion{ + assertSource(source.AllLayersScope), + assertSuccessfulReturnCode, + }, + }, + { + name: "packages-scope-env-binding", + env: map[string]string{ + "SYFT_PACKAGES_SCOPE": "all-layers", + }, + args: []string{"packages", "-o", "json", request}, + assertions: []traitAssertion{ + assertSource(source.AllLayersScope), + assertSuccessfulReturnCode, + }, + }, + { + name: "attempt-upload-on-cli-switches", + args: []string{"packages", "-vv", "-H", "localhost:8080", "-u", "the-username", "-d", "test-fixtures/image-pkg-coverage/Dockerfile", "--overwrite-existing-image", request}, + env: map[string]string{ + "SYFT_ANCHORE_PATH": "path/to/api", + "SYFT_ANCHORE_PASSWORD": "the-password", + }, + assertions: []traitAssertion{ + // we cannot easily assert a successful upload behavior, so instead we are doing the next best thing + // and asserting that the parsed configuration has the expected values and we see log entries + // indicating an upload attempt. + assertNotInOutput("the-username"), + assertNotInOutput("the-password"), + assertInOutput("uploading results to localhost:8080"), + assertInOutput(`dockerfile: test-fixtures/image-pkg-coverage/Dockerfile`), + assertInOutput(`overwrite-existing-image: true`), + assertInOutput(`path: path/to/api`), + assertInOutput(`host: localhost:8080`), + assertFailingReturnCode, // upload can't go anywhere, so if this passes that would be surprising + }, + }, + { + name: "dockerfile-without-upload-is-invalid", + args: []string{"packages", "-vv", "-d", "test-fixtures/image-pkg-coverage/Dockerfile", request}, + assertions: []traitAssertion{ + + assertNotInOutput("uploading results to localhost:8080"), + assertInOutput("invalid application config: cannot provide dockerfile option without enabling upload"), + assertFailingReturnCode, + }, + }, + { + name: "attempt-upload-with-env-host-set", + args: []string{"packages", "-vv", request}, + env: map[string]string{ + "SYFT_ANCHORE_HOST": "localhost:8080", + }, + assertions: []traitAssertion{ + assertInOutput("uploading results to localhost:8080"), + assertFailingReturnCode, // upload can't go anywhere, so if this passes that would be surprising + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd, stdout, stderr := runSyftCommand(t, test.env, test.args...) + for _, traitFn := range test.assertions { + traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode()) + } + if t.Failed() { + t.Log("STDOUT:\n", stdout) + t.Log("STDERR:\n", stderr) + t.Log("COMMAND:", strings.Join(cmd.Args, " ")) + } + }) + } +} diff --git a/test/cli/power_user_cmd_test.go b/test/cli/power_user_cmd_test.go new file mode 100644 index 000000000..5bc6aa982 --- /dev/null +++ b/test/cli/power_user_cmd_test.go @@ -0,0 +1,49 @@ +package cli + +import ( + "strings" + "testing" +) + +func TestPowerUserCmdFlags(t *testing.T) { + request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage") + + tests := []struct { + name string + args []string + env map[string]string + assertions []traitAssertion + }{ + { + name: "json-output-flag-fails", + args: []string{"power-user", "-o", "json", request}, + assertions: []traitAssertion{ + assertFailingReturnCode, + }, + }, + { + name: "default-results", + args: []string{"power-user", request}, + assertions: []traitAssertion{ + assertInOutput(`"type": "regularFile"`), // proof of file-metadata data + assertInOutput(`"algorithm": "sha256"`), // proof of file-metadata default digest algorithm of sha256 + assertInOutput(`"metadataType": "ApkMetadata"`), // proof of package artifacts data + assertSuccessfulReturnCode, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd, stdout, stderr := runSyftCommand(t, test.env, test.args...) + for _, traitFn := range test.assertions { + traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode()) + } + if t.Failed() { + t.Log("STDOUT:\n", stdout) + t.Log("STDERR:\n", stderr) + t.Log("COMMAND:", strings.Join(cmd.Args, " ")) + } + }) + } +} diff --git a/test/cli/root_cmd_test.go b/test/cli/root_cmd_test.go new file mode 100644 index 000000000..9c58ec299 --- /dev/null +++ b/test/cli/root_cmd_test.go @@ -0,0 +1,91 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/sergi/go-diff/diffmatchpatch" +) + +func TestRootCmdAliasesToPackagesSubcommand(t *testing.T) { + request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage") + deprecationWarning := "The root command is deprecated" + + _, aliasStdout, aliasStderr := runSyftCommand(t, nil, request) + + if !strings.Contains(aliasStderr, deprecationWarning) { + t.Errorf("missing root-packages alias deprecation warning") + } + + _, pkgsStdout, pkgsStderr := runSyftCommand(t, nil, "packages", request) + + if strings.Contains(pkgsStderr, deprecationWarning) { + t.Errorf("packages command should not have deprecation warning") + } + + if aliasStdout != pkgsStdout { + t.Errorf("packages and root command should have same report output but do not!") + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(aliasStdout, pkgsStdout, true) + t.Error(dmp.DiffPrettyText(diffs)) + } +} + +func TestPersistentFlags(t *testing.T) { + request := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage") + + tests := []struct { + name string + args []string + env map[string]string + assertions []traitAssertion + }{ + { + name: "quiet-flag", + // note: the root command will always show the deprecation warning, so the packages command is used instead + args: []string{"packages", "-q", request}, + assertions: []traitAssertion{ + func(tb testing.TB, stdout, stderr string, rc int) { + // ensure there is no status + if len(stderr) != 0 { + tb.Errorf("should have seen no stderr output, got %d bytes", len(stderr)) + } + // ensure there is still a report + if len(stdout) == 0 { + tb.Errorf("should have seen a report on stdout, got nothing") + } + }, + }, + }, + { + name: "info-log-flag", + args: []string{"-v", request}, + assertions: []traitAssertion{ + assertLoggingLevel("info"), + assertSuccessfulReturnCode, + }, + }, + { + name: "debug-log-flag", + args: []string{"-vv", request}, + assertions: []traitAssertion{ + assertLoggingLevel("debug"), + assertSuccessfulReturnCode, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd, stdout, stderr := runSyftCommand(t, test.env, test.args...) + for _, traitFn := range test.assertions { + traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode()) + } + if t.Failed() { + t.Log("STDOUT:\n", stdout) + t.Log("STDERR:\n", stderr) + t.Log("COMMAND:", strings.Join(cmd.Args, " ")) + } + }) + } +} diff --git a/test/cli/test-fixtures/image-pkg-coverage b/test/cli/test-fixtures/image-pkg-coverage new file mode 120000 index 000000000..155332274 --- /dev/null +++ b/test/cli/test-fixtures/image-pkg-coverage @@ -0,0 +1 @@ +../../integration/test-fixtures/image-pkg-coverage \ No newline at end of file diff --git a/test/cli/trait_assertions_test.go b/test/cli/trait_assertions_test.go new file mode 100644 index 000000000..1eb1b807e --- /dev/null +++ b/test/cli/trait_assertions_test.go @@ -0,0 +1,82 @@ +package cli + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "testing" + + "github.com/acarl005/stripansi" + "github.com/anchore/syft/syft/source" +) + +type traitAssertion func(tb testing.TB, stdout, stderr string, rc int) + +func assertJsonReport(tb testing.TB, stdout, _ string, _ int) { + var data interface{} + + if err := json.Unmarshal([]byte(stdout), &data); err != nil { + tb.Errorf("expected to find a JSON report, but was unmarshalable: %+v", err) + } +} + +func assertTableReport(tb testing.TB, stdout, _ string, _ int) { + if !strings.Contains(stdout, "NAME") || !strings.Contains(stdout, "VERSION") || !strings.Contains(stdout, "TYPE") { + tb.Errorf("expected to find a table report, but did not") + } +} + +func assertSource(scope source.Scope) traitAssertion { + return func(tb testing.TB, stdout, stderr string, rc int) { + // we can only verify source with the json report + assertJsonReport(tb, stdout, stderr, rc) + + if !strings.Contains(stdout, fmt.Sprintf(`"scope": "%s"`, scope.String())) { + tb.Errorf("JSON report did not indicate the %q scope", scope) + } + } +} + +func assertLoggingLevel(level string) traitAssertion { + // match examples: + // "[0000] INFO" + // "[0012] DEBUG" + logPattern := regexp.MustCompile(`(?m)^\[\d\d\d\d\]\s+` + strings.ToUpper(level)) + return func(tb testing.TB, _, stderr string, _ int) { + if !logPattern.MatchString(stripansi.Strip(stderr)) { + tb.Errorf("output did not indicate the %q logging level", level) + } + } +} + +func assertNotInOutput(data string) traitAssertion { + return func(tb testing.TB, stdout, stderr string, _ int) { + if strings.Contains(stripansi.Strip(stderr), data) { + tb.Errorf("data=%q was found in stderr, but should not have been there", data) + } + if strings.Contains(stripansi.Strip(stdout), data) { + tb.Errorf("data=%q was found in stdout, but should not have been there", data) + } + } +} + +func assertInOutput(data string) traitAssertion { + return func(tb testing.TB, stdout, stderr string, _ int) { + if !strings.Contains(stripansi.Strip(stderr), data) && !strings.Contains(stripansi.Strip(stdout), data) { + tb.Errorf("data=%q was NOT found in any output, but should have been there", data) + } + } +} + +func assertFailingReturnCode(tb testing.TB, _, _ string, rc int) { + if rc == 0 { + tb.Errorf("expected a failure but got rc=%d", rc) + } +} + +func assertSuccessfulReturnCode(tb testing.TB, _, _ string, rc int) { + if rc != 0 { + tb.Errorf("expected no failure but got rc=%d", rc) + } +} diff --git a/test/cli/utils_test.go b/test/cli/utils_test.go new file mode 100644 index 000000000..a2b6bf0c7 --- /dev/null +++ b/test/cli/utils_test.go @@ -0,0 +1,73 @@ +package cli + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/anchore/stereoscope/pkg/imagetest" +) + +func getFixtureImage(t testing.TB, fixtureImageName string) string { + imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) + return imagetest.GetFixtureImageTarPath(t, fixtureImageName) +} + +func runSyftCommand(t testing.TB, env map[string]string, args ...string) (*exec.Cmd, string, string) { + cmd := getSyftCommand(t, args...) + if env != nil { + var envList []string + for key, val := range env { + if key == "" { + continue + } + envList = append(envList, fmt.Sprintf("%s=%s", key, val)) + } + cmd.Env = envList + } + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + // ignore errors since this may be what the test expects + cmd.Run() + + return cmd, stdout.String(), stderr.String() +} + +func getSyftCommand(t testing.TB, args ...string) *exec.Cmd { + + var binaryLocation string + if os.Getenv("SYFT_BINARY_LOCATION") != "" { + // SYFT_BINARY_LOCATION is relative to the repository root. (e.g., "snapshot/syft-linux_amd64/syft") + // This value is transformed due to the CLI tests' need for a path relative to the test directory. + binaryLocation = path.Join(repoRoot(t), os.Getenv("SYFT_BINARY_LOCATION")) + } else { + os := runtime.GOOS + if os == "darwin" { + os = "macos_darwin" + } + + binaryLocation = path.Join(repoRoot(t), fmt.Sprintf("snapshot/syft-%s_%s/syft", os, runtime.GOARCH)) + } + return exec.Command(binaryLocation, args...) +} + +func repoRoot(t testing.TB) string { + t.Helper() + root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + t.Fatalf("unable to find repo root dir: %+v", err) + } + absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root))) + if err != nil { + t.Fatal("unable to get abs path to repo root:", err) + } + return absRepoRoot +}