From 03ee4fdf5e87907c5a49ae353c44682894bb411c Mon Sep 17 00:00:00 2001 From: Christopher Angelo Phillips <32073428+spiffcs@users.noreply.github.com> Date: Thu, 12 May 2022 12:56:04 -0400 Subject: [PATCH] add integration tests for validating CycloneDX output using cyclonedx-cli (#1000) --- .../cyclonedxhelpers/external_references.go | 19 ++- test/cli/cyclonedx_valid_test.go | 142 ++++++++++++++++++ test/cli/utils_test.go | 23 ++- 3 files changed, 179 insertions(+), 5 deletions(-) create mode 100644 test/cli/cyclonedx_valid_test.go diff --git a/internal/formats/common/cyclonedxhelpers/external_references.go b/internal/formats/common/cyclonedxhelpers/external_references.go index e94a60111..c7f952717 100644 --- a/internal/formats/common/cyclonedxhelpers/external_references.go +++ b/internal/formats/common/cyclonedxhelpers/external_references.go @@ -56,7 +56,7 @@ func encodeExternalReferences(p pkg.Package) *[]cyclonedx.ExternalReference { URL: "", Type: cyclonedx.ERTypeBuildMeta, Hashes: &[]cyclonedx.Hash{{ - Algorithm: cyclonedx.HashAlgorithm(digest.Algorithm), + Algorithm: toCycloneDXAlgorithm(digest.Algorithm), Value: digest.Value, }}, }) @@ -81,6 +81,21 @@ func encodeExternalReferences(p pkg.Package) *[]cyclonedx.ExternalReference { return nil } +// supported algorithm in cycloneDX as of 1.4 +// "MD5", "SHA-1", "SHA-256", "SHA-384", "SHA-512", +// "SHA3-256", "SHA3-384", "SHA3-512", "BLAKE2b-256", "BLAKE2b-384", "BLAKE2b-512", "BLAKE3" +// syft supported digests: cmd/syft/cli/eventloop/tasks.go +// MD5, SHA1, SHA256 +func toCycloneDXAlgorithm(algorithm string) cyclonedx.HashAlgorithm { + validMap := map[string]cyclonedx.HashAlgorithm{ + "sha1": cyclonedx.HashAlgorithm("SHA-1"), + "md5": cyclonedx.HashAlgorithm("MD5"), + "sha256": cyclonedx.HashAlgorithm("SHA-256"), + } + + return validMap[algorithm] +} + func decodeExternalReferences(c *cyclonedx.Component, metadata interface{}) { if c.ExternalReferences == nil { return @@ -101,7 +116,7 @@ func decodeExternalReferences(c *cyclonedx.Component, metadata interface{}) { if ref.Hashes != nil { for _, hash := range *ref.Hashes { digests = append(digests, syftFile.Digest{ - Algorithm: string(hash.Algorithm), + Algorithm: syftFile.CleanDigestAlgorithmName(string(hash.Algorithm)), Value: hash.Value, }) } diff --git a/test/cli/cyclonedx_valid_test.go b/test/cli/cyclonedx_valid_test.go new file mode 100644 index 000000000..f72a18a80 --- /dev/null +++ b/test/cli/cyclonedx_valid_test.go @@ -0,0 +1,142 @@ +package cli + +import ( + "os" + "strings" + "testing" + + "github.com/anchore/stereoscope/pkg/imagetest" +) + +// We have schema validation mechanims in schema/cyclonedx/ +// This test allows us to double check that validation against the cyclonedx-cli tool +func TestValidCycloneDX(t *testing.T) { + imageFixture := func(t *testing.T) string { + fixtureImageName := "image-pkg-coverage" + imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) + tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) + return "docker-archive:" + tarPath + } + + // TODO update image to exercise entire cyclonedx schema + tests := []struct { + name string + subcommand string + args []string + fixture func(*testing.T) string + assertions []traitAssertion + }{ + { + name: "validate cyclonedx output", + subcommand: "packages", + args: []string{"-o", "cyclonedx-json"}, + fixture: imageFixture, + assertions: []traitAssertion{ + assertSuccessfulReturnCode, + assertValidCycloneDX, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixtureRef := test.fixture(t) + args := []string{ + test.subcommand, fixtureRef, "-q", + } + for _, a := range test.args { + args = append(args, a) + } + + cmd, stdout, stderr := runSyft(t, nil, args...) + for _, traitFn := range test.assertions { + traitFn(t, stdout, stderr, cmd.ProcessState.ExitCode()) + } + if t.Failed() { + t.Log("STDOUT:\n", stdout) + t.Log("STDERR:\n", stderr) + t.Log("COMMAND:", strings.Join(cmd.Args, " ")) + } + + validateCycloneDXJSON(t, stdout) + }) + } +} + +func assertValidCycloneDX(tb testing.TB, stdout, stderr string, rc int) { + tb.Helper() + f, err := os.CreateTemp("", "tmpfile-") + if err != nil { + tb.Fatal(err) + } + + // close and remove the temporary file at the end of the program + defer f.Close() + defer os.Remove(f.Name()) + + data := []byte(stdout) + + if _, err := f.Write(data); err != nil { + tb.Fatal(err) + } + + args := []string{ + "validate", + "--input-format", + "json", + "--input-version", + "v1_4", + "--input-file", + "/sbom", + } + + cmd, stdout, stderr := runCycloneDXInDocker(tb, nil, "cyclonedx/cyclonedx-cli", f, args...) + if cmd.ProcessState.ExitCode() != 0 { + tb.Errorf("expected no validation failures for cyclonedx-cli but got rc=%d", rc) + } + + if tb.Failed() { + tb.Log("STDOUT:\n", stdout) + tb.Log("STDERR:\n", stderr) + tb.Log("COMMAND:", strings.Join(cmd.Args, " ")) + } +} + +// validate --input-format json --input-version v1_4 --input-file bom.json +func validateCycloneDXJSON(t *testing.T, stdout string) { + f, err := os.CreateTemp("", "tmpfile-") + if err != nil { + t.Fatal(err) + } + + // close and remove the temporary file at the end of the program + defer f.Close() + defer os.Remove(f.Name()) + + data := []byte(stdout) + + if _, err := f.Write(data); err != nil { + t.Fatal(err) + } + + args := []string{ + "validate", + "--input-format", + "json", + "--input-version", + "v1_4", + "--input-file", + "/sbom", + } + + cmd, stdout, stderr := runCycloneDXInDocker(t, nil, "cyclonedx/cyclonedx-cli", f, args...) + if strings.Contains(stdout, "BOM is not valid") { + t.Errorf("expected no validation failures for cyclonedx-cli but found invalid BOM") + } + + if t.Failed() { + t.Log("STDOUT:\n", stdout) + t.Log("STDERR:\n", stderr) + t.Log("COMMAND:", strings.Join(cmd.Args, " ")) + } +} diff --git a/test/cli/utils_test.go b/test/cli/utils_test.go index 7897f38c6..e985cc983 100644 --- a/test/cli/utils_test.go +++ b/test/cli/utils_test.go @@ -66,6 +66,23 @@ func pullDockerImage(t testing.TB, image string) { } } +// docker run -v $(pwd)/sbom:/sbom cyclonedx/cyclonedx-cli:latest validate --input-format json --input-version v1_4 --input-file /sbom +func runCycloneDXInDocker(t testing.TB, env map[string]string, image string, f *os.File, args ...string) (*exec.Cmd, string, string) { + allArgs := append( + []string{ + "run", + "-t", + "-v", + fmt.Sprintf("%s:/sbom", f.Name()), + image, + }, + args..., + ) + cmd := exec.Command("docker", allArgs...) + stdout, stderr, _ := runCommand(cmd, env) + return cmd, stdout, stderr +} + func runSyftInDocker(t testing.TB, env map[string]string, image string, args ...string) (*exec.Cmd, string, string) { allArgs := append( []string{ @@ -150,7 +167,7 @@ func runSyftCommand(t testing.TB, env map[string]string, expectError bool, args } func runCosign(t testing.TB, env map[string]string, args ...string) (*exec.Cmd, string, string) { - cmd := getCosignCommand(t, args...) + cmd := getCommand(t, ".tmp/cosign", args...) if env == nil { env = make(map[string]string) } @@ -164,8 +181,8 @@ func runCosign(t testing.TB, env map[string]string, args ...string) (*exec.Cmd, return cmd, stdout, stderr } -func getCosignCommand(t testing.TB, args ...string) *exec.Cmd { - return exec.Command(filepath.Join(repoRoot(t), ".tmp/cosign"), args...) +func getCommand(t testing.TB, location string, args ...string) *exec.Cmd { + return exec.Command(filepath.Join(repoRoot(t), location), args...) } func runCommand(cmd *exec.Cmd, env map[string]string) (string, string, error) {