diff --git a/cmd/syft/internal/commands/convert.go b/cmd/syft/internal/commands/convert.go index 0e411281e..a6189b847 100644 --- a/cmd/syft/internal/commands/convert.go +++ b/cmd/syft/internal/commands/convert.go @@ -1,24 +1,34 @@ package commands import ( + "context" "fmt" "io" "os" + "path/filepath" + "strings" "github.com/spf13/cobra" "github.com/anchore/clio" + "github.com/anchore/go-homedir" "github.com/anchore/syft/cmd/syft/internal/options" "github.com/anchore/syft/cmd/syft/internal/ui" "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/spillbuf" + "github.com/anchore/syft/internal/tmpdir" "github.com/anchore/syft/syft/format" + "github.com/anchore/syft/syft/format/syftjson" + "github.com/anchore/syft/syft/sbom" ) const ( convertExample = ` {{.appName}} {{.command}} img.syft.json -o spdx-json convert a syft SBOM to spdx-json, output goes to stdout {{.appName}} {{.command}} img.syft.json -o cyclonedx-json=img.cdx.json convert a syft SBOM to CycloneDX, output is written to the file "img.cdx.json" {{.appName}} {{.command}} - -o spdx-json convert an SBOM from STDIN to spdx-json + SYFT_CONVERT_PASSTHROUGH_EXACT_FORMAT=true {{.appName}} {{.command}} img.syft.json -o syft-json=out.syft.json copy the input to "out.syft.json" unchanged if it is already syft-json at the current schema version ` ) @@ -26,6 +36,7 @@ type ConvertOptions struct { options.Config `yaml:",inline" mapstructure:",squash"` options.Output `yaml:",inline" mapstructure:",squash"` options.UpdateCheck `yaml:",inline" mapstructure:",squash"` + Convert options.Convert `yaml:"convert" json:"convert" mapstructure:"convert"` } func Convert(app clio.Application) *cobra.Command { @@ -34,6 +45,7 @@ func Convert(app clio.Application) *cobra.Command { opts := &ConvertOptions{ UpdateCheck: options.DefaultUpdateCheck(), Output: options.DefaultOutput(), + Convert: options.DefaultConvert(), } return app.SetupCommand(&cobra.Command{ @@ -46,11 +58,11 @@ func Convert(app clio.Application) *cobra.Command { }), Args: validateConvertArgs, PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck), - RunE: func(_ *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { restoreStdout := ui.CaptureStdoutToTraceLog() defer restoreStdout() - return RunConvert(opts, args[0]) + return RunConvert(cmd.Context(), opts, args[0]) }, }, opts) } @@ -59,30 +71,63 @@ func validateConvertArgs(cmd *cobra.Command, args []string) error { return validateArgs(cmd, args, "an SBOM argument is required") } -func RunConvert(opts *ConvertOptions, userInput string) error { +func RunConvert(ctx context.Context, opts *ConvertOptions, userInput string) error { log.Warn("convert is an experimental feature, run `syft convert -h` for help") - writer, err := opts.SBOMWriter() + // a temp dir is only needed when STDIN is a pipe and has to be spilled to disk; it is created lazily on + // first use (under the user's TMPDIR), so a file input never touches it. Mirrors syft.CreateSBOM, which + // also honors a TempDir already attached to the context. + td := tmpdir.FromContext(ctx) + if td == nil { + _, td = tmpdir.Root(ctx, "syft-convert") + defer func() { + _ = td.Cleanup() + }() + } + + reader, inputInfo, err := openConvertInput(userInput, td) if err != nil { return err } + defer func() { + _ = reader.Close() + }() - var reader io.ReadSeekCloser + outputs := opts.Outputs + var unchanged []string + if opts.Convert.PassthroughExactFormat { + log.Warn("convert.passthrough-exact-format is enabled: syft-json input that already exactly matches a requested output format and schema version will be copied to that output unchanged (this does not apply to any other input format)") - if userInput == "-" { - // though os.Stdin is an os.File, it does not support seeking - // you will get errors such as "seek /dev/stdin: illegal seek". - // We need to buffer what we read. - reader = internal.NewBufferedSeeker(os.Stdin) - } else { - f, err := os.Open(userInput) + unchanged, outputs, err = partitionOutputsBySourceFormat(opts.Output, reader) if err != nil { - return fmt.Errorf("failed to open SBOM file: %w", err) + return err } - defer func() { - _ = f.Close() - }() - reader = f + } + + // resolve (and validate) every output that needs converting before writing anything, so that + // a bad -o value fails the command without leaving partial results behind + var writer sbom.Writer + if len(outputs) > 0 { + outputOpts := opts.Output + outputOpts.Outputs = outputs + writer, err = outputOpts.SBOMWriter() + if err != nil { + return err + } + } + + for _, output := range unchanged { + if err := writeUnchanged(output, opts.LegacyFile, inputInfo, reader); err != nil { + return err + } + } + + if writer == nil { + return nil + } + + if _, err := reader.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("unable to seek to start of SBOM: %w", err) } s, _, _, err := format.Decode(reader) @@ -100,3 +145,171 @@ func RunConvert(opts *ConvertOptions, userInput string) error { return nil } + +// openConvertInput opens the SBOM document at the given path, or STDIN when "-". The returned reader is seekable so +// the document can be identified, copied, and decoded in turn without being loaded into memory. The returned +// os.FileInfo describes the file the document is being read from, so that writing back over it can be recognized; +// it is nil when the input is not a file (a pipe or a terminal on STDIN). +func openConvertInput(userInput string, td *tmpdir.TempDir) (io.ReadSeekCloser, os.FileInfo, error) { + if userInput == "-" { + return openStdin(td) + } + + f, err := os.Open(userInput) + if err != nil { + return nil, nil, fmt.Errorf("failed to open SBOM file: %w", err) + } + + info, err := f.Stat() + if err != nil { + // the document is readable, we just cannot recognize it as an output destination + log.WithFields("error", err, "path", userInput).Debug("unable to stat SBOM file") + } + return f, info, nil +} + +// openStdin returns a seekable view of STDIN. When STDIN is a regular file (such as a shell redirect) it is read in +// place. A pipe or terminal cannot seek (you will get errors such as "seek /dev/stdin: illegal seek"), so its +// contents are drained into a spill buffer: a small head stays in memory and everything beyond it goes to a temp +// file under td, so an arbitrarily large piped SBOM never has to be held in memory to be made seekable. +func openStdin(td *tmpdir.TempDir) (io.ReadSeekCloser, os.FileInfo, error) { + if info, err := os.Stdin.Stat(); err == nil && info.Mode().IsRegular() { + if start, err := os.Stdin.Seek(0, io.SeekCurrent); err == nil { + // a redirect reads the file in place, so it is just as much an input file as a path argument is + return readSeekNopCloser{io.NewSectionReader(os.Stdin, start, info.Size()-start)}, info, nil + } + } + + buf := spillbuf.New(td) + if _, err := io.Copy(io.NewOffsetWriter(buf, 0), os.Stdin); err != nil { + _ = buf.Close() + return nil, nil, fmt.Errorf("failed to read SBOM from STDIN: %w", err) + } + + return &spilledReader{ + SectionReader: io.NewSectionReader(buf, 0, buf.Size()), + buf: buf, + }, nil, nil +} + +type readSeekNopCloser struct { + io.ReadSeeker +} + +func (readSeekNopCloser) Close() error { + return nil +} + +// spilledReader is a seekable view over a spill buffer; closing it releases the buffer's memory and spill file. +type spilledReader struct { + *io.SectionReader + buf *spillbuf.Buffer +} + +func (r *spilledReader) Close() error { + return r.buf.Close() +} + +// partitionOutputsBySourceFormat splits the requested outputs (in "[@][=]" form) into those +// that exactly match the format and schema version of a syft-json input document (which can be written out +// unchanged) and those that still require conversion. Input in any other format is always converted. Outputs that +// cannot be resolved to an encoder are left for the writer to report on. +func partitionOutputsBySourceFormat(output options.Output, reader io.ReadSeeker) (unchanged []string, toConvert []string, err error) { + if _, err := reader.Seek(0, io.SeekStart); err != nil { + return nil, nil, fmt.Errorf("unable to seek to start of SBOM: %w", err) + } + + id, version := format.Identify(reader) + if id == "" || version == "" { + // let decoding surface the appropriate error + return nil, output.Outputs, nil + } + + if id != syftjson.ID { + log.WithFields("format", id, "version", version).Debug("input is not syft-json, passthrough does not apply") + return nil, output.Outputs, nil + } + + encoders, err := output.Encoders() + if err != nil { + return nil, nil, err + } + collection := format.NewEncoderCollection(encoders...) + + for _, out := range output.Outputs { + name, _, _ := strings.Cut(strings.TrimSpace(out), "=") + enc := collection.GetByString(name) + if enc != nil && enc.ID() == id && enc.Version() == version { + log.WithFields("format", id, "version", version, "output", out).Warn("input already exactly matches the requested output format and schema version, copying it unchanged instead of converting it") + unchanged = append(unchanged, out) + continue + } + toConvert = append(toConvert, out) + } + + return unchanged, toConvert, nil +} + +// writeUnchanged copies the SBOM document as-is to the destination described by the given "[=]" +// output value, falling back to the (deprecated) --file path, and finally to the report bus (STDOUT). When the +// destination is the input file itself there is nothing to do. +func writeUnchanged(output string, defaultFile string, inputInfo os.FileInfo, reader io.ReadSeeker) error { + if _, err := reader.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("unable to seek to start of SBOM: %w", err) + } + + _, path, _ := strings.Cut(strings.TrimSpace(output), "=") + if path == "" { + path = defaultFile + } + + if path == "" { + // the report bus only carries strings, so STDOUT output must be materialized in memory + content, err := io.ReadAll(reader) //nolint:gocritic // the whole SBOM is the report; there is no meaningful bound + if err != nil { + return fmt.Errorf("unable to read SBOM: %w", err) + } + bus.Report(string(content)) + return nil + } + + expandedPath, err := homedir.Expand(path) + if err != nil { + log.Warnf("could not expand given writer output path=%q: %w", path, err) + expandedPath = path + } + + if isSameFile(inputInfo, expandedPath) { + log.WithFields("path", expandedPath).Info("output is the input file, leaving it as-is") + return nil + } + + if err := os.MkdirAll(filepath.Dir(expandedPath), 0755); err != nil { + return fmt.Errorf("unable to create report directory: %w", err) + } + + f, err := os.OpenFile(expandedPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("unable to create report file: %w", err) + } + + if _, err := io.Copy(f, reader); err != nil { + _ = f.Close() + return fmt.Errorf("unable to write report file: %w", err) + } + + return f.Close() +} + +// isSameFile reports whether the given path names the file the input is being read from. Comparing the input's +// os.FileInfo rather than its path covers a shell redirect onto STDIN, which has no path to compare. +func isSameFile(inputInfo os.FileInfo, path string) bool { + if inputInfo == nil { + return false + } + pathInfo, err := os.Stat(path) + if err != nil { + return false + } + return os.SameFile(inputInfo, pathInfo) +} diff --git a/cmd/syft/internal/commands/convert_test.go b/cmd/syft/internal/commands/convert_test.go new file mode 100644 index 000000000..c98b0f3dc --- /dev/null +++ b/cmd/syft/internal/commands/convert_test.go @@ -0,0 +1,448 @@ +package commands + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/syft/cmd/syft/internal/options" + "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/spillbuf" + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/format" + "github.com/anchore/syft/syft/format/cyclonedxjson" + "github.com/anchore/syft/syft/format/spdxjson" + "github.com/anchore/syft/syft/format/syftjson" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/sbom" + "github.com/anchore/syft/syft/source" +) + +func convertTestSBOM() sbom.SBOM { + catalog := pkg.NewCollection() + catalog.Add(pkg.Package{ + Name: "musl-utils", + Version: "1.2.3-r4", + Type: pkg.ApkPkg, + FoundBy: "apk-db-cataloger", + Locations: file.NewLocationSet( + file.NewLocation("/lib/apk/db/installed"), + ), + PURL: "pkg:apk/alpine/musl-utils@1.2.3-r4", + }) + + return sbom.SBOM{ + Artifacts: sbom.Artifacts{ + Packages: catalog, + }, + Source: source.Description{ + ID: "some-id", + Name: "some-dir", + Version: "latest", + Metadata: source.DirectoryMetadata{ + Path: "/some/path", + }, + }, + Descriptor: sbom.Descriptor{ + Name: "syft", + Version: "v0.0.0-test", + }, + } +} + +func mustConvertEncoder(enc sbom.FormatEncoder, err error) sbom.FormatEncoder { + if err != nil { + panic(err) + } + return enc +} + +func encodeConvertTestSBOM(t *testing.T, enc sbom.FormatEncoder) []byte { + buf := &bytes.Buffer{} + require.NoError(t, enc.Encode(buf, convertTestSBOM())) + return buf.Bytes() +} + +// prettySyftJSON returns the test SBOM as pretty-printed syft-json. Pretty input makes a verbatim copy +// distinguishable from a re-encode, since re-encoding always produces compact output here. +func prettySyftJSON(t *testing.T) []byte { + cfg := syftjson.DefaultEncoderConfig() + cfg.Pretty = true + return encodeConvertTestSBOM(t, mustConvertEncoder(syftjson.NewFormatEncoderWithConfig(cfg))) +} + +func Test_partitionOutputsBySourceFormat(t *testing.T) { + syftEncoder := mustConvertEncoder(syftjson.NewFormatEncoderWithConfig(syftjson.DefaultEncoderConfig())) + syftVersion := syftEncoder.Version() + syftContent := encodeConvertTestSBOM(t, syftEncoder) + + // the same document claiming an older schema version + olderSyftContent := bytes.ReplaceAll(syftContent, []byte(`"version":"`+syftVersion+`"`), []byte(`"version":"16.0.0"`)) + require.NotEqual(t, syftContent, olderSyftContent) + + cdxContent := encodeConvertTestSBOM(t, mustConvertEncoder(cyclonedxjson.NewFormatEncoderWithConfig(cyclonedxjson.DefaultEncoderConfig()))) + + tests := []struct { + name string + content []byte + outputs []string + wantUnchanged []string + wantToConvert []string + }{ + { + name: "syft-json matches the bare format name", + content: syftContent, + outputs: []string{"syft-json"}, + wantUnchanged: []string{"syft-json"}, + }, + { + name: "syft-json matches its alias", + content: syftContent, + outputs: []string{"json"}, + wantUnchanged: []string{"json"}, + }, + { + name: "syft-json matches an explicit version and file path", + content: syftContent, + outputs: []string{"syft-json@" + syftVersion + "=out/some.syft.json"}, + wantUnchanged: []string{"syft-json@" + syftVersion + "=out/some.syft.json"}, + }, + { + name: "syft-json at an older schema version requires conversion", + content: olderSyftContent, + outputs: []string{"syft-json"}, + wantToConvert: []string{"syft-json"}, + }, + { + name: "a different output format requires conversion", + content: syftContent, + outputs: []string{"spdx-json"}, + wantToConvert: []string{"spdx-json"}, + }, + { + name: "mixed outputs are partitioned", + content: syftContent, + outputs: []string{"syft-json=a.json", "spdx-json=b.json", "cyclonedx-xml"}, + wantUnchanged: []string{"syft-json=a.json"}, + wantToConvert: []string{"spdx-json=b.json", "cyclonedx-xml"}, + }, + { + name: "unknown format is left for the writer to report", + content: syftContent, + outputs: []string{"bogus"}, + wantToConvert: []string{"bogus"}, + }, + { + name: "non syft-json input is always converted, even when it matches the output", + content: cdxContent, + outputs: []string{"cyclonedx-json"}, + wantToConvert: []string{"cyclonedx-json"}, + }, + { + name: "unidentifiable content is left for the decoder to report", + content: []byte("definitely not an sbom"), + outputs: []string{"syft-json"}, + wantToConvert: []string{"syft-json"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + output := options.DefaultOutput() + output.Outputs = tt.outputs + + unchanged, toConvert, err := partitionOutputsBySourceFormat(output, bytes.NewReader(tt.content)) + require.NoError(t, err) + assert.Equal(t, tt.wantUnchanged, unchanged) + assert.Equal(t, tt.wantToConvert, toConvert) + }) + } +} + +func Test_RunConvert_passthroughExactFormat(t *testing.T) { + syftContent := prettySyftJSON(t) + + newOpts := func(passthrough bool, outputs ...string) *ConvertOptions { + opts := &ConvertOptions{ + Output: options.DefaultOutput(), + Convert: options.Convert{ + PassthroughExactFormat: passthrough, + }, + } + opts.Outputs = outputs + return opts + } + + writeInput := func(t *testing.T, content []byte) string { + input := filepath.Join(t.TempDir(), "input.json") + require.NoError(t, os.WriteFile(input, content, 0600)) + return input + } + + t.Run("matching output is copied unchanged while others are converted", func(t *testing.T) { + dir := t.TempDir() + syftOut := filepath.Join(dir, "out.syft.json") + spdxOut := filepath.Join(dir, "out.spdx.json") + + opts := newOpts(true, "syft-json="+syftOut, "spdx-json="+spdxOut) + require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, syftContent))) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + + spdx, err := os.ReadFile(spdxOut) + require.NoError(t, err) + id, _ := format.Identify(bytes.NewReader(spdx)) + assert.Equal(t, spdxjson.ID, id) + }) + + t.Run("STDIN input is copied unchanged while others are converted", func(t *testing.T) { + dir := t.TempDir() + syftOut := filepath.Join(dir, "out.syft.json") + spdxOut := filepath.Join(dir, "out.spdx.json") + + stdin, err := os.Open(writeInput(t, syftContent)) + require.NoError(t, err) + defer stdin.Close() + + originalStdin := os.Stdin + os.Stdin = stdin + t.Cleanup(func() { os.Stdin = originalStdin }) + + opts := newOpts(true, "syft-json="+syftOut, "spdx-json="+spdxOut) + require.NoError(t, RunConvert(context.Background(), opts, "-")) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + + spdx, err := os.ReadFile(spdxOut) + require.NoError(t, err) + id, _ := format.Identify(bytes.NewReader(spdx)) + assert.Equal(t, spdxjson.ID, id) + }) + + t.Run("piped STDIN input is copied unchanged while others are converted", func(t *testing.T) { + dir := t.TempDir() + syftOut := filepath.Join(dir, "out.syft.json") + spdxOut := filepath.Join(dir, "out.spdx.json") + + pipeReader, pipeWriter, err := os.Pipe() + require.NoError(t, err) + defer pipeReader.Close() + go func() { + defer pipeWriter.Close() + _, _ = pipeWriter.Write(syftContent) + }() + + originalStdin := os.Stdin + os.Stdin = pipeReader + t.Cleanup(func() { os.Stdin = originalStdin }) + + opts := newOpts(true, "syft-json="+syftOut, "spdx-json="+spdxOut) + require.NoError(t, RunConvert(context.Background(), opts, "-")) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + + spdx, err := os.ReadFile(spdxOut) + require.NoError(t, err) + id, _ := format.Identify(bytes.NewReader(spdx)) + assert.Equal(t, spdxjson.ID, id) + }) + + t.Run("without the option the matching output is re-encoded", func(t *testing.T) { + syftOut := filepath.Join(t.TempDir(), "out.syft.json") + + opts := newOpts(false, "syft-json="+syftOut) + require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, syftContent))) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + id, _ := format.Identify(bytes.NewReader(got)) + assert.Equal(t, syftjson.ID, id) + // the input was pretty-printed and the encoder is not configured to be, so a re-encode cannot reproduce it + assert.NotEqual(t, syftContent, got) + }) + + t.Run("non syft-json input is re-encoded even when it matches the output", func(t *testing.T) { + cdxContent := encodeConvertTestSBOM(t, mustConvertEncoder(cyclonedxjson.NewFormatEncoderWithConfig(cyclonedxjson.DefaultEncoderConfig()))) + cdxOut := filepath.Join(t.TempDir(), "out.cdx.json") + + opts := newOpts(true, "cyclonedx-json="+cdxOut) + require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, cdxContent))) + + got, err := os.ReadFile(cdxOut) + require.NoError(t, err) + id, _ := format.Identify(bytes.NewReader(got)) + assert.Equal(t, cyclonedxjson.ID, id) + // a fresh serial number is minted on every encode, so a round trip never reproduces the input + assert.NotEqual(t, cdxContent, got) + }) + + t.Run("the deprecated --file path is honored", func(t *testing.T) { + syftOut := filepath.Join(t.TempDir(), "out.syft.json") + + opts := newOpts(true, "syft-json") + opts.LegacyFile = syftOut + require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, syftContent))) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + }) + + t.Run("writing over the input file leaves it intact", func(t *testing.T) { + input := writeInput(t, syftContent) + + opts := newOpts(true, "syft-json="+input) + require.NoError(t, RunConvert(context.Background(), opts, input)) + + got, err := os.ReadFile(input) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + }) + + t.Run("writing over a redirected STDIN input leaves it intact", func(t *testing.T) { + input := writeInput(t, syftContent) + + stdin, err := os.Open(input) + require.NoError(t, err) + defer stdin.Close() + + originalStdin := os.Stdin + os.Stdin = stdin + t.Cleanup(func() { os.Stdin = originalStdin }) + + opts := newOpts(true, "syft-json="+input) + require.NoError(t, RunConvert(context.Background(), opts, "-")) + + got, err := os.ReadFile(input) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + }) + + t.Run("an invalid output fails before anything is written", func(t *testing.T) { + syftOut := filepath.Join(t.TempDir(), "out.syft.json") + + opts := newOpts(true, "syft-json="+syftOut, "bogus") + require.Error(t, RunConvert(context.Background(), opts, writeInput(t, syftContent))) + + assert.NoFileExists(t, syftOut) + }) + + t.Run("a missing input file is reported", func(t *testing.T) { + syftOut := filepath.Join(t.TempDir(), "out.syft.json") + + opts := newOpts(true, "syft-json="+syftOut) + err := RunConvert(context.Background(), opts, filepath.Join(t.TempDir(), "does-not-exist.json")) + require.ErrorContains(t, err, "failed to open SBOM file") + + assert.NoFileExists(t, syftOut) + }) + + t.Run("unrecognized input is left to the decoder to reject", func(t *testing.T) { + syftOut := filepath.Join(t.TempDir(), "out.syft.json") + + opts := newOpts(true, "syft-json="+syftOut) + err := RunConvert(context.Background(), opts, writeInput(t, []byte("definitely not an sbom"))) + require.ErrorContains(t, err, "failed to decode SBOM") + }) +} + +// largeSyftJSON returns a pretty-printed syft-json document of at least minBytes, built from many small packages. +func largeSyftJSON(t *testing.T, minBytes int64) []byte { + catalog := pkg.NewCollection() + for i := 0; i < int(minBytes/100); i++ { + catalog.Add(pkg.Package{ + Name: fmt.Sprintf("pkg-%d", i), + Version: "1.0.0", + Type: pkg.ApkPkg, + PURL: fmt.Sprintf("pkg:apk/alpine/pkg-%d@1.0.0", i), + }) + } + s := convertTestSBOM() + s.Artifacts.Packages = catalog + + cfg := syftjson.DefaultEncoderConfig() + cfg.Pretty = true + enc := mustConvertEncoder(syftjson.NewFormatEncoderWithConfig(cfg)) + + buf := &bytes.Buffer{} + require.NoError(t, enc.Encode(buf, s)) + require.Greater(t, int64(buf.Len()), minBytes) + return buf.Bytes() +} + +func Test_RunConvert_pipedInputLargerThanMemoryLimitSpillsToDisk(t *testing.T) { + // a piped document bigger than the spill buffer's in-memory tier is written to a temp file under TMPDIR; + // the conversion must identify, copy and clean up from there rather than holding the document in memory + dir := t.TempDir() + syftOut := filepath.Join(dir, "out.syft.json") + spillRoot := filepath.Join(dir, "spill") + require.NoError(t, os.Mkdir(spillRoot, 0o755)) + t.Setenv("TMPDIR", spillRoot) + + content := largeSyftJSON(t, 2*spillbuf.DefaultMemLimit) + + pipeReader, pipeWriter, err := os.Pipe() + require.NoError(t, err) + defer pipeReader.Close() + go func() { + defer pipeWriter.Close() + _, _ = pipeWriter.Write(content) + }() + + originalStdin := os.Stdin + os.Stdin = pipeReader + t.Cleanup(func() { os.Stdin = originalStdin }) + + opts := &ConvertOptions{ + Output: options.DefaultOutput(), + Convert: options.Convert{ + PassthroughExactFormat: true, + }, + } + opts.Outputs = []string{"syft-json=" + syftOut} + require.NoError(t, RunConvert(context.Background(), opts, "-")) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + assert.Equal(t, content, got) + + entries, err := os.ReadDir(spillRoot) + require.NoError(t, err) + assert.Empty(t, entries, "the spill directory must be removed once the conversion finishes") +} + +func Test_writeUnchanged_stdout(t *testing.T) { + // with no output path the document is published on the report bus, which is how the CLI writes to STDOUT + b := partybus.NewBus() + subscription := b.Subscribe() + bus.Set(b) + t.Cleanup(func() { + bus.Set(nil) + }) + + content := []byte(`{"some":"document"}`) + require.NoError(t, writeUnchanged("syft-json", "", nil, bytes.NewReader(content))) + + select { + case e := <-subscription.Events(): + assert.Equal(t, event.CLIReport, e.Type) + assert.Equal(t, string(content), e.Value) + case <-time.After(time.Second): + t.Fatal("no report was published") + } +} diff --git a/cmd/syft/internal/options/convert.go b/cmd/syft/internal/options/convert.go new file mode 100644 index 000000000..c86852170 --- /dev/null +++ b/cmd/syft/internal/options/convert.go @@ -0,0 +1,27 @@ +package options + +import ( + "github.com/anchore/clio" +) + +var _ clio.FieldDescriber = (*Convert)(nil) + +// Convert holds the options specific to the convert command. These are deliberately configuration-only (config file +// or environment) rather than flags: they change how the command behaves for every invocation in a pipeline and are +// not expected to vary from one call to the next. +type Convert struct { + PassthroughExactFormat bool `yaml:"passthrough-exact-format" json:"passthrough-exact-format" mapstructure:"passthrough-exact-format"` +} + +func DefaultConvert() Convert { + return Convert{ + PassthroughExactFormat: false, + } +} + +func (o *Convert) DescribeFields(descriptions clio.FieldDescriptionSet) { + descriptions.Add(&o.PassthroughExactFormat, `when a syft-json input SBOM already exactly matches the requested output format and schema version, copy it to that output unchanged instead of decoding and re-encoding it. +Note: this only applies to syft-json input, and only when the document's schema version is identical to the schema version this version of syft produces. Any other input is always converted. +Note: output written this way is a byte-for-byte copy of the input, so format options (such as 'format.pretty' or 'format.json.legacy') are not applied to it. +Note: to keep memory use minimal on large documents, give the input as a file path (or a shell redirect) and write the output with '-o ='. Piped input, or output to STDOUT, requires the whole document to be held in memory.`) +} diff --git a/syft/format/decoders.go b/syft/format/decoders.go index 48dba03e9..fe8bdda29 100644 --- a/syft/format/decoders.go +++ b/syft/format/decoders.go @@ -21,6 +21,9 @@ func init() { func Decoders() []sbom.FormatDecoder { return []sbom.FormatDecoder{ + // syft-json must remain the first decoder tried: it is by far the most common input, and its decoder can + // identify a syft document from the tail of the file alone. Trying anything else first would pay for a full + // parse of a large document by another decoder before reaching the common path. syftjson.NewFormatDecoder(), cyclonedxxml.NewFormatDecoder(), cyclonedxjson.NewFormatDecoder(), diff --git a/syft/format/internal/stream/seekable_reader.go b/syft/format/internal/stream/seekable_reader.go index 5df51a881..4a59ce860 100644 --- a/syft/format/internal/stream/seekable_reader.go +++ b/syft/format/internal/stream/seekable_reader.go @@ -72,8 +72,21 @@ func (o *offsetReadSeeker) Seek(offset int64, whence int) (int64, error) { } newOffset, err := o.rdr.Seek(offset, io.SeekCurrent) return newOffset - o.offset, err + case io.SeekEnd: + newOffset, err := o.rdr.Seek(offset, io.SeekEnd) + if err != nil { + return 0, fmt.Errorf("cannot seek end: %w", err) + } + if newOffset < o.offset { + // landed before the start of this view; move back to the start so the position stays valid + if _, err := o.rdr.Seek(o.offset, io.SeekStart); err != nil { + return 0, fmt.Errorf("cannot seek end: %w", err) + } + return 0, fmt.Errorf("cannot seek < 0") + } + return newOffset - o.offset, nil } - return 0, fmt.Errorf("only SeekStart and SeekCurrent supported") + return 0, fmt.Errorf("only SeekStart, SeekCurrent, and SeekEnd supported") } var _ io.ReadSeeker = (*offsetReadSeeker)(nil) diff --git a/syft/format/internal/stream/seekable_reader_test.go b/syft/format/internal/stream/seekable_reader_test.go index 035325305..3b4646c8f 100644 --- a/syft/format/internal/stream/seekable_reader_test.go +++ b/syft/format/internal/stream/seekable_reader_test.go @@ -179,11 +179,25 @@ func Test_offsetReadSeeker(t *testing.T) { whence: io.SeekCurrent, expected: "4", }, + { + name: "seek end", + input: abcd1234offset(2), + seek: -4, + whence: io.SeekEnd, + expected: "1234", + }, + { + name: "seek end before the start of the view", + input: abcd1234offset(2), + seek: -7, + whence: io.SeekEnd, + wantErr: require.Error, // would be < offset, which is an error + }, { name: "bad whence", input: abcd1234, seek: 1, - whence: io.SeekEnd, + whence: 99, wantErr: require.Error, }, } diff --git a/syft/format/syftjson/decoder.go b/syft/format/syftjson/decoder.go index de9b51433..5e3413094 100644 --- a/syft/format/syftjson/decoder.go +++ b/syft/format/syftjson/decoder.go @@ -1,7 +1,9 @@ package syftjson import ( + "bytes" "encoding/json" + "errors" "fmt" "io" "strings" @@ -15,6 +17,12 @@ import ( "github.com/anchore/syft/syft/sbom" ) +// identifyTailSize is how much of the end of a document is inspected when identifying it from its trailing schema +// block, before falling back to parsing the whole document. +const identifyTailSize = 2 * 1024 * 1024 + +var schemaKey = []byte(`"schema"`) + var _ sbom.FormatDecoder = (*decoder)(nil) type decoder struct{} @@ -45,6 +53,12 @@ func (d decoder) Decode(r io.Reader) (*sbom.SBOM, sbom.FormatID, string, error) return nil, "", "", fmt.Errorf("unable to decode syft-json document: %w", err) } + if doc.Schema.Version == "" { + // Identify found a schema block that the parsed document does not have, so it was describing something + // else in the stream; returning the empty document it decoded into would be a silently wrong answer + return nil, "", "", fmt.Errorf("not a syft-json document") + } + if err := checkSupportedSchema(doc.Schema.Version, internal.JSONSchemaVersion); err != nil { log.Warn(err) } @@ -57,6 +71,15 @@ func (d decoder) Identify(r io.Reader) (sbom.FormatID, string) { return "", "" } + // fast path: syft writes the schema block as the last key of the document (see model.Document), so for seekable + // input the format and version can be read from the tail without parsing a potentially very large document. + // Anything the tail cannot vouch for falls through to the full parse below. + if rs, ok := r.(io.ReadSeeker); ok { + if id, version, ok := identifyFromTail(rs); ok { + return id, version + } + } + type Document struct { Schema model.Schema `json:"schema"` } @@ -78,6 +101,160 @@ func (d decoder) Identify(r io.Reader) (sbom.FormatID, string) { return ID, doc.Schema.Version } +// identifyFromTail reads at most identifyTailSize bytes from the end of the reader and attempts to identify the +// document from a top-level schema block found there. The reader is returned to its original position afterwards. +func identifyFromTail(rs io.ReadSeeker) (sbom.FormatID, string, bool) { + start, err := rs.Seek(0, io.SeekCurrent) + if err != nil { + return "", "", false + } + + id, version, ok := readTail(rs, start) + + if _, err := rs.Seek(start, io.SeekStart); err != nil { + // without the original position the caller cannot safely parse the document afterwards + log.WithFields("error", err).Debug("unable to restore reader position after inspecting document tail") + return "", "", false + } + + return id, version, ok +} + +// readTail does the seeking and reading for identifyFromTail, leaving the reader wherever it ends up. +func readTail(rs io.ReadSeeker, start int64) (sbom.FormatID, string, bool) { + end, err := rs.Seek(0, io.SeekEnd) + if err != nil || end <= start { + return "", "", false + } + + // the tail only describes the document the reader is pointed at if that document starts where the reader is, + // so check the first byte: bytes ahead of it (a byte order mark, a log line prepended by a wrapper) would + // otherwise be accepted silently, leaving Identify claiming a document that cannot be decoded + if !opensObject(rs, start) { + return "", "", false + } + + tailLen := min(end-start, int64(identifyTailSize)) + tail := make([]byte, tailLen) + if _, err := rs.Seek(end-tailLen, io.SeekStart); err != nil { + return "", "", false + } + if _, err := io.ReadFull(rs, tail); err != nil { + return "", "", false + } + + return identifyFromTailBytes(tail) +} + +// opensObject reports whether the document at the given offset begins with a JSON object. +func opensObject(rs io.ReadSeeker, start int64) bool { + if _, err := rs.Seek(start, io.SeekStart); err != nil { + return false + } + + head := make([]byte, 64) + n, err := io.ReadFull(rs, head) + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { + return false + } + + head = bytes.TrimLeft(head[:n], " \t\r\n") + return len(head) > 0 && head[0] == '{' +} + +// identifyFromTailBytes looks for a syft schema block among the trailing bytes of a JSON document and, when one is +// found directly within the top-level object, returns the format and schema version it declares. The bytes may +// begin anywhere within the document, but must run to its very end. +func identifyFromTailBytes(tail []byte) (sbom.FormatID, string, bool) { + // find the last "schema" key. An unescaped quote cannot occur inside a JSON string, so a match whose opening + // quote is not preceded by a backslash is a genuine string token (and a key if a colon follows it). + idx := len(tail) + for { + idx = bytes.LastIndex(tail[:idx], schemaKey) + if idx <= 0 { + // not found, or found at the very start of the tail where we cannot tell whether it is escaped + return "", "", false + } + if tail[idx-1] != '\\' { + break + } + } + + rest := bytes.TrimLeft(tail[idx+len(schemaKey):], " \t\r\n") + if len(rest) == 0 || rest[0] != ':' { + // a string value that happens to read "schema", not a key + return "", "", false + } + rest = rest[1:] + + var schema model.Schema + dec := json.NewDecoder(bytes.NewReader(rest)) + if err := dec.Decode(&schema); err != nil { + return "", "", false + } + + // the schema block only identifies the document when it belongs to the top-level object + if !closesTopLevelObject(rest[dec.InputOffset():]) { + return "", "", false + } + + if schema.Version == "" || !strings.Contains(schema.URL, "anchore/syft") { + return "", "", false + } + + return ID, schema.Version, true +} + +// closesTopLevelObject reports whether the given bytes, which follow a value somewhere within a JSON document, run +// to the end of that document closing exactly one enclosing object (and nothing else) at the very end. When true, +// the value they follow sits directly within the top-level object. Braces inside strings are ignored. +func closesTopLevelObject(b []byte) bool { + var ( + depth int + inString bool + escaped bool + closed bool + ) + for _, c := range b { + if closed { + // only whitespace may follow the top-level object + if c == ' ' || c == '\t' || c == '\r' || c == '\n' { + continue + } + return false + } + if inString { + switch { + case escaped: + escaped = false + case c == '\\': + escaped = true + case c == '"': + inString = false + } + continue + } + switch c { + case '"': + inString = true + case '{', '[': + depth++ + case '}': + depth-- + if depth == -1 { + closed = true + } + case ']': + depth-- + if depth == -1 { + // the enclosing container is an array, not the top-level object + return false + } + } + } + return closed +} + func checkSupportedSchema(documentVersion string, parserVersion string) error { documentV, err := semver.NewVersion(documentVersion) if err != nil { diff --git a/syft/format/syftjson/decoder_identify_test.go b/syft/format/syftjson/decoder_identify_test.go new file mode 100644 index 000000000..8d60f3c1b --- /dev/null +++ b/syft/format/syftjson/decoder_identify_test.go @@ -0,0 +1,340 @@ +package syftjson + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/format/internal/stream" + "github.com/anchore/syft/syft/format/internal/testutil" +) + +const ( + testIdentifySchemaURL = "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-16.0.0.json" + testIdentifySchemaBlock = `"schema":{"version":"16.0.0","url":"` + testIdentifySchemaURL + `"}` +) + +func Test_identifyFromTailBytes(t *testing.T) { + tests := []struct { + name string + doc string + wantVersion string + wantOK bool + }{ + { + name: "syft key order (schema is the last key)", + doc: `{"artifacts":[],"descriptor":{"name":"syft"},` + testIdentifySchemaBlock + `}`, + wantVersion: "16.0.0", + wantOK: true, + }, + { + name: "alphabetical key order (schema is followed by source)", + doc: `{"artifacts":[],` + testIdentifySchemaBlock + `,"source":{"id":"x","name":"img","type":"image","metadata":{"tags":["a","b"],"layers":[{"digest":"sha256:1"}]}}}`, + wantVersion: "16.0.0", + wantOK: true, + }, + { + name: "pretty printed with a trailing newline", + doc: "{\n \"artifacts\": [],\n \"schema\": {\n \"version\": \"16.0.0\",\n \"url\": \"" + testIdentifySchemaURL + "\"\n }\n}\n", + wantVersion: "16.0.0", + wantOK: true, + }, + { + name: "nested schema key is not mistaken for the document schema", + doc: `{"artifacts":[{"metadata":{` + testIdentifySchemaBlock + `}}],"descriptor":{}}`, + wantOK: false, + }, + { + name: "top-level schema wins over an earlier nested one", + doc: `{"artifacts":[{"metadata":{"schema":{"version":"1","url":"anchore/syft"}}}],` + testIdentifySchemaBlock + `}`, + wantVersion: "16.0.0", + wantOK: true, + }, + { + name: "escaped schema text (with braces) inside a trailing string is ignored", + doc: `{"artifacts":[],` + testIdentifySchemaBlock + `,"source":{"name":"contains \"schema\":{\"version\":\"9\"} text"}}`, + wantVersion: "16.0.0", + wantOK: true, + }, + { + name: "array wrapped document", + doc: `[{"artifacts":[],` + testIdentifySchemaBlock + `}]`, + wantOK: false, + }, + { + name: "truncated document", + doc: `{"artifacts":[],` + testIdentifySchemaBlock, + wantOK: false, + }, + { + name: "trailing content after the document", + doc: `{"artifacts":[],` + testIdentifySchemaBlock + `}{"another":"document"}`, + wantOK: false, + }, + { + name: "schema value is not an object", + doc: `{"artifacts":[],"schema":"16.0.0"}`, + wantOK: false, + }, + { + name: "schema is a string value rather than a key", + doc: `{"artifacts":[],"kind":"schema"}`, + wantOK: false, + }, + { + name: "not a syft schema url", + doc: `{"artifacts":[],"schema":{"version":"1.0","url":"https://example.com/schema.json"}}`, + wantOK: false, + }, + { + name: "no schema key at all", + doc: `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[]}`, + wantOK: false, + }, + { + name: "schema key at the very start of the tail cannot be vouched for", + doc: testIdentifySchemaBlock + `}`, + wantOK: false, + }, + { + name: "empty", + doc: "", + wantOK: false, + }, + { + // the key's text is: he said "schema + name: "an escaped quote does not make a string into the schema key", + doc: `{"he said \"schema":{"version":"1.0","url":"https://example.com/anchore/syft"}}`, + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + id, version, ok := identifyFromTailBytes([]byte(tt.doc)) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.wantVersion, version) + if tt.wantOK { + assert.Equal(t, ID, id) + } else { + assert.Empty(t, id) + } + }) + } +} + +func Test_Identify_largeDocument(t *testing.T) { + // build a document comfortably larger than the tail window so only the tail path could avoid a full parse + unit := `{"name":"pkg","version":"1.0"},` + padding := strings.Repeat(unit, (identifyTailSize/len(unit))+1024) + doc := `{"artifacts":[` + padding + `{"name":"last"}],` + testIdentifySchemaBlock + `}` + require.Greater(t, len(doc), identifyTailSize) + + t.Run("seekable reader", func(t *testing.T) { + r := strings.NewReader(doc) + + id, version := NewFormatDecoder().Identify(r) + assert.Equal(t, ID, id) + assert.Equal(t, "16.0.0", version) + + // the reader is left where it started so callers can go on to read the document + pos, err := r.Seek(0, io.SeekCurrent) + require.NoError(t, err) + assert.Equal(t, int64(0), pos) + }) + + t.Run("non-seekable reader falls back to a full parse", func(t *testing.T) { + r := struct{ io.Reader }{strings.NewReader(doc)} + + id, version := NewFormatDecoder().Identify(r) + assert.Equal(t, ID, id) + assert.Equal(t, "16.0.0", version) + }) + + t.Run("reader positioned at a later document", func(t *testing.T) { + first := `{"artifacts":[],"schema":{"version":"1.0.0","url":"` + testIdentifySchemaURL + `"}}` + r := strings.NewReader(first + "\n" + doc) + _, err := r.Seek(int64(len(first)+1), io.SeekStart) + require.NoError(t, err) + + // mirror what the decoder collection does: seeks become relative to the current position + sr, err := stream.SeekableReader(r) + require.NoError(t, err) + + id, version := NewFormatDecoder().Identify(sr) + assert.Equal(t, ID, id) + assert.Equal(t, "16.0.0", version) + + pos, err := sr.Seek(0, io.SeekCurrent) + require.NoError(t, err) + assert.Equal(t, int64(0), pos) + }) +} + +func Test_Identify_agreesWithFullParse(t *testing.T) { + // the tail is only a shortcut: whatever it decides must match what parsing the whole document says, including + // for documents it has to decline. + // + // note: a stream holding several concatenated documents is deliberately not covered. Identifying the document + // at the reader's position would mean proving no other document ends before the stream does, which means + // reading all of it, which is the cost the tail path exists to avoid. Decode still reports the document it + // actually parsed (see Test_Decode_rejectsADocumentTheTailDidNotDescribe). + syftDoc := `{"artifacts":[],` + testIdentifySchemaBlock + `}` + + tests := []struct { + name string + doc string + wantVersion string + }{ + {name: "plain document", doc: syftDoc, wantVersion: "16.0.0"}, + {name: "byte order mark before the document", doc: "\xef\xbb\xbf" + syftDoc}, + {name: "content before the document", doc: "WARN something happened\n" + syftDoc}, + {name: "not a syft document", doc: `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[]}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := NewFormatDecoder() + + // a bytes.Reader is seekable, so this takes the tail path + fastID, fastVersion := d.Identify(bytes.NewReader([]byte(tt.doc))) + + // readerOnly hides Seek, so this takes the full parse + slowID, slowVersion := d.Identify(readerOnly{strings.NewReader(tt.doc)}) + + assert.Equal(t, slowID, fastID, "fast and slow paths disagree on the format") + assert.Equal(t, slowVersion, fastVersion, "fast and slow paths disagree on the version") + assert.Equal(t, tt.wantVersion, fastVersion) + }) + } +} + +// readerOnly hides any Seek method the underlying reader has, forcing the full parse. +type readerOnly struct { + io.Reader +} + +func Test_Decode_rejectsADocumentTheTailDidNotDescribe(t *testing.T) { + // the tail can find a schema block belonging to a later document in the stream; whatever it decides, Decode + // must not hand back the empty SBOM that parsing the first document into model.Document would produce + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[{"type":"library","name":"pkg-a"}]}` + + "\n" + `{"artifacts":[],` + testIdentifySchemaBlock + `}` + + s, _, _, err := NewFormatDecoder().Decode(bytes.NewReader([]byte(doc))) + require.Error(t, err) + assert.Nil(t, s) +} + +func Test_encoderWritesSchemaLast(t *testing.T) { + // identification reads the schema block from the tail of the document (see model.Document), which depends on + // the encoder always writing it as the final key + s := testutil.DirectoryInput(t, t.TempDir()) + + for _, pretty := range []bool{false, true} { + cfg := DefaultEncoderConfig() + cfg.Pretty = pretty + + enc, err := NewFormatEncoderWithConfig(cfg) + require.NoError(t, err) + + var buf bytes.Buffer + require.NoError(t, enc.Encode(&buf, s)) + + keys := topLevelKeys(t, buf.Bytes()) + require.NotEmpty(t, keys) + assert.Equal(t, "schema", keys[len(keys)-1], "pretty=%v", pretty) + + id, version, ok := identifyFromTailBytes(buf.Bytes()) + assert.True(t, ok, "pretty=%v", pretty) + assert.Equal(t, ID, id) + assert.Equal(t, internal.JSONSchemaVersion, version) + } +} + +// noEndSeeker behaves like a reader that buffers a stream as it goes: it can seek within what it has, but cannot +// seek relative to the end. +type noEndSeeker struct { + io.ReadSeeker +} + +func (s noEndSeeker) Seek(offset int64, whence int) (int64, error) { + if whence == io.SeekEnd { + return 0, errors.New("SeekEnd is not supported") + } + return s.ReadSeeker.Seek(offset, whence) +} + +func Test_Identify_fallsBackWhenTailIsUnavailable(t *testing.T) { + doc := `{"artifacts":[],` + testIdentifySchemaBlock + `}` + r := noEndSeeker{strings.NewReader(doc)} + + id, version := NewFormatDecoder().Identify(r) + assert.Equal(t, ID, id) + assert.Equal(t, "16.0.0", version) +} + +func Test_identifyFromTail_readerEdgeCases(t *testing.T) { + doc := `{"artifacts":[],` + testIdentifySchemaBlock + `}` + + t.Run("empty reader", func(t *testing.T) { + r := strings.NewReader("") + + _, _, ok := identifyFromTail(r) + assert.False(t, ok) + }) + + t.Run("reader already at the end has nothing to inspect and keeps its position", func(t *testing.T) { + r := strings.NewReader(doc) + _, err := r.Seek(0, io.SeekEnd) + require.NoError(t, err) + + _, _, ok := identifyFromTail(r) + assert.False(t, ok) + + pos, err := r.Seek(0, io.SeekCurrent) + require.NoError(t, err) + assert.Equal(t, int64(len(doc)), pos) + }) + + t.Run("whitespace around the colon and CRLF line endings", func(t *testing.T) { + crlf := "{\r\n \"artifacts\": [],\r\n \"schema\" : {\r\n \"version\": \"16.0.0\",\r\n \"url\": \"" + testIdentifySchemaURL + "\"\r\n }\r\n}\r\n" + + id, version, ok := identifyFromTailBytes([]byte(crlf)) + require.True(t, ok) + assert.Equal(t, ID, id) + assert.Equal(t, "16.0.0", version) + }) + + t.Run("document ending inside a string is not trusted", func(t *testing.T) { + truncated := `{"artifacts":[],` + testIdentifySchemaBlock + `,"source":{"name":"unterminated` + + _, _, ok := identifyFromTailBytes([]byte(truncated)) + assert.False(t, ok) + }) +} + +func topLevelKeys(t *testing.T, doc []byte) []string { + dec := json.NewDecoder(bytes.NewReader(doc)) + + tok, err := dec.Token() + require.NoError(t, err) + require.Equal(t, json.Delim('{'), tok) + + var keys []string + for dec.More() { + tok, err := dec.Token() + require.NoError(t, err) + keys = append(keys, tok.(string)) + + var value json.RawMessage + require.NoError(t, dec.Decode(&value)) + } + return keys +} diff --git a/syft/format/syftjson/model/document.go b/syft/format/syftjson/model/document.go index 9672ac3e5..e733978d5 100644 --- a/syft/format/syftjson/model/document.go +++ b/syft/format/syftjson/model/document.go @@ -5,6 +5,14 @@ import ( "fmt" ) +// Field order matters: encoding/json writes struct fields in declaration order, and Schema must remain the last +// field so that the schema block is always the final key of the encoded document. Format identification relies on +// finding the schema block by reading only the tail of a (potentially very large) document; moving Schema earlier +// would not break decoding but would silently force a full parse of the whole document to identify it. +// +// (This block is deliberately separated from the doc comment below: the JSON schema generator lifts doc comments +// into schema descriptions, and changing the description would alter the published schema.) + // Document represents the syft cataloging findings as a JSON document type Document struct { Artifacts []Package `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog diff --git a/test/cli/convert_cmd_test.go b/test/cli/convert_cmd_test.go index 04df14910..402e120cf 100644 --- a/test/cli/convert_cmd_test.go +++ b/test/cli/convert_cmd_test.go @@ -12,6 +12,7 @@ import ( "github.com/anchore/syft/syft/format/cyclonedxxml" "github.com/anchore/syft/syft/format/spdxjson" "github.com/anchore/syft/syft/format/spdxtagvalue" + "github.com/anchore/syft/syft/format/syftjson" "github.com/anchore/syft/syft/sbom" ) @@ -64,6 +65,93 @@ func TestConvertCmd(t *testing.T) { } } +func TestConvertCmd_PassthroughExactFormat(t *testing.T) { + // pretty-printed input makes a verbatim copy distinguishable from a re-encode (which is compact) + sbomArgs := []string{"dir:./testdata/image-pkg-coverage", "-o", "syft-json"} + cmd, input, stderr := runSyft(t, map[string]string{"SYFT_FORMAT_PRETTY": "true"}, sbomArgs...) + if cmd.ProcessState.ExitCode() != 0 { + t.Log("STDOUT:\n", input) + t.Log("STDERR:\n", stderr) + t.Log("COMMAND:", strings.Join(cmd.Args, " ")) + t.Fatalf("failure executing syft creating an sbom") + return + } + + inputID, inputVersion := format.Identify(strings.NewReader(input)) + require.Equal(t, syftjson.ID, inputID) + require.NotEmpty(t, inputVersion) + + const ( + enabledWarning = "convert.passthrough-exact-format is enabled" + copiedWarning = "copying it unchanged instead of converting it" + ) + + tests := []struct { + name string + env map[string]string + to string + wantUnchanged bool + wantID sbom.FormatID + wantVersion string + wantInStderr []string + wantNotStderr []string + }{ + { + name: "matching syft-json is passed through unchanged", + env: map[string]string{"SYFT_CONVERT_PASSTHROUGH_EXACT_FORMAT": "true"}, + to: "syft-json", + wantUnchanged: true, + wantID: syftjson.ID, + wantVersion: inputVersion, + wantInStderr: []string{enabledWarning, copiedWarning}, + }, + { + name: "different format is converted", + env: map[string]string{"SYFT_CONVERT_PASSTHROUGH_EXACT_FORMAT": "true"}, + to: "cyclonedx-json", + wantID: cyclonedxjson.ID, + wantVersion: mustEncoder(cyclonedxjson.NewFormatEncoderWithConfig(cyclonedxjson.DefaultEncoderConfig())).Version(), + wantInStderr: []string{enabledWarning}, + wantNotStderr: []string{copiedWarning}, + }, + { + name: "option off by default, matching syft-json is re-encoded", + to: "syft-json", + wantID: syftjson.ID, + wantVersion: inputVersion, + wantNotStderr: []string{enabledWarning, copiedWarning}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := getSyftCommand(t, "convert", "-", "-o", test.to) + cmd.Stdin = strings.NewReader(input) + stdout, stderr := runCommandObj(t, cmd, test.env, false) + + assertSuccessfulReturnCode(t, stdout, stderr, cmd.ProcessState.ExitCode()) + logOutputOnFailure(t, cmd, stdout, stderr) + + foundID, foundVersion := format.Identify(strings.NewReader(stdout)) + require.Equal(t, test.wantID, foundID) + require.Equal(t, test.wantVersion, foundVersion) + + if test.wantUnchanged { + require.Equal(t, strings.TrimSpace(input), strings.TrimSpace(stdout)) + } else { + require.NotEqual(t, strings.TrimSpace(input), strings.TrimSpace(stdout)) + } + + for _, want := range test.wantInStderr { + require.Contains(t, stderr, want) + } + for _, notWant := range test.wantNotStderr { + require.NotContains(t, stderr, notWant) + } + }) + } +} + func mustEncoder(enc sbom.FormatEncoder, err error) sbom.FormatEncoder { if err != nil { panic(err)