diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml index aad6d6090..51304dbec 100644 --- a/.github/actions/bootstrap/action.yaml +++ b/.github/actions/bootstrap/action.yaml @@ -30,7 +30,7 @@ runs: using: "composite" steps: - name: Setup go + go-make tooling - uses: anchore/go-make/.github/actions/setup@430e2175bb166bfd138ef75466adbcc8ddab875f # v0.8.0 + uses: anchore/go-make/.github/actions/setup@fa5421b3bf24c9d18f11a736c5eb4172f2b7d683 # v0.8.1 with: go-version: ${{ inputs.go-version }} cache-key-prefix: ${{ inputs.cache-key-prefix }} @@ -47,7 +47,17 @@ runs: APT_PACKAGES: ${{ inputs.bootstrap-apt-packages }} run: | IFS=' ' read -ra packages <<< "$APT_PACKAGES" - DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y "${packages[@]}" + # the AMI already ships most of these, and `apt update` costs ~40MB of index (minutes, + # when archive.ubuntu.com throttles us) to find that out. only touch apt if we must. + missing=() + for pkg in "${packages[@]}"; do + dpkg -s "$pkg" >/dev/null 2>&1 || missing+=("$pkg") + done + if [ ${#missing[@]} -eq 0 ]; then + echo "all apt packages already present: ${packages[*]}" + exit 0 + fi + DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y "${missing[@]}" # ORAS cache: restore-only on non-default branches / forks - name: Restore ORAS cache from github actions diff --git a/.github/workflows/validate-github-actions.yaml b/.github/workflows/validate-github-actions.yaml index 92d1a0da1..52f427ff1 100644 --- a/.github/workflows/validate-github-actions.yaml +++ b/.github/workflows/validate-github-actions.yaml @@ -25,7 +25,7 @@ jobs: persist-credentials: false - name: "Run zizmor" - uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 + uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4 with: # there is a pass/fail gate as a repo ruleset (if there is no ruleset configured then the action will pass by default) advanced-security: true diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index b4bb10ec2..2656ab1f1 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -21,10 +21,13 @@ jobs: name: "Static analysis" # runs-on.com: memory & general purpose instances for testing # spot enabled: ok to interrupt non-production workloads - # tmpfs: faster io-intensive workflows + # tmpfs intentionally omitted -- it RAM-backs /home/runner, /tmp and /var/lib/docker at + # 100% of instance memory, starving `make unit` of headroom to link with -race. + # volume: the caches and docker images tmpfs used to hold. gp3 baseline throughput is + # enough; a 4 vCPU instance can't pull much more than that anyway. # note: s3-cache intentionally omitted -- PR runs are untrusted and must not write to the # shared cache backend that the trusted release workflow reads from (cache poisoning). - runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=32+128/family=r5+r6+r7+r8+m4+m5+m6+m7+m8/spot=price-capacity-optimized/extras=tmpfs" + runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=16+32/family=m5+m6+m7+m8+c5+c6+c7+c8/spot=price-capacity-optimized/volume=100gb" permissions: contents: read steps: diff --git a/.gitignore b/.gitignore index 907f14e2d..f69feb3e4 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ bin/ /.tool /.task /generate +/spec /specs mise.toml .make/.make diff --git a/.golangci.yaml b/.golangci.yaml index 422a7448a..6b9bf74b0 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -32,6 +32,10 @@ linters: funlen: lines: 70 statements: 50 + goconst: + # the default of 3 turns every repeated literal in a lookup table or a test fixture into a finding. + # 5 keeps the ones worth hoisting without flagging data that reads better written out. + min-occurrences: 5 gocritic: enabled-checks: - deferInLoop @@ -54,6 +58,12 @@ linters: - linters: - revive text: "var-naming: avoid package names that conflict" + # cpegenerate is almost entirely lookup tables mapping artifact names to vendors, products, and + # maven group ids. The repeated strings are the data itself, so hoisting them to constants would + # replace a readable table with indirection for no gain. + - linters: + - goconst + path: syft/pkg/cataloger/internal/cpegenerate/ paths: - third_party$ - builtin$ diff --git a/.make/go.mod b/.make/go.mod index 787e6d0cc..b67a4d3a0 100644 --- a/.make/go.mod +++ b/.make/go.mod @@ -2,11 +2,11 @@ module github.com/anchore/syft/.make go 1.25.8 -require github.com/anchore/go-make v0.8.0 +require github.com/anchore/go-make v0.8.1 require ( github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/goccy/go-yaml v1.19.2 // indirect - golang.org/x/mod v0.37.0 // indirect - golang.org/x/sys v0.46.0 // indirect + golang.org/x/mod v0.40.0 // indirect + golang.org/x/sys v0.47.0 // indirect ) diff --git a/.make/go.sum b/.make/go.sum index 72d002cfc..f8fd03437 100644 --- a/.make/go.sum +++ b/.make/go.sum @@ -1,10 +1,10 @@ -github.com/anchore/go-make v0.8.0 h1:3XAD4WNHIbqY4kwFyCQJ7kKiYLzSzjB0UwzDVwRqlUY= -github.com/anchore/go-make v0.8.0/go.mod h1:4M6TnArb5w693VyWsgr5dCWrk2BLNu/ed4JUcsrzS34= +github.com/anchore/go-make v0.8.1 h1:GAhPXR+9qVQhFB1pfVlHFOzxAt3dTDTUIA7Tp7nx9tg= +github.com/anchore/go-make v0.8.1/go.mod h1:xALbVsPsDoJnJxdkjPlxOLZV7GbAdT5Pv3gd0kP1Lz0= github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/cmd/syft/internal/commands/cataloger_info.go b/cmd/syft/internal/commands/cataloger_info.go index 57e0f2744..2a9db6280 100644 --- a/cmd/syft/internal/commands/cataloger_info.go +++ b/cmd/syft/internal/commands/cataloger_info.go @@ -349,8 +349,8 @@ func renderCatalogerInfoTable(_ *capabilities.Document, catalogers []capabilitie })), tablewriter.WithConfig(tablewriter.Config{ Row: tw.CellConfig{ - Formatting: tw.CellFormatting{MergeMode: tw.MergeHierarchical}, - Alignment: tw.CellAlignment{Global: tw.AlignLeft}, + Merging: tw.CellMerging{Mode: tw.MergeHierarchical}, + Alignment: tw.CellAlignment{Global: tw.AlignLeft}, }, }), ) diff --git a/cmd/syft/internal/commands/convert.go b/cmd/syft/internal/commands/convert.go index 0e411281e..a6189b847 100644 --- a/cmd/syft/internal/commands/convert.go +++ b/cmd/syft/internal/commands/convert.go @@ -1,24 +1,34 @@ package commands import ( + "context" "fmt" "io" "os" + "path/filepath" + "strings" "github.com/spf13/cobra" "github.com/anchore/clio" + "github.com/anchore/go-homedir" "github.com/anchore/syft/cmd/syft/internal/options" "github.com/anchore/syft/cmd/syft/internal/ui" "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/spillbuf" + "github.com/anchore/syft/internal/tmpdir" "github.com/anchore/syft/syft/format" + "github.com/anchore/syft/syft/format/syftjson" + "github.com/anchore/syft/syft/sbom" ) const ( convertExample = ` {{.appName}} {{.command}} img.syft.json -o spdx-json convert a syft SBOM to spdx-json, output goes to stdout {{.appName}} {{.command}} img.syft.json -o cyclonedx-json=img.cdx.json convert a syft SBOM to CycloneDX, output is written to the file "img.cdx.json" {{.appName}} {{.command}} - -o spdx-json convert an SBOM from STDIN to spdx-json + SYFT_CONVERT_PASSTHROUGH_EXACT_FORMAT=true {{.appName}} {{.command}} img.syft.json -o syft-json=out.syft.json copy the input to "out.syft.json" unchanged if it is already syft-json at the current schema version ` ) @@ -26,6 +36,7 @@ type ConvertOptions struct { options.Config `yaml:",inline" mapstructure:",squash"` options.Output `yaml:",inline" mapstructure:",squash"` options.UpdateCheck `yaml:",inline" mapstructure:",squash"` + Convert options.Convert `yaml:"convert" json:"convert" mapstructure:"convert"` } func Convert(app clio.Application) *cobra.Command { @@ -34,6 +45,7 @@ func Convert(app clio.Application) *cobra.Command { opts := &ConvertOptions{ UpdateCheck: options.DefaultUpdateCheck(), Output: options.DefaultOutput(), + Convert: options.DefaultConvert(), } return app.SetupCommand(&cobra.Command{ @@ -46,11 +58,11 @@ func Convert(app clio.Application) *cobra.Command { }), Args: validateConvertArgs, PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck), - RunE: func(_ *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { restoreStdout := ui.CaptureStdoutToTraceLog() defer restoreStdout() - return RunConvert(opts, args[0]) + return RunConvert(cmd.Context(), opts, args[0]) }, }, opts) } @@ -59,30 +71,63 @@ func validateConvertArgs(cmd *cobra.Command, args []string) error { return validateArgs(cmd, args, "an SBOM argument is required") } -func RunConvert(opts *ConvertOptions, userInput string) error { +func RunConvert(ctx context.Context, opts *ConvertOptions, userInput string) error { log.Warn("convert is an experimental feature, run `syft convert -h` for help") - writer, err := opts.SBOMWriter() + // a temp dir is only needed when STDIN is a pipe and has to be spilled to disk; it is created lazily on + // first use (under the user's TMPDIR), so a file input never touches it. Mirrors syft.CreateSBOM, which + // also honors a TempDir already attached to the context. + td := tmpdir.FromContext(ctx) + if td == nil { + _, td = tmpdir.Root(ctx, "syft-convert") + defer func() { + _ = td.Cleanup() + }() + } + + reader, inputInfo, err := openConvertInput(userInput, td) if err != nil { return err } + defer func() { + _ = reader.Close() + }() - var reader io.ReadSeekCloser + outputs := opts.Outputs + var unchanged []string + if opts.Convert.PassthroughExactFormat { + log.Warn("convert.passthrough-exact-format is enabled: syft-json input that already exactly matches a requested output format and schema version will be copied to that output unchanged (this does not apply to any other input format)") - if userInput == "-" { - // though os.Stdin is an os.File, it does not support seeking - // you will get errors such as "seek /dev/stdin: illegal seek". - // We need to buffer what we read. - reader = internal.NewBufferedSeeker(os.Stdin) - } else { - f, err := os.Open(userInput) + unchanged, outputs, err = partitionOutputsBySourceFormat(opts.Output, reader) if err != nil { - return fmt.Errorf("failed to open SBOM file: %w", err) + return err } - defer func() { - _ = f.Close() - }() - reader = f + } + + // resolve (and validate) every output that needs converting before writing anything, so that + // a bad -o value fails the command without leaving partial results behind + var writer sbom.Writer + if len(outputs) > 0 { + outputOpts := opts.Output + outputOpts.Outputs = outputs + writer, err = outputOpts.SBOMWriter() + if err != nil { + return err + } + } + + for _, output := range unchanged { + if err := writeUnchanged(output, opts.LegacyFile, inputInfo, reader); err != nil { + return err + } + } + + if writer == nil { + return nil + } + + if _, err := reader.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("unable to seek to start of SBOM: %w", err) } s, _, _, err := format.Decode(reader) @@ -100,3 +145,171 @@ func RunConvert(opts *ConvertOptions, userInput string) error { return nil } + +// openConvertInput opens the SBOM document at the given path, or STDIN when "-". The returned reader is seekable so +// the document can be identified, copied, and decoded in turn without being loaded into memory. The returned +// os.FileInfo describes the file the document is being read from, so that writing back over it can be recognized; +// it is nil when the input is not a file (a pipe or a terminal on STDIN). +func openConvertInput(userInput string, td *tmpdir.TempDir) (io.ReadSeekCloser, os.FileInfo, error) { + if userInput == "-" { + return openStdin(td) + } + + f, err := os.Open(userInput) + if err != nil { + return nil, nil, fmt.Errorf("failed to open SBOM file: %w", err) + } + + info, err := f.Stat() + if err != nil { + // the document is readable, we just cannot recognize it as an output destination + log.WithFields("error", err, "path", userInput).Debug("unable to stat SBOM file") + } + return f, info, nil +} + +// openStdin returns a seekable view of STDIN. When STDIN is a regular file (such as a shell redirect) it is read in +// place. A pipe or terminal cannot seek (you will get errors such as "seek /dev/stdin: illegal seek"), so its +// contents are drained into a spill buffer: a small head stays in memory and everything beyond it goes to a temp +// file under td, so an arbitrarily large piped SBOM never has to be held in memory to be made seekable. +func openStdin(td *tmpdir.TempDir) (io.ReadSeekCloser, os.FileInfo, error) { + if info, err := os.Stdin.Stat(); err == nil && info.Mode().IsRegular() { + if start, err := os.Stdin.Seek(0, io.SeekCurrent); err == nil { + // a redirect reads the file in place, so it is just as much an input file as a path argument is + return readSeekNopCloser{io.NewSectionReader(os.Stdin, start, info.Size()-start)}, info, nil + } + } + + buf := spillbuf.New(td) + if _, err := io.Copy(io.NewOffsetWriter(buf, 0), os.Stdin); err != nil { + _ = buf.Close() + return nil, nil, fmt.Errorf("failed to read SBOM from STDIN: %w", err) + } + + return &spilledReader{ + SectionReader: io.NewSectionReader(buf, 0, buf.Size()), + buf: buf, + }, nil, nil +} + +type readSeekNopCloser struct { + io.ReadSeeker +} + +func (readSeekNopCloser) Close() error { + return nil +} + +// spilledReader is a seekable view over a spill buffer; closing it releases the buffer's memory and spill file. +type spilledReader struct { + *io.SectionReader + buf *spillbuf.Buffer +} + +func (r *spilledReader) Close() error { + return r.buf.Close() +} + +// partitionOutputsBySourceFormat splits the requested outputs (in "[@][=]" form) into those +// that exactly match the format and schema version of a syft-json input document (which can be written out +// unchanged) and those that still require conversion. Input in any other format is always converted. Outputs that +// cannot be resolved to an encoder are left for the writer to report on. +func partitionOutputsBySourceFormat(output options.Output, reader io.ReadSeeker) (unchanged []string, toConvert []string, err error) { + if _, err := reader.Seek(0, io.SeekStart); err != nil { + return nil, nil, fmt.Errorf("unable to seek to start of SBOM: %w", err) + } + + id, version := format.Identify(reader) + if id == "" || version == "" { + // let decoding surface the appropriate error + return nil, output.Outputs, nil + } + + if id != syftjson.ID { + log.WithFields("format", id, "version", version).Debug("input is not syft-json, passthrough does not apply") + return nil, output.Outputs, nil + } + + encoders, err := output.Encoders() + if err != nil { + return nil, nil, err + } + collection := format.NewEncoderCollection(encoders...) + + for _, out := range output.Outputs { + name, _, _ := strings.Cut(strings.TrimSpace(out), "=") + enc := collection.GetByString(name) + if enc != nil && enc.ID() == id && enc.Version() == version { + log.WithFields("format", id, "version", version, "output", out).Warn("input already exactly matches the requested output format and schema version, copying it unchanged instead of converting it") + unchanged = append(unchanged, out) + continue + } + toConvert = append(toConvert, out) + } + + return unchanged, toConvert, nil +} + +// writeUnchanged copies the SBOM document as-is to the destination described by the given "[=]" +// output value, falling back to the (deprecated) --file path, and finally to the report bus (STDOUT). When the +// destination is the input file itself there is nothing to do. +func writeUnchanged(output string, defaultFile string, inputInfo os.FileInfo, reader io.ReadSeeker) error { + if _, err := reader.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("unable to seek to start of SBOM: %w", err) + } + + _, path, _ := strings.Cut(strings.TrimSpace(output), "=") + if path == "" { + path = defaultFile + } + + if path == "" { + // the report bus only carries strings, so STDOUT output must be materialized in memory + content, err := io.ReadAll(reader) //nolint:gocritic // the whole SBOM is the report; there is no meaningful bound + if err != nil { + return fmt.Errorf("unable to read SBOM: %w", err) + } + bus.Report(string(content)) + return nil + } + + expandedPath, err := homedir.Expand(path) + if err != nil { + log.Warnf("could not expand given writer output path=%q: %w", path, err) + expandedPath = path + } + + if isSameFile(inputInfo, expandedPath) { + log.WithFields("path", expandedPath).Info("output is the input file, leaving it as-is") + return nil + } + + if err := os.MkdirAll(filepath.Dir(expandedPath), 0755); err != nil { + return fmt.Errorf("unable to create report directory: %w", err) + } + + f, err := os.OpenFile(expandedPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("unable to create report file: %w", err) + } + + if _, err := io.Copy(f, reader); err != nil { + _ = f.Close() + return fmt.Errorf("unable to write report file: %w", err) + } + + return f.Close() +} + +// isSameFile reports whether the given path names the file the input is being read from. Comparing the input's +// os.FileInfo rather than its path covers a shell redirect onto STDIN, which has no path to compare. +func isSameFile(inputInfo os.FileInfo, path string) bool { + if inputInfo == nil { + return false + } + pathInfo, err := os.Stat(path) + if err != nil { + return false + } + return os.SameFile(inputInfo, pathInfo) +} diff --git a/cmd/syft/internal/commands/convert_test.go b/cmd/syft/internal/commands/convert_test.go new file mode 100644 index 000000000..c98b0f3dc --- /dev/null +++ b/cmd/syft/internal/commands/convert_test.go @@ -0,0 +1,448 @@ +package commands + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-partybus" + + "github.com/anchore/syft/cmd/syft/internal/options" + "github.com/anchore/syft/internal/bus" + "github.com/anchore/syft/internal/spillbuf" + "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/format" + "github.com/anchore/syft/syft/format/cyclonedxjson" + "github.com/anchore/syft/syft/format/spdxjson" + "github.com/anchore/syft/syft/format/syftjson" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/sbom" + "github.com/anchore/syft/syft/source" +) + +func convertTestSBOM() sbom.SBOM { + catalog := pkg.NewCollection() + catalog.Add(pkg.Package{ + Name: "musl-utils", + Version: "1.2.3-r4", + Type: pkg.ApkPkg, + FoundBy: "apk-db-cataloger", + Locations: file.NewLocationSet( + file.NewLocation("/lib/apk/db/installed"), + ), + PURL: "pkg:apk/alpine/musl-utils@1.2.3-r4", + }) + + return sbom.SBOM{ + Artifacts: sbom.Artifacts{ + Packages: catalog, + }, + Source: source.Description{ + ID: "some-id", + Name: "some-dir", + Version: "latest", + Metadata: source.DirectoryMetadata{ + Path: "/some/path", + }, + }, + Descriptor: sbom.Descriptor{ + Name: "syft", + Version: "v0.0.0-test", + }, + } +} + +func mustConvertEncoder(enc sbom.FormatEncoder, err error) sbom.FormatEncoder { + if err != nil { + panic(err) + } + return enc +} + +func encodeConvertTestSBOM(t *testing.T, enc sbom.FormatEncoder) []byte { + buf := &bytes.Buffer{} + require.NoError(t, enc.Encode(buf, convertTestSBOM())) + return buf.Bytes() +} + +// prettySyftJSON returns the test SBOM as pretty-printed syft-json. Pretty input makes a verbatim copy +// distinguishable from a re-encode, since re-encoding always produces compact output here. +func prettySyftJSON(t *testing.T) []byte { + cfg := syftjson.DefaultEncoderConfig() + cfg.Pretty = true + return encodeConvertTestSBOM(t, mustConvertEncoder(syftjson.NewFormatEncoderWithConfig(cfg))) +} + +func Test_partitionOutputsBySourceFormat(t *testing.T) { + syftEncoder := mustConvertEncoder(syftjson.NewFormatEncoderWithConfig(syftjson.DefaultEncoderConfig())) + syftVersion := syftEncoder.Version() + syftContent := encodeConvertTestSBOM(t, syftEncoder) + + // the same document claiming an older schema version + olderSyftContent := bytes.ReplaceAll(syftContent, []byte(`"version":"`+syftVersion+`"`), []byte(`"version":"16.0.0"`)) + require.NotEqual(t, syftContent, olderSyftContent) + + cdxContent := encodeConvertTestSBOM(t, mustConvertEncoder(cyclonedxjson.NewFormatEncoderWithConfig(cyclonedxjson.DefaultEncoderConfig()))) + + tests := []struct { + name string + content []byte + outputs []string + wantUnchanged []string + wantToConvert []string + }{ + { + name: "syft-json matches the bare format name", + content: syftContent, + outputs: []string{"syft-json"}, + wantUnchanged: []string{"syft-json"}, + }, + { + name: "syft-json matches its alias", + content: syftContent, + outputs: []string{"json"}, + wantUnchanged: []string{"json"}, + }, + { + name: "syft-json matches an explicit version and file path", + content: syftContent, + outputs: []string{"syft-json@" + syftVersion + "=out/some.syft.json"}, + wantUnchanged: []string{"syft-json@" + syftVersion + "=out/some.syft.json"}, + }, + { + name: "syft-json at an older schema version requires conversion", + content: olderSyftContent, + outputs: []string{"syft-json"}, + wantToConvert: []string{"syft-json"}, + }, + { + name: "a different output format requires conversion", + content: syftContent, + outputs: []string{"spdx-json"}, + wantToConvert: []string{"spdx-json"}, + }, + { + name: "mixed outputs are partitioned", + content: syftContent, + outputs: []string{"syft-json=a.json", "spdx-json=b.json", "cyclonedx-xml"}, + wantUnchanged: []string{"syft-json=a.json"}, + wantToConvert: []string{"spdx-json=b.json", "cyclonedx-xml"}, + }, + { + name: "unknown format is left for the writer to report", + content: syftContent, + outputs: []string{"bogus"}, + wantToConvert: []string{"bogus"}, + }, + { + name: "non syft-json input is always converted, even when it matches the output", + content: cdxContent, + outputs: []string{"cyclonedx-json"}, + wantToConvert: []string{"cyclonedx-json"}, + }, + { + name: "unidentifiable content is left for the decoder to report", + content: []byte("definitely not an sbom"), + outputs: []string{"syft-json"}, + wantToConvert: []string{"syft-json"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + output := options.DefaultOutput() + output.Outputs = tt.outputs + + unchanged, toConvert, err := partitionOutputsBySourceFormat(output, bytes.NewReader(tt.content)) + require.NoError(t, err) + assert.Equal(t, tt.wantUnchanged, unchanged) + assert.Equal(t, tt.wantToConvert, toConvert) + }) + } +} + +func Test_RunConvert_passthroughExactFormat(t *testing.T) { + syftContent := prettySyftJSON(t) + + newOpts := func(passthrough bool, outputs ...string) *ConvertOptions { + opts := &ConvertOptions{ + Output: options.DefaultOutput(), + Convert: options.Convert{ + PassthroughExactFormat: passthrough, + }, + } + opts.Outputs = outputs + return opts + } + + writeInput := func(t *testing.T, content []byte) string { + input := filepath.Join(t.TempDir(), "input.json") + require.NoError(t, os.WriteFile(input, content, 0600)) + return input + } + + t.Run("matching output is copied unchanged while others are converted", func(t *testing.T) { + dir := t.TempDir() + syftOut := filepath.Join(dir, "out.syft.json") + spdxOut := filepath.Join(dir, "out.spdx.json") + + opts := newOpts(true, "syft-json="+syftOut, "spdx-json="+spdxOut) + require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, syftContent))) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + + spdx, err := os.ReadFile(spdxOut) + require.NoError(t, err) + id, _ := format.Identify(bytes.NewReader(spdx)) + assert.Equal(t, spdxjson.ID, id) + }) + + t.Run("STDIN input is copied unchanged while others are converted", func(t *testing.T) { + dir := t.TempDir() + syftOut := filepath.Join(dir, "out.syft.json") + spdxOut := filepath.Join(dir, "out.spdx.json") + + stdin, err := os.Open(writeInput(t, syftContent)) + require.NoError(t, err) + defer stdin.Close() + + originalStdin := os.Stdin + os.Stdin = stdin + t.Cleanup(func() { os.Stdin = originalStdin }) + + opts := newOpts(true, "syft-json="+syftOut, "spdx-json="+spdxOut) + require.NoError(t, RunConvert(context.Background(), opts, "-")) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + + spdx, err := os.ReadFile(spdxOut) + require.NoError(t, err) + id, _ := format.Identify(bytes.NewReader(spdx)) + assert.Equal(t, spdxjson.ID, id) + }) + + t.Run("piped STDIN input is copied unchanged while others are converted", func(t *testing.T) { + dir := t.TempDir() + syftOut := filepath.Join(dir, "out.syft.json") + spdxOut := filepath.Join(dir, "out.spdx.json") + + pipeReader, pipeWriter, err := os.Pipe() + require.NoError(t, err) + defer pipeReader.Close() + go func() { + defer pipeWriter.Close() + _, _ = pipeWriter.Write(syftContent) + }() + + originalStdin := os.Stdin + os.Stdin = pipeReader + t.Cleanup(func() { os.Stdin = originalStdin }) + + opts := newOpts(true, "syft-json="+syftOut, "spdx-json="+spdxOut) + require.NoError(t, RunConvert(context.Background(), opts, "-")) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + + spdx, err := os.ReadFile(spdxOut) + require.NoError(t, err) + id, _ := format.Identify(bytes.NewReader(spdx)) + assert.Equal(t, spdxjson.ID, id) + }) + + t.Run("without the option the matching output is re-encoded", func(t *testing.T) { + syftOut := filepath.Join(t.TempDir(), "out.syft.json") + + opts := newOpts(false, "syft-json="+syftOut) + require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, syftContent))) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + id, _ := format.Identify(bytes.NewReader(got)) + assert.Equal(t, syftjson.ID, id) + // the input was pretty-printed and the encoder is not configured to be, so a re-encode cannot reproduce it + assert.NotEqual(t, syftContent, got) + }) + + t.Run("non syft-json input is re-encoded even when it matches the output", func(t *testing.T) { + cdxContent := encodeConvertTestSBOM(t, mustConvertEncoder(cyclonedxjson.NewFormatEncoderWithConfig(cyclonedxjson.DefaultEncoderConfig()))) + cdxOut := filepath.Join(t.TempDir(), "out.cdx.json") + + opts := newOpts(true, "cyclonedx-json="+cdxOut) + require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, cdxContent))) + + got, err := os.ReadFile(cdxOut) + require.NoError(t, err) + id, _ := format.Identify(bytes.NewReader(got)) + assert.Equal(t, cyclonedxjson.ID, id) + // a fresh serial number is minted on every encode, so a round trip never reproduces the input + assert.NotEqual(t, cdxContent, got) + }) + + t.Run("the deprecated --file path is honored", func(t *testing.T) { + syftOut := filepath.Join(t.TempDir(), "out.syft.json") + + opts := newOpts(true, "syft-json") + opts.LegacyFile = syftOut + require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, syftContent))) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + }) + + t.Run("writing over the input file leaves it intact", func(t *testing.T) { + input := writeInput(t, syftContent) + + opts := newOpts(true, "syft-json="+input) + require.NoError(t, RunConvert(context.Background(), opts, input)) + + got, err := os.ReadFile(input) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + }) + + t.Run("writing over a redirected STDIN input leaves it intact", func(t *testing.T) { + input := writeInput(t, syftContent) + + stdin, err := os.Open(input) + require.NoError(t, err) + defer stdin.Close() + + originalStdin := os.Stdin + os.Stdin = stdin + t.Cleanup(func() { os.Stdin = originalStdin }) + + opts := newOpts(true, "syft-json="+input) + require.NoError(t, RunConvert(context.Background(), opts, "-")) + + got, err := os.ReadFile(input) + require.NoError(t, err) + assert.Equal(t, syftContent, got) + }) + + t.Run("an invalid output fails before anything is written", func(t *testing.T) { + syftOut := filepath.Join(t.TempDir(), "out.syft.json") + + opts := newOpts(true, "syft-json="+syftOut, "bogus") + require.Error(t, RunConvert(context.Background(), opts, writeInput(t, syftContent))) + + assert.NoFileExists(t, syftOut) + }) + + t.Run("a missing input file is reported", func(t *testing.T) { + syftOut := filepath.Join(t.TempDir(), "out.syft.json") + + opts := newOpts(true, "syft-json="+syftOut) + err := RunConvert(context.Background(), opts, filepath.Join(t.TempDir(), "does-not-exist.json")) + require.ErrorContains(t, err, "failed to open SBOM file") + + assert.NoFileExists(t, syftOut) + }) + + t.Run("unrecognized input is left to the decoder to reject", func(t *testing.T) { + syftOut := filepath.Join(t.TempDir(), "out.syft.json") + + opts := newOpts(true, "syft-json="+syftOut) + err := RunConvert(context.Background(), opts, writeInput(t, []byte("definitely not an sbom"))) + require.ErrorContains(t, err, "failed to decode SBOM") + }) +} + +// largeSyftJSON returns a pretty-printed syft-json document of at least minBytes, built from many small packages. +func largeSyftJSON(t *testing.T, minBytes int64) []byte { + catalog := pkg.NewCollection() + for i := 0; i < int(minBytes/100); i++ { + catalog.Add(pkg.Package{ + Name: fmt.Sprintf("pkg-%d", i), + Version: "1.0.0", + Type: pkg.ApkPkg, + PURL: fmt.Sprintf("pkg:apk/alpine/pkg-%d@1.0.0", i), + }) + } + s := convertTestSBOM() + s.Artifacts.Packages = catalog + + cfg := syftjson.DefaultEncoderConfig() + cfg.Pretty = true + enc := mustConvertEncoder(syftjson.NewFormatEncoderWithConfig(cfg)) + + buf := &bytes.Buffer{} + require.NoError(t, enc.Encode(buf, s)) + require.Greater(t, int64(buf.Len()), minBytes) + return buf.Bytes() +} + +func Test_RunConvert_pipedInputLargerThanMemoryLimitSpillsToDisk(t *testing.T) { + // a piped document bigger than the spill buffer's in-memory tier is written to a temp file under TMPDIR; + // the conversion must identify, copy and clean up from there rather than holding the document in memory + dir := t.TempDir() + syftOut := filepath.Join(dir, "out.syft.json") + spillRoot := filepath.Join(dir, "spill") + require.NoError(t, os.Mkdir(spillRoot, 0o755)) + t.Setenv("TMPDIR", spillRoot) + + content := largeSyftJSON(t, 2*spillbuf.DefaultMemLimit) + + pipeReader, pipeWriter, err := os.Pipe() + require.NoError(t, err) + defer pipeReader.Close() + go func() { + defer pipeWriter.Close() + _, _ = pipeWriter.Write(content) + }() + + originalStdin := os.Stdin + os.Stdin = pipeReader + t.Cleanup(func() { os.Stdin = originalStdin }) + + opts := &ConvertOptions{ + Output: options.DefaultOutput(), + Convert: options.Convert{ + PassthroughExactFormat: true, + }, + } + opts.Outputs = []string{"syft-json=" + syftOut} + require.NoError(t, RunConvert(context.Background(), opts, "-")) + + got, err := os.ReadFile(syftOut) + require.NoError(t, err) + assert.Equal(t, content, got) + + entries, err := os.ReadDir(spillRoot) + require.NoError(t, err) + assert.Empty(t, entries, "the spill directory must be removed once the conversion finishes") +} + +func Test_writeUnchanged_stdout(t *testing.T) { + // with no output path the document is published on the report bus, which is how the CLI writes to STDOUT + b := partybus.NewBus() + subscription := b.Subscribe() + bus.Set(b) + t.Cleanup(func() { + bus.Set(nil) + }) + + content := []byte(`{"some":"document"}`) + require.NoError(t, writeUnchanged("syft-json", "", nil, bytes.NewReader(content))) + + select { + case e := <-subscription.Events(): + assert.Equal(t, event.CLIReport, e.Type) + assert.Equal(t, string(content), e.Value) + case <-time.After(time.Second): + t.Fatal("no report was published") + } +} diff --git a/cmd/syft/internal/options/convert.go b/cmd/syft/internal/options/convert.go new file mode 100644 index 000000000..c86852170 --- /dev/null +++ b/cmd/syft/internal/options/convert.go @@ -0,0 +1,27 @@ +package options + +import ( + "github.com/anchore/clio" +) + +var _ clio.FieldDescriber = (*Convert)(nil) + +// Convert holds the options specific to the convert command. These are deliberately configuration-only (config file +// or environment) rather than flags: they change how the command behaves for every invocation in a pipeline and are +// not expected to vary from one call to the next. +type Convert struct { + PassthroughExactFormat bool `yaml:"passthrough-exact-format" json:"passthrough-exact-format" mapstructure:"passthrough-exact-format"` +} + +func DefaultConvert() Convert { + return Convert{ + PassthroughExactFormat: false, + } +} + +func (o *Convert) DescribeFields(descriptions clio.FieldDescriptionSet) { + descriptions.Add(&o.PassthroughExactFormat, `when a syft-json input SBOM already exactly matches the requested output format and schema version, copy it to that output unchanged instead of decoding and re-encoding it. +Note: this only applies to syft-json input, and only when the document's schema version is identical to the schema version this version of syft produces. Any other input is always converted. +Note: output written this way is a byte-for-byte copy of the input, so format options (such as 'format.pretty' or 'format.json.legacy') are not applied to it. +Note: to keep memory use minimal on large documents, give the input as a file path (or a shell redirect) and write the output with '-o ='. Piped input, or output to STDOUT, requires the whole document to be held in memory.`) +} diff --git a/cmd/syft/internal/test/integration/multiplatform_oci_test.go b/cmd/syft/internal/test/integration/multiplatform_oci_test.go index 94879a190..3373d8f36 100644 --- a/cmd/syft/internal/test/integration/multiplatform_oci_test.go +++ b/cmd/syft/internal/test/integration/multiplatform_oci_test.go @@ -60,6 +60,8 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) { // The raw config of the selected image must match the requested platform... assertConfigPlatform(t, meta.RawConfig, "linux", arch) + assert.Equal(t, "linux", meta.OS) + assert.Equal(t, arch, meta.Architecture) // ...and it must be the exact per-platform image from the multi-platform index. assert.Equal(t, expectedDigest[arch], meta.ID) }) @@ -125,6 +127,8 @@ func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) { // The raw config of the selected image must match the current platform assertConfigPlatform(t, meta.RawConfig, "linux", runtime.GOARCH) + assert.Equal(t, "linux", meta.OS) + assert.Equal(t, runtime.GOARCH, meta.Architecture) }) } } diff --git a/cmd/syft/internal/test/integration/single_platform_oci_test.go b/cmd/syft/internal/test/integration/single_platform_oci_test.go new file mode 100644 index 000000000..b028aaaa4 --- /dev/null +++ b/cmd/syft/internal/test/integration/single_platform_oci_test.go @@ -0,0 +1,64 @@ +package integration + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft" + "github.com/anchore/syft/syft/source" +) + +// TestSinglePlatformOCIImage covers the single-image (non-index) OCI layouts, where syft used to report +// empty os/architecture metadata and silently accept any --platform. Both now come from the image config. +func TestSinglePlatformOCIImage(t *testing.T) { + // a platform this fixture is never built for, so the mismatch is not host dependent + unavailablePlatform, err := image.NewPlatform("linux/ppc64le") + require.NoError(t, err) + + sources := map[string]image.Source{ + "oci-dir": image.OciDirectorySource, + "oci-archive": image.OciTarballSource, + } + + for from, imageSource := range sources { + t.Run(from, func(t *testing.T) { + localPath := strings.TrimPrefix(imagetest.PrepareFixtureImage(t, imageSource, "image-distro-id"), string(imageSource)+":") + + t.Run("reports the platform from the image config", func(t *testing.T) { + src, err := syft.GetSource( + context.Background(), + localPath, + syft.DefaultGetSourceConfig().WithSources(from), + ) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + + meta, ok := src.Describe().Metadata.(source.ImageMetadata) + require.True(t, ok, "expected image metadata, got %T", src.Describe().Metadata) + + // the fixture is built for whatever the local docker daemon runs, so assert against the + // config rather than a fixed platform + assert.NotEmpty(t, meta.OS) + assert.NotEmpty(t, meta.Architecture) + assertConfigPlatform(t, meta.RawConfig, meta.OS, meta.Architecture) + }) + + t.Run("rejects a platform the image does not match", func(t *testing.T) { + _, err := syft.GetSource( + context.Background(), + localPath, + syft.DefaultGetSourceConfig().WithSources(from).WithPlatform(unavailablePlatform), + ) + require.ErrorContains(t, err, "linux/ppc64le") + }) + }) + } +} diff --git a/cmd/syft/internal/test/integration/testdata/image-java-no-main-package/Dockerfile b/cmd/syft/internal/test/integration/testdata/image-java-no-main-package/Dockerfile index 3271f14cf..9356e4f8b 100644 --- a/cmd/syft/internal/test/integration/testdata/image-java-no-main-package/Dockerfile +++ b/cmd/syft/internal/test/integration/testdata/image-java-no-main-package/Dockerfile @@ -8,8 +8,6 @@ RUN mkdir tmp WORKDIR /usr/share/jenkins/tmp -RUN apt-get update 2>&1 > /dev/null && apt-get install -y less zip 2>&1 > /dev/null - RUN unzip ../jenkins.war 2>&1 > /dev/null RUN rm -rf ./META-INF/MANIFEST.MF ./WEB-INF ./jsbundles ./scripts ./css @@ -18,7 +16,8 @@ WORKDIR /usr/share/jenkins RUN rm -rf jenkins.war -RUN cd ./tmp && zip -r ../jenkins.war . && cd .. +# use jar (from the JDK in the base image) to rezip -- M keeps it manifest-less, which is the point of this fixture +RUN cd ./tmp && jar cfM ../jenkins.war . && cd .. RUN rm -rf ./tmp diff --git a/cmd/syft/internal/test/integration/testdata/image-pkg-coverage/Dockerfile b/cmd/syft/internal/test/integration/testdata/image-pkg-coverage/Dockerfile index 17ad2a6f4..bd14516d4 100644 --- a/cmd/syft/internal/test/integration/testdata/image-pkg-coverage/Dockerfile +++ b/cmd/syft/internal/test/integration/testdata/image-pkg-coverage/Dockerfile @@ -16,7 +16,7 @@ RUN unxz /lib/modules/6.0.7-301.fc37.x86_64/kernel/drivers/tty/ttynull.ko.xz # dotnet pkg coverage # https://nuget.info/packages/DocuSign.eSign.dll/6.8.0 # https://github.com/docusign/docusign-esign-csharp-client/blob/master/LICENSE -RUN curl -LO https://www.nuget.org/api/v2/package/DocuSign.eSign.dll/6.8.0 +RUN curl -fLO https://www.nuget.org/api/v2/package/DocuSign.eSign.dll/6.8.0 RUN unzip 6.8.0 RUN chmod 600 lib/net462/DocuSign.eSign.dll RUN rm 6.8.0 diff --git a/go.mod b/go.mod index 3107bb98d..8c49b84ff 100644 --- a/go.mod +++ b/go.mod @@ -19,10 +19,10 @@ require ( github.com/anchore/go-logger v0.1.1 github.com/anchore/go-macholibre v0.1.1 github.com/anchore/go-rpmdb v0.2.0 - github.com/anchore/go-sync v0.1.1 + github.com/anchore/go-sync v0.1.2 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b github.com/anchore/packageurl-go v0.2.0 - github.com/anchore/stereoscope v0.3.1 + github.com/anchore/stereoscope v0.3.2 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be github.com/aquasecurity/go-pep440-version v0.0.1 github.com/bitnami/go-version v0.0.0-20250131085805-b1f57a8634ef @@ -48,13 +48,13 @@ require ( github.com/goccy/go-yaml v1.19.2 github.com/gohugoio/hashstructure v0.6.0 github.com/google/go-cmp v0.7.0 - github.com/google/go-containerregistry v0.22.0 + github.com/google/go-containerregistry v0.22.1 github.com/google/licensecheck v0.3.1 github.com/google/uuid v1.6.0 github.com/gookit/color v1.6.1 - github.com/gpustack/gguf-parser-go v0.26.0 + github.com/gpustack/gguf-parser-go v0.26.3 github.com/hashicorp/go-cleanhttp v0.5.2 - github.com/hashicorp/go-getter v1.8.8 + github.com/hashicorp/go-getter v1.8.9 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/hcl/v2 v2.24.0 github.com/iancoleman/strcase v0.3.0 @@ -62,7 +62,7 @@ require ( github.com/jedib0t/go-pretty/v6 v6.8.3 github.com/jinzhu/copier v0.4.0 github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 - github.com/klauspost/compress v1.19.2 + github.com/klauspost/compress v1.20.0 github.com/magiconair/properties v1.18.11 github.com/mholt/archives v0.1.5 github.com/moby/sys/mountinfo v0.7.2 @@ -93,12 +93,12 @@ require ( go.uber.org/goleak v1.3.0 go.yaml.in/yaml/v3 v3.0.5 golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f - golang.org/x/mod v0.40.0 - golang.org/x/net v0.58.0 - golang.org/x/time v0.15.0 - golang.org/x/tools v0.49.0 + golang.org/x/mod v0.41.0 + golang.org/x/net v0.59.0 + golang.org/x/time v0.16.0 + golang.org/x/tools v0.50.0 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.57.0 + modernc.org/sqlite v1.58.0 ) require ( @@ -110,12 +110,12 @@ require ( require ( cel.dev/expr v0.25.2 // indirect cloud.google.com/go v0.123.0 // indirect - cloud.google.com/go/auth v0.22.0 // indirect + cloud.google.com/go/auth v0.23.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.11.0 // indirect - cloud.google.com/go/monitoring v1.29.0 // indirect - cloud.google.com/go/storage v1.64.0 // indirect + cloud.google.com/go/iam v1.12.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect + cloud.google.com/go/storage v1.65.1 // indirect dario.cat/mergo v1.0.2 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect @@ -133,24 +133,24 @@ require ( github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/aquasecurity/go-version v0.0.1 // indirect github.com/atotto/clipboard v0.1.4 // indirect - github.com/aws/aws-sdk-go-v2 v1.43.4 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.16 // indirect - github.com/aws/aws-sdk-go-v2/config v1.32.35 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.19.34 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.36 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.15 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.28 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.35 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.36 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.106.5 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.5.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.33.4 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.45.4 // indirect - github.com/aws/smithy-go v1.27.6 // indirect + github.com/aws/aws-sdk-go-v2 v1.44.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.40 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.39 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.34.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.46.0 // indirect + github.com/aws/smithy-go v1.28.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/becheran/wildmatch-go v1.0.0 // indirect @@ -170,7 +170,7 @@ require ( github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/containerd/cgroups/v3 v3.1.3 // indirect github.com/containerd/containerd/api v1.11.1 // indirect - github.com/containerd/containerd/v2 v2.3.4 // indirect + github.com/containerd/containerd/v2 v2.3.5 // indirect github.com/containerd/continuity v0.5.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect @@ -181,7 +181,7 @@ require ( github.com/containerd/ttrpc v1.2.8 // indirect github.com/containerd/typeurl/v2 v2.2.3 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect - github.com/docker/cli v29.7.2+incompatible // indirect + github.com/docker/cli v29.8.0+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.5 // indirect github.com/docker/go-connections v0.8.1 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -198,15 +198,15 @@ require ( github.com/gkampitakis/ciinfo v0.3.4 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-restruct/restruct v1.2.0-alpha // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.19 // indirect - github.com/googleapis/gax-go/v2 v2.23.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.20 // indirect + github.com/googleapis/gax-go/v2 v2.24.0 // indirect github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.74 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-version v1.9.0 // indirect @@ -236,8 +236,8 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect - github.com/moby/moby/api v1.55.0 // indirect - github.com/moby/moby/client v0.5.1 // indirect + github.com/moby/moby/api v1.56.0 // indirect + github.com/moby/moby/client v0.6.0 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/signal v0.7.1 // indirect github.com/moby/sys/user v0.4.0 // indirect @@ -299,31 +299,31 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect - go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/otel/sdk v1.44.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.55.0 // indirect + golang.org/x/crypto v0.57.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/term v0.45.0 // indirect - golang.org/x/text v0.41.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/term v0.46.0 // indirect + golang.org/x/text v0.42.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gonum.org/v1/gonum v0.17.0 // indirect - google.golang.org/api v0.292.0 // indirect - google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/grpc v1.83.1 // indirect - google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + google.golang.org/api v0.294.0 // indirect + google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - modernc.org/libc v1.74.4 // indirect + modernc.org/libc v1.75.6 // indirect modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.11.0 // indirect + modernc.org/memory v1.12.1 // indirect ) retract ( diff --git a/go.sum b/go.sum index f94639fb4..99c006c5d 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,8 @@ cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0c cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.22.0 h1:Xp9wAKkLoeaYb5pYZZoQGz4E9sdPxIbzS3gywZE3ciQ= -cloud.google.com/go/auth v0.22.0/go.mod h1:M9o2Oz+YI2jAfxewJgb1vyI3vceHF+eohmxyzmrl+9s= +cloud.google.com/go/auth v0.23.2 h1:pxSCpfiji41hpzpPdMCftEUCezpgpqmmDdYiAjCKXxo= +cloud.google.com/go/auth v0.23.2/go.mod h1:4DhBRcqvtljQN3dJ57qtqbib5ZGCYE5f2crfiiC2EM0= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -46,14 +46,14 @@ cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCB cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= -cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= -cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= -cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k= -cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= +cloud.google.com/go/iam v1.12.0 h1:Aki3bX9aHUDKPHfnRJfDcTdVedvy6quGBQcTqx3DRXk= +cloud.google.com/go/iam v1.12.0/go.mod h1:FEZ4lXpADAC2AIpQY7LANNjjwyQ2jK439CI2VaD+sLY= +cloud.google.com/go/logging v1.19.0 h1:NCqhdVUg3wQ8Cobdf16FDSuTGi3+6+hdSBHrY5TsR6Q= +cloud.google.com/go/logging v1.19.0/go.mod h1:i40NZCHC9Gqvod4yE+yQfDWwlgwW/SrshkkGibCHxcA= cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= -cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY= -cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -63,8 +63,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.64.0 h1:KLpxI/oX9LxeRsNqn877d2WyeT3ryiEwnGt8pwcSPZg= -cloud.google.com/go/storage v1.64.0/go.mod h1:lWyAtwvDZHdL3k68WVKbESP6bmWaV23ZJJ/JEVw/ZaQ= +cloud.google.com/go/storage v1.65.1 h1:LRRpBJUTf+OXDPX9jZUKZ3mSLIsz3htG+qUpeNZovyA= +cloud.google.com/go/storage v1.65.1/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM= cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= @@ -140,14 +140,14 @@ github.com/anchore/go-rpmdb v0.2.0 h1:ixcyMg/guhBL9XjBVwhTv9CG48xO07Qkg2uDX2+zx/ github.com/anchore/go-rpmdb v0.2.0/go.mod h1:ATsRlpCXstnoYfzqBfhwGw0U2dolx1BBQ9rAU8jWBAw= github.com/anchore/go-struct-converter v0.2.0-rc2 h1:q+859fW2/jbHJHB2etbNfRlFwYpknyvbqqk1hUdamQ4= github.com/anchore/go-struct-converter v0.2.0-rc2/go.mod h1:cDBA5vhcR62nXWo8QH9/Kk2807o65ISaHPNPX66L+Uw= -github.com/anchore/go-sync v0.1.1 h1:91SZ+YqUIIHmf2jPAYZPuHqM/ZqnK1pVJDWtET6+AJE= -github.com/anchore/go-sync v0.1.1/go.mod h1:3haGsk2BnaoVhsfqae8Kd0FKIAILE1Hw69P4Xo5iVBw= +github.com/anchore/go-sync v0.1.2 h1:RDD5RCanrWqhbtBulqPSZpERadUlgcYjpybYzYd3Uk8= +github.com/anchore/go-sync v0.1.2/go.mod h1:grjZH059bCHM6f6jh+Y7VYvX/q8YcjkDJqG6nplHGkE= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= github.com/anchore/packageurl-go v0.2.0 h1:CkrM4RMUwrEGAiE1OVlxaZNzWj0TuHRey7o4T/EAErk= github.com/anchore/packageurl-go v0.2.0/go.mod h1:2JCgOQMIsqZ7TmliXG4PnUthPJAKE3mWQbsW2XHjAOE= -github.com/anchore/stereoscope v0.3.1 h1:SFLeRyi+3L0fVDTcd4sjT6Xalvi9pju7DegeUMmxna8= -github.com/anchore/stereoscope v0.3.1/go.mod h1:KHLeUz03/lpsxM8jvs7gFSWRHCXB9MLnxpusCIPe3ME= +github.com/anchore/stereoscope v0.3.2 h1:A2FDMSU56m3rErO1e0CtWAVcDS8tc2iX4UXTw1hyPnI= +github.com/anchore/stereoscope v0.3.2/go.mod h1:/IUDVDka+4gufc8kDw+JO3gFl9PkpCjJ2r3bd9UWouM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= @@ -169,42 +169,42 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/aws/aws-sdk-go-v2 v1.43.4 h1:b9FTvbRwy+JCsfp2Wp6wV/KbOx3Aj7nkoFb2cRX0IhE= -github.com/aws/aws-sdk-go-v2 v1.43.4/go.mod h1:70vwSy16txshwG+g55WkpgPKDIByzHI8ccBsOteo3bQ= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.16 h1:aiuaKlDweRC5qExJondpWjOgyzMHpofpwspGXUtwn4c= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.16/go.mod h1:nG/LOlmox9BDe9HvQnXWzgcK8uKbgBMZ/Hp5pVt/21I= -github.com/aws/aws-sdk-go-v2/config v1.32.35 h1:UEzXuET8E42lxBPijuACu/tEK7v5lFPlk0Q+GT5WD9E= -github.com/aws/aws-sdk-go-v2/config v1.32.35/go.mod h1:KaMtJpFa2JlL2BStjjHQVwQpzZEmw+ND/EgVrfFoo2g= -github.com/aws/aws-sdk-go-v2/credentials v1.19.34 h1:y6GkSmcv5myd1ngrYbGmiLlwQqB6TQhOuN/tbSSuWDY= -github.com/aws/aws-sdk-go-v2/credentials v1.19.34/go.mod h1:w3dTcnDVoQIewjo7JG45hduAToikiIFLC4FIO7fndvw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.35 h1:+S7kbJoLDDQ5tE+lHrUBgMkzC8NLgsaioS2F3dVoFAE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.35/go.mod h1:Ak7xXviIARfFdNUJ9Etb0bdVDt/KAvKjMGJVLWXDzik= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35 h1:kzVuGlatQtYinwBJEEyLAbggepCoavosiaHHX9+fD+c= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35/go.mod h1:0yLx0yEI+SfqeJMPvOtIEFoZbiQYXMGszBueiutQyaI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35 h1:WK6CjihTuLisCjSKKbildJ79sGZZgbBz3iNa7VsKIhU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35/go.mod h1:KYleN57luLoe97R7vTnx8PMcVrr9gAcRECtOjl91DNg= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.36 h1:jbGY4CXLzZElOXgGsexlC3Hi+3YM0rSmk4opFXKqg/k= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.36/go.mod h1:uBu/9aKsS/UQGc72RAt3y54kjgYQxmhut8ZD2dXCDNE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.15 h1:JJLBQxwY+AFwuPAi5ivGc1ChnTdUt4cXMv7e76m2c/Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.15/go.mod h1:lQknBIe78MVL0cQOQDlag8KGflMbMEVFx9mB6O8ENvk= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.28 h1:Q1TF1J9jVD+vFo0LzNnmNdQ9EAt52TS+MQlq9Ir+Yxo= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.28/go.mod h1:4KqXXC/p1hrotmouDFbrRoWaLy962b9PMUReCG6+uWo= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.35 h1:BBEElKh4a+rKshvjrfpajTe9CbpZvrbb4Jkg2PB7RzA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.35/go.mod h1:zaZk983w//8beSruBVec/mr4CmDwgZitW/qzGhAAX0g= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.36 h1:EUIwBoN+q7UmhAejxgD27APiRjh1vwCFo53gSqdT0BM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.36/go.mod h1:6u00gmlTGR6W0b2k9NBrld7MnOEmf1Spqx0VVt6AqyE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.106.5 h1:HpN6GgZ3T8pSvRp81ZsgumNjlvRsa+9M0ZL2o6W4uLY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.106.5/go.mod h1:5FTZoQxhmLEiCAtYVk6V+t0iS/B5yGZVLZ3Wq5FDJZI= -github.com/aws/aws-sdk-go-v2/service/signin v1.5.4 h1:cOJELVNrq5Q3Udry2GLuHUM7MhwpeaQRdYaoa6GI/yI= -github.com/aws/aws-sdk-go-v2/service/signin v1.5.4/go.mod h1:f4LxzKBtaTxD7xh3PiVg3CE1tchQemfmghaJr+NbK2c= -github.com/aws/aws-sdk-go-v2/service/sso v1.33.4 h1:AMW7a7S8iQaHjBYZdU3PCq4GKRPijTPRAc7e6XtEThY= -github.com/aws/aws-sdk-go-v2/service/sso v1.33.4/go.mod h1:QQNsFV1DVXoXcZt18FS8lI8rtUrlDyAuWZLQ5shunv4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.4 h1:AsbZcJAQPRmHDJG8K1N0pof/1zPWjVT8TFlTWuGLSvo= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.4/go.mod h1:6imqztH0//t0mKbl6yWl7swSEl7F/w32oAmqB3vP1ag= -github.com/aws/aws-sdk-go-v2/service/sts v1.45.4 h1:w/AryDYMjSUANSQ2uoZxJovUsMTwWJNTv3IMex30Y+4= -github.com/aws/aws-sdk-go-v2/service/sts v1.45.4/go.mod h1:WeBiAa67azG7Su9Vf+ChGDBLiAozJCXzdjXiPBUwtbc= -github.com/aws/smithy-go v1.27.6 h1:0zjT8jgK3jbrTT7JJ3EE6JsMhX8JTrZ+f1sEndYDXrA= -github.com/aws/smithy-go v1.27.6/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2 v1.44.0 h1:4IbaHhtzy+4h37z4JQyO9a2QsiCml3CNYHtq5hIHigo= +github.com/aws/aws-sdk-go-v2 v1.44.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= +github.com/aws/aws-sdk-go-v2/config v1.32.40 h1:lAVC9gMmKusmqDRe32dPtgKl/BWvJmMJoWELKHCAObw= +github.com/aws/aws-sdk-go-v2/config v1.32.40/go.mod h1:8xOJLbe/hOj1g4PVsfJYV7O2byq+UGET1onDdUgbwqc= +github.com/aws/aws-sdk-go-v2/credentials v1.19.39 h1:XOg8LC3Kgnsa3WiPQjc7Bi8k5IBN92cPYfIV9XMFss0= +github.com/aws/aws-sdk-go-v2/credentials v1.19.39/go.mod h1:GonTDBQ+mTpCVNwaHjj0PagspfrYYMEqOx7FehoEP/I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40 h1:r5aGipEVgI9aT/tAGjdrPbDQvIAKdTrS3rUPQtG4Rmo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40/go.mod h1:vOD3CnPxAdkL6MWZeROkZsTlskklMFfgVFkHzx/oZpY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40 h1:UIXlbijuB2XK1Kr57fo8iIxCuaSHJzwZ1uo+2tbEYIk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40/go.mod h1:wcEsL6jscjZjVUinb0Q5qD/GXOG1yT3GNfmT9HuDwzU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 h1:xLQVRDs2NddDmK9BEyh5KSlJ1Gpy5/GIJXrV6WcVGAE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40/go.mod h1:XRXnpFVFGLaEVK+olDdFIM1vNa04ETW452oFGEPUxAo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41 h1:nv/ILuCY0yXACzMQwvtt/HbqDDjemZiI0AeDbxGQlnU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41/go.mod h1:dzvOSpxaPqQ3j0xS6Lc1vyVuWW0RBj7s/QqYpzu3Q/0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0 h1:U8/A0RRBaEspzH1uul3JHLbypXwEGUkRkvoT9f0ATcM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.10.0/go.mod h1:UELStX5KwtJNtQxa+UuF8dc3z4UYc40e8yHYJSozNwY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 h1:gr3Fw1cxZXNCdeo/lQ7isHEHzvHVM7z75qb2zW9aMjw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40/go.mod h1:8z/9CmfnQhiuXD7Ykbcg4a/whSWsniE0ODSx9uwVzfk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41 h1:Q9DIKDuJix/oJnQxFpQ26L0EwVa/YNo4k2kbktrjQjE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.41/go.mod h1:x+TuqkOIG1SZS0+yN54sExGA9ZpjhPO6vPdYnpTFX1M= +github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0 h1:Yp+x5PKXEmoqHsgP/pAkBy5Tyq1UlXAzM0OInh0vxWw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.108.0/go.mod h1:locV6DtXyp7Xzr2BG6jtsbeBi3YAWJ/CY4xUThYmIwQ= +github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 h1:agcr0j8YeFEzdXNo17Rg9MbbjLRjrimabwNtji4e+lU= +github.com/aws/aws-sdk-go-v2/service/signin v1.6.0/go.mod h1:qU5PxgQ4JiUOOMotzfO3+5oUda5W+8JDVKyLQqlrJik= +github.com/aws/aws-sdk-go-v2/service/sso v1.34.0 h1:FxaN8/sn61DTXNI6Gt678tFJUY8iUsCchm6Y/F/RjaA= +github.com/aws/aws-sdk-go-v2/service/sso v1.34.0/go.mod h1:vu4OY6s8LJtT8BtYG2LD6BGSZMptkYn3o5hvCPB22jc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0 h1:crWKPeGYTBTuBxQ3p73kjfJvt4brUIsr+Fuypko8FxY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0/go.mod h1:HjjZVhaBz0JBR/kbWKThmNDhFKS7y6EURuk493tJk9Y= +github.com/aws/aws-sdk-go-v2/service/sts v1.46.0 h1:IZ63JdogSNNjex/jsODNv7jGDcO/xJYd9FsgyfCsp1g= +github.com/aws/aws-sdk-go-v2/service/sts v1.46.0/go.mod h1:I+rwAf3spG5dITBaAo3xXRowk8kiOhtU1kYxfvCTC44= +github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= +github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= @@ -294,8 +294,8 @@ github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6 github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= github.com/containerd/containerd/api v1.11.1 h1:h8nfoDW9+fNsC/9TwiAHj8B1GzXKtR4eFtkhi/X5RLU= github.com/containerd/containerd/api v1.11.1/go.mod h1:CaQFRu+N1MtbgL6JDOJLUB1hCKESU1lD6MuTJhgtdlw= -github.com/containerd/containerd/v2 v2.3.4 h1:c2PJo/9UGVdiiw8SwrxuLxWGY+9b3jQ6Xp9zntneIvI= -github.com/containerd/containerd/v2 v2.3.4/go.mod h1:a30D8fWZJ1Uzx/2WpjLbLsxBkq9He41pe8ENW+QZ3LY= +github.com/containerd/containerd/v2 v2.3.5 h1:9MYlI81gUcOZ0WsCkSMtvOU7rTR3hqAoa2eCzhoLlkA= +github.com/containerd/containerd/v2 v2.3.5/go.mod h1:RXDyLPaI3zoO7dFdAW9/54W4cix+z3A6larieufC9mg= github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg= github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -337,8 +337,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= -github.com/docker/cli v29.7.2+incompatible h1:dlkwallR8XqfeVnA2ELEhdwvb4lsSwuB4IgsG8Q9cLY= -github.com/docker/cli v29.7.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v29.8.0+incompatible h1:ih0c2jq/nN7QfES8zIfwSzIhRScjs4ehER+kZ60aeSk= +github.com/docker/cli v29.8.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY= github.com/docker/docker-credential-helpers v0.9.5/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c= github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M= @@ -432,8 +432,8 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-restruct/restruct v1.2.0-alpha h1:2Lp474S/9660+SJjpVxoKuWX09JsXHSrdV7Nv3/gkvc= @@ -507,8 +507,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-containerregistry v0.22.0 h1:eGbCiPeYxAH/7WLLq6zTBALP0tUIFsoyRauhxXDJ53I= -github.com/google/go-containerregistry v0.22.0/go.mod h1:bJR35SK8XgisYmhg/FMQ/5RK0S/XrOAqLBV5/LR2XE0= +github.com/google/go-containerregistry v0.22.1 h1:RZuuSYhTvlDvtsK+NkutoCZ//C0X2ebLK8X8l3ULs84= +github.com/google/go-containerregistry v0.22.1/go.mod h1:bJR35SK8XgisYmhg/FMQ/5RK0S/XrOAqLBV5/LR2XE0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/licensecheck v0.3.1 h1:QoxgoDkaeC4nFrtGN1jV7IPmDCHFNIVh54e5hSt6sPs= github.com/google/licensecheck v0.3.1/go.mod h1:ORkR35t/JjW+emNKtfJDII0zlciG9JgbT7SmsohlHmY= @@ -543,21 +543,21 @@ github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.19 h1:mMOE7DN2+p76/EdIrmAy9B9bH+yC4563vmnJ34QR8i4= -github.com/googleapis/enterprise-certificate-proxy v0.3.19/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/enterprise-certificate-proxy v0.3.20 h1:t/xL64VUoN69MuMRQuJETqYGOw4Z9mSRJK9epIEtwFk= +github.com/googleapis/enterprise-certificate-proxy v0.3.20/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= -github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= +github.com/googleapis/gax-go/v2 v2.24.0 h1:myMaPYyF9MecEmvQqMqomIwn9t/4KCZN9qnwsS76wlg= +github.com/googleapis/gax-go/v2 v2.24.0/go.mod h1:IaTHBDd7NHxSCiu0vEs8pQZu4dGZrWwuSoxCnk16OFM= github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0= github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E= github.com/gookit/color v1.2.5/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg= github.com/gookit/color v1.6.1 h1:KoTnDxJPRgrL0SoX0f8rCFg2zI0t4E3GZZBMo2nN8LU= github.com/gookit/color v1.6.1/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= -github.com/gpustack/gguf-parser-go v0.26.0 h1:klBqNj3w5oyY2GWC6/aQJ6MtPUYtHyYRlEyzEf00cxA= -github.com/gpustack/gguf-parser-go v0.26.0/go.mod h1:y4TwTtDqFWTK+xvprOjRUh+dowgU2TKCX37vRKvGiZ0= +github.com/gpustack/gguf-parser-go v0.26.3 h1:F63PlUPIW56HGU5k4Crv0JAG9ojtz/TuwzgMYlb82Ec= +github.com/gpustack/gguf-parser-go v0.26.3/go.mod h1:y4TwTtDqFWTK+xvprOjRUh+dowgU2TKCX37vRKvGiZ0= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.74 h1:mymLUKThnV9wFvogOK8NnsMP9/vlhnjXY98gr2QIGW8= github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.74/go.mod h1:Bh9qYL8ehmDxSg14Tk8oxFCP90XHfs6NxV1D84884xA= @@ -570,8 +570,8 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.8.8 h1:sRakhf+EH6s0LZLO2IgBwZ6hAFp+fZOZMNREwVELV80= -github.com/hashicorp/go-getter v1.8.8/go.mod h1:fqFlibKpwfns/s4oljLB3upJspyfFLQhS7031PlfUDc= +github.com/hashicorp/go-getter v1.8.9 h1:1AOTMUmz/S/GuqOTE6+bg6nwPXTEbKr3Tc/T/lVS7is= +github.com/hashicorp/go-getter v1.8.9/go.mod h1:qI7vH/m552bXutKe4UkWptOJl4bo2KeO/CSOoh4s0ps= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -643,8 +643,8 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= -github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -721,10 +721,10 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= -github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= -github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw= -github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM= +github.com/moby/moby/api v1.56.0 h1:GQzua3NA599ASSIICx0iFgiJeO9YkdDARvQsm23ZZuQ= +github.com/moby/moby/api v1.56.0/go.mod h1:sZ+THbVWkjOmBPPfbnzdD/G1LuIexWhqlSHHPTDQ1Uk= +github.com/moby/moby/client v0.6.0 h1:AJjEB21QPbXSXjDsZorFBoDZPhMrfbpaPLgSMAW9Bgs= +github.com/moby/moby/client v0.6.0/go.mod h1:OCo00wNRyA3m4lmJ228W3JbyCN4ZNNYjpOXiJydBdcQ= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= @@ -1002,20 +1002,20 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.6 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0 h1:hqxVTu/GtBF+vJ8d1fzW7fRxZFvgoDjWcxwwCaFDYpU= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0/go.mod h1:z5fVEF4X5v0ESvlJqBrrFlBVoj5EQuefZpzsu7R+x5Q= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= -go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -1041,8 +1041,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1082,8 +1082,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= -golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1128,8 +1128,8 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1161,8 +1161,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1237,13 +1237,13 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= -golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1254,13 +1254,13 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1316,8 +1316,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= -golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= +golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1358,8 +1358,8 @@ google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdr google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= -google.golang.org/api v0.292.0 h1:Ewiwo/GTtiaPZSNAZQUcWLh8AYDEoPmIXyJfeoTSMHU= -google.golang.org/api v0.292.0/go.mod h1:07kjmMnFGm2RQuCza2EZM/5N68G/fVvFb1xKjWqoFA0= +google.golang.org/api v0.294.0 h1:8gASjJxdtcIieB3OqbkLcF0FfbXVNqKtU5iozD1ssvA= +google.golang.org/api v0.294.0/go.mod h1:02qB8+Ox1ZFzcaKFMguy1nQLJmSIyvV6Ff4txJEXtl4= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1430,12 +1430,12 @@ google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 h1:YJjbgu+dkp5kUJLfpMyCLfBIWZb/FcJyuLeo1gVBOuo= -google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:RRHjglSYABVCWpQ7USCpdfhcd9t4PkajvVwyynZizTc= -google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= -google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d h1:C9v1o0/4quuhOAfmRXA2j+we0PqZIp8traLdeogF3Ms= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d/go.mod h1:Wz2wFJntZFmLGo7pLDXZ3wYk5hyc0Mb+SkHhDDXT+lU= +google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d h1:QwnJwPte4XXAkhPu26LTDIahnsMSUV0kK8HkxbC+Pc4= +google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d/go.mod h1:WRrQ7/7N19PypuT0fxLOL5Lq0waoiRri4FbtHDEKrGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1463,8 +1463,8 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= -google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1479,8 +1479,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1514,30 +1514,30 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM= howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= -modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= -modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= -modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= -modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= -modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus= +modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg= -modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/sqlite v1.58.0 h1:38u40/bwkfM7f0Myhosl+SEMltSDxnGdQf8o6Kjmys0= +modernc.org/sqlite v1.58.0/go.mod h1:rsD2CckafgObKC4DhBlGBf+RiHxkc3hINGt1Xw32tVY= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/internal/file/reader_size.go b/internal/file/reader_size.go new file mode 100644 index 000000000..9637ac056 --- /dev/null +++ b/internal/file/reader_size.go @@ -0,0 +1,66 @@ +package file + +import "io" + +// ReaderSize reports the number of bytes behind r. The bool is false when that cannot be determined, and +// callers must branch on it rather than treating the zero as a size. +// +// Binary parsers read sizes out of the files they are parsing and allocate against them. Those sizes are +// 32- and 64-bit fields under the control of whoever produced the file, so a truncated or hostile input +// can declare gigabytes that are not there. The read would fail afterwards either way; what matters is +// refusing before the allocation, and that needs a real byte count to weigh the claim against. +// +// The bool is the whole point of the signature. An earlier version returned 0 for every failure mode, and +// a caller that forgot to special-case it got a bound of "nothing is allowed" or, more often, silently +// fell back to an unbounded path. Making the caller name the failure keeps that from happening by +// omission. +// +// Both *bytes.Reader and *io.SectionReader answer directly. A file handle only has Seek, so fall back to +// a save-and-restore seek, which leaves the caller's cursor where it found it. That fallback is not +// atomic: it moves the cursor and puts it back, so a reader being read concurrently can observe the +// intermediate position. +// +// Neither answer is taken on faith, because a reader can report a length it cannot deliver: an +// *io.SectionReader answers with the length it was constructed with, and debug/elf and friends construct +// theirs with a nominal one, so io.NewSectionReader(r, 0, 1<<63-1) would otherwise report a bound that +// permits everything as if it had been measured. The last byte is read back to confirm the count, which is +// one ReadAt whatever the size, and the read is what the caller was going to weigh anyway. +// +// A wrapper that embeds io.ReaderAt as an interface promotes only ReadAt, so it answers no size here +// however sizable the reader underneath it is. A type meant to be bounded needs to forward Size() itself. +func ReaderSize(r io.ReaderAt) (int64, bool) { + size, ok := reportedSize(r) + if !ok { + return 0, false + } + // ReadAt may return io.EOF alongside a full read, so the count is what says the byte is there + var last [1]byte + if n, _ := r.ReadAt(last[:], size-1); n != len(last) { + return 0, false + } + return size, true +} + +// reportedSize is what the reader says about itself, before ReaderSize checks whether it can back it up. +func reportedSize(r io.ReaderAt) (int64, bool) { + if sr, ok := r.(interface{ Size() int64 }); ok { + size := sr.Size() + return size, size > 0 + } + s, ok := r.(io.Seeker) + if !ok { + return 0, false + } + cur, err := s.Seek(0, io.SeekCurrent) + if err != nil { + return 0, false + } + size, err := s.Seek(0, io.SeekEnd) + if err != nil { + return 0, false + } + if _, err := s.Seek(cur, io.SeekStart); err != nil { + return 0, false + } + return size, size > 0 +} diff --git a/internal/file/reader_size_test.go b/internal/file/reader_size_test.go new file mode 100644 index 000000000..f090225cc --- /dev/null +++ b/internal/file/reader_size_test.go @@ -0,0 +1,140 @@ +package file + +import ( + "bytes" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReaderSize(t *testing.T) { + tests := []struct { + name string + r io.ReaderAt + want int64 + wantOK bool + }{ + { + name: "bytes.Reader answers directly", + r: bytes.NewReader(make([]byte, 1234)), + want: 1234, + wantOK: true, + }, + { + // an empty reader is not a size to bound against, so it reports not-ok along with everything + // else that cannot be measured + name: "empty bytes.Reader", + r: bytes.NewReader(nil), + want: 0, + }, + { + name: "SectionReader over a real length", + r: io.NewSectionReader(bytes.NewReader(make([]byte, 500)), 100, 300), + want: 300, + wantOK: true, + }, + { + name: "seek-only reader falls back to seeking", + r: seekOnly{bytes.NewReader(make([]byte, 4096))}, + want: 4096, + wantOK: true, + }, + { + name: "reader with neither Size nor Seek cannot be measured", + r: readAtOnly{bytes.NewReader(make([]byte, 4096))}, + want: 0, + }, + { + // the shape debug/elf, debug/pe and debug/macho build internally. Taking this at its word hands + // back a bound that permits everything, reported as if it had been measured. + name: "SectionReader over a nominal length reports what it can deliver", + r: io.NewSectionReader(bytes.NewReader(make([]byte, 500)), 0, 1<<63-1), + want: 0, + }, + { + // a Size() a type simply got wrong is the same failure: the bytes are what decides + name: "a size the reader cannot back up is refused", + r: overstatedSize{ReaderAt: bytes.NewReader(make([]byte, 10)), size: 1 << 30}, + want: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + size, ok := ReaderSize(tt.r) + assert.Equal(t, tt.want, size) + assert.Equal(t, tt.wantOK, ok) + }) + } +} + +func TestReaderSize_FileHandle(t *testing.T) { + path := filepath.Join(t.TempDir(), "f.bin") + require.NoError(t, os.WriteFile(path, make([]byte, 7777), 0o600)) + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + + // the cursor has to come back where it was found: the caller may be mid-read, and an *os.File is the + // one input here that carries a shared position + _, err = f.Seek(101, io.SeekStart) + require.NoError(t, err) + + size, ok := ReaderSize(f) + assert.True(t, ok) + assert.Equal(t, int64(7777), size) + + at, err := f.Seek(0, io.SeekCurrent) + require.NoError(t, err) + assert.Equal(t, int64(101), at, "measuring the file must not move the caller's cursor") +} + +// TestReaderSize_WrapperMustForwardSize pins the shape that made every bound expressed against this +// helper go silently inert. A type that embeds io.ReaderAt as an interface promotes only ReadAt, so it +// answers 0 here however sizable the reader underneath is, and a caller that treats 0 as "fall back to +// unbounded" then stops bounding anything. Forwarding Size() is what fixes it, and this is the assertion +// that a new wrapper has to satisfy. +func TestReaderSize_WrapperMustForwardSize(t *testing.T) { + inner := bytes.NewReader(make([]byte, 2048)) + + size, ok := ReaderSize(embedsReaderAt{inner}) + assert.False(t, ok, "a wrapper that only embeds the interface cannot be measured") + assert.Equal(t, int64(0), size) + + size, ok = ReaderSize(forwardsSize{inner}) + assert.True(t, ok, "a wrapper meant to be bounded has to forward Size itself") + assert.Equal(t, int64(2048), size) +} + +type seekOnly struct{ inner *bytes.Reader } + +func (s seekOnly) ReadAt(p []byte, off int64) (int, error) { return s.inner.ReadAt(p, off) } +func (s seekOnly) Seek(off int64, whence int) (int64, error) { + return s.inner.Seek(off, whence) +} + +type readAtOnly struct{ inner *bytes.Reader } + +func (r readAtOnly) ReadAt(p []byte, off int64) (int, error) { return r.inner.ReadAt(p, off) } + +type embedsReaderAt struct{ io.ReaderAt } + +type forwardsSize struct{ io.ReaderAt } + +func (f forwardsSize) Size() int64 { + size, _ := ReaderSize(f.ReaderAt) + return size +} + +// overstatedSize answers with a size larger than the bytes behind it, the way a wrapper that returns a +// declared length rather than a measured one would. +type overstatedSize struct { + io.ReaderAt + size int64 +} + +func (o overstatedSize) Size() int64 { return o.size } diff --git a/internal/spillbuf/buffer.go b/internal/spillbuf/buffer.go new file mode 100644 index 000000000..814c16ba3 --- /dev/null +++ b/internal/spillbuf/buffer.go @@ -0,0 +1,330 @@ +// Package spillbuf provides a sparse, offset-addressed byte buffer that keeps a bounded amount in memory +// and spills the rest to a temp file. +// +// It exists for the shape where output arrives out of order, at offsets the input itself declares: a +// decompressor placing extents, an archive rebuilding a file from chunks. A plain []byte cannot do that +// without either pre-sizing to a length the input claims or growing to the furthest offset it names, and +// both hand a hostile input an allocation knob. +// +// The buffer reports only what it actually stored. Size is the contiguous run written from offset zero, +// and a read past it is io.EOF rather than the zeros an unwritten region would otherwise hand back. That +// distinction is the whole point: sparse storage serves holes as zeros for free, so a reader that reported +// a length covering them would let a write offset stand in for real output. Anything sizing an allocation +// against this reader (debug/elf's saferio does exactly that) would then allocate bytes nothing produced. +// +// Allocation is bounded by the memory limit rather than by how much is written: filling 1MB and filling +// 64MB cost about the same, because everything past the limit goes to the file. Only memory is bounded +// here: how much ends up on disk is whatever the caller writes, so bounding that stays the caller's job. +package spillbuf + +import ( + "errors" + "fmt" + "io" + "math" + "os" + "slices" + + "github.com/anchore/syft/internal/tmpdir" +) + +// DefaultMemLimit is how much a buffer keeps in memory before it spills, when the caller does not say. +// Small on purpose: the memory tier saves a temp file for the common small case, it is not a place to +// hold output. A limit large enough to matter is a limit large enough to be an amplification path. +const DefaultMemLimit int64 = 1 << 20 // 1MB + +// ErrNoTempDir is returned when a buffer needs to spill and has nowhere to spill to. +var ErrNoTempDir = errors.New("no temp dir available to spill to") + +// Buffer is a sparse, offset-addressed buffer. The zero value is not usable; call New. +// +// Storage is two tiers with a fixed boundary: bytes below memLimit live in mem, bytes at or past it live +// in file. Nothing ever moves between them, so an offset belongs to exactly one tier for the life of the +// buffer and the only case needing care is a range that straddles the boundary. +// +// Not safe for concurrent use. Every consumer writes from a single goroutine, and guarding it would cost +// each write an uncontended lock for no reader. +type Buffer struct { + td *tmpdir.TempDir + memLimit int64 + + // mem holds [0, memLimit). Grown geometrically to fit what has been written, never past the limit. + mem []byte + + // file holds [memLimit, ...), created on the first write that reaches there, so a buffer that stays + // small never touches disk. remove deletes it on Close. + file *os.File + remove func() + + // written is the set of ranges stored, sorted by start and merged, so runs of adjacent writes collapse + // to one entry rather than growing per call. Everything the buffer reports comes from it. + written []extent + + closed bool +} + +// extent is a half-open byte range that has been written. +// +// Unexported on purpose. What a caller needs to know about the contents is answered by Size and FirstGap; +// handing out the bookkeeping invites re-deriving those at the call site and getting them subtly wrong, +// which is the bug this package exists to prevent. +type extent struct { + start, end int64 +} + +// Option configures a Buffer. +type Option func(*Buffer) + +// WithMemLimit sets how many bytes are held in memory before the buffer spills to disk. Zero sends every +// write to the temp file. Negative is treated as zero. +// +// Peak allocation is a small multiple of this, since growing the tier holds the old copy alongside the +// new one and the new one may be rounded up past what was asked for. +func WithMemLimit(n int64) Option { + return func(b *Buffer) { + b.memLimit = max(n, 0) + } +} + +// New returns a buffer that spills into td. +// +// td may be nil only when every write is known to stay inside the memory limit; a write past it then +// fails with ErrNoTempDir rather than allocating. Nothing is created on disk until a write needs it. +func New(td *tmpdir.TempDir, opts ...Option) *Buffer { + b := &Buffer{td: td, memLimit: DefaultMemLimit} + for _, opt := range opts { + opt(b) + } + return b +} + +// split divides a range of n bytes starting at off across the two tiers, reporting how many bytes fall in +// each. The file portion starts at off+inMem. +func (b *Buffer) split(off, n int64) (inMem, inFile int64) { + if off >= b.memLimit { + return 0, n + } + inMem = min(n, b.memLimit-off) + return inMem, n - inMem +} + +// fileOffset translates a buffer offset at or past the limit into an offset within the spill file. +func (b *Buffer) fileOffset(off int64) int64 { return off - b.memLimit } + +// WriteAt stores p at off. Writes may land anywhere, in any order, and may overlap. A write that fails +// leaves the buffer as it was. +func (b *Buffer) WriteAt(p []byte, off int64) (int, error) { + if b.closed { + return 0, fmt.Errorf("spillbuf: write to a closed buffer: %w", os.ErrClosed) + } + if off < 0 { + return 0, fmt.Errorf("spillbuf: negative offset %d", off) + } + if len(p) == 0 { + return 0, nil + } + // subtraction form so off+len(p) cannot wrap into a small in-range end + if off > math.MaxInt64-int64(len(p)) { + return 0, fmt.Errorf("spillbuf: write of %d bytes at %d overflows", len(p), off) + } + + inMem, inFile := b.split(off, int64(len(p))) + // the fallible half goes first: once the memory tier is stamped the old bytes are gone, so a file + // write that fails afterward would report failure over a buffer it had already changed + if inFile > 0 { + if err := b.writeFile(p[inMem:], b.fileOffset(off+inMem)); err != nil { + return 0, err + } + } + if inMem > 0 { + b.writeMem(p[:inMem], off) + } + + b.record(extent{start: off, end: off + int64(len(p))}) + return len(p), nil +} + +func (b *Buffer) writeMem(p []byte, off int64) { + if need := off + int64(len(p)); int64(len(b.mem)) < need { + // geometric, capped at the limit. Sizing to exactly what each write needs looks tidier and is + // quadratic: a caller streaming a block in 32KB chunks reallocates and copies the whole tier every + // chunk, so filling 1MB costs ~16MB of allocation. + grown := min(b.memLimit, max(need, 2*int64(len(b.mem)))) + // sized to exactly the geometric target rather than grown through append: append rounds the new + // capacity up past what was asked for, and the tier is already capped, so the slack is pure waste + mem := make([]byte, grown) + copy(mem, b.mem) + b.mem = mem + } + copy(b.mem[off:], p) +} + +func (b *Buffer) writeFile(p []byte, off int64) error { + if b.file == nil { + if b.td == nil { + return ErrNoTempDir + } + f, remove, err := b.td.NewFile("syft-spill-*.bin") //nolint:gocritic // removal happens in Close + if err != nil { + return fmt.Errorf("spillbuf: unable to create spill file: %w", err) + } + b.file, b.remove = f, remove + } + if _, err := b.file.WriteAt(p, off); err != nil { + return fmt.Errorf("spillbuf: unable to write to spill file: %w", err) + } + return nil +} + +// ReadAt returns bytes this buffer actually stored. A read starting at or past Size is io.EOF, and one +// running past it is short, because everything beyond is a hole the buffer never held. +func (b *Buffer) ReadAt(p []byte, off int64) (int, error) { + if b.closed { + return 0, fmt.Errorf("spillbuf: read from a closed buffer: %w", os.ErrClosed) + } + if off < 0 { + return 0, fmt.Errorf("spillbuf: negative offset %d", off) + } + if len(p) == 0 { + return 0, nil + } + size := b.Size() + if off >= size { + return 0, io.EOF + } + + // bounded by Size, so every byte in [off, off+want) was written and both tiers really hold theirs + want := min(int64(len(p)), size-off) + inMem, inFile := b.split(off, want) + + // guarded rather than relying on a zero-length copy: when off is past the limit the slice expression + // itself is out of range, since mem is only as long as what was written into it + if inMem > 0 { + copy(p[:inMem], b.mem[off:off+inMem]) + } + if inFile > 0 { + if err := readFullAt(b.file, p[inMem:want], b.fileOffset(off+inMem)); err != nil { + return int(inMem), fmt.Errorf("spillbuf: unable to read spill file: %w", err) + } + } + + if want < int64(len(p)) { + return int(want), io.EOF + } + return int(want), nil +} + +// readFullAt fills p from r. A short read is an error rather than a short return: the caller has already +// bounded the range by what was written, so the file failing to deliver means it is not what we left. +func readFullAt(r io.ReaderAt, p []byte, off int64) error { + for read := 0; read < len(p); { + n, err := r.ReadAt(p[read:], off+int64(read)) + read += n + if err != nil { + return err + } + } + return nil +} + +// Size reports the contiguous run written from offset zero, which is the length this buffer is willing to +// stand behind. Deliberately not the furthest offset written: see the package doc for why the two are not +// interchangeable. +// +// record keeps the set merged and sorted, so the run from zero is at most the first two entries and the +// loop stops there: the cost this adds to every ReadAt is effectively constant, not a scan of the set. +func (b *Buffer) Size() int64 { + var covered int64 + for _, e := range b.written { + if e.start > covered { + break + } + covered = max(covered, e.end) + } + return covered +} + +// FirstGap returns the offset of the earliest unwritten run of at least size bytes lying entirely within +// [0, within), and whether there is one. Callers that place output at offsets of their own choosing use it +// to fill what they left behind, without needing the buffer's bookkeeping. +// +// The arithmetic is in subtraction form throughout: within comes from caller input and the result is used +// as a write offset, so a wrap here would be a write nobody bounded. +func (b *Buffer) FirstGap(size, within int64) (int64, bool) { + if b.closed || size < 0 || within < 0 { + return 0, false + } + + var at int64 + for _, e := range b.written { + // once the cursor is at the bound there is nothing left to offer, and it only moves forward + if at >= within { + return 0, false + } + // the gap ahead of this extent ends at whichever comes first, the extent or the bound + end := min(e.start, within) + if end > at && end-at >= size { + return at, true + } + at = max(at, e.end) + } + if at <= within && within-at >= size { + return at, true + } + return 0, false +} + +// record merges e into the written set, keeping it sorted by start with no overlapping or adjacent pairs. +func (b *Buffer) record(e extent) { + if e.end <= e.start { + return + } + + // inserted in place rather than appended and re-sorted: the set is already ordered, so a run of + // ascending writes (the common case) stays linear + i, _ := slices.BinarySearchFunc(b.written, e, func(x, y extent) int { + return cmpInt64(x.start, y.start) + }) + b.written = slices.Insert(b.written, i, e) + + merged := b.written[:1] + for _, cur := range b.written[1:] { + last := &merged[len(merged)-1] + if cur.start <= last.end { + last.end = max(last.end, cur.end) + continue + } + merged = append(merged, cur) + } + b.written = merged +} + +func cmpInt64(x, y int64) int { + switch { + case x < y: + return -1 + case x > y: + return 1 + } + return 0 +} + +// Close releases the spill file, if one was ever created, and drops the memory tier. Safe on a nil +// receiver and safe to call more than once. Reads and writes after it fail with os.ErrClosed. +func (b *Buffer) Close() error { + if b == nil { + return nil + } + b.mem = nil + b.written = nil + b.closed = true + if b.file == nil { + return nil + } + err := b.file.Close() + if b.remove != nil { + b.remove() + } + b.file, b.remove = nil, nil + return err +} diff --git a/internal/spillbuf/buffer_test.go b/internal/spillbuf/buffer_test.go new file mode 100644 index 000000000..d71255f75 --- /dev/null +++ b/internal/spillbuf/buffer_test.go @@ -0,0 +1,864 @@ +package spillbuf + +import ( + "bytes" + "errors" + "io" + "math" + "math/rand" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/testutils" + "github.com/anchore/syft/internal/tmpdir" +) + +func newTestBuffer(t *testing.T, opts ...Option) *Buffer { + t.Helper() + b := New(tmpdir.FromPath(t.TempDir()), opts...) + t.Cleanup(func() { _ = b.Close() }) + return b +} + +func mustWrite(t *testing.T, b *Buffer, off int64, p []byte) { + t.Helper() + n, err := b.WriteAt(p, off) + require.NoError(t, err) + require.Equal(t, len(p), n, "WriteAt must report the full write") +} + +// readAll returns everything the buffer is willing to hand back. +func readAll(t *testing.T, b *Buffer) []byte { + t.Helper() + out := make([]byte, b.Size()) + if len(out) == 0 { + return nil + } + n, err := b.ReadAt(out, 0) + require.NoError(t, err, "a read of exactly Size bytes is not short") + require.Equal(t, len(out), n) + return out +} + +func repeat(b byte, n int) []byte { return bytes.Repeat([]byte{b}, n) } + +// --- the core property: report only what was stored ------------------------------------------------- + +// TestSizeIsTheContiguousPrefix is the property the package exists for. A sparse buffer hands its holes +// back as zeros it never stored, so reporting a length that covers them lets a write offset stand in for +// real output: whatever sizes an allocation against this reader then allocates bytes nothing produced. +func TestSizeIsTheContiguousPrefix(t *testing.T) { + tests := []struct { + name string + writes []extent // start is the offset, end-start the length + wantSize int64 + }{ + { + name: "nothing written", + wantSize: 0, + }, + { + name: "one run from zero", + writes: []extent{{0, 100}}, + wantSize: 100, + }, + { + name: "a run that does not start at zero covers nothing", + writes: []extent{{10, 100}}, + wantSize: 0, + }, + { + name: "adjacent runs join", + writes: []extent{{0, 50}, {50, 100}}, + wantSize: 100, + }, + { + name: "a hole stops the prefix", + writes: []extent{{0, 50}, {60, 100}}, + wantSize: 50, + }, + { + name: "the hole is filled later", + writes: []extent{{0, 50}, {60, 100}, {50, 60}}, + wantSize: 100, + }, + { + name: "out of order still resolves", + writes: []extent{{60, 100}, {0, 50}, {50, 60}}, + wantSize: 100, + }, + { + name: "overlapping runs", + writes: []extent{{0, 60}, {40, 100}}, + wantSize: 100, + }, + { + name: "a run fully inside another", + writes: []extent{{0, 100}, {20, 40}}, + wantSize: 100, + }, + { + name: "far placement is not coverage", + writes: []extent{{0, 64}, {1 << 20, 1<<20 + 64}}, + wantSize: 64, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := newTestBuffer(t) + for _, w := range tt.writes { + mustWrite(t, b, w.start, repeat('x', int(w.end-w.start))) + } + assert.Equal(t, tt.wantSize, b.Size(), "Size is the contiguous prefix") + }) + } +} + +// TestReadPastSizeIsEOFNotZeros is the other half of the same property. Sparse storage returns zeros for +// a hole for free; handing those out would make the hole indistinguishable from real content. +func TestReadPastSizeIsEOFNotZeros(t *testing.T) { + b := newTestBuffer(t) + mustWrite(t, b, 0, repeat('A', 64)) + mustWrite(t, b, 4096, repeat('B', 64)) // past a hole + + require.Equal(t, int64(64), b.Size()) + + t.Run("a read starting in the hole is EOF", func(t *testing.T) { + p := make([]byte, 16) + n, err := b.ReadAt(p, 100) + assert.Zero(t, n) + assert.ErrorIs(t, err, io.EOF) + }) + + t.Run("a read starting at the far extent is EOF, not the bytes there", func(t *testing.T) { + p := make([]byte, 64) + n, err := b.ReadAt(p, 4096) + assert.Zero(t, n) + assert.ErrorIs(t, err, io.EOF) + assert.Equal(t, make([]byte, 64), p, "nothing may be copied out") + }) + + t.Run("a read running off the end is short and says so", func(t *testing.T) { + p := make([]byte, 128) + n, err := b.ReadAt(p, 0) + assert.Equal(t, 64, n, "only the stored prefix comes back") + assert.ErrorIs(t, err, io.EOF) + assert.Equal(t, repeat('A', 64), p[:64]) + assert.Equal(t, make([]byte, 64), p[64:], "the rest is untouched") + }) +} + +// TestSparsePlacementIsNotAnAllocationKnob pins the amplification directly: a tiny payload parked at a +// huge offset must not cost memory proportional to the offset. +func TestSparsePlacementIsNotAnAllocationKnob(t *testing.T) { + const far = 512 << 20 // 512MB out + + b := newTestBuffer(t, WithMemLimit(1<<20)) + mustWrite(t, b, 0, repeat('A', 64)) + + allocated := testutils.MeasureAlloc(t, func() { + mustWrite(t, b, far, repeat('B', 64)) + }) + t.Logf("writing 64 bytes at offset %d allocated %d bytes", far, allocated) + assert.Less(t, allocated, uint64(1<<20), + "a far placement must cost its own bytes, not its offset") + assert.Equal(t, int64(64), b.Size(), "and it must not count toward what the buffer will deliver") +} + +// --- tiers ------------------------------------------------------------------------------------------ + +func TestMemoryTierNeverTouchesDisk(t *testing.T) { + dir := t.TempDir() + b := New(tmpdir.FromPath(dir), WithMemLimit(4096)) + t.Cleanup(func() { _ = b.Close() }) + + mustWrite(t, b, 0, repeat('A', 4096)) // exactly the limit + + assert.Equal(t, repeat('A', 4096), readAll(t, b)) + assert.Nil(t, b.file, "a buffer that stays inside the limit must not create a file") + assert.Empty(t, dirEntries(t, dir), "and must leave nothing on disk") +} + +func TestSpillCreatesTheFileOnlyWhenNeeded(t *testing.T) { + dir := t.TempDir() + b := New(tmpdir.FromPath(dir), WithMemLimit(4096)) + t.Cleanup(func() { _ = b.Close() }) + + mustWrite(t, b, 0, repeat('A', 4096)) + require.Empty(t, dirEntries(t, dir)) + + mustWrite(t, b, 4096, repeat('B', 1)) // one byte past the limit + assert.NotNil(t, b.file) + assert.Len(t, dirEntries(t, dir), 1, "exactly one spill file") +} + +func TestWriteStraddlingTheLimit(t *testing.T) { + const limit = 1024 + b := newTestBuffer(t, WithMemLimit(limit)) + + payload := make([]byte, 2048) + for i := range payload { + payload[i] = byte(i % 251) + } + // starts below the limit and ends well past it, so the write is split across both tiers + mustWrite(t, b, limit-512, payload) + + require.Equal(t, int64(0), b.Size(), "nothing was written at zero yet") + mustWrite(t, b, 0, repeat('Z', limit-512)) + + got := readAll(t, b) + require.Len(t, got, limit-512+2048) + assert.Equal(t, repeat('Z', limit-512), got[:limit-512], "the memory-only part") + assert.Equal(t, payload, got[limit-512:], "the straddling part reads back whole") +} + +func TestMemLimitZeroSendsEverythingToDisk(t *testing.T) { + dir := t.TempDir() + b := New(tmpdir.FromPath(dir), WithMemLimit(0)) + t.Cleanup(func() { _ = b.Close() }) + + mustWrite(t, b, 0, repeat('A', 16)) + assert.Equal(t, repeat('A', 16), readAll(t, b)) + assert.Len(t, dirEntries(t, dir), 1, "with no memory tier the first byte spills") + assert.Nil(t, b.mem) +} + +func TestNegativeMemLimitIsTreatedAsZero(t *testing.T) { + b := newTestBuffer(t, WithMemLimit(-1)) + mustWrite(t, b, 0, repeat('A', 8)) + assert.Equal(t, repeat('A', 8), readAll(t, b)) +} + +func TestDefaultMemLimitApplies(t *testing.T) { + b := New(nil) + t.Cleanup(func() { _ = b.Close() }) + assert.Equal(t, DefaultMemLimit, b.memLimit) +} + +// --- no temp dir ------------------------------------------------------------------------------------ + +func TestNoTempDir(t *testing.T) { + t.Run("a write inside the memory limit needs no temp dir", func(t *testing.T) { + b := New(nil, WithMemLimit(4096)) + t.Cleanup(func() { _ = b.Close() }) + + mustWrite(t, b, 0, repeat('A', 4096)) + assert.Equal(t, repeat('A', 4096), readAll(t, b)) + }) + + t.Run("a write past it fails rather than allocating", func(t *testing.T) { + b := New(nil, WithMemLimit(4096)) + t.Cleanup(func() { _ = b.Close() }) + + _, err := b.WriteAt(repeat('B', 1), 4096) + require.Error(t, err) + assert.ErrorIs(t, err, ErrNoTempDir) + }) +} + +// --- input validation ------------------------------------------------------------------------------- + +func TestWriteAtRejectsBadInput(t *testing.T) { + b := newTestBuffer(t) + + t.Run("negative offset", func(t *testing.T) { + _, err := b.WriteAt([]byte("x"), -1) + assert.Error(t, err) + }) + + t.Run("an offset plus length that would overflow", func(t *testing.T) { + _, err := b.WriteAt(repeat('x', 16), math.MaxInt64-8) + require.Error(t, err) + assert.Contains(t, err.Error(), "overflow", + "the check must be a subtraction, not off+len wrapping into an in-range end") + }) + + t.Run("an empty write is a no-op, not an extent", func(t *testing.T) { + n, err := b.WriteAt(nil, 500) + require.NoError(t, err) + assert.Zero(t, n) + assert.Empty(t, b.written, "a zero-length write records nothing") + }) +} + +func TestReadAtRejectsBadInput(t *testing.T) { + b := newTestBuffer(t) + mustWrite(t, b, 0, repeat('A', 16)) + + t.Run("negative offset", func(t *testing.T) { + _, err := b.ReadAt(make([]byte, 4), -1) + assert.Error(t, err) + }) + + t.Run("empty read", func(t *testing.T) { + n, err := b.ReadAt(nil, 0) + assert.NoError(t, err) + assert.Zero(t, n) + }) + + t.Run("read from an empty buffer", func(t *testing.T) { + empty := newTestBuffer(t) + n, err := empty.ReadAt(make([]byte, 4), 0) + assert.Zero(t, n) + assert.ErrorIs(t, err, io.EOF) + }) +} + +// TestReadAtHonorsTheReaderAtContract pins what io.ReaderAt requires: a short read must come with a +// non-nil error, and a full read must not invent one. Parsers built on io.ReaderAt (debug/elf among them) +// rely on this to tell "the file ends here" from "try again". +func TestReadAtHonorsTheReaderAtContract(t *testing.T) { + b := newTestBuffer(t, WithMemLimit(64)) + mustWrite(t, b, 0, repeat('A', 200)) // spans both tiers + + for _, size := range []int{1, 63, 64, 65, 199, 200} { + p := make([]byte, size) + n, err := b.ReadAt(p, 0) + assert.Equal(t, size, n, "size=%d", size) + assert.NoError(t, err, "a read fully inside the buffer must not report EOF (size=%d)", size) + } + + p := make([]byte, 201) + n, err := b.ReadAt(p, 0) + assert.Equal(t, 200, n) + assert.ErrorIs(t, err, io.EOF, "one byte past the end is short and must say so") +} + +func TestReadAtEveryOffsetAndLength(t *testing.T) { + const limit = 32 + b := newTestBuffer(t, WithMemLimit(limit)) + + want := make([]byte, 100) + for i := range want { + want[i] = byte(i) + } + mustWrite(t, b, 0, want) + + // every offset x length pair, so a tier-boundary off-by-one cannot hide + for off := 0; off <= len(want); off++ { + for l := 0; l <= len(want)-off+2; l++ { + p := make([]byte, l) + n, err := b.ReadAt(p, int64(off)) + + expected := len(want) - off + if l < expected { + expected = l + } + assert.Equal(t, expected, n, "off=%d len=%d", off, l) + assert.Equal(t, want[off:off+expected], p[:expected], "off=%d len=%d", off, l) + + switch { + case l == 0: + assert.NoError(t, err, "off=%d len=0", off) + case off >= len(want): + assert.ErrorIs(t, err, io.EOF, "off=%d len=%d", off, l) + case l > expected: + assert.ErrorIs(t, err, io.EOF, "off=%d len=%d", off, l) + default: + assert.NoError(t, err, "off=%d len=%d", off, l) + } + } + } +} + +// --- overwrite semantics ---------------------------------------------------------------------------- + +func TestOverwrite(t *testing.T) { + tests := []struct { + name string + memLimit int64 + }{ + {"in memory", 1 << 20}, + {"on disk", 0}, + {"across the boundary", 8}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := newTestBuffer(t, WithMemLimit(tt.memLimit)) + mustWrite(t, b, 0, repeat('A', 16)) + mustWrite(t, b, 4, repeat('B', 8)) + + want := append(append(repeat('A', 4), repeat('B', 8)...), repeat('A', 4)...) + assert.Equal(t, want, readAll(t, b), "the later write wins over the range it covers") + assert.Equal(t, int64(16), b.Size(), "an overwrite does not extend the buffer") + }) + } +} + +// --- extents ---------------------------------------------------------------------------------------- + +func TestWrittenExtentsAreSortedAndMerged(t *testing.T) { + tests := []struct { + name string + writes []extent + want []extent + }{ + { + name: "disjoint stay separate, in order", + writes: []extent{{100, 150}, {0, 50}}, + want: []extent{{0, 50}, {100, 150}}, + }, + { + name: "adjacent merge", + writes: []extent{{0, 50}, {50, 100}}, + want: []extent{{0, 100}}, + }, + { + name: "overlapping merge", + writes: []extent{{0, 60}, {40, 100}}, + want: []extent{{0, 100}}, + }, + { + name: "a write bridging two runs collapses all three", + writes: []extent{{0, 20}, {80, 100}, {20, 80}}, + want: []extent{{0, 100}}, + }, + { + name: "a contained write changes nothing", + writes: []extent{{0, 100}, {20, 40}}, + want: []extent{{0, 100}}, + }, + { + name: "identical writes collapse", + writes: []extent{{0, 10}, {0, 10}, {0, 10}}, + want: []extent{{0, 10}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := newTestBuffer(t) + for _, w := range tt.writes { + mustWrite(t, b, w.start, repeat('x', int(w.end-w.start))) + } + assert.Equal(t, tt.want, b.written) + }) + } +} + +func TestWrittenExtentsStayCompactUnderManyAdjacentWrites(t *testing.T) { + b := newTestBuffer(t) + for i := int64(0); i < 1000; i++ { + mustWrite(t, b, i*8, repeat('A', 8)) + } + assert.Len(t, b.written, 1, "a run of adjacent writes must collapse to one extent, not 1000") + assert.Equal(t, int64(8000), b.Size()) +} + +// --- lifecycle -------------------------------------------------------------------------------------- + +func TestCloseRemovesTheSpillFile(t *testing.T) { + dir := t.TempDir() + b := New(tmpdir.FromPath(dir), WithMemLimit(0)) + mustWrite(t, b, 0, repeat('A', 32)) + require.Len(t, dirEntries(t, dir), 1) + + require.NoError(t, b.Close()) + assert.Empty(t, dirEntries(t, dir), "Close must take the spill file with it") +} + +func TestCloseIsIdempotent(t *testing.T) { + b := New(tmpdir.FromPath(t.TempDir()), WithMemLimit(0)) + mustWrite(t, b, 0, repeat('A', 32)) + + require.NoError(t, b.Close()) + assert.NoError(t, b.Close(), "a second Close is a no-op, not a double-close") + assert.NoError(t, b.Close()) +} + +func TestCloseOnNilReceiver(t *testing.T) { + var b *Buffer + assert.NotPanics(t, func() { + assert.NoError(t, b.Close()) + }) +} + +func TestCloseWithoutASpillFile(t *testing.T) { + b := New(nil, WithMemLimit(1<<20)) + mustWrite(t, b, 0, repeat('A', 32)) + assert.NoError(t, b.Close(), "a memory-only buffer closes cleanly with no file to remove") +} + +func TestWriteAfterCloseFails(t *testing.T) { + b := New(tmpdir.FromPath(t.TempDir())) + mustWrite(t, b, 0, repeat('A', 8)) + require.NoError(t, b.Close()) + + _, err := b.WriteAt(repeat('B', 8), 0) + require.Error(t, err, "a closed buffer must refuse writes rather than resurrect its file") + assert.ErrorIs(t, err, os.ErrClosed) +} + +// TestReadAfterCloseFails pins that a closed buffer reads as closed rather than as empty. Close drops the +// extents, so without the check every read would come back io.EOF and a caller could not tell a buffer it +// still owns from one somebody already released. +func TestReadAfterCloseFails(t *testing.T) { + b := New(tmpdir.FromPath(t.TempDir())) + mustWrite(t, b, 0, repeat('A', 8)) + require.NoError(t, b.Close()) + + n, err := b.ReadAt(make([]byte, 8), 0) + assert.Zero(t, n) + assert.ErrorIs(t, err, os.ErrClosed) + + at, ok := b.FirstGap(8, 100) + assert.False(t, ok, "and a closed buffer offers nowhere to write") + assert.Zero(t, at) +} + +// TestWriteFailureLeavesTheBufferUnchanged pins the ordering inside WriteAt: the fallible half runs first, +// so a spill file that cannot be created does not take the memory tier down with it. +func TestWriteFailureLeavesTheBufferUnchanged(t *testing.T) { + // FromPath takes the directory as given, so a path that does not exist makes file creation fail + b := New(tmpdir.FromPath(filepath.Join(t.TempDir(), "no-such-dir")), WithMemLimit(16)) + t.Cleanup(func() { _ = b.Close() }) + + mustWrite(t, b, 0, repeat('A', 8)) + + _, err := b.WriteAt(repeat('B', 32), 0) // straddles the limit, so it needs the spill file + require.Error(t, err) + + assert.Equal(t, int64(8), b.Size(), "a failed write records nothing") + assert.Nil(t, b.file, "and leaves no half-made spill file behind") + + p := make([]byte, 8) + n, err := b.ReadAt(p, 0) + require.NoError(t, err) + require.Equal(t, 8, n) + assert.Equal(t, repeat('A', 8), p, "the memory tier still holds what it held") + + mustWrite(t, b, 8, repeat('C', 8)) // inside the limit, so it still needs no file + assert.Equal(t, append(repeat('A', 8), repeat('C', 8)...), readAll(t, b)) +} + +// --- interface compliance --------------------------------------------------------------------------- + +func TestBufferSatisfiesTheStandardInterfaces(t *testing.T) { + b := newTestBuffer(t) + var ( + _ io.WriterAt = b + _ io.ReaderAt = b + _ io.Closer = b + ) + assert.Implements(t, (*io.ReaderAt)(nil), b) + assert.Implements(t, (*io.WriterAt)(nil), b) +} + +// TestWorksThroughIOOffsetWriter pins the composition every consumer uses: stream a decoded block into +// the buffer at a chosen offset without the writer knowing where it lands. +func TestWorksThroughIOOffsetWriter(t *testing.T) { + b := newTestBuffer(t, WithMemLimit(16)) + + w := io.NewOffsetWriter(b, 8) + n, err := io.Copy(w, bytes.NewReader(repeat('B', 32))) + require.NoError(t, err) + require.Equal(t, int64(32), n) + + mustWrite(t, b, 0, repeat('A', 8)) + + got := readAll(t, b) + assert.Equal(t, append(repeat('A', 8), repeat('B', 32)...), got) +} + +// TestWorksWithIOSectionReader pins the other direction: parsers wrap an io.ReaderAt in a SectionReader, +// and it must see a consistent length. +func TestWorksWithIOSectionReader(t *testing.T) { + b := newTestBuffer(t) + mustWrite(t, b, 0, repeat('A', 100)) + + sr := io.NewSectionReader(b, 0, b.Size()) + got, err := io.ReadAll(sr) + require.NoError(t, err) + assert.Equal(t, repeat('A', 100), got) +} + +// --- model-based randomized testing ----------------------------------------------------------------- + +// TestAgainstAReferenceModel drives random writes through the buffer and a dumb reference at the same +// time, then compares every readable byte. This is what catches tier-boundary and merge bugs that +// hand-written cases miss. +func TestAgainstAReferenceModel(t *testing.T) { + const universe = 8192 + + for _, limit := range []int64{0, 1, 64, 1000, 4096, universe * 2} { + t.Run("memLimit="+itoa(limit), func(t *testing.T) { + for seed := int64(0); seed < 40; seed++ { + rng := rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic test input, not crypto + b := newTestBuffer(t, WithMemLimit(limit)) + + model := make([]byte, universe) + written := make([]bool, universe) + + for op := 0; op < 40; op++ { + off := rng.Int63n(universe) + l := rng.Int63n(256) + 1 + if off+l > universe { + l = universe - off + } + payload := make([]byte, l) + for i := range payload { + payload[i] = byte(rng.Intn(256)) + } + + mustWrite(t, b, off, payload) + copy(model[off:], payload) + for i := off; i < off+l; i++ { + written[i] = true + } + } + + // the reference contiguous prefix + var want int64 + for want < universe && written[want] { + want++ + } + + require.Equal(t, want, b.Size(), "seed=%d limit=%d", seed, limit) + if want == 0 { + continue + } + assert.Equal(t, model[:want], readAll(t, b), "seed=%d limit=%d", seed, limit) + } + }) + } +} + +// TestRandomReadsMatchTheModel checks partial reads at arbitrary offsets, which the whole-buffer +// comparison above cannot reach. +func TestRandomReadsMatchTheModel(t *testing.T) { + rng := rand.New(rand.NewSource(7)) //nolint:gosec // deterministic test input, not crypto + b := newTestBuffer(t, WithMemLimit(97)) + + model := make([]byte, 1000) + for i := range model { + model[i] = byte(rng.Intn(256)) + } + // written in random-sized chunks, in order, so the whole range is covered + for off := 0; off < len(model); { + l := rng.Intn(50) + 1 + if off+l > len(model) { + l = len(model) - off + } + mustWrite(t, b, int64(off), model[off:off+l]) + off += l + } + require.Equal(t, int64(len(model)), b.Size()) + + for i := 0; i < 500; i++ { + off := rng.Intn(len(model)) + l := rng.Intn(len(model)-off) + 1 + p := make([]byte, l) + n, err := b.ReadAt(p, int64(off)) + require.NoError(t, err) + require.Equal(t, l, n) + require.Equal(t, model[off:off+l], p, "off=%d len=%d", off, l) + } +} + +// --- helpers ---------------------------------------------------------------------------------------- + +func dirEntries(t *testing.T, dir string) []os.DirEntry { + t.Helper() + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil + } + require.NoError(t, err) + + // tmpdir.NewFile creates the spill file inside the root it is given + var files []os.DirEntry + for _, e := range entries { + if e.IsDir() { + sub, err := os.ReadDir(filepath.Join(dir, e.Name())) + require.NoError(t, err) + for _, s := range sub { + files = append(files, s) + } + continue + } + files = append(files, e) + } + return files +} + +func itoa(n int64) string { + if n == 0 { + return "0" + } + var digits []byte + for n > 0 { + digits = append([]byte{byte('0' + n%10)}, digits...) + n /= 10 + } + return string(digits) +} + +// TestMemoryTierGrowsAmortized is the regression test for quadratic growth. Sizing the tier to exactly +// what each write needs reallocates and copies the whole thing per call, so a caller streaming a block in +// small chunks (which is what io.Copy through an OffsetWriter does) pays O(n^2): filling a 1MB tier in +// 32KB chunks cost ~16MB of allocation and showed up as a 19MB spike decompressing a 16MB payload. +func TestMemoryTierGrowsAmortized(t *testing.T) { + const ( + limit = 1 << 20 + chunk = 32 << 10 + ) + b := newTestBuffer(t, WithMemLimit(limit)) + + allocated := testutils.MeasureAlloc(t, func() { + for off := int64(0); off < limit; off += chunk { + mustWrite(t, b, off, repeat('A', chunk)) + } + }) + t.Logf("filling a %d byte tier in %d byte chunks allocated %d bytes", limit, chunk, allocated) + assert.Less(t, allocated, uint64(4*limit), + "growth has to be amortized, not a fresh copy of the whole tier per write") + assert.Equal(t, int64(limit), b.Size()) +} + +// TestMemoryTierNeverExceedsTheLimit pins the cap that geometric growth could otherwise overshoot. +func TestMemoryTierNeverExceedsTheLimit(t *testing.T) { + const limit = 1000 + b := newTestBuffer(t, WithMemLimit(limit)) + + for off := int64(0); off < 4*limit; off += 100 { + mustWrite(t, b, off, repeat('A', 100)) + assert.LessOrEqual(t, int64(len(b.mem)), int64(limit), + "the memory tier may never grow past its own limit (off=%d)", off) + } + assert.Equal(t, int64(4*limit), b.Size()) +} + +// TestFirstGap covers the placement question a caller writing at offsets of its own choosing has to ask. +// It moved here from the UPX cataloger, which used to walk an exported extent list to answer it itself. +func TestFirstGap(t *testing.T) { + tests := []struct { + name string + writes []extent + size int64 + within int64 + wantAt int64 + wantOK bool + }{ + { + name: "an empty buffer starts at zero", + size: 16, + within: 100, + wantAt: 0, wantOK: true, + }, + { + name: "the gap between two runs comes first", + writes: []extent{{0, 1000}, {1016, 2000}}, + size: 16, + within: 3000, + wantAt: 1000, wantOK: true, + }, + { + name: "with the gap filled the tail is what is left", + writes: []extent{{0, 2000}}, + size: 1000, + within: 3000, + wantAt: 2000, wantOK: true, + }, + { + name: "a gap too small is skipped for a later one", + writes: []extent{{0, 100}, {108, 200}}, + size: 50, + within: 1000, + wantAt: 200, wantOK: true, + }, + { + name: "nowhere left to fit is reported rather than squeezed in", + writes: []extent{{0, 1000}, {1016, 2000}}, + size: 1001, + within: 3000, + wantOK: false, + }, + { + name: "a run past the declared total must not wrap the remaining-space subtraction", + writes: []extent{{0, 200}}, + size: 8, + within: 100, + wantOK: false, + }, + { + name: "exactly filling the tail is allowed", + writes: []extent{{0, 900}}, + size: 100, + within: 1000, + wantAt: 900, wantOK: true, + }, + { + name: "a gap between two runs is still cut off by the bound", + writes: []extent{{0, 10}, {1000, 1010}}, + size: 45, + within: 50, + wantOK: false, + }, + { + name: "a gap between two runs that the bound still leaves room in", + writes: []extent{{0, 10}, {1000, 1010}}, + size: 30, + within: 50, + wantAt: 10, wantOK: true, + }, + { + name: "a negative size is refused rather than treated as zero", + size: -1, + within: 100, + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := newTestBuffer(t) + for _, w := range tt.writes { + mustWrite(t, b, w.start, repeat('x', int(w.end-w.start))) + } + + at, ok := b.FirstGap(tt.size, tt.within) + assert.Equal(t, tt.wantOK, ok) + if tt.wantOK { + assert.Equal(t, tt.wantAt, at) + } + }) + } +} + +// TestFirstGapNeverOverlapsWhatWasWritten is the property behind the table: whatever offset comes back has +// to be usable, so writing size bytes there must not land on top of anything already stored. +func TestFirstGapNeverOverlapsWhatWasWritten(t *testing.T) { + rng := rand.New(rand.NewSource(11)) //nolint:gosec // deterministic test input, not crypto + const universe = 4096 + + for seed := 0; seed < 200; seed++ { + b := newTestBuffer(t) + occupied := make([]bool, universe) + + for op := 0; op < 6; op++ { + // deliberately reaches past universe too, so the bound has to cut a between-extents gap short + off := rng.Int63n(2 * universe) + l := rng.Int63n(200) + 1 + mustWrite(t, b, off, repeat('x', int(l))) + for i := off; i < off+l && i < universe; i++ { + occupied[i] = true + } + } + + size := rng.Int63n(100) + 1 + at, ok := b.FirstGap(size, universe) + if !ok { + continue + } + require.LessOrEqual(t, at+size, int64(universe), "the result must fit inside the declared total") + for i := at; i < at+size; i++ { + require.False(t, occupied[i], "FirstGap returned %d for %d bytes, but %d is taken", at, size, i) + } + } +} diff --git a/internal/testutils/alloc.go b/internal/testutils/alloc.go new file mode 100644 index 000000000..c84200f10 --- /dev/null +++ b/internal/testutils/alloc.go @@ -0,0 +1,31 @@ +package testutils + +import ( + "runtime" + "testing" +) + +// MeasureAlloc reports the bytes allocated while fn ran. +// +// This exists for guards whose whole point is "a small input cannot make us reserve a large buffer". +// Only a byte count states that property: asserting that an error comes back would keep passing if the +// allocation were hoisted above the check, which is the bug these guards were written for. +// +// Two things to know when using it: +// +// - TotalAlloc is process-wide, so a test calling this must not call t.Parallel. Another test's +// allocations would land in the measurement. +// - a budget assertion is only meaningful if the fixture would really blow past it unguarded. Prefer +// measuring the unguarded path too and asserting it exceeds the declared size, so a fixture that +// quietly stops being a bomb fails instead of passing. +func MeasureAlloc(t testing.TB, fn func()) uint64 { + t.Helper() + + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + fn() + runtime.ReadMemStats(&after) + + return after.TotalAlloc - before.TotalAlloc +} diff --git a/syft/file/cataloger/executable/testdata/elf/Dockerfile b/syft/file/cataloger/executable/testdata/elf/Dockerfile index 70af60649..dd56c71c6 100644 --- a/syft/file/cataloger/executable/testdata/elf/Dockerfile +++ b/syft/file/cataloger/executable/testdata/elf/Dockerfile @@ -19,4 +19,4 @@ RUN printf 'deb http://archive.debian.org/debian bullseye main\ndeb http://archi #RUN cd selfrando && make -j`nprocs --all` #RUN cd selfrando && make install -RUN curl -o /bin/checksec https://raw.githubusercontent.com/slimm609/checksec.sh/2.6.0/checksec && chmod +x /bin/checksec +RUN curl -f -o /bin/checksec https://raw.githubusercontent.com/slimm609/checksec.sh/2.6.0/checksec && chmod +x /bin/checksec diff --git a/syft/format/common/spdxhelpers/find_root_packages_test.go b/syft/format/common/spdxhelpers/find_root_packages_test.go new file mode 100644 index 000000000..4ea4e48de --- /dev/null +++ b/syft/format/common/spdxhelpers/find_root_packages_test.go @@ -0,0 +1,181 @@ +package spdxhelpers + +import ( + "fmt" + "testing" + "time" + + "github.com/spdx/tools-golang/spdx" + "github.com/spdx/tools-golang/spdx/v2/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The tests in this file cover findRootPackages, which locates the packages an SPDX document declares it DESCRIBES +// (or which declare themselves DESCRIBED_BY the document). Until it was rewritten, this function compared every +// package against every relationship, so its cost grew with packages x relationships. On real documents that made +// SPDX decoding quadratic in document size: a 119MB SPDX JSON document (70k packages, 153k relationships) spent +// 72s here, a 238MB document 266s, and a 1.3GB document did not finish within 90 minutes. Encoding the same +// documents, and decoding equivalently sized syft-json or CycloneDX documents, is linear and takes seconds. +// +// Test_findRootPackages pins the exact semantics of the original implementation, including the corner cases a +// rewrite could easily change: result order follows package order, an element described twice appears twice, and +// only edges that touch the DOCUMENT element count. Test_findRootPackages_scalesLinearly is the regression guard +// for the performance fix itself. + +func Test_findRootPackages(t *testing.T) { + pkgA := &spdx.Package{PackageSPDXIdentifier: "Package-a"} + pkgB := &spdx.Package{PackageSPDXIdentifier: "Package-b"} + pkgC := &spdx.Package{PackageSPDXIdentifier: "Package-c"} + + describes := func(id common.ElementID) *spdx.Relationship { + return &spdx.Relationship{ + RefA: common.DocElementID{ElementRefID: "DOCUMENT"}, + RefB: common.DocElementID{ElementRefID: id}, + Relationship: spdx.RelationshipDescribes, + } + } + describedBy := func(id common.ElementID) *spdx.Relationship { + return &spdx.Relationship{ + RefA: common.DocElementID{ElementRefID: id}, + RefB: common.DocElementID{ElementRefID: "DOCUMENT"}, + Relationship: spdx.RelationshipDescribedBy, + } + } + + tests := []struct { + name string + packages []*spdx.Package + relationships []*spdx.Relationship + want []*spdx.Package + }{ + { + name: "no relationships", + packages: []*spdx.Package{pkgA, pkgB}, + relationships: nil, + want: nil, + }, + { + name: "document describes one package", + packages: []*spdx.Package{pkgA, pkgB, pkgC}, + relationships: []*spdx.Relationship{ + {RefA: common.DocElementID{ElementRefID: "Package-a"}, RefB: common.DocElementID{ElementRefID: "Package-b"}, Relationship: spdx.RelationshipContains}, + describes("Package-b"), + }, + want: []*spdx.Package{pkgB}, + }, + { + name: "package described by the document", + packages: []*spdx.Package{pkgA, pkgB, pkgC}, + relationships: []*spdx.Relationship{ + describedBy("Package-c"), + }, + want: []*spdx.Package{pkgC}, + }, + { + name: "multiple roots are returned in package order", + packages: []*spdx.Package{pkgA, pkgB, pkgC}, + relationships: []*spdx.Relationship{ + describes("Package-c"), + describedBy("Package-a"), + }, + want: []*spdx.Package{pkgA, pkgC}, + }, + { + name: "duplicate edges yield duplicate roots (preserved behavior)", + packages: []*spdx.Package{pkgA, pkgB}, + relationships: []*spdx.Relationship{ + describes("Package-a"), + describedBy("Package-a"), + }, + want: []*spdx.Package{pkgA, pkgA}, + }, + { + name: "describes edges not from the document are ignored", + packages: []*spdx.Package{pkgA, pkgB}, + relationships: []*spdx.Relationship{ + {RefA: common.DocElementID{ElementRefID: "Package-a"}, RefB: common.DocElementID{ElementRefID: "Package-b"}, Relationship: spdx.RelationshipDescribes}, + {RefA: common.DocElementID{ElementRefID: "Package-b"}, RefB: common.DocElementID{ElementRefID: "Package-a"}, Relationship: spdx.RelationshipDescribedBy}, + }, + want: nil, + }, + { + name: "described element that is not a package is ignored", + packages: []*spdx.Package{pkgA}, + relationships: []*spdx.Relationship{ + describes("File-1"), + }, + want: nil, + }, + { + name: "nil packages and relationships are skipped", + packages: []*spdx.Package{nil, pkgA}, + relationships: []*spdx.Relationship{ + nil, + describes("Package-a"), + }, + want: []*spdx.Package{pkgA}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := findRootPackages(&spdx.Document{ + Packages: tt.packages, + Relationships: tt.relationships, + }) + assert.Equal(t, tt.want, got) + }) + } +} + +// Test_findRootPackages_scalesLinearly guards against reintroducing a packages x relationships scan. At 50k +// packages and 100k relationships (5e9 comparisons) the previous implementation needs on the order of a minute on +// a fast laptop and several minutes on a CI runner, whereas the indexed implementation completes in about a +// millisecond. The 10s bound is therefore loose enough never to flake on slow hardware yet far below what any +// quadratic implementation could achieve. +func Test_findRootPackages_scalesLinearly(t *testing.T) { + doc := largeSPDXDocument(50_000, 100_000) + + start := time.Now() + got := findRootPackages(doc) + elapsed := time.Since(start) + + require.Len(t, got, 1) + assert.Equal(t, common.ElementID("Package-0"), got[0].PackageSPDXIdentifier) + assert.Less(t, elapsed, 10*time.Second, "findRootPackages took %s on %d packages and %d relationships", elapsed, len(doc.Packages), len(doc.Relationships)) +} + +// Benchmark_findRootPackages measures the same 50k x 100k document as the scaling test so the two can be compared +// directly: run with `go test ./syft/format/common/spdxhelpers/ -run xxx -bench findRootPackages`. +func Benchmark_findRootPackages(b *testing.B) { + doc := largeSPDXDocument(50_000, 100_000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = findRootPackages(doc) + } +} + +// largeSPDXDocument builds a document with the given number of packages and relationships where the document +// describes exactly the first package and every other relationship links packages to one another. +func largeSPDXDocument(packages int, relationships int) *spdx.Document { + doc := &spdx.Document{} + for i := 0; i < packages; i++ { + doc.Packages = append(doc.Packages, &spdx.Package{ + PackageSPDXIdentifier: common.ElementID(fmt.Sprintf("Package-%d", i)), + }) + } + doc.Relationships = append(doc.Relationships, &spdx.Relationship{ + RefA: common.DocElementID{ElementRefID: "DOCUMENT"}, + RefB: common.DocElementID{ElementRefID: "Package-0"}, + Relationship: spdx.RelationshipDescribes, + }) + for i := 1; i < relationships; i++ { + doc.Relationships = append(doc.Relationships, &spdx.Relationship{ + RefA: common.DocElementID{ElementRefID: common.ElementID(fmt.Sprintf("Package-%d", i%packages))}, + RefB: common.DocElementID{ElementRefID: common.ElementID(fmt.Sprintf("Package-%d", (i+1)%packages))}, + Relationship: spdx.RelationshipDependsOn, + }) + } + return doc +} diff --git a/syft/format/common/spdxhelpers/to_syft_model.go b/syft/format/common/spdxhelpers/to_syft_model.go index c375114bb..59a40957d 100644 --- a/syft/format/common/spdxhelpers/to_syft_model.go +++ b/syft/format/common/spdxhelpers/to_syft_model.go @@ -83,20 +83,30 @@ func removeRelationships(relationships []*spdx.Relationship, spdxID spdx.Element } func findRootPackages(doc *spdx.Document) (out []*spdx.Package) { + // index the DESCRIBES / DESCRIBED_BY relationships once, counting how many + // times each element is described by the document, so that the result + // matches the previous packages x relationships scan exactly (one entry per + // matching relationship) without the quadratic cost. + describedCount := make(map[spdx.ElementID]int) + for _, r := range doc.Relationships { + if r == nil { + continue + } + switch { + case r.RefA.ElementRefID == "DOCUMENT" && r.Relationship == spdx.RelationshipDescribes: + describedCount[r.RefB.ElementRefID]++ + case r.RefB.ElementRefID == "DOCUMENT" && r.Relationship == spdx.RelationshipDescribedBy: + describedCount[r.RefA.ElementRefID]++ + } + } + if len(describedCount) == 0 { + return nil + } for _, p := range doc.Packages { - for _, r := range doc.Relationships { - describes := r.RefA.ElementRefID == "DOCUMENT" && - r.Relationship == spdx.RelationshipDescribes && - r.RefB.ElementRefID == p.PackageSPDXIdentifier - - describedBy := r.RefB.ElementRefID == "DOCUMENT" && - r.Relationship == spdx.RelationshipDescribedBy && - r.RefA.ElementRefID == p.PackageSPDXIdentifier - - if !describes && !describedBy { - continue - } - + if p == nil { + continue + } + for i := 0; i < describedCount[p.PackageSPDXIdentifier]; i++ { out = append(out, p) } } diff --git a/syft/format/decoders.go b/syft/format/decoders.go index 48dba03e9..fe8bdda29 100644 --- a/syft/format/decoders.go +++ b/syft/format/decoders.go @@ -21,6 +21,9 @@ func init() { func Decoders() []sbom.FormatDecoder { return []sbom.FormatDecoder{ + // syft-json must remain the first decoder tried: it is by far the most common input, and its decoder can + // identify a syft document from the tail of the file alone. Trying anything else first would pay for a full + // parse of a large document by another decoder before reaching the common path. syftjson.NewFormatDecoder(), cyclonedxxml.NewFormatDecoder(), cyclonedxjson.NewFormatDecoder(), diff --git a/syft/format/internal/cyclonedxutil/helpers/property_encoder.go b/syft/format/internal/cyclonedxutil/helpers/property_encoder.go index dd442f6e3..2aeb8b6d0 100644 --- a/syft/format/internal/cyclonedxutil/helpers/property_encoder.go +++ b/syft/format/internal/cyclonedxutil/helpers/property_encoder.go @@ -93,7 +93,7 @@ func encode(out map[string]string, value reflect.Value, prefix string, fn FieldN typ := value.Type() switch typ.Kind() { - case reflect.Ptr: + case reflect.Pointer: if value.IsNil() { return } @@ -158,7 +158,7 @@ func fieldName(f reflect.StructField, prefix string, fn FieldName) (string, bool // Decode based on the given type, applies all values to hydrate a new instance func Decode(typ reflect.Type, values map[string]string, prefix string, fn FieldName) any { isPtr := false - for typ.Kind() == reflect.Ptr { + for typ.Kind() == reflect.Pointer { typ = typ.Elem() isPtr = true } @@ -188,7 +188,7 @@ func Decode(typ reflect.Type, values map[string]string, prefix string, fn FieldN func DecodeInto(obj any, values map[string]string, prefix string, fn FieldName) { value := reflect.ValueOf(obj) - for value.Type().Kind() == reflect.Ptr { + for value.Type().Kind() == reflect.Pointer { value = value.Elem() } @@ -205,7 +205,7 @@ func decode(vals map[string]string, value reflect.Value, prefix string, fn Field incoming, valid := vals[prefix] switch typ.Kind() { - case reflect.Ptr: + case reflect.Pointer: t := typ.Elem() v := value if v.IsNil() { @@ -269,13 +269,13 @@ func decode(vals map[string]string, value reflect.Value, prefix string, fn Field str := fmt.Sprintf("%s:%d", prefix, idx) // create new placeholder and decode values newType := t - if t.Kind() == reflect.Ptr { + if t.Kind() == reflect.Pointer { newType = t.Elem() } v := reflect.New(newType) if decode(vals, v.Elem(), str, fn) { // append to slice - if t.Kind() != reflect.Ptr { + if t.Kind() != reflect.Pointer { v = v.Elem() } slice = reflect.Append(slice, v) @@ -304,7 +304,7 @@ func decode(vals map[string]string, value reflect.Value, prefix string, fn Field keyVals[key] = after // create new placeholder and decode key newKeyType := keyType - if keyType.Kind() == reflect.Ptr { + if keyType.Kind() == reflect.Pointer { newKeyType = keyType.Elem() } k := reflect.New(newKeyType) @@ -312,18 +312,18 @@ func decode(vals map[string]string, value reflect.Value, prefix string, fn Field log.Debugf("unable to decode key for: %s", key) continue } - if keyType.Kind() != reflect.Ptr { + if keyType.Kind() != reflect.Pointer { k = k.Elem() } // create new placeholder and decode value newValueType := valueType - if valueType.Kind() == reflect.Ptr { + if valueType.Kind() == reflect.Pointer { newValueType = valueType.Elem() } v := reflect.New(newValueType) if decode(vals, v.Elem(), key, fn) { - if valueType.Kind() != reflect.Ptr { + if valueType.Kind() != reflect.Pointer { v = v.Elem() } @@ -367,7 +367,7 @@ func PtrToStruct(ptr any) any { return nil } switch v.Type().Kind() { - case reflect.Ptr: + case reflect.Pointer: return PtrToStruct(v.Elem().Interface()) case reflect.Interface: return PtrToStruct(v.Elem().Interface()) diff --git a/syft/format/internal/stream/seekable_reader.go b/syft/format/internal/stream/seekable_reader.go index 5df51a881..4a59ce860 100644 --- a/syft/format/internal/stream/seekable_reader.go +++ b/syft/format/internal/stream/seekable_reader.go @@ -72,8 +72,21 @@ func (o *offsetReadSeeker) Seek(offset int64, whence int) (int64, error) { } newOffset, err := o.rdr.Seek(offset, io.SeekCurrent) return newOffset - o.offset, err + case io.SeekEnd: + newOffset, err := o.rdr.Seek(offset, io.SeekEnd) + if err != nil { + return 0, fmt.Errorf("cannot seek end: %w", err) + } + if newOffset < o.offset { + // landed before the start of this view; move back to the start so the position stays valid + if _, err := o.rdr.Seek(o.offset, io.SeekStart); err != nil { + return 0, fmt.Errorf("cannot seek end: %w", err) + } + return 0, fmt.Errorf("cannot seek < 0") + } + return newOffset - o.offset, nil } - return 0, fmt.Errorf("only SeekStart and SeekCurrent supported") + return 0, fmt.Errorf("only SeekStart, SeekCurrent, and SeekEnd supported") } var _ io.ReadSeeker = (*offsetReadSeeker)(nil) diff --git a/syft/format/internal/stream/seekable_reader_test.go b/syft/format/internal/stream/seekable_reader_test.go index 035325305..3b4646c8f 100644 --- a/syft/format/internal/stream/seekable_reader_test.go +++ b/syft/format/internal/stream/seekable_reader_test.go @@ -179,11 +179,25 @@ func Test_offsetReadSeeker(t *testing.T) { whence: io.SeekCurrent, expected: "4", }, + { + name: "seek end", + input: abcd1234offset(2), + seek: -4, + whence: io.SeekEnd, + expected: "1234", + }, + { + name: "seek end before the start of the view", + input: abcd1234offset(2), + seek: -7, + whence: io.SeekEnd, + wantErr: require.Error, // would be < offset, which is an error + }, { name: "bad whence", input: abcd1234, seek: 1, - whence: io.SeekEnd, + whence: 99, wantErr: require.Error, }, } diff --git a/syft/format/internal/testutil/directory_input.go b/syft/format/internal/testutil/directory_input.go index 22bb91d63..8d4a23460 100644 --- a/syft/format/internal/testutil/directory_input.go +++ b/syft/format/internal/testutil/directory_input.go @@ -16,6 +16,7 @@ import ( "github.com/anchore/syft/syft/source/directorysource" ) +//nolint:goconst // fixture data: the repeated names and versions are what the golden files assert against, so they read better spelled out than behind constants func DirectoryInput(t testing.TB, dir string) sbom.SBOM { catalog := newDirectoryCatalog() @@ -97,6 +98,7 @@ func DirectoryInputWithAuthorField(t testing.TB) sbom.SBOM { } } +//nolint:goconst // fixture data: the repeated names and versions are what the golden files assert against, so they read better spelled out than behind constants func newDirectoryCatalog() *pkg.Collection { catalog := pkg.NewCollection() ctx := context.TODO() diff --git a/syft/format/internal/testutil/image_input.go b/syft/format/internal/testutil/image_input.go index f9ab7e06d..881c0952a 100644 --- a/syft/format/internal/testutil/image_input.go +++ b/syft/format/internal/testutil/image_input.go @@ -19,6 +19,7 @@ import ( "github.com/anchore/syft/syft/source/stereoscopesource" ) +//nolint:goconst // fixture data: the repeated names and versions are what the golden files assert against, so they read better spelled out than behind constants func ImageInput(t testing.TB, testImage string, options ...ImageOption) sbom.SBOM { t.Helper() catalog := pkg.NewCollection() @@ -95,6 +96,7 @@ func changeToDirectoryWithGoldenFixture(t testing.TB, testImage string) func() { return fn } +//nolint:goconst // fixture data: the repeated names and versions are what the golden files assert against, so they read better spelled out than behind constants func populateImageCatalog(catalog *pkg.Collection, img *image.Image) { // TODO: this helper function is coupled to the image-simple fixture, which seems like a bad idea _, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks) diff --git a/syft/format/spdxjson/testdata/snapshot/TestSPDXJSONImageEncoder.golden b/syft/format/spdxjson/testdata/snapshot/TestSPDXJSONImageEncoder.golden index 3e55bda38..9f8da17a8 100644 --- a/syft/format/spdxjson/testdata/snapshot/TestSPDXJSONImageEncoder.golden +++ b/syft/format/spdxjson/testdata/snapshot/TestSPDXJSONImageEncoder.golden @@ -81,7 +81,7 @@ { "referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", - "referenceLocator": "pkg:oci/user-image-input@sha256%3A2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368?arch=" + "referenceLocator": "pkg:oci/user-image-input@sha256%3A2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368?arch=amd64" } ], "primaryPackagePurpose": "CONTAINER" diff --git a/syft/format/spdxjson/testdata/snapshot/TestSPDXRelationshipOrder.golden b/syft/format/spdxjson/testdata/snapshot/TestSPDXRelationshipOrder.golden index 2edf48c7e..844fc5a42 100644 --- a/syft/format/spdxjson/testdata/snapshot/TestSPDXRelationshipOrder.golden +++ b/syft/format/spdxjson/testdata/snapshot/TestSPDXRelationshipOrder.golden @@ -81,7 +81,7 @@ { "referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", - "referenceLocator": "pkg:oci/user-image-input@sha256%3A2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368?arch=" + "referenceLocator": "pkg:oci/user-image-input@sha256%3A2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368?arch=amd64" } ], "primaryPackagePurpose": "CONTAINER" diff --git a/syft/format/spdxtagvalue/testdata/snapshot/TestSPDXRelationshipOrder.golden b/syft/format/spdxtagvalue/testdata/snapshot/TestSPDXRelationshipOrder.golden index fb5ffb8e9..41535f1d3 100644 --- a/syft/format/spdxtagvalue/testdata/snapshot/TestSPDXRelationshipOrder.golden +++ b/syft/format/spdxtagvalue/testdata/snapshot/TestSPDXRelationshipOrder.golden @@ -71,7 +71,7 @@ PackageChecksum: SHA256: 2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf4 PackageLicenseConcluded: NOASSERTION PackageLicenseDeclared: NOASSERTION PackageCopyrightText: NOASSERTION -ExternalRef: PACKAGE-MANAGER purl pkg:oci/user-image-input@sha256%3A2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368?arch= +ExternalRef: PACKAGE-MANAGER purl pkg:oci/user-image-input@sha256%3A2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368?arch=amd64 ##### Package: package-2 diff --git a/syft/format/spdxtagvalue/testdata/snapshot/TestSPDXTagValueImageEncoder.golden b/syft/format/spdxtagvalue/testdata/snapshot/TestSPDXTagValueImageEncoder.golden index 54afe3558..cd97d7217 100644 --- a/syft/format/spdxtagvalue/testdata/snapshot/TestSPDXTagValueImageEncoder.golden +++ b/syft/format/spdxtagvalue/testdata/snapshot/TestSPDXTagValueImageEncoder.golden @@ -21,7 +21,7 @@ PackageChecksum: SHA256: 2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf4 PackageLicenseConcluded: NOASSERTION PackageLicenseDeclared: NOASSERTION PackageCopyrightText: NOASSERTION -ExternalRef: PACKAGE-MANAGER purl pkg:oci/user-image-input@sha256%3A2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368?arch= +ExternalRef: PACKAGE-MANAGER purl pkg:oci/user-image-input@sha256%3A2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368?arch=amd64 ##### Package: package-2 diff --git a/syft/format/syftjson/decoder.go b/syft/format/syftjson/decoder.go index de9b51433..5e3413094 100644 --- a/syft/format/syftjson/decoder.go +++ b/syft/format/syftjson/decoder.go @@ -1,7 +1,9 @@ package syftjson import ( + "bytes" "encoding/json" + "errors" "fmt" "io" "strings" @@ -15,6 +17,12 @@ import ( "github.com/anchore/syft/syft/sbom" ) +// identifyTailSize is how much of the end of a document is inspected when identifying it from its trailing schema +// block, before falling back to parsing the whole document. +const identifyTailSize = 2 * 1024 * 1024 + +var schemaKey = []byte(`"schema"`) + var _ sbom.FormatDecoder = (*decoder)(nil) type decoder struct{} @@ -45,6 +53,12 @@ func (d decoder) Decode(r io.Reader) (*sbom.SBOM, sbom.FormatID, string, error) return nil, "", "", fmt.Errorf("unable to decode syft-json document: %w", err) } + if doc.Schema.Version == "" { + // Identify found a schema block that the parsed document does not have, so it was describing something + // else in the stream; returning the empty document it decoded into would be a silently wrong answer + return nil, "", "", fmt.Errorf("not a syft-json document") + } + if err := checkSupportedSchema(doc.Schema.Version, internal.JSONSchemaVersion); err != nil { log.Warn(err) } @@ -57,6 +71,15 @@ func (d decoder) Identify(r io.Reader) (sbom.FormatID, string) { return "", "" } + // fast path: syft writes the schema block as the last key of the document (see model.Document), so for seekable + // input the format and version can be read from the tail without parsing a potentially very large document. + // Anything the tail cannot vouch for falls through to the full parse below. + if rs, ok := r.(io.ReadSeeker); ok { + if id, version, ok := identifyFromTail(rs); ok { + return id, version + } + } + type Document struct { Schema model.Schema `json:"schema"` } @@ -78,6 +101,160 @@ func (d decoder) Identify(r io.Reader) (sbom.FormatID, string) { return ID, doc.Schema.Version } +// identifyFromTail reads at most identifyTailSize bytes from the end of the reader and attempts to identify the +// document from a top-level schema block found there. The reader is returned to its original position afterwards. +func identifyFromTail(rs io.ReadSeeker) (sbom.FormatID, string, bool) { + start, err := rs.Seek(0, io.SeekCurrent) + if err != nil { + return "", "", false + } + + id, version, ok := readTail(rs, start) + + if _, err := rs.Seek(start, io.SeekStart); err != nil { + // without the original position the caller cannot safely parse the document afterwards + log.WithFields("error", err).Debug("unable to restore reader position after inspecting document tail") + return "", "", false + } + + return id, version, ok +} + +// readTail does the seeking and reading for identifyFromTail, leaving the reader wherever it ends up. +func readTail(rs io.ReadSeeker, start int64) (sbom.FormatID, string, bool) { + end, err := rs.Seek(0, io.SeekEnd) + if err != nil || end <= start { + return "", "", false + } + + // the tail only describes the document the reader is pointed at if that document starts where the reader is, + // so check the first byte: bytes ahead of it (a byte order mark, a log line prepended by a wrapper) would + // otherwise be accepted silently, leaving Identify claiming a document that cannot be decoded + if !opensObject(rs, start) { + return "", "", false + } + + tailLen := min(end-start, int64(identifyTailSize)) + tail := make([]byte, tailLen) + if _, err := rs.Seek(end-tailLen, io.SeekStart); err != nil { + return "", "", false + } + if _, err := io.ReadFull(rs, tail); err != nil { + return "", "", false + } + + return identifyFromTailBytes(tail) +} + +// opensObject reports whether the document at the given offset begins with a JSON object. +func opensObject(rs io.ReadSeeker, start int64) bool { + if _, err := rs.Seek(start, io.SeekStart); err != nil { + return false + } + + head := make([]byte, 64) + n, err := io.ReadFull(rs, head) + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { + return false + } + + head = bytes.TrimLeft(head[:n], " \t\r\n") + return len(head) > 0 && head[0] == '{' +} + +// identifyFromTailBytes looks for a syft schema block among the trailing bytes of a JSON document and, when one is +// found directly within the top-level object, returns the format and schema version it declares. The bytes may +// begin anywhere within the document, but must run to its very end. +func identifyFromTailBytes(tail []byte) (sbom.FormatID, string, bool) { + // find the last "schema" key. An unescaped quote cannot occur inside a JSON string, so a match whose opening + // quote is not preceded by a backslash is a genuine string token (and a key if a colon follows it). + idx := len(tail) + for { + idx = bytes.LastIndex(tail[:idx], schemaKey) + if idx <= 0 { + // not found, or found at the very start of the tail where we cannot tell whether it is escaped + return "", "", false + } + if tail[idx-1] != '\\' { + break + } + } + + rest := bytes.TrimLeft(tail[idx+len(schemaKey):], " \t\r\n") + if len(rest) == 0 || rest[0] != ':' { + // a string value that happens to read "schema", not a key + return "", "", false + } + rest = rest[1:] + + var schema model.Schema + dec := json.NewDecoder(bytes.NewReader(rest)) + if err := dec.Decode(&schema); err != nil { + return "", "", false + } + + // the schema block only identifies the document when it belongs to the top-level object + if !closesTopLevelObject(rest[dec.InputOffset():]) { + return "", "", false + } + + if schema.Version == "" || !strings.Contains(schema.URL, "anchore/syft") { + return "", "", false + } + + return ID, schema.Version, true +} + +// closesTopLevelObject reports whether the given bytes, which follow a value somewhere within a JSON document, run +// to the end of that document closing exactly one enclosing object (and nothing else) at the very end. When true, +// the value they follow sits directly within the top-level object. Braces inside strings are ignored. +func closesTopLevelObject(b []byte) bool { + var ( + depth int + inString bool + escaped bool + closed bool + ) + for _, c := range b { + if closed { + // only whitespace may follow the top-level object + if c == ' ' || c == '\t' || c == '\r' || c == '\n' { + continue + } + return false + } + if inString { + switch { + case escaped: + escaped = false + case c == '\\': + escaped = true + case c == '"': + inString = false + } + continue + } + switch c { + case '"': + inString = true + case '{', '[': + depth++ + case '}': + depth-- + if depth == -1 { + closed = true + } + case ']': + depth-- + if depth == -1 { + // the enclosing container is an array, not the top-level object + return false + } + } + } + return closed +} + func checkSupportedSchema(documentVersion string, parserVersion string) error { documentV, err := semver.NewVersion(documentVersion) if err != nil { diff --git a/syft/format/syftjson/decoder_identify_test.go b/syft/format/syftjson/decoder_identify_test.go new file mode 100644 index 000000000..8d60f3c1b --- /dev/null +++ b/syft/format/syftjson/decoder_identify_test.go @@ -0,0 +1,340 @@ +package syftjson + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/format/internal/stream" + "github.com/anchore/syft/syft/format/internal/testutil" +) + +const ( + testIdentifySchemaURL = "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-16.0.0.json" + testIdentifySchemaBlock = `"schema":{"version":"16.0.0","url":"` + testIdentifySchemaURL + `"}` +) + +func Test_identifyFromTailBytes(t *testing.T) { + tests := []struct { + name string + doc string + wantVersion string + wantOK bool + }{ + { + name: "syft key order (schema is the last key)", + doc: `{"artifacts":[],"descriptor":{"name":"syft"},` + testIdentifySchemaBlock + `}`, + wantVersion: "16.0.0", + wantOK: true, + }, + { + name: "alphabetical key order (schema is followed by source)", + doc: `{"artifacts":[],` + testIdentifySchemaBlock + `,"source":{"id":"x","name":"img","type":"image","metadata":{"tags":["a","b"],"layers":[{"digest":"sha256:1"}]}}}`, + wantVersion: "16.0.0", + wantOK: true, + }, + { + name: "pretty printed with a trailing newline", + doc: "{\n \"artifacts\": [],\n \"schema\": {\n \"version\": \"16.0.0\",\n \"url\": \"" + testIdentifySchemaURL + "\"\n }\n}\n", + wantVersion: "16.0.0", + wantOK: true, + }, + { + name: "nested schema key is not mistaken for the document schema", + doc: `{"artifacts":[{"metadata":{` + testIdentifySchemaBlock + `}}],"descriptor":{}}`, + wantOK: false, + }, + { + name: "top-level schema wins over an earlier nested one", + doc: `{"artifacts":[{"metadata":{"schema":{"version":"1","url":"anchore/syft"}}}],` + testIdentifySchemaBlock + `}`, + wantVersion: "16.0.0", + wantOK: true, + }, + { + name: "escaped schema text (with braces) inside a trailing string is ignored", + doc: `{"artifacts":[],` + testIdentifySchemaBlock + `,"source":{"name":"contains \"schema\":{\"version\":\"9\"} text"}}`, + wantVersion: "16.0.0", + wantOK: true, + }, + { + name: "array wrapped document", + doc: `[{"artifacts":[],` + testIdentifySchemaBlock + `}]`, + wantOK: false, + }, + { + name: "truncated document", + doc: `{"artifacts":[],` + testIdentifySchemaBlock, + wantOK: false, + }, + { + name: "trailing content after the document", + doc: `{"artifacts":[],` + testIdentifySchemaBlock + `}{"another":"document"}`, + wantOK: false, + }, + { + name: "schema value is not an object", + doc: `{"artifacts":[],"schema":"16.0.0"}`, + wantOK: false, + }, + { + name: "schema is a string value rather than a key", + doc: `{"artifacts":[],"kind":"schema"}`, + wantOK: false, + }, + { + name: "not a syft schema url", + doc: `{"artifacts":[],"schema":{"version":"1.0","url":"https://example.com/schema.json"}}`, + wantOK: false, + }, + { + name: "no schema key at all", + doc: `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[]}`, + wantOK: false, + }, + { + name: "schema key at the very start of the tail cannot be vouched for", + doc: testIdentifySchemaBlock + `}`, + wantOK: false, + }, + { + name: "empty", + doc: "", + wantOK: false, + }, + { + // the key's text is: he said "schema + name: "an escaped quote does not make a string into the schema key", + doc: `{"he said \"schema":{"version":"1.0","url":"https://example.com/anchore/syft"}}`, + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + id, version, ok := identifyFromTailBytes([]byte(tt.doc)) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.wantVersion, version) + if tt.wantOK { + assert.Equal(t, ID, id) + } else { + assert.Empty(t, id) + } + }) + } +} + +func Test_Identify_largeDocument(t *testing.T) { + // build a document comfortably larger than the tail window so only the tail path could avoid a full parse + unit := `{"name":"pkg","version":"1.0"},` + padding := strings.Repeat(unit, (identifyTailSize/len(unit))+1024) + doc := `{"artifacts":[` + padding + `{"name":"last"}],` + testIdentifySchemaBlock + `}` + require.Greater(t, len(doc), identifyTailSize) + + t.Run("seekable reader", func(t *testing.T) { + r := strings.NewReader(doc) + + id, version := NewFormatDecoder().Identify(r) + assert.Equal(t, ID, id) + assert.Equal(t, "16.0.0", version) + + // the reader is left where it started so callers can go on to read the document + pos, err := r.Seek(0, io.SeekCurrent) + require.NoError(t, err) + assert.Equal(t, int64(0), pos) + }) + + t.Run("non-seekable reader falls back to a full parse", func(t *testing.T) { + r := struct{ io.Reader }{strings.NewReader(doc)} + + id, version := NewFormatDecoder().Identify(r) + assert.Equal(t, ID, id) + assert.Equal(t, "16.0.0", version) + }) + + t.Run("reader positioned at a later document", func(t *testing.T) { + first := `{"artifacts":[],"schema":{"version":"1.0.0","url":"` + testIdentifySchemaURL + `"}}` + r := strings.NewReader(first + "\n" + doc) + _, err := r.Seek(int64(len(first)+1), io.SeekStart) + require.NoError(t, err) + + // mirror what the decoder collection does: seeks become relative to the current position + sr, err := stream.SeekableReader(r) + require.NoError(t, err) + + id, version := NewFormatDecoder().Identify(sr) + assert.Equal(t, ID, id) + assert.Equal(t, "16.0.0", version) + + pos, err := sr.Seek(0, io.SeekCurrent) + require.NoError(t, err) + assert.Equal(t, int64(0), pos) + }) +} + +func Test_Identify_agreesWithFullParse(t *testing.T) { + // the tail is only a shortcut: whatever it decides must match what parsing the whole document says, including + // for documents it has to decline. + // + // note: a stream holding several concatenated documents is deliberately not covered. Identifying the document + // at the reader's position would mean proving no other document ends before the stream does, which means + // reading all of it, which is the cost the tail path exists to avoid. Decode still reports the document it + // actually parsed (see Test_Decode_rejectsADocumentTheTailDidNotDescribe). + syftDoc := `{"artifacts":[],` + testIdentifySchemaBlock + `}` + + tests := []struct { + name string + doc string + wantVersion string + }{ + {name: "plain document", doc: syftDoc, wantVersion: "16.0.0"}, + {name: "byte order mark before the document", doc: "\xef\xbb\xbf" + syftDoc}, + {name: "content before the document", doc: "WARN something happened\n" + syftDoc}, + {name: "not a syft document", doc: `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[]}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := NewFormatDecoder() + + // a bytes.Reader is seekable, so this takes the tail path + fastID, fastVersion := d.Identify(bytes.NewReader([]byte(tt.doc))) + + // readerOnly hides Seek, so this takes the full parse + slowID, slowVersion := d.Identify(readerOnly{strings.NewReader(tt.doc)}) + + assert.Equal(t, slowID, fastID, "fast and slow paths disagree on the format") + assert.Equal(t, slowVersion, fastVersion, "fast and slow paths disagree on the version") + assert.Equal(t, tt.wantVersion, fastVersion) + }) + } +} + +// readerOnly hides any Seek method the underlying reader has, forcing the full parse. +type readerOnly struct { + io.Reader +} + +func Test_Decode_rejectsADocumentTheTailDidNotDescribe(t *testing.T) { + // the tail can find a schema block belonging to a later document in the stream; whatever it decides, Decode + // must not hand back the empty SBOM that parsing the first document into model.Document would produce + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[{"type":"library","name":"pkg-a"}]}` + + "\n" + `{"artifacts":[],` + testIdentifySchemaBlock + `}` + + s, _, _, err := NewFormatDecoder().Decode(bytes.NewReader([]byte(doc))) + require.Error(t, err) + assert.Nil(t, s) +} + +func Test_encoderWritesSchemaLast(t *testing.T) { + // identification reads the schema block from the tail of the document (see model.Document), which depends on + // the encoder always writing it as the final key + s := testutil.DirectoryInput(t, t.TempDir()) + + for _, pretty := range []bool{false, true} { + cfg := DefaultEncoderConfig() + cfg.Pretty = pretty + + enc, err := NewFormatEncoderWithConfig(cfg) + require.NoError(t, err) + + var buf bytes.Buffer + require.NoError(t, enc.Encode(&buf, s)) + + keys := topLevelKeys(t, buf.Bytes()) + require.NotEmpty(t, keys) + assert.Equal(t, "schema", keys[len(keys)-1], "pretty=%v", pretty) + + id, version, ok := identifyFromTailBytes(buf.Bytes()) + assert.True(t, ok, "pretty=%v", pretty) + assert.Equal(t, ID, id) + assert.Equal(t, internal.JSONSchemaVersion, version) + } +} + +// noEndSeeker behaves like a reader that buffers a stream as it goes: it can seek within what it has, but cannot +// seek relative to the end. +type noEndSeeker struct { + io.ReadSeeker +} + +func (s noEndSeeker) Seek(offset int64, whence int) (int64, error) { + if whence == io.SeekEnd { + return 0, errors.New("SeekEnd is not supported") + } + return s.ReadSeeker.Seek(offset, whence) +} + +func Test_Identify_fallsBackWhenTailIsUnavailable(t *testing.T) { + doc := `{"artifacts":[],` + testIdentifySchemaBlock + `}` + r := noEndSeeker{strings.NewReader(doc)} + + id, version := NewFormatDecoder().Identify(r) + assert.Equal(t, ID, id) + assert.Equal(t, "16.0.0", version) +} + +func Test_identifyFromTail_readerEdgeCases(t *testing.T) { + doc := `{"artifacts":[],` + testIdentifySchemaBlock + `}` + + t.Run("empty reader", func(t *testing.T) { + r := strings.NewReader("") + + _, _, ok := identifyFromTail(r) + assert.False(t, ok) + }) + + t.Run("reader already at the end has nothing to inspect and keeps its position", func(t *testing.T) { + r := strings.NewReader(doc) + _, err := r.Seek(0, io.SeekEnd) + require.NoError(t, err) + + _, _, ok := identifyFromTail(r) + assert.False(t, ok) + + pos, err := r.Seek(0, io.SeekCurrent) + require.NoError(t, err) + assert.Equal(t, int64(len(doc)), pos) + }) + + t.Run("whitespace around the colon and CRLF line endings", func(t *testing.T) { + crlf := "{\r\n \"artifacts\": [],\r\n \"schema\" : {\r\n \"version\": \"16.0.0\",\r\n \"url\": \"" + testIdentifySchemaURL + "\"\r\n }\r\n}\r\n" + + id, version, ok := identifyFromTailBytes([]byte(crlf)) + require.True(t, ok) + assert.Equal(t, ID, id) + assert.Equal(t, "16.0.0", version) + }) + + t.Run("document ending inside a string is not trusted", func(t *testing.T) { + truncated := `{"artifacts":[],` + testIdentifySchemaBlock + `,"source":{"name":"unterminated` + + _, _, ok := identifyFromTailBytes([]byte(truncated)) + assert.False(t, ok) + }) +} + +func topLevelKeys(t *testing.T, doc []byte) []string { + dec := json.NewDecoder(bytes.NewReader(doc)) + + tok, err := dec.Token() + require.NoError(t, err) + require.Equal(t, json.Delim('{'), tok) + + var keys []string + for dec.More() { + tok, err := dec.Token() + require.NoError(t, err) + keys = append(keys, tok.(string)) + + var value json.RawMessage + require.NoError(t, dec.Decode(&value)) + } + return keys +} diff --git a/syft/format/syftjson/model/document.go b/syft/format/syftjson/model/document.go index 9672ac3e5..e733978d5 100644 --- a/syft/format/syftjson/model/document.go +++ b/syft/format/syftjson/model/document.go @@ -5,6 +5,14 @@ import ( "fmt" ) +// Field order matters: encoding/json writes struct fields in declaration order, and Schema must remain the last +// field so that the schema block is always the final key of the encoded document. Format identification relies on +// finding the schema block by reading only the tail of a (potentially very large) document; moving Schema earlier +// would not break decoding but would silently force a full parse of the whole document to identify it. +// +// (This block is deliberately separated from the doc comment below: the JSON schema generator lifts doc comments +// into schema descriptions, and changing the description would alter the published schema.) + // Document represents the syft cataloging findings as a JSON document type Document struct { Artifacts []Package `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog diff --git a/syft/format/syftjson/testdata/snapshot/TestImageEncoder.golden b/syft/format/syftjson/testdata/snapshot/TestImageEncoder.golden index 35984b5be..9a8c244bc 100644 --- a/syft/format/syftjson/testdata/snapshot/TestImageEncoder.golden +++ b/syft/format/syftjson/testdata/snapshot/TestImageEncoder.golden @@ -105,8 +105,8 @@ "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NTgsImRpZ2VzdCI6InNoYTI1NjozNWE2NjU4ZTI0ZmFiOTJlYWU5ZWM2ZmMyNTJkZWM1ODk4NmM0YzAwNzg5MTc1OGQ0ZDM3YzdlNDNmYmJlMGM1In0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjpkZmVmZTYxOGM4OWIwOGZlZjBmOWM3ZjFhMjY4MjUyMWRkZGJlMDNkNjY3OGY0YTlmYjliMDc4MzgxZDhlYjQ1In0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2OjM4ZGRjMjg0N2ZiNmJjYWZkNzQwMWI0YmYyN2MxMDAxNGI1ZDYwZTI0MDBiYzE4ODg5MGM3Y2I3Y2RkN2NkNmMifV19", "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8ifSwiY3JlYXRlZCI6IjIwMjQtMTEtMDRUMDQ6MTM6NDMuMzQwNTA3MTc1WiIsImhpc3RvcnkiOlt7ImNyZWF0ZWQiOiIyMDI0LTExLTA0VDA0OjEzOjQzLjMyMDg2OTk2OFoiLCJjcmVhdGVkX2J5IjoiQUREIGZpbGUtMS50eHQgL3NvbWVmaWxlLTEudHh0ICMgYnVpbGRraXQiLCJjb21tZW50IjoiYnVpbGRraXQuZG9ja2VyZmlsZS52MCJ9LHsiY3JlYXRlZCI6IjIwMjQtMTEtMDRUMDQ6MTM6NDMuMzQwNTA3MTc1WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0yLnR4dCAvc29tZWZpbGUtMi50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6ZGZlZmU2MThjODliMDhmZWYwZjljN2YxYTI2ODI1MjFkZGRiZTAzZDY2NzhmNGE5ZmI5YjA3ODM4MWQ4ZWI0NSIsInNoYTI1NjozOGRkYzI4NDdmYjZiY2FmZDc0MDFiNGJmMjdjMTAwMTRiNWQ2MGUyNDAwYmMxODg4OTBjN2NiN2NkZDdjZDZjIl19fQ==", "repoDigests": [], - "architecture": "", - "os": "" + "architecture": "amd64", + "os": "linux" } }, "distro": { diff --git a/syft/internal/elfutil/elfutil.go b/syft/internal/elfutil/elfutil.go index 4870466f3..5017c944c 100644 --- a/syft/internal/elfutil/elfutil.go +++ b/syft/internal/elfutil/elfutil.go @@ -35,6 +35,7 @@ package elfutil import ( "debug/elf" "encoding/binary" + "errors" "fmt" "io" "math" @@ -55,6 +56,15 @@ import ( // sit just under it. That is deliberate, since the sections syft actually reads are few. const maxDeclaredSectionSize uint64 = 128 * intFile.MB +// ErrDeclaredSizeExceeded marks a file this package refused to parse because a section declared more +// decompressed bytes than maxDeclaredSectionSize allows. +// +// Exported so a caller can tell "syft declined to expand this" apart from "this file is broken". The +// distinction matters where the two are reported differently: the refusal is a real gap in the SBOM and +// worth surfacing, while a corrupt or non-Go executable is neither, and the golang cataloger sees far +// more of the latter than the former. +var ErrDeclaredSizeExceeded = errors.New("declared decompressed size over the limit") + // legacyZlibHeaderSize is the size of the .zdebug header: the "ZLIB" magic plus a big-endian size. const legacyZlibHeaderSize = 12 @@ -102,6 +112,39 @@ func NewFile(r io.ReaderAt) (*elf.File, error) { return f, nil } +// CheckAllSections rejects ELF decompression bombs before debug/elf can expand one, without keeping the +// parse. +// +// The attack it stops: a compressed section's header declares its own decompressed size, and debug/elf +// believes that number, allocating it up front when the section is opened. Nothing forces the declared +// size to match what the compressed bytes actually yield, so a small file can name an enormous one. In +// the case this was written for, a 260KB ELF declaring a compressed .symtab drove 1.3GB of allocation. +// This walks the section headers and refuses any reachable section declaring more than +// maxDeclaredSectionSize, returning ErrDeclaredSizeExceeded. +// +// It is a strict superset of CheckSectionNameTable, which is what "All" in the name marks: reach for that +// narrower one only where a full elf.NewFile parse on every file is not worth paying for. Like it, this +// is for a caller whose debug/elf call is made inside another package, but it is the whole check rather +// than the eager half: goversion reads .symtab and the string table it links, and those are expanded +// lazily, so the name table bound alone leaves them unbounded. +// +// Anything that is not an ELF this package understands passes through untouched, since a caller reaching +// for this parses other containers too (goversion takes PE and Mach-O). A file elf.NewFile cannot parse +// passes through as well: describing a malformed file is the caller's job, not this gate's. +func CheckAllSections(r io.ReaderAt) error { + if _, _, ok := identify(r); !ok { + return nil + } + if err := CheckSectionNameTable(r); err != nil { + return err + } + f, err := elf.NewFile(r) + if err != nil { + return nil //nolint:nilerr // the caller's own parse reports the format error + } + return checkReachableSections(f) +} + // checkReachableSections bounds every section syft can drive debug/elf into decompressing. This runs // after the parse because (*Section).Open expands lazily, so nothing has been allocated yet. func checkReachableSections(f *elf.File) error { @@ -109,8 +152,8 @@ func checkReachableSections(f *elf.File) error { s := f.Sections[i] declared, claimed := declaredSectionSize(s) if claimed && declared > maxDeclaredSectionSize { - return fmt.Errorf("elf section %q declares %d decompressed bytes, over the %d byte limit", - s.Name, declared, maxDeclaredSectionSize) + return fmt.Errorf("%w: elf section %q declares %d decompressed bytes, over the %d byte limit", + ErrDeclaredSizeExceeded, s.Name, declared, maxDeclaredSectionSize) } } return nil @@ -233,6 +276,10 @@ func zdebugDeclaredSize(s *elf.Section) (uint64, bool) { // debug/elf call is made for them inside another package, debug/buildinfo being the one syft reaches: // gate the reader on this and the eager section-name table read is bounded, which is everything that // package expands today, since it reaches .go.buildinfo through the program headers instead. +// +// This is deliberately the eager half only, so it is the wrong gate for a caller whose parser goes on to +// read sections of its own. Use CheckAllSections for those: a package that reads symbols expands sections +// this never looks at, and the difference is the whole reason both exist. func CheckSectionNameTable(r io.ReaderAt) error { class, order, ok := identify(r) if !ok { @@ -252,8 +299,8 @@ func CheckSectionNameTable(r io.ReaderAt) error { return nil //nolint:nilerr // truncated section; elf.NewFile will say so } if declared > maxDeclaredSectionSize { - return fmt.Errorf("elf section name table declares %d decompressed bytes, over the %d byte limit", - declared, maxDeclaredSectionSize) + return fmt.Errorf("%w: elf section name table declares %d decompressed bytes, over the %d byte limit", + ErrDeclaredSizeExceeded, declared, maxDeclaredSectionSize) } return nil } diff --git a/syft/internal/elfutil/elfutil_test.go b/syft/internal/elfutil/elfutil_test.go index 3d70b69f0..8d45026a1 100644 --- a/syft/internal/elfutil/elfutil_test.go +++ b/syft/internal/elfutil/elfutil_test.go @@ -7,11 +7,12 @@ import ( "encoding/binary" "fmt" "io" - "runtime" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/testutils" ) // section describes one section to place in a fixture. The compression headers are deliberately @@ -341,6 +342,9 @@ func TestNewFile_RejectsOversizedReachableSections(t *testing.T) { // the fixtures are hand-assembled ELF bytes, so a bad one would satisfy require.Error on // its own. Naming the section proves the rejection is the one we set up, and naming the // limit proves it came from this package rather than from debug/elf. + // the sentinel is what the callers key their reporting on, so it is asserted here rather + // than only from the packages downstream that consume it + assert.ErrorIs(t, err, ErrDeclaredSizeExceeded) assert.Contains(t, err.Error(), tt.wantErr) assert.Contains(t, err.Error(), fmt.Sprint(maxDeclaredSectionSize)) }) @@ -540,13 +544,10 @@ func TestNewFile_BombDoesNotAllocate(t *testing.T) { {name: ".strtab", typ: elf.SHT_STRTAB}, }, buildOpts{}) - var before, after runtime.MemStats - runtime.GC() - runtime.ReadMemStats(&before) - - _, err := NewFile(bytes.NewReader(data)) - - runtime.ReadMemStats(&after) + var err error + allocated := testutils.MeasureAlloc(t, func() { + _, err = NewFile(bytes.NewReader(data)) + }) require.Error(t, err) assert.Contains(t, err.Error(), "over the") @@ -554,7 +555,7 @@ func TestNewFile_BombDoesNotAllocate(t *testing.T) { // the guard has to reject before the allocation, not after it. The margin is loose on purpose: this // is asserting "nothing near the declared size was allocated", not a precise budget. const margin = 32 * 1024 * 1024 - assert.Less(t, after.TotalAlloc-before.TotalAlloc, uint64(margin), + assert.Less(t, allocated, uint64(margin), "parsing allocated far more than the input warrants") } @@ -610,6 +611,87 @@ func TestNewFile_ExtendedShnumOverflowStillChecks(t *testing.T) { require.Error(t, err, "a bogus section count must not disable the check") } +// TestCheckAllSections covers the standalone gate, used by callers whose debug/elf parse happens inside +// another package (goversion, here) rather than through NewFile. +func TestCheckAllSections(t *testing.T) { + tests := []struct { + name string + data func(t *testing.T) []byte + // wantErr is empty when CheckAllSections must return nil. + wantErr string + // nameTableMisses marks a fixture CheckSectionNameTable alone does not reject, since the oversized + // section here is one debug/elf only expands lazily, after the parse CheckAllSections goes on to do. + nameTableMisses bool + }{ + { + name: "non-ELF reader", + data: func(t *testing.T) []byte { return []byte("not an elf") }, + }, + { + name: "valid ident but a body elf.NewFile cannot parse", + data: func(t *testing.T) []byte { + full := buildELF(t, elf.ELFCLASS64, binary.LittleEndian, nil, buildOpts{}) + truncated := full[:binary.Size(elf.Header64{})+4] + _, err := elf.NewFile(bytes.NewReader(truncated)) + require.Error(t, err, "fixture is supposed to be rejected by debug/elf") + return truncated + }, + }, + { + name: "clean small ELF", + data: func(t *testing.T) []byte { + data := buildELF(t, elf.ELFCLASS64, binary.LittleEndian, + []section{{name: ".text", typ: elf.SHT_PROGBITS, flags: elf.SHF_ALLOC}}, buildOpts{}) + _, err := elf.NewFile(bytes.NewReader(data)) + require.NoError(t, err, "fixture is supposed to be a file debug/elf accepts") + return data + }, + }, + { + name: "oversized section name table", + data: func(t *testing.T) []byte { + return buildELF(t, elf.ELFCLASS64, binary.LittleEndian, nil, + buildOpts{nameTable: §ion{compressed: true, declaredSize: overLimit}}) + }, + wantErr: "section name table", + }, + { + name: "oversized .symtab, expanded lazily rather than during the parse", + data: func(t *testing.T) []byte { + data := buildELF(t, elf.ELFCLASS64, binary.LittleEndian, []section{ + {name: ".symtab", typ: elf.SHT_SYMTAB, link: 2, compressed: true, declaredSize: overLimit}, + {name: ".strtab", typ: elf.SHT_STRTAB}, + }, buildOpts{}) + _, err := elf.NewFile(bytes.NewReader(data)) + require.NoError(t, err, "fixture is supposed to be a file debug/elf accepts") + return data + }, + wantErr: `".symtab"`, + nameTableMisses: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data := tt.data(t) + + if tt.nameTableMisses { + require.NoError(t, CheckSectionNameTable(bytes.NewReader(data)), + "the name table check alone doesn't reach a section debug/elf only expands lazily") + } + + err := CheckAllSections(bytes.NewReader(data)) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.Error(t, err) + assert.ErrorIs(t, err, ErrDeclaredSizeExceeded) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + func TestNewFile_PassesThroughNonELF(t *testing.T) { tests := []struct { name string diff --git a/syft/pkg/cataloger/ai/parse_safetensors_test.go b/syft/pkg/cataloger/ai/parse_safetensors_test.go index 56d50151a..c210bfbe0 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_test.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/anchore/syft/internal/testutils" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -1200,3 +1201,60 @@ func TestDockerAIModelConfigMediaTypes(t *testing.T) { }) } } + +// TestReadSafeTensorsHeader_capBoundsHeap pins maxSafeTensorsHeaderSize as bytes rather than as an +// error string. The existing over-cap case omits the body precisely because the guard rejects before +// allocating, which means it would keep passing if the guard moved after the read. This one delivers +// the bytes, so moving the check shows up as heap. +func TestReadSafeTensorsHeader_capBoundsHeap(t *testing.T) { + const declared = maxSafeTensorsHeaderSize + 1 + + body := bytes.Repeat([]byte{'x'}, declared) + file := make([]byte, 8, 8+len(body)) + binary.LittleEndian.PutUint64(file, uint64(declared)) + file = append(file, body...) + + // the unguarded shape: honor the declared length without weighing it against a ceiling first. + // This is what readSafeTensorsHeader does once the cap is passed, so it is what the cap prevents. + unbounded := testutils.MeasureAlloc(t, func() { + b, err := io.ReadAll(io.LimitReader(bytes.NewReader(file[8:]), declared)) + require.NoError(t, err) + require.Len(t, b, declared) + }) + require.Greater(t, unbounded, uint64(declared), + "fixture did not actually cost more than its own size; it is no longer a bomb") + + bounded := testutils.MeasureAlloc(t, func() { + _, err := readSafeTensorsHeader(bytes.NewReader(file)) + require.ErrorContains(t, err, "exceeds maximum") + }) + + t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded) + assert.Less(t, bounded, uint64(8*1024*1024), + "the declared-size check has to reject before the body is read, not after") +} + +// TestCopyHeader_boundIsTheCallersLimitedReader pins where the GGUF bound actually lives. copyHeader +// io.Copy's whatever it is handed, so maxHeaderSize only holds because every caller wraps the reader +// first. A caller that stops doing that reads the whole file, and nothing in copyHeader would object. +// +// Bytes retained is asserted directly rather than through testutils.MeasureAlloc: the destination is a +// bytes.Buffer, whose doubling growth costs a multiple of what it holds and lands on a different power +// of two depending on the platform. buf.Len() is the same property without that noise. +func TestCopyHeader_boundIsTheCallersLimitedReader(t *testing.T) { + const payload = 2*maxHeaderSize + 1 + + file := make([]byte, 24, 24+payload) + binary.LittleEndian.PutUint32(file[0:4], ggufMagicNumber) + file = append(file, bytes.Repeat([]byte{'x'}, payload)...) + + var unbounded bytes.Buffer + require.NoError(t, copyHeader(&unbounded, bytes.NewReader(file))) + require.Equal(t, len(file), unbounded.Len(), + "an unwrapped reader is copied whole; copyHeader applies no limit of its own") + + var bounded bytes.Buffer + require.NoError(t, copyHeader(&bounded, &io.LimitedReader{R: bytes.NewReader(file), N: maxHeaderSize})) + assert.Equal(t, maxHeaderSize, bounded.Len(), + "a wrapped reader must cap the copy at maxHeaderSize") +} diff --git a/syft/pkg/cataloger/arch/cataloger_test.go b/syft/pkg/cataloger/arch/cataloger_test.go index dd014086c..56e00ab9d 100644 --- a/syft/pkg/cataloger/arch/cataloger_test.go +++ b/syft/pkg/cataloger/arch/cataloger_test.go @@ -1,10 +1,14 @@ package arch import ( + "bytes" "context" + "io" + "os" "testing" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/stretchr/testify/require" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -12,7 +16,23 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) +const bombFixture = "testdata/installed/var/lib/pacman/local/bombpkg-1.0-1/mtree" + +// writeBombFixture generates bombpkg's mtree: one line over maxMtreeLines, so the cataloger rejects +// it while walking testdata/installed. Generated rather than committed so the size that makes it a +// bomb tracks the constant instead of hiding in a gzip blob. +func writeBombFixture(t *testing.T) { + t.Helper() + + listing := append([]byte("#mtree\n"), bytes.Repeat([]byte("./f\n"), maxMtreeLines+1)...) + compressed, err := io.ReadAll(gzipOf(t, listing)) + require.NoError(t, err) + require.NoError(t, os.WriteFile(bombFixture, compressed, 0o600)) +} + func TestAlpmUnknowns(t *testing.T) { + writeBombFixture(t) + pkgtest.NewCatalogTester(). FromDirectory(t, "testdata/installed"). WithCompareOptions(cmpopts.IgnoreFields(pkg.AlpmFileRecord{}, "Time")). @@ -21,11 +41,14 @@ func TestAlpmUnknowns(t *testing.T) { } func TestAlpmCataloger(t *testing.T) { + writeBombFixture(t) + gmpDbLocation := file.NewLocation("var/lib/pacman/local/gmp-6.2.1-2/desc") treeSitterDbLocation := file.NewLocation("var/lib/pacman/local/tree-sitter-0.22.6-1/desc") emacsDbLocation := file.NewLocation("var/lib/pacman/local/emacs-29.3-3/desc") fuzzyDbLocation := file.NewLocation("var/lib/pacman/local/fuzzy-1.2-3/desc") madeupDbLocation := file.NewLocation("var/lib/pacman/local/madeup-20.30-4/desc") + bombDbLocation := file.NewLocation("var/lib/pacman/local/bombpkg-1.0-1/desc") ctx := context.TODO() treeSitterPkg := pkg.Package{ @@ -119,6 +142,26 @@ func TestAlpmCataloger(t *testing.T) { }, } + // bombPkg's listing exceeds maxMtreeLines, so it emits with an empty file list, not dropped + bombPkg := pkg.Package{ + Name: "bombpkg", + Version: "1.0-1", + Type: pkg.AlpmPkg, + FoundBy: "alpm-db-cataloger", + Locations: file.NewLocationSet(bombDbLocation), + Metadata: pkg.AlpmDBEntry{ + BasePackage: "bombpkg", + Package: "bombpkg", + Version: "1.0-1", + Description: "Package whose mtree listing exceeds the parser's line bound", + Architecture: "x86_64", + Size: 1, + Reason: 1, + Files: []pkg.AlpmFileRecord{}, + Backup: []pkg.AlpmFileRecord{}, + }, + } + gmpPkg := pkg.Package{ Name: "gmp", Version: "6.2.1-2", @@ -283,6 +326,7 @@ func TestAlpmCataloger(t *testing.T) { fuzzyPkg, madeupPkg, gmpPkg, + bombPkg, } expectedRelationships := []artifact.Relationship{ diff --git a/syft/pkg/cataloger/arch/parse_alpm_db.go b/syft/pkg/cataloger/arch/parse_alpm_db.go index 6e409824b..fe9762087 100644 --- a/syft/pkg/cataloger/arch/parse_alpm_db.go +++ b/syft/pkg/cataloger/arch/parse_alpm_db.go @@ -4,6 +4,7 @@ import ( "bufio" "compress/gzip" "context" + "errors" "fmt" "io" "path" @@ -16,6 +17,7 @@ import ( "github.com/vbatts/go-mtree" "github.com/anchore/syft/internal" + intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" @@ -216,6 +218,10 @@ func parseDatabase(b *bufio.Scanner) (*parsedData, error) { var backup []map[string]any for _, f := range strings.Split(value, "\n") { fields := strings.SplitN(f, "\t", 2) + // without the tab there is no digest, and fields[1] would panic + if len(fields) < 2 { + continue + } p := fmt.Sprintf("/%s", fields[0]) if ok := ignoredFiles[p]; !ok { backup = append(backup, map[string]any{ @@ -241,6 +247,12 @@ func parseDatabase(b *bufio.Scanner) (*parsedData, error) { } } + // a field over the scanner's 1MB token cap ends the scan early, which would otherwise look + // like a complete package + if err := b.Err(); err != nil { + return nil, err + } + return parsePkgFiles(pkgFields) } @@ -277,15 +289,107 @@ func parsePkgFiles(pkgFields map[string]any) (*parsedData, error) { return &entry, nil } +// maxMtreeSize bounds the decompressed listing. texlive-fontsextra, the largest package Arch +// ships, runs 16 to 27MB. +// +// this caps bytes, not memory: go-mtree allocates per field, so a crafted listing can still cost +// about a gigabyte, times cataloger parallelism. +const maxMtreeSize = 64 * intFile.MB + +// maxMtreeLines bounds entry count, which the byte cap does not: go-mtree keeps a ~128 byte entry +// per line, blanks included. 300k is ~2.8x the largest package Arch ships. +const maxMtreeLines = 300_000 + +var ( + errMtreeTooLarge = errors.New("mtree file is larger than the max allowed size") + errTooManyMtreeLines = errors.New("mtree file has more lines than allowed") + + // errMtreeLineContinued rejects a line ending in a backslash, which readers treat as "join with + // the next line". We refuse the file instead, for two reasons. + // + // cost: go-mtree joins with a quadratic string concat, so work grows as lines squared. 100k + // continued lines is a 1.2MB listing costing ~3.7s and ~52GB of churn. The bounds above are + // linear and never see it, and no line cap helps, since real packages reach ~107k lines. + // + // spec: mtree(5) says "Each line is interpreted independently" and documents no continuation, + // nor does Arch's ALPM-MTREEv2(5). Readers accept them anyway (BSD mtree via fparseln(3), + // libarchive in readline()), but no writer emits one: libarchive escapes a literal backslash as + // \134, and Arch builds .MTREE with it. So a continuation here is crafted, not a package. + errMtreeLineContinued = errors.New("mtree file uses line continuations") +) + +// lineLimitedReader fails the read once the stream carries more than max newlines, or a line +// continuation. Counting while the bytes flow is what bounds entries: the parser builds one per +// line before returning any of them. +// +// continuations are refused rather than joined, unlike every other mtree reader. See +// errMtreeLineContinued. Both \n and \r\n count, since go-mtree strips a trailing carriage return +// before testing for the backslash. +// +// a tripped bound is latched, so reading past the first error does not come back clean. +type lineLimitedReader struct { + reader io.Reader + lines int + max int + lastByte byte + err error +} + +func (l *lineLimitedReader) Read(p []byte) (int, error) { + if l.err != nil { + return 0, l.err + } + + n, err := l.reader.Read(p) + + // prev carries over the last byte of the previous read, so a backslash split from its line + // ending is still caught + prev := l.lastByte + for _, c := range p[:n] { + if (c == '\n' || c == '\r') && prev == '\\' { + l.err = errMtreeLineContinued + return n, l.err + } + if c == '\n' { + l.lines++ + } + prev = c + } + l.lastByte = prev + + if l.lines > l.max { + l.err = fmt.Errorf("%w (max %d)", errTooManyMtreeLines, l.max) + return n, l.err + } + + return n, err +} + func parseMtree(r io.Reader) ([]pkg.AlpmFileRecord, error) { - var err error + return parseMtreeWithLimits(r, maxMtreeSize, maxMtreeLines) +} + +// parseMtreeWithLimits lets the boundary tests use small exact limits instead of the shipped caps. +// maxSize must be below math.MaxInt64, since the limiter carries a byte of headroom. +func parseMtreeWithLimits(r io.Reader, maxSize int64, maxLines int) ([]pkg.AlpmFileRecord, error) { var entries []pkg.AlpmFileRecord - r, err = gzip.NewReader(r) + gzReader, err := gzip.NewReader(r) if err != nil { return nil, err } - specDh, err := mtree.ParseSpec(r) + defer internal.CloseAndLogError(gzReader, "mtree") + + // the line cap trips mid-stream, the byte cap cannot: io.LimitedReader reports a drained budget + // as io.EOF, which the parser reads as a complete listing, so it is checked after the parse. The + // extra byte tells a drained budget from a listing ending exactly at the cap. + sizeLimited := &io.LimitedReader{R: gzReader, N: maxSize + 1} + specDh, err := mtree.ParseSpec(&lineLimitedReader{reader: sizeLimited, max: maxLines}) + + // before err: a drained budget means the parser saw only part of the listing + if sizeLimited.N <= 0 { + return nil, fmt.Errorf("%w (%d bytes)", errMtreeTooLarge, maxSize) + } if err != nil { return nil, err } diff --git a/syft/pkg/cataloger/arch/parse_alpm_db_test.go b/syft/pkg/cataloger/arch/parse_alpm_db_test.go index a046fff88..6b26209e8 100644 --- a/syft/pkg/cataloger/arch/parse_alpm_db_test.go +++ b/syft/pkg/cataloger/arch/parse_alpm_db_test.go @@ -2,13 +2,21 @@ package arch import ( "bufio" + "bytes" + "compress/gzip" + "fmt" + "io" "os" + "strings" "testing" "time" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/testutils" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" ) @@ -155,6 +163,26 @@ func TestDatabaseParser(t *testing.T) { } } +func Test_parseAlpmDBEntry_malformedDatabase(t *testing.T) { + t.Run("backup line with no tab", func(t *testing.T) { + // this used to panic, which drops every package rather than the one + entry, err := parseAlpmDBEntry(strings.NewReader("%NAME%\nfoo\n\n%BACKUP%\nnotabhere\n\n")) + + require.NoError(t, err) + require.NotNil(t, entry) + require.Empty(t, entry.Backup) + }) + + t.Run("field over the scanner's token cap", func(t *testing.T) { + // everything after the oversize field is lost, so this has to error rather than look complete + db := "%NAME%\nfoo\n\n%DESC%\n" + strings.Repeat("x", 2*intFile.MB) + "\n\n%ARCH%\nx86_64\n\n" + + _, err := parseAlpmDBEntry(strings.NewReader(db)) + + require.ErrorIs(t, err, bufio.ErrTooLong) + }) +} + func parseTime(stime string) time.Time { t, _ := time.Parse(time.RFC3339, stime) return t @@ -207,6 +235,7 @@ func TestMtreeParse(t *testing.T) { reader := bufio.NewReader(f) + // a normal listing must not trip any of the bounds entry, err := parseMtree(reader) require.NoError(t, err) @@ -217,3 +246,319 @@ func TestMtreeParse(t *testing.T) { } } + +// mtreeSpec builds a valid listing of n files. Boundary tests need the parser to reach the end of +// it, which filler bytes never do. +func mtreeSpec(n int) []byte { + var buf bytes.Buffer + buf.WriteString("#mtree\n") + buf.WriteString("/set type=file uid=0 gid=0 mode=644\n") + for i := range n { + fmt.Fprintf(&buf, "./file%d time=1649595592.0 size=10 sha256digest=%064x\n", i, i) + } + return buf.Bytes() +} + +func gzipOf(t *testing.T, data []byte) io.Reader { + t.Helper() + + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + _, err := w.Write(data) + require.NoError(t, err) + require.NoError(t, w.Close()) + + return bytes.NewReader(buf.Bytes()) +} + +// gzipOfRepeated builds a gzip member decompressing to n bytes of one repeated byte, so a few KB of +// input expands to whatever size the caller asks for. +func gzipOfRepeated(t *testing.T, payload byte, n int64) *bytes.Reader { + t.Helper() + + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + chunk := bytes.Repeat([]byte{payload}, 32*1024) + for remaining := n; remaining > 0; { + size := min(remaining, int64(len(chunk))) + written, err := w.Write(chunk[:size]) + require.NoError(t, err) + remaining -= int64(written) + } + require.NoError(t, w.Close()) + + return bytes.NewReader(buf.Bytes()) +} + +func Test_parseMtree_boundsDecompressedSize(t *testing.T) { + // limits come from the fixture, so both sides of the boundary are exact + spec := mtreeSpec(50) + + t.Run("a listing at the cap parses whole", func(t *testing.T) { + records, err := parseMtreeWithLimits(gzipOf(t, spec), int64(len(spec)), maxMtreeLines) + + // records, not just no error: landing exactly on the cap must not truncate + require.NoError(t, err) + require.Len(t, records, 50) + require.Equal(t, "/file0", records[0].Path) + require.Equal(t, "/file49", records[49].Path) + }) + + t.Run("rejects a listing one byte past the cap", func(t *testing.T) { + _, err := parseMtreeWithLimits(gzipOf(t, spec), int64(len(spec))-1, maxMtreeLines) + + require.ErrorIs(t, err, errMtreeTooLarge) + }) +} + +// Test_parseMtree_shippedConstants pins the production values. The boundary tests parameterize the +// limits, so they would pass unchanged if a constant regressed. Driving a real 64MB listing through +// the parser instead cost 12s of a 16s race run. +func Test_parseMtree_shippedConstants(t *testing.T) { + require.Equal(t, int64(64*intFile.MB), int64(maxMtreeSize)) + require.Equal(t, 300_000, maxMtreeLines) +} + +func Test_parseMtree_boundsLineCount(t *testing.T) { + // the byte cap does not bound memory: the parser keeps an entry per line, blanks included + spec := mtreeSpec(50) + lines := bytes.Count(spec, []byte("\n")) // the two header lines get entries of their own + + t.Run("a listing at the cap parses whole", func(t *testing.T) { + records, err := parseMtreeWithLimits(gzipOf(t, spec), maxMtreeSize, lines) + + require.NoError(t, err) + require.Len(t, records, 50) + }) + + t.Run("rejects a listing one line past the cap", func(t *testing.T) { + _, err := parseMtreeWithLimits(gzipOf(t, spec), maxMtreeSize, lines-1) + + require.ErrorIs(t, err, errTooManyMtreeLines) + }) + + t.Run("rejects a bomb at the production limits without reading all of it", func(t *testing.T) { + // what the cap exists for: a few KB expanding past the line cap while inside the byte cap + bomb := gzipOfRepeated(t, '\n', 4*1024*1024) + require.Less(t, bomb.Size(), int64(64*1024), "payload should be small enough to be worth rejecting") + + counted := &countingReader{reader: bomb} + + _, err := parseMtree(counted) + + require.ErrorIs(t, err, errTooManyMtreeLines) + + // what matters is giving up mid-read, not after building the whole listing. Bytes consumed is + // the stable way to check that; an allocation delta drifts with GC and parallelism. + require.Less(t, counted.n, bomb.Size(), + "the whole bomb was read, so the cap tripped after the parse rather than during it") + }) +} + +// countingReader records how many bytes were pulled through it. +type countingReader struct { + reader io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.reader.Read(p) + c.n += int64(n) + return n, err +} + +// Test_parseMtree_sizeCapTakesPrecedenceOverLineCap pins which error wins when one read exceeds both +// bounds. No other test exceeds both at once, so a reorder would go unnoticed. +func Test_parseMtree_sizeCapTakesPrecedenceOverLineCap(t *testing.T) { + spec := mtreeSpec(50) + + // maxLines=0 trips on the first newline; maxSize=20 is drained by that same read + _, err := parseMtreeWithLimits(gzipOf(t, spec), 20, 0) + + require.ErrorIs(t, err, errMtreeTooLarge) + require.NotErrorIs(t, err, errTooManyMtreeLines) +} + +func Test_parseMtree_rejectsLineContinuations(t *testing.T) { + // go-mtree strips a trailing carriage return before testing for the backslash, so CRLF continues + // a line too + for _, lineEnding := range []string{"\\\n", "\\\r\n"} { + t.Run(fmt.Sprintf("%q", lineEnding), func(t *testing.T) { + spec := mtreeSpec(5) + spec = bytes.Replace(spec, []byte("size=10 sha256digest="), []byte("size=10 "+lineEnding+"sha256digest="), 1) + + _, err := parseMtree(gzipOf(t, spec)) + + require.ErrorIs(t, err, errMtreeLineContinued) + }) + } +} + +// scriptedReader replays a fixed sequence of Read results, driving the straddling case directly. +type scriptedReader struct { + chunks []string +} + +func (r *scriptedReader) Read(p []byte) (int, error) { + if len(r.chunks) == 0 { + return 0, io.EOF + } + n := copy(p, r.chunks[0]) + // keep what did not fit, so a short buffer still obeys io.Reader + if n < len(r.chunks[0]) { + r.chunks[0] = r.chunks[0][n:] + } else { + r.chunks = r.chunks[1:] + } + return n, nil +} + +func Test_lineLimitedReader_rejectsStraddlingContinuation(t *testing.T) { + // the backslash and its line ending land in separate Read calls, which happens for real: the + // io.LimitedReader truncates the buffer and flate returns short reads. + tests := []struct { + name string + chunks []string + }{ + {name: "lf", chunks: []string{"...\\", "\n..."}}, + {name: "crlf", chunks: []string{"...\\", "\r\n..."}}, + {name: "crlf split at the carriage return", chunks: []string{"...\\\r", "\n..."}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lr := &lineLimitedReader{reader: &scriptedReader{chunks: tt.chunks}, max: 100} + + var err error + buf := make([]byte, 16) + for err == nil { + _, err = lr.Read(buf) + } + + require.ErrorIs(t, err, errMtreeLineContinued) + }) + } +} + +func Test_lineLimitedReader_latchesTheError(t *testing.T) { + // a tripped bound must not come back clean on the next read + lr := &lineLimitedReader{reader: &scriptedReader{chunks: []string{"a\\\nb", "clean"}}, max: 100} + + buf := make([]byte, 16) + _, first := lr.Read(buf) + require.ErrorIs(t, first, errMtreeLineContinued) + + n, second := lr.Read(buf) + + require.ErrorIs(t, second, errMtreeLineContinued) + require.Zero(t, n) +} + +func Test_parseMtree_malformedInput(t *testing.T) { + t.Run("not gzip", func(t *testing.T) { + _, err := parseMtree(bytes.NewReader(mtreeSpec(2))) + + require.ErrorIs(t, err, gzip.ErrHeader) + }) + + t.Run("truncated gzip", func(t *testing.T) { + spec := mtreeSpec(50) + var buf bytes.Buffer + _, err := io.Copy(&buf, gzipOf(t, spec)) + require.NoError(t, err) + + // the cap sits just above what the truncated stream delivers, so the size check and the + // truncation actually race. The shipped 64MB cap could never fire here. + _, err = parseMtreeWithLimits(bytes.NewReader(buf.Bytes()[:buf.Len()/2]), int64(len(spec)), maxMtreeLines) + + // must fail, but not as a size-bound trip + require.Error(t, err) + require.NotErrorIs(t, err, errMtreeTooLarge) + }) + + t.Run("single line over the scanner's token cap", func(t *testing.T) { + // the third bound is go-mtree's scanner, at the default 64KB token limit. Nothing else here + // would catch a dependency bump that calls scanner.Buffer. + spec := append([]byte("#mtree\n./big "), bytes.Repeat([]byte("a"), 128*1024)...) + + _, err := parseMtree(gzipOf(t, append(spec, '\n'))) + + require.ErrorContains(t, err, "token too long") + }) +} + +// Test_parseMtree_lineCapBoundsHeap pins the measurement that maxMtreeLines exists for. The byte cap +// alone does not bound this path: go-mtree keeps an entry per line, so a listing well inside 64MB of +// bytes still costs multiples of its own size in heap. Asserting the error would keep passing if the +// line cap were removed, because the byte cap would still reject a large enough input eventually. +func Test_parseMtree_lineCapBoundsHeap(t *testing.T) { + // newline-only lines are the worst case: minimum bytes per entry, so the most entries per byte + const payloadSize = 16 * intFile.MB + spec := bytes.Repeat([]byte{'\n'}, payloadSize) + + // well inside the byte cap, so only the line cap can stop this + require.Less(t, int64(payloadSize), int64(maxMtreeSize)) + + unbounded := testutils.MeasureAlloc(t, func() { + // the same listing with the line cap lifted, to prove the fixture is really a bomb + _, err := parseMtreeWithLimits(gzipOf(t, spec), maxMtreeSize, payloadSize+1) + t.Logf("unbounded parse err: %v", err) + }) + require.Greater(t, unbounded, uint64(payloadSize), + "fixture did not actually cost more than its own size; it is no longer a bomb") + + bounded := testutils.MeasureAlloc(t, func() { + _, err := parseMtreeWithLimits(gzipOf(t, spec), maxMtreeSize, maxMtreeLines) + require.ErrorIs(t, err, errTooManyMtreeLines) + }) + + t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded) + + // measured at ~238MB against ~16.5GB uncapped, so the cap is doing its job, but note what it + // actually buys: 300k entries still cost a few hundred MB, and that is per concurrent cataloger. + // The budget is set to catch the cap being removed or raised by an order of magnitude, not to pin + // the exact figure. + assert.Less(t, bounded, uint64(512*intFile.MB), + "the line cap has to stop the parse before it builds an entry per line") +} + +// Test_parseMtree_sizeCapBoundsHeap is the byte-cap counterpart to the line-cap budget above. +// Test_parseMtree_boundsDecompressedSize already covers the cap behaviorally, but with a spec small +// enough that buffering the whole listing would pass anyway. This one expands well past the cap, so a +// bound checked after the read instead of during it shows up as heap. +// +// The line length matters: go-mtree runs its own bufio.Scanner, which refuses a token past 64KB, so a +// single enormous line fails as "token too long" before the byte cap is ever consulted. Lines have to +// be ordinary for the byte cap to be the thing under test, and there have to be few enough of them +// that the line cap does not fire first. +func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) { + const ( + lineLen = 1024 + numLines = 96 * 1024 // ~96MB across ~98k lines: past the 64MB cap, well under the 300k line cap + ) + + var raw bytes.Buffer + line := append(bytes.Repeat([]byte{'a'}, lineLen-1), '\n') + for range numLines { + raw.Write(line) + } + require.Greater(t, int64(raw.Len()), int64(maxMtreeSize), "fixture must exceed the byte cap") + require.Less(t, numLines, maxMtreeLines, "and must not trip the line cap first") + + unbounded := testutils.MeasureAlloc(t, func() { + // the same listing with the byte cap lifted past it, to prove the fixture is really a bomb + _, err := parseMtreeWithLimits(gzipOf(t, raw.Bytes()), int64(raw.Len())*2, maxMtreeLines) + t.Logf("unbounded parse err: %v", err) + }) + require.Greater(t, unbounded, uint64(raw.Len()), + "fixture did not actually cost more than its own size; it is no longer a bomb") + + bounded := testutils.MeasureAlloc(t, func() { + _, err := parseMtreeWithLimits(gzipOf(t, raw.Bytes()), maxMtreeSize, maxMtreeLines) + require.ErrorIs(t, err, errMtreeTooLarge) + }) + + t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded) + assert.Less(t, bounded, uint64(512*intFile.MB), + "the size cap has to stop the read at the cap, not after the member is drained") +} diff --git a/syft/pkg/cataloger/arch/testdata/installed/var/lib/pacman/local/bombpkg-1.0-1/.gitignore b/syft/pkg/cataloger/arch/testdata/installed/var/lib/pacman/local/bombpkg-1.0-1/.gitignore new file mode 100644 index 000000000..dfa63e683 --- /dev/null +++ b/syft/pkg/cataloger/arch/testdata/installed/var/lib/pacman/local/bombpkg-1.0-1/.gitignore @@ -0,0 +1,2 @@ +# generated by writeBombFixture, see cataloger_test.go +/mtree diff --git a/syft/pkg/cataloger/arch/testdata/installed/var/lib/pacman/local/bombpkg-1.0-1/desc b/syft/pkg/cataloger/arch/testdata/installed/var/lib/pacman/local/bombpkg-1.0-1/desc new file mode 100644 index 000000000..abd0ad98d --- /dev/null +++ b/syft/pkg/cataloger/arch/testdata/installed/var/lib/pacman/local/bombpkg-1.0-1/desc @@ -0,0 +1,20 @@ +%NAME% +bombpkg + +%VERSION% +1.0-1 + +%BASE% +bombpkg + +%DESC% +Package whose mtree listing exceeds the parser's line bound + +%ARCH% +x86_64 + +%SIZE% +1 + +%REASON% +1 diff --git a/syft/pkg/cataloger/binary/classifiers.go b/syft/pkg/cataloger/binary/classifiers.go index 5fc3c7b80..17834ab0c 100644 --- a/syft/pkg/cataloger/binary/classifiers.go +++ b/syft/pkg/cataloger/binary/classifiers.go @@ -8,6 +8,9 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/internal/binutils" ) +// mysqlClusterPackage is the package name shared by every MySQL Cluster classifier below. +const mysqlClusterPackage = "mysql-cluster" + // in both binaries and shared libraries, the version pattern is [NUL]3.11.2[NUL] var pythonVersionTemplate = `(?m)\x00(?P{{ .version }}[-._a-zA-Z0-9]*)\x00` @@ -374,7 +377,7 @@ func DefaultClassifiers() []binutils.Classifier { Class: "mysqld-mysql-cluster-legacy-binary", EvidenceMatcher: m.FileContentsVersionMatcher( `cluster-gpl\x00[0-9]+(\.[0-9]+)?(\.[0-9]+)?\-ndb\-(?P[0-9]+(\.[0-9]+)?(\.[0-9]+)?)`), - Package: "mysql-cluster", + Package: mysqlClusterPackage, PURL: mustPURL("pkg:generic/mysql-cluster@version"), CPEs: []cpe.CPE{ cpe.Must("cpe:2.3:a:oracle:mysql_cluster:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), @@ -385,7 +388,7 @@ func DefaultClassifiers() []binutils.Classifier { Class: "mysqld-mysql-cluster-binary", EvidenceMatcher: m.FileContentsVersionMatcher( `/mysql-cluster-gpl-(?P[0-9]+(\.[0-9]+)?(\.[0-9]+)?(alpha[0-9]|beta[0-9]|rc[0-9])?)/`), - Package: "mysql-cluster", + Package: mysqlClusterPackage, PURL: mustPURL("pkg:generic/mysql-cluster@version"), CPEs: []cpe.CPE{ cpe.Must("cpe:2.3:a:oracle:mysql:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), @@ -412,7 +415,7 @@ func DefaultClassifiers() []binutils.Classifier { FileGlob: "**/ndbd", EvidenceMatcher: m.FileContentsVersionMatcher( `/mysql-cluster-gpl-(?P[0-9]+(\.[0-9]+)?(\.[0-9]+)?(alpha[0-9]|beta[0-9]|rc[0-9])?)/`), - Package: "mysql-cluster", + Package: mysqlClusterPackage, PURL: mustPURL("pkg:generic/mysql-cluster@version"), CPEs: []cpe.CPE{ cpe.Must("cpe:2.3:a:oracle:mysql_cluster:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), @@ -423,7 +426,7 @@ func DefaultClassifiers() []binutils.Classifier { FileGlob: "**/ndbmtd", EvidenceMatcher: m.FileContentsVersionMatcher( `/mysql-cluster-gpl-(?P[0-9]+(\.[0-9]+)?(\.[0-9]+)?(alpha[0-9]|beta[0-9]|rc[0-9])?)/`), - Package: "mysql-cluster", + Package: mysqlClusterPackage, PURL: mustPURL("pkg:generic/mysql-cluster@version"), CPEs: []cpe.CPE{ cpe.Must("cpe:2.3:a:oracle:mysql_cluster:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), @@ -434,7 +437,7 @@ func DefaultClassifiers() []binutils.Classifier { FileGlob: "**/ndb_mgmd", EvidenceMatcher: m.FileContentsVersionMatcher( `/mysql-cluster-gpl-(?P[0-9]+(\.[0-9]+)?(\.[0-9]+)?(alpha[0-9]|beta[0-9]|rc[0-9])?)/`), - Package: "mysql-cluster", + Package: mysqlClusterPackage, PURL: mustPURL("pkg:generic/mysql-cluster@version"), CPEs: []cpe.CPE{ cpe.Must("cpe:2.3:a:oracle:mysql_cluster:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), diff --git a/syft/pkg/cataloger/binary/pe_package_cataloger.go b/syft/pkg/cataloger/binary/pe_package_cataloger.go index 74abb379d..05410ab97 100644 --- a/syft/pkg/cataloger/binary/pe_package_cataloger.go +++ b/syft/pkg/cataloger/binary/pe_package_cataloger.go @@ -36,5 +36,11 @@ func parsePE(_ context.Context, _ file.Resolver, _ *generic.Environment, reader p := newPEPackage(f.VersionResources, f.Location) + // the file was usable but not fully parsed, which for this cataloger means the package name and version + // may have come from the filename rather than from version resources. Worth reporting alongside it. + if f.ParseErr != nil { + return []pkg.Package{p}, nil, unknown.New(reader, f.ParseErr) + } + return []pkg.Package{p}, nil, nil } diff --git a/syft/pkg/cataloger/debian/cataloger_test.go b/syft/pkg/cataloger/debian/cataloger_test.go index 4d5deb9fb..8d38c7b33 100644 --- a/syft/pkg/cataloger/debian/cataloger_test.go +++ b/syft/pkg/cataloger/debian/cataloger_test.go @@ -285,6 +285,7 @@ func TestDpkgArchiveCataloger(t *testing.T) { pkgtest.NewCatalogTester(). WithImageResolver(t, tt.name). IgnoreLocationLayer(). // this fixture can be rebuilt, thus the layer ID will change + WithErrorAssertion(require.NoError). Expects(tt.expected, nil). TestCataloger(t, c) }) diff --git a/syft/pkg/cataloger/debian/package.go b/syft/pkg/cataloger/debian/package.go index 364963516..8ad569f0f 100644 --- a/syft/pkg/cataloger/debian/package.go +++ b/syft/pkg/cataloger/debian/package.go @@ -202,7 +202,11 @@ func getAdditionalFileListing(resolver file.Resolver, dbLocation file.Location, if md5Reader != nil && md5Location != nil { defer internal.CloseAndLogError(md5Reader, md5Location.AccessPath) // attach the file list - files = append(files, parseDpkgMD5Info(md5Reader)...) + records, err := parseDpkgMD5Info(md5Reader) + if err != nil { + log.Debugf("dpkg md5sums file %q: %v", md5Location.RealPath, err) + } + files = append(files, records...) // keep a record of the file where this was discovered locations = append(locations, *md5Location) @@ -213,7 +217,11 @@ func getAdditionalFileListing(resolver file.Resolver, dbLocation file.Location, if conffilesReader != nil && conffilesLocation != nil { defer internal.CloseAndLogError(conffilesReader, conffilesLocation.AccessPath) // attach the file list - files = append(files, parseDpkgConffileInfo(conffilesReader)...) + records, err := parseDpkgConffileInfo(conffilesReader) + if err != nil { + log.Debugf("dpkg conffiles file %q: %v", conffilesLocation.RealPath, err) + } + files = append(files, records...) // keep a record of the file where this was discovered locations = append(locations, *conffilesLocation) diff --git a/syft/pkg/cataloger/debian/parse_deb_archive.go b/syft/pkg/cataloger/debian/parse_deb_archive.go index b3b5f923f..4036e93d4 100644 --- a/syft/pkg/cataloger/debian/parse_deb_archive.go +++ b/syft/pkg/cataloger/debian/parse_deb_archive.go @@ -3,8 +3,10 @@ package debian import ( "archive/tar" "context" + "errors" "fmt" "io" + "math" "path/filepath" "regexp" "strings" @@ -13,6 +15,7 @@ import ( "github.com/mholt/archives" "github.com/anchore/syft/internal" + intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -33,6 +36,7 @@ func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environmen var metadata *pkg.DpkgArchiveEntry var licenses []string var unknownErr error + var sawControl, sawData bool for { header, err := arReader.Next() if err == io.EOF { @@ -43,19 +47,28 @@ func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environmen } switch { - case strings.HasPrefix(header.Name, "control.tar"): + // a .deb has exactly one control.tar.* and one data.tar.* member; anything past the first of + // each is attacker-supplied filler that would otherwise buy a fresh decompression budget + case !sawControl && strings.HasPrefix(header.Name, "control.tar"): + sawControl = true // Decompress the control.tar.* file - dcReader, err := decompressionStream(ctx, arReader, header.Name) + dcReader, err := decompressionStream(ctx, arReader, header.Name, maxControlTarSize) if err != nil { return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to decompress control.tar.* file: %w", err)) } metadata, err = processControlTar(dcReader) - if err != nil { + switch { + case err != nil && metadata == nil: return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to process control.tar.* file: %w", err)) + case err != nil: + // metadata came back usable but incomplete (a clipped file listing); keep the package and + // record why it is partial + unknownErr = unknown.Append(unknownErr, reader.Location, fmt.Errorf("partial control.tar.* file: %w", err)) } - case strings.HasPrefix(header.Name, "data.tar"): + case !sawData && strings.HasPrefix(header.Name, "data.tar"): + sawData = true // Decompress the data.tar.* file - dcReader, err := decompressionStream(ctx, arReader, header.Name) + dcReader, err := decompressionStream(ctx, arReader, header.Name, maxDataTarSize) if err != nil { return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to decompress data.tar.* file: %w", err)) } @@ -64,15 +77,21 @@ func parseDebArchive(ctx context.Context, _ file.Resolver, _ *generic.Environmen unknownErr = unknown.Append(unknownErr, reader.Location, fmt.Errorf("failed to process data.tar.* file: %w", err)) } } + + if sawControl && sawData { + break + } } if metadata == nil { return nil, nil, unknown.New(reader.Location, fmt.Errorf("no application found described in .dpkg archive")) } + // a partial parse still yields a usable package, so report what went wrong alongside it rather than + // dropping either one return []pkg.Package{ newDebArchivePackage(ctx, reader.Location, *metadata, licenses), - }, nil, nil + }, nil, unknownErr } // this is the pattern you'd expect to see in a tar header for a debian package license file () @@ -81,6 +100,7 @@ var archiveHeaderLicensePathPattern = regexp.MustCompile(`^\.?/usr/share/doc/[^/ func processDataTar(dcReader io.ReadCloser) ([]string, error) { defer internal.CloseAndLogError(dcReader, "") var licenses []string + var copyrightFiles int tarReader := tar.NewReader(dcReader) for { @@ -96,6 +116,10 @@ func processDataTar(dcReader io.ReadCloser) ([]string, error) { // TODO: in the future we can add archive sub indexes to the locations to see where within // the dpkg archive the license was found if archiveHeaderLicensePathPattern.MatchString(header.Name) { + copyrightFiles++ + if copyrightFiles > maxCopyrightFiles { + return licenses, fmt.Errorf("copyright file count exceeds %d; remaining files not read", maxCopyrightFiles) + } licenses = append(licenses, parseLicensesFromCopyright(tarReader)...) } } @@ -103,6 +127,10 @@ func processDataTar(dcReader io.ReadCloser) ([]string, error) { return licenses, nil } +// processControlTar always returns whatever metadata it managed to parse, even alongside a non-nil +// error. A non-nil error with non-nil metadata means the package is usable but incomplete (e.g. a +// clipped file listing); nil metadata means nothing usable was found and the caller should treat it +// as fatal. func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) { defer internal.CloseAndLogError(dcReader, "") @@ -111,6 +139,7 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) { var metadata *pkg.DpkgArchiveEntry var files []pkg.DpkgFileRecord var confFileRecords []pkg.DpkgFileRecord + var listingErr error for { header, err := tarReader.Next() @@ -118,7 +147,7 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) { break } if err != nil { - return nil, fmt.Errorf("failed to read control tar: %w", err) + return metadata, fmt.Errorf("failed to read control tar: %w", err) } switch filepath.Base(header.Name) { @@ -134,11 +163,15 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) { entry := pkg.DpkgArchiveEntry(entries[0].toDpkgEntry()) metadata = &entry case "md5sums": - // parseDpkgMD5Info already streams via bufio.Scanner - files = parseDpkgMD5Info(tarReader) + // parseDpkgMD5Info streams via bufio.Scanner and reports its own clipping/scan errors + var err error + files, err = parseDpkgMD5Info(tarReader) + listingErr = errors.Join(listingErr, err) case "conffiles": - // parseDpkgConffileInfo already streams via bufio.Scanner - confFileRecords = parseDpkgConffileInfo(tarReader) + // parseDpkgConffileInfo streams via bufio.Scanner and reports its own clipping/scan errors + var err error + confFileRecords, err = parseDpkgConffileInfo(tarReader) + listingErr = errors.Join(listingErr, err) } } @@ -159,10 +192,36 @@ func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) { } metadata.Files = files - return metadata, nil + + // a clipped or unreadable file listing leaves the package usable, so hand both back and let the + // caller decide + return metadata, listingErr } -func decompressionStream(ctx context.Context, r io.Reader, filePath string) (io.ReadCloser, error) { +// a .deb's member sizes are bounded by the archive, but what they decompress to is not: xz and zstd +// both exceed gzip's ~1032:1, so a few KB member can expand without limit. The two members get +// different budgets because they hold different things. +const ( + // control.tar holds only metadata (control, md5sums, conffiles). Even a package shipping a hundred + // thousand files lands around 12MB of md5sums text. + maxControlTarSize int64 = 16 * intFile.MB + + // data.tar is the package payload, which is legitimately large (texlive and cuda ship hundreds of + // MB), so this is only here to stop the tar walk from spinning forever on a bomb. It mirrors the + // perFileReadLimit already used for archive extraction in internal/file/copy.go. + maxDataTarSize int64 = 2 * intFile.GB + + // a real .deb ships exactly one copyright file per package. + maxCopyrightFiles = 1000 +) + +// sentinels so callers (and tests) can match with errors.Is instead of substring matching. +var ( + errDecompressedTooLarge = errors.New("decompressed stream is larger than the max allowed size") + errClippedFileListing = errors.New("file listing exceeds the max allowed entries") +) + +func decompressionStream(ctx context.Context, r io.Reader, filePath string, maxDecompressed int64) (io.ReadCloser, error) { format, stream, err := archives.Identify(ctx, filePath, r) if err != nil { return nil, fmt.Errorf("failed to identify compression format: %w", err) @@ -178,5 +237,45 @@ func decompressionStream(ctx context.Context, r io.Reader, filePath string) (io. return nil, fmt.Errorf("failed to create decompression reader: %w", err) } - return rc, nil + return newBoundedReadCloser(rc, maxDecompressed), nil +} + +// boundedReadCloser fails the read past a byte budget rather than truncating. +// +// io.LimitedReader is the obvious thing to reach for here and is wrong for this: past its limit it +// just stops returning bytes, and both consumers of this are tar readers, which read a truncated +// stream as a clean end of archive. Together that emits a package silently missing files, which is +// the exact failure the bound exists to catch. The signal is recoverable by checking +// LimitedReader.N after the walk finishes (the same shape as safeCopy in internal/file/copy.go, +// which counts bytes off io.Copy), but nothing obliges a caller to write that check and a missed +// one is silent all over again. Erroring at the Read is the version that cannot be forgotten. +// +// net/http's MaxBytesReader has precisely these semantics and is stdlib, but reaching into the HTTP +// package to parse a .deb leaves the next reader wondering what it is doing here. +type boundedReadCloser struct { + io.ReadCloser + remaining int64 + limit int64 +} + +func newBoundedReadCloser(rc io.ReadCloser, limit int64) *boundedReadCloser { + // budget is one byte past the cap so a stream of exactly limit bytes still reaches EOF normally, + // and only the byte after the cap trips the error. Clamp so a limit near math.MaxInt64 can't wrap + // remaining negative and fail every read immediately. + if limit < 0 || limit > math.MaxInt64-1 { + limit = math.MaxInt64 - 1 + } + return &boundedReadCloser{ReadCloser: rc, remaining: limit + 1, limit: limit} +} + +func (b *boundedReadCloser) Read(p []byte) (int, error) { + if b.remaining <= 0 { + return 0, fmt.Errorf("%w (%d bytes)", errDecompressedTooLarge, b.limit) + } + if int64(len(p)) > b.remaining { + p = p[:b.remaining] + } + n, err := b.ReadCloser.Read(p) + b.remaining -= int64(n) + return n, err } diff --git a/syft/pkg/cataloger/debian/parse_deb_archive_test.go b/syft/pkg/cataloger/debian/parse_deb_archive_test.go index c9baee738..7afa0a455 100644 --- a/syft/pkg/cataloger/debian/parse_deb_archive_test.go +++ b/syft/pkg/cataloger/debian/parse_deb_archive_test.go @@ -3,11 +3,24 @@ package debian import ( "archive/tar" "bytes" + "compress/gzip" + "context" + "errors" + "fmt" "io" + "strings" "testing" + "testing/iotest" + "github.com/blakesmith/ar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/testutils" + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg/cataloger/generic" ) func TestProcessControlTar(t *testing.T) { @@ -85,3 +98,319 @@ func writeTarEntry(t *testing.T, tw *tar.Writer, name, content string) { _, err := tw.Write([]byte(content)) require.NoError(t, err) } + +func Test_boundedReadCloser(t *testing.T) { + payload := []byte("0123456789") + + t.Run("a stream at exactly the cap reads through whole", func(t *testing.T) { + // the positive assertion is the point: it proves the one-byte-past budget does not truncate + // at-cap content, which a bare "did not error" check would not catch + r := newBoundedReadCloser(io.NopCloser(bytes.NewReader(payload)), int64(len(payload))) + + got, err := io.ReadAll(r) + + require.NoError(t, err) + assert.Equal(t, payload, got) + }) + + t.Run("a stream past the cap errors rather than truncating", func(t *testing.T) { + r := newBoundedReadCloser(io.NopCloser(bytes.NewReader(payload)), int64(len(payload)-1)) + + _, err := io.ReadAll(r) + + require.ErrorIs(t, err, errDecompressedTooLarge) + }) + + t.Run("the underlying read error is not masked", func(t *testing.T) { + r := newBoundedReadCloser(io.NopCloser(iotest.ErrReader(errBoom)), 1024) + + _, err := io.ReadAll(r) + + require.ErrorIs(t, err, errBoom) + }) +} + +var errBoom = errors.New("boom") + +func Test_decompressionStream_boundsDecompressedSize(t *testing.T) { + // a highly compressible payload standing in for a decompression bomb: tiny compressed, large out + var raw bytes.Buffer + raw.Write(make([]byte, 512*1024)) + + var gz bytes.Buffer + gw := gzip.NewWriter(&gz) + _, err := gw.Write(raw.Bytes()) + require.NoError(t, err) + require.NoError(t, gw.Close()) + + require.Less(t, gz.Len(), 4096, "compressed payload should be tiny relative to what it expands to") + + rc, err := decompressionStream(context.Background(), bytes.NewReader(gz.Bytes()), "control.tar.gz", 64*1024) + require.NoError(t, err, "the bound is enforced on read, not on open") + t.Cleanup(func() { _ = rc.Close() }) + + _, err = io.ReadAll(rc) + + require.ErrorIs(t, err, errDecompressedTooLarge) +} + +func Test_processControlTar_rejectsABombedControlMember(t *testing.T) { + // end to end through the real consumer: a tar reader treats a truncated stream as a clean end of + // archive, so this asserts the bomb surfaces as an error instead of a package missing its files. + // "control" parses fully before the md5sums stream dies, so this also covers the Fix C contract: a + // control tar read failure must not drop the package that was already parsed. + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + writeTarEntry(t, tw, "control", "Package: test-package\nVersion: 1.0.0\n") + writeTarEntry(t, tw, "md5sums", strings.Repeat("d41d8cd98f00b204e9800998ecf8427e usr/bin/x\n", 20000)) + require.NoError(t, tw.Close()) + + metadata, err := processControlTar(newBoundedReadCloser(io.NopCloser(bytes.NewReader(tarBuf.Bytes())), 4096)) + + require.ErrorIs(t, err, errDecompressedTooLarge) + require.NotNil(t, metadata, "a control tar read failure after \"control\" was parsed must not drop the package") + assert.Equal(t, "test-package", metadata.Package) +} + +func Test_processControlTar_clippedListingIsUsableButReported(t *testing.T) { + // the record cap must not drop the package: metadata comes back usable AND an error explains why the + // file list is short, so the caller can record it as an unknown instead of discarding the package + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + writeTarEntry(t, tw, "control", "Package: test-package\nVersion: 1.0.0\n") + writeTarEntry(t, tw, "md5sums", strings.Repeat("d41d8cd98f00b204e9800998ecf8427e usr/bin/x\n", maxDpkgFileRecords+10)) + require.NoError(t, tw.Close()) + + metadata, err := processControlTar(io.NopCloser(bytes.NewReader(tarBuf.Bytes()))) + + require.Error(t, err, "clipping must be reported") + require.ErrorIs(t, err, errClippedFileListing) + require.NotNil(t, metadata, "the package must survive a clipped file listing") + assert.Equal(t, "test-package", metadata.Package) + assert.Len(t, metadata.Files, maxDpkgFileRecords) +} + +// buildDeb assembles a minimal .deb (an ar archive of debian-binary + control.tar.gz + data.tar.gz). +func buildDeb(t *testing.T, controlTar, dataTar []byte) []byte { + t.Helper() + + gzipped := func(b []byte) []byte { + var out bytes.Buffer + w := gzip.NewWriter(&out) + _, err := w.Write(b) + require.NoError(t, err) + require.NoError(t, w.Close()) + return out.Bytes() + } + + var buf bytes.Buffer + aw := ar.NewWriter(&buf) + require.NoError(t, aw.WriteGlobalHeader()) + + for _, m := range []struct { + name string + body []byte + }{ + {"debian-binary", []byte("2.0\n")}, + {"control.tar.gz", gzipped(controlTar)}, + {"data.tar.gz", gzipped(dataTar)}, + } { + require.NoError(t, aw.WriteHeader(&ar.Header{Name: m.name, Size: int64(len(m.body))})) + _, err := aw.Write(m.body) + require.NoError(t, err) + } + + return buf.Bytes() +} + +func Test_parseDebArchive_surfacesPartialParseAsUnknown(t *testing.T) { + // guards a silent failure: unknownErr was accumulated and then dropped on return, so a data.tar that + // failed to parse produced a clean-looking SBOM with no record that anything went wrong. The property + // that actually matters is that the returned error is a *unknown.CoordinateError: that's what decides + // between "recorded as an unknown" (this case) and "aborts the entire scan" (anything else), per + // internal/task/executor.go and syft/create_sbom.go. A plain substring match on "data.tar" would pass + // just as well if unknown.Append were swapped for errors.Join, which is the regression that matters. + var controlTar bytes.Buffer + ctw := tar.NewWriter(&controlTar) + writeTarEntry(t, ctw, "control", "Package: test-package\nVersion: 1.0.0\nArchitecture: all\n") + require.NoError(t, ctw.Close()) + + // a data.tar that is not a valid tar at all, so processDataTar errors + dataTar := []byte(strings.Repeat("not a tar file at all", 100)) + + debBytes := buildDeb(t, controlTar.Bytes(), dataTar) + + loc := file.NewVirtualLocation("/test.deb", "/test.deb") + pkgs, _, err := parseDebArchive(context.Background(), nil, &generic.Environment{}, + file.LocationReadCloser{ + Location: loc, + ReadCloser: io.NopCloser(bytes.NewReader(debBytes)), + }) + + require.Len(t, pkgs, 1, "the package must still be emitted") + assert.Equal(t, "test-package", pkgs[0].Name) + require.Error(t, err, "a failed data.tar parse must not be swallowed") + + coordinateErrors, remaining := unknown.ExtractCoordinateErrors(err) + require.NoError(t, remaining, "no non-coordinate errors should remain in the chain") + require.Len(t, coordinateErrors, 1) + assert.Equal(t, loc.Coordinates, coordinateErrors[0].Coordinates) + assert.Contains(t, coordinateErrors[0].Reason.Error(), "data.tar") +} + +func Test_parseDebArchive_controlTarBombRejectedByRealSizeBound(t *testing.T) { + // end to end through the real entrypoint using the real maxControlTarSize (16MB), not a swapped-in + // test value: nothing else in this suite exercises that constant at its actual call site. The bomb + // entry is written before "control", so "control" is never reached and processControlTar returns nil + // metadata alongside the error -- this exercises parseDebArchive's fatal branch (metadata == nil). + var controlTar bytes.Buffer + ctw := tar.NewWriter(&controlTar) + writeTarEntry(t, ctw, "bomb", strings.Repeat("\x00", 20*1024*1024)) + writeTarEntry(t, ctw, "control", "Package: test-package\nVersion: 1.0.0\n") + require.NoError(t, ctw.Close()) + + debBytes := buildDeb(t, controlTar.Bytes(), nil) + require.Less(t, len(debBytes), 64*1024, "compressed .deb should stay tiny relative to what it expands to") + + loc := file.NewVirtualLocation("/bomb.deb", "/bomb.deb") + pkgs, _, err := parseDebArchive(context.Background(), nil, &generic.Environment{}, + file.LocationReadCloser{ + Location: loc, + ReadCloser: io.NopCloser(bytes.NewReader(debBytes)), + }) + + require.Empty(t, pkgs, "no package should be emitted when the control tar never parses") + require.Error(t, err) + + coordinateErrors, remaining := unknown.ExtractCoordinateErrors(err) + require.NoError(t, remaining, "no non-coordinate errors should remain in the chain") + require.Len(t, coordinateErrors, 1) + require.ErrorIs(t, coordinateErrors[0].Reason, errDecompressedTooLarge) +} + +// arMember is one named, already-encoded ar entry body, for tests that need control over the raw +// member list (order, duplicates) that buildDeb's fixed three-member shape does not allow. +type arMember struct { + name string + body []byte +} + +// buildDebWithMembers is buildDeb generalized to an arbitrary member list. +func buildDebWithMembers(t *testing.T, members []arMember) []byte { + t.Helper() + + var buf bytes.Buffer + aw := ar.NewWriter(&buf) + require.NoError(t, aw.WriteGlobalHeader()) + + for _, m := range members { + require.NoError(t, aw.WriteHeader(&ar.Header{Name: m.name, Size: int64(len(m.body))})) + _, err := aw.Write(m.body) + require.NoError(t, err) + } + + return buf.Bytes() +} + +func Test_parseDebArchive_ignoresDuplicateArMembers(t *testing.T) { + // a .deb has exactly one control.tar.* and one data.tar.*; a second data.tar.* member is + // attacker-supplied filler that would otherwise buy a fresh decompression budget, and here would also + // fail to parse as a tar at all -- so this proves the walk stops after the first of each kind rather + // than merely surviving a well-formed duplicate. + gzipped := func(b []byte) []byte { + var out bytes.Buffer + w := gzip.NewWriter(&out) + _, err := w.Write(b) + require.NoError(t, err) + require.NoError(t, w.Close()) + return out.Bytes() + } + + var controlTar bytes.Buffer + ctw := tar.NewWriter(&controlTar) + writeTarEntry(t, ctw, "control", "Package: test-package\nVersion: 1.0.0\n") + require.NoError(t, ctw.Close()) + + var firstDataTar bytes.Buffer + dtw := tar.NewWriter(&firstDataTar) + require.NoError(t, dtw.Close()) + + debBytes := buildDebWithMembers(t, []arMember{ + {"debian-binary", []byte("2.0\n")}, + // the duplicate sits before control.tar.gz on purpose: the loop cannot have stopped yet at that + // point, so only the per-member guard can keep it from being read + {"data.tar.gz", gzipped(firstDataTar.Bytes())}, + {"data.tar.gz", []byte("not a tar file at all")}, // not even gzipped; would error if ever read + {"control.tar.gz", gzipped(controlTar.Bytes())}, + }) + + pkgs, _, err := parseDebArchive(context.Background(), nil, &generic.Environment{}, + file.LocationReadCloser{ + Location: file.NewVirtualLocation("/test.deb", "/test.deb"), + ReadCloser: io.NopCloser(bytes.NewReader(debBytes)), + }) + + require.NoError(t, err, "the second data.tar.* member must never be reached") + require.Len(t, pkgs, 1) +} + +func Test_processDataTar_capsCopyrightFiles(t *testing.T) { + // a real .deb ships exactly one copyright file per package; this asserts the walk stops and errors + // rather than accumulating an unbounded number of license reads + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + for i := 0; i < maxCopyrightFiles+10; i++ { + writeTarEntry(t, tw, fmt.Sprintf("/usr/share/doc/pkg%d/copyright", i), "License: MIT\n") + } + require.NoError(t, tw.Close()) + + _, err := processDataTar(io.NopCloser(bytes.NewReader(tarBuf.Bytes()))) + + require.Error(t, err) +} + +// Test_decompressionStream_failsEarlyInsteadOfBuffering pins that the bound is spent on read rather +// than after the fact. The existing bounds test uses a payload small enough that buffering the whole +// thing would pass anyway; this one is large enough that the difference shows up as bytes. +func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) { + const expanded = 512 * intFile.MB + + // a few KB of input expanding to 512MB, so heap is what separates a streaming bound from a + // bound checked after the read completes + var gz bytes.Buffer + gw := gzip.NewWriter(&gz) + chunk := make([]byte, 32*1024) + for remaining := int64(expanded); remaining > 0; { + n, err := gw.Write(chunk[:min(remaining, int64(len(chunk)))]) + require.NoError(t, err) + remaining -= int64(n) + } + require.NoError(t, gw.Close()) + require.Less(t, gz.Len(), 1024*1024, "compressed payload should be tiny relative to what it expands to") + + // both arms use io.ReadAll on purpose. The stream itself never holds much whatever the cap says, + // so copying to io.Discard measures nothing; what the cap actually protects is a consumer that + // buffers, and the real one (a tar reader building entries) does. + unbounded := testutils.MeasureAlloc(t, func() { + rc, err := decompressionStream(context.Background(), bytes.NewReader(gz.Bytes()), "control.tar.gz", expanded*2) + require.NoError(t, err) + defer rc.Close() + b, err := io.ReadAll(rc) + require.NoError(t, err) + require.Len(t, b, expanded) + }) + require.Greater(t, unbounded, uint64(expanded), + "fixture did not actually cost more than its own size; it is no longer a bomb") + + bounded := testutils.MeasureAlloc(t, func() { + rc, err := decompressionStream(context.Background(), bytes.NewReader(gz.Bytes()), "control.tar.gz", 64*1024) + require.NoError(t, err) + defer rc.Close() + _, err = io.ReadAll(rc) + require.ErrorIs(t, err, errDecompressedTooLarge) + }) + + t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded) + assert.Less(t, bounded, uint64(8*intFile.MB), + "a bounded stream must stop at the cap, not buffer the whole member and check afterwards") +} diff --git a/syft/pkg/cataloger/debian/parse_dpkg_db.go b/syft/pkg/cataloger/debian/parse_dpkg_db.go index be9f04448..daea1b541 100644 --- a/syft/pkg/cataloger/debian/parse_dpkg_db.go +++ b/syft/pkg/cataloger/debian/parse_dpkg_db.go @@ -141,7 +141,12 @@ func (d *dpkgExtractedMetadata) toDpkgEntry() pkg.DpkgDBEntry { // there may be an optional conffiles section that we should persist as files if d.Conffiles != "" { - entry.Files = parseDpkgConffileInfo(strings.NewReader(d.Conffiles)) + var err error + entry.Files, err = parseDpkgConffileInfo(strings.NewReader(d.Conffiles)) + if err != nil { + // toDpkgEntry has no error return, a log line is all that's available here + log.Debugf("dpkg status conffiles field for package %q: %v", d.Package, err) + } } if entry.Files == nil { diff --git a/syft/pkg/cataloger/debian/parse_dpkg_info_files.go b/syft/pkg/cataloger/debian/parse_dpkg_info_files.go index d9d6802ba..b194184ba 100644 --- a/syft/pkg/cataloger/debian/parse_dpkg_info_files.go +++ b/syft/pkg/cataloger/debian/parse_dpkg_info_files.go @@ -2,6 +2,7 @@ package debian import ( "bufio" + "fmt" "io" "strings" @@ -9,10 +10,27 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func parseDpkgMD5Info(reader io.Reader) (findings []pkg.DpkgFileRecord) { +// maxDpkgFileRecords bounds the file lists parsed out of md5sums and conffiles. A byte cap on the +// enclosing stream is not enough on its own: the shortest line that still yields a record is 4 bytes, +// so 16MB of control.tar buys millions of records. +// +// The number comes from measuring what a record actually retains, not from the struct size. A +// DpkgFileRecord is 32 bytes, but a populated one also heap-allocates a file.Digest and three string +// backing arrays: 121 bytes each with realistic Debian paths, 70 with minimal ones. At this bound that +// is ~24MB per package, and the effective ceiling is that times cataloger parallelism (NumCPU*4). +// The largest real Debian packages ship on the order of 10^4 files, so this leaves ~5x headroom. +const maxDpkgFileRecords = 200_000 + +// parseDpkgMD5Info returns the records it managed to parse, plus a non-nil error when the record bound +// was hit (wrapping errClippedFileListing) or the scanner failed partway through. +func parseDpkgMD5Info(reader io.Reader) ([]pkg.DpkgFileRecord, error) { + var findings []pkg.DpkgFileRecord scanner := bufio.NewScanner(reader) for scanner.Scan() { + if len(findings) >= maxDpkgFileRecords { + return findings, fmt.Errorf("%w: dpkg md5sums listing exceeds %d entries", errClippedFileListing, maxDpkgFileRecords) + } line := scanner.Text() fields := strings.SplitN(line, " ", 2) if len(fields) == 2 { @@ -29,13 +47,22 @@ func parseDpkgMD5Info(reader io.Reader) (findings []pkg.DpkgFileRecord) { }) } } - return findings + if err := scanner.Err(); err != nil { + return findings, fmt.Errorf("failed to scan dpkg md5sums: %w", err) + } + return findings, nil } -func parseDpkgConffileInfo(reader io.Reader) (findings []pkg.DpkgFileRecord) { +// parseDpkgConffileInfo returns the records it managed to parse, plus a non-nil error when the record +// bound was hit (wrapping errClippedFileListing) or the scanner failed partway through. +func parseDpkgConffileInfo(reader io.Reader) ([]pkg.DpkgFileRecord, error) { + var findings []pkg.DpkgFileRecord scanner := bufio.NewScanner(reader) for scanner.Scan() { + if len(findings) >= maxDpkgFileRecords { + return findings, fmt.Errorf("%w: dpkg conffiles listing exceeds %d entries", errClippedFileListing, maxDpkgFileRecords) + } line := strings.Trim(scanner.Text(), " \n") fields := strings.SplitN(line, " ", 2) @@ -70,5 +97,8 @@ func parseDpkgConffileInfo(reader io.Reader) (findings []pkg.DpkgFileRecord) { findings = append(findings, record) } } - return findings + if err := scanner.Err(); err != nil { + return findings, fmt.Errorf("failed to scan dpkg conffiles: %w", err) + } + return findings, nil } diff --git a/syft/pkg/cataloger/debian/parse_dpkg_info_files_test.go b/syft/pkg/cataloger/debian/parse_dpkg_info_files_test.go index 90aafd904..613883e6b 100644 --- a/syft/pkg/cataloger/debian/parse_dpkg_info_files_test.go +++ b/syft/pkg/cataloger/debian/parse_dpkg_info_files_test.go @@ -1,10 +1,13 @@ package debian import ( + "bufio" "os" + "strings" "testing" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/anchore/syft/syft/file" @@ -45,7 +48,8 @@ func TestMD5SumInfoParsing(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, f.Close()) }) - actual := parseDpkgMD5Info(f) + actual, err := parseDpkgMD5Info(f) + require.NoError(t, err) if diff := cmp.Diff(test.expected, actual); diff != "" { t.Errorf("unexpected md5 files (-want +got):\n%s", diff) @@ -79,7 +83,8 @@ func TestConffileInfoParsing(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, f.Close()) }) - actual := parseDpkgConffileInfo(f) + actual, err := parseDpkgConffileInfo(f) + require.NoError(t, err) if diff := cmp.Diff(test.expected, actual); diff != "" { t.Errorf("unexpected md5 files (-want +got):\n%s", diff) @@ -88,3 +93,49 @@ func TestConffileInfoParsing(t *testing.T) { }) } } + +func Test_parseDpkgInfoFiles_boundRecordCount(t *testing.T) { + // a byte cap on the enclosing stream cannot bound these on its own, since the shortest line that + // still yields a record is a few bytes. exercises the real maxDpkgFileRecords const directly, since + // the limit is no longer injectable. + t.Run("md5sums stops at the record bound", func(t *testing.T) { + buf := strings.Repeat("d41d8cd98f00b204e9800998ecf8427e usr/bin/x\n", maxDpkgFileRecords+10) + + got, err := parseDpkgMD5Info(strings.NewReader(buf)) + + assert.Len(t, got, maxDpkgFileRecords) + require.ErrorIs(t, err, errClippedFileListing) + }) + + t.Run("conffiles stops at the record bound", func(t *testing.T) { + buf := strings.Repeat("/etc/x.conf\n", maxDpkgFileRecords+10) + + got, err := parseDpkgConffileInfo(strings.NewReader(buf)) + + assert.Len(t, got, maxDpkgFileRecords) + require.ErrorIs(t, err, errClippedFileListing) + }) + + t.Run("a listing under the bound is returned whole", func(t *testing.T) { + // guards against the bound clipping legitimate input + got, err := parseDpkgMD5Info(strings.NewReader( + "d41d8cd98f00b204e9800998ecf8427e usr/bin/a\n" + + "d41d8cd98f00b204e9800998ecf8427e usr/bin/b\n")) + + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "/usr/bin/a", got[0].Path) + assert.Equal(t, "/usr/bin/b", got[1].Path) + }) +} + +func Test_parseDpkgMD5Info_scannerError(t *testing.T) { + // a single line longer than bufio.MaxScanTokenSize with no trailing newline trips the scanner's + // internal error rather than yielding a clean (and silently incomplete) empty list + line := strings.Repeat("a", bufio.MaxScanTokenSize+1) + + got, err := parseDpkgMD5Info(strings.NewReader(line)) + + assert.Empty(t, got) + require.Error(t, err) +} diff --git a/syft/pkg/cataloger/debian/testdata/image-single-dpkg/Dockerfile b/syft/pkg/cataloger/debian/testdata/image-single-dpkg/Dockerfile index dae0145f5..c367f4d8c 100644 --- a/syft/pkg/cataloger/debian/testdata/image-single-dpkg/Dockerfile +++ b/syft/pkg/cataloger/debian/testdata/image-single-dpkg/Dockerfile @@ -2,7 +2,8 @@ FROM ubuntu:22.04 AS downloader RUN apt-get update && apt-get install -y curl -RUN curl -o zlib1g.deb http://archive.ubuntu.com/ubuntu/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb +# launchpad's +files URLs are immutable, unlike pool/main which only keeps the current version +RUN curl -fsSL -o zlib1g.deb https://launchpad.net/ubuntu/+archive/primary/+files/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb FROM scratch diff --git a/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go b/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go index 15f19de22..cde56fd44 100644 --- a/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go +++ b/syft/pkg/cataloger/dotnet/deps_binary_cataloger.go @@ -503,6 +503,11 @@ func findPEFiles(resolver file.Resolver) ([]logicalPE, error, error) { if ldpe == nil { continue } + if ldpe.ParseErr != nil { + // the file parsed well enough to catalog but not completely, so record what we could not read + // and still keep the package + unknownErr = unknown.Append(unknownErr, loc, ldpe.ParseErr) + } peFiles = append(peFiles, *ldpe) } diff --git a/syft/pkg/cataloger/dotnet/testdata/image-net8-app/Dockerfile b/syft/pkg/cataloger/dotnet/testdata/image-net8-app/Dockerfile index 8afe7e726..28cdca64d 100644 --- a/syft/pkg/cataloger/dotnet/testdata/image-net8-app/Dockerfile +++ b/syft/pkg/cataloger/dotnet/testdata/image-net8-app/Dockerfile @@ -72,9 +72,6 @@ RUN dotnet publish -r $RUNTIME --no-restore -o /app # > Humanizer.Core.zh-Hans 2.14.1 # > Humanizer.Core.zh-Hant 2.14.1 -# lets pull in a file that is not related at all and in fact is not a .NET binary either (this should be ignored) -RUN wget -O /app/jruby_windows_9_3_15_0.exe https://github.com/jruby/jruby/releases/download/9.3.15.0/jruby_windows_9_3_15_0.exe - FROM busybox WORKDIR /app COPY --from=build /app . diff --git a/syft/pkg/cataloger/golang/cataloger_test.go b/syft/pkg/cataloger/golang/cataloger_test.go index 96c05e37d..860171f70 100644 --- a/syft/pkg/cataloger/golang/cataloger_test.go +++ b/syft/pkg/cataloger/golang/cataloger_test.go @@ -4,7 +4,11 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) @@ -15,6 +19,11 @@ func Test_PackageCataloger_Binary(t *testing.T) { fixture string expectedPkgs []string expectedRels []string + // wantErr is set where the fixture is expected to catalog cleanly. Without it the tester discards + // the cataloger's error, so an unknown newly attached to every file in the image cannot fail this + // table: packages and relationships still match. The packed fixture is the one that needs it, since + // the UPX reporting policy decides per file whether a gap is worth an unknown. + wantErr require.ErrorAssertionFunc }{ { name: "simple module with dependencies", @@ -50,6 +59,7 @@ func Test_PackageCataloger_Binary(t *testing.T) { { name: "upx compressed binary", fixture: "image-small-upx", + wantErr: require.NoError, expectedPkgs: []string{ "anchore.io/not/real @ v1.0.0 (/run-me)", "github.com/andybalholm/brotli @ v1.1.1 (/run-me)", @@ -115,16 +125,63 @@ func Test_PackageCataloger_Binary(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - pkgtest.NewCatalogTester(). + tester := pkgtest.NewCatalogTester(). WithImageResolver(t, test.fixture). ExpectsPackageStrings(test.expectedPkgs). - ExpectsRelationshipStrings(test.expectedRels). - TestCataloger(t, NewGoModuleBinaryCataloger(DefaultCatalogerConfig())) + ExpectsRelationshipStrings(test.expectedRels) + if test.wantErr != nil { + tester = tester.WithErrorAssertion(test.wantErr) + } + tester.TestCataloger(t, NewGoModuleBinaryCataloger(DefaultCatalogerConfig())) }) } } +// Test_PackageCataloger_Binary_SymbolsFromPackedBinary covers what the packed path used to miss: the +// pclntab is compressed along with everything else, so a UPX binary reported its packages with no symbols +// at all until the unpacked contents were threaded through the rest of the scan instead of only the +// build info read. +func Test_PackageCataloger_Binary_SymbolsFromPackedBinary(t *testing.T) { + cfg := DefaultCatalogerConfig() + cfg.CaptureSymbols = cataloging.SymbolScopeAll + + symbolCounts := func(t *testing.T, pkgs []pkg.Package, _ []artifact.Relationship) map[string]int { + t.Helper() + counts := make(map[string]int) + for _, p := range pkgs { + meta, ok := p.Metadata.(pkg.GolangBinaryBuildinfoEntry) + require.True(t, ok, "unexpected metadata on %s", p.Name) + for _, names := range meta.Symbols { + counts[p.Name] += len(names) + } + } + return counts + } + + var packed, unpacked map[string]int + pkgtest.NewCatalogTester(). + WithImageResolver(t, "image-small"). + ExpectsAssertion(func(t *testing.T, pkgs []pkg.Package, rels []artifact.Relationship) { + unpacked = symbolCounts(t, pkgs, rels) + }). + TestCataloger(t, NewGoModuleBinaryCataloger(cfg)) + + pkgtest.NewCatalogTester(). + WithImageResolver(t, "image-small-upx"). + // a real `upx --best --lzma` binary must unpack without leaving a gap behind. The reconstruction is + // truncated to the contiguous extent rebuilt, and a chain that stops short of p_filesize is now a + // reported partial, so this is the assertion that catches that firing on well-formed output. + WithErrorAssertion(require.NoError). + ExpectsAssertion(func(t *testing.T, pkgs []pkg.Package, rels []artifact.Relationship) { + packed = symbolCounts(t, pkgs, rels) + }). + TestCataloger(t, NewGoModuleBinaryCataloger(cfg)) + + require.NotEmpty(t, unpacked, "the unpacked fixture is the control: it must carry symbols") + assert.Equal(t, unpacked, packed, "the packed binary must report the same symbols as the one it was packed from") +} + func Test_Mod_Cataloger_Globs(t *testing.T) { tests := []struct { name string diff --git a/syft/pkg/cataloger/golang/internal/README.md b/syft/pkg/cataloger/golang/internal/README.md deleted file mode 100644 index 58cdf17ce..000000000 --- a/syft/pkg/cataloger/golang/internal/README.md +++ /dev/null @@ -1,4 +0,0 @@ -xcoff ------ - -The code in this package comes from: https://github.com/golang/go/tree/master/src/internal/xcoff -- it was copied over to add support for [xcoff](https://en.wikipedia.org/wiki/XCOFF) binaries. Golang keeps this package as internal, forbidding its external use. diff --git a/syft/pkg/cataloger/golang/internal/xcoff/file.go b/syft/pkg/cataloger/golang/internal/xcoff/file.go deleted file mode 100644 index 9fae1d4d7..000000000 --- a/syft/pkg/cataloger/golang/internal/xcoff/file.go +++ /dev/null @@ -1,688 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package xcoff implements access to XCOFF (Extended Common Object File Format) files. - -//nolint:all -package xcoff - -import ( - "encoding/binary" - "fmt" - "io" - "os" - "strings" -) - -// SectionHeader holds information about an XCOFF section header. -type SectionHeader struct { - Name string - VirtualAddress uint64 - Size uint64 - Type uint32 - Relptr uint64 - Nreloc uint32 -} - -type Section struct { - SectionHeader - Relocs []Reloc - io.ReaderAt - sr *io.SectionReader -} - -// AuxiliaryCSect holds information about an XCOFF symbol in an AUX_CSECT entry. -type AuxiliaryCSect struct { - Length int64 - StorageMappingClass int - SymbolType int -} - -// AuxiliaryFcn holds information about an XCOFF symbol in an AUX_FCN entry. -type AuxiliaryFcn struct { - Size int64 -} - -type Symbol struct { - Name string - Value uint64 - SectionNumber int - StorageClass int - AuxFcn AuxiliaryFcn - AuxCSect AuxiliaryCSect -} - -type Reloc struct { - VirtualAddress uint64 - Symbol *Symbol - Signed bool - InstructionFixed bool - Length uint8 - Type uint8 -} - -// ImportedSymbol holds information about an imported XCOFF symbol. -type ImportedSymbol struct { - Name string - Library string -} - -// FileHeader holds information about an XCOFF file header. -type FileHeader struct { - TargetMachine uint16 -} - -// A File represents an open XCOFF file. -type File struct { - FileHeader - Sections []*Section - Symbols []*Symbol - StringTable []byte - LibraryPaths []string - - closer io.Closer -} - -// Open opens the named file using os.Open and prepares it for use as an XCOFF binary. -func Open(name string) (*File, error) { - f, err := os.Open(name) - if err != nil { - return nil, err - } - ff, err := NewFile(f) - if err != nil { - f.Close() - return nil, err - } - ff.closer = f - return ff, nil -} - -// Close closes the File. -// If the File was created using NewFile directly instead of Open, -// Close has no effect. -func (f *File) Close() error { - var err error - if f.closer != nil { - err = f.closer.Close() - f.closer = nil - } - return err -} - -// Section returns the first section with the given name, or nil if no such -// section exists. -// Xcoff have section's name limited to 8 bytes. Some sections like .gosymtab -// can be trunked but this method will still find them. -func (f *File) Section(name string) *Section { - for _, s := range f.Sections { - if s.Name == name || (len(name) > 8 && s.Name == name[:8]) { - return s - } - } - return nil -} - -// SectionByType returns the first section in f with the -// given type, or nil if there is no such section. -func (f *File) SectionByType(typ uint32) *Section { - for _, s := range f.Sections { - if s.Type == typ { - return s - } - } - return nil -} - -// cstring converts ASCII byte sequence b to string. -// It stops once it finds 0 or reaches end of b. -func cstring(b []byte) string { - var i int - for i = 0; i < len(b) && b[i] != 0; i++ { - } - return string(b[:i]) -} - -// getString extracts a string from an XCOFF string table. -func getString(st []byte, offset uint32) (string, bool) { - if offset < 4 || int(offset) >= len(st) { - return "", false - } - return cstring(st[offset:]), true -} - -// NewFile creates a new File for accessing an XCOFF binary in an underlying reader. -func NewFile(r io.ReaderAt) (*File, error) { - sr := io.NewSectionReader(r, 0, 1<<63-1) - // Read XCOFF target machine - var magic uint16 - if err := binary.Read(sr, binary.BigEndian, &magic); err != nil { - return nil, err - } - if magic != U802TOCMAGIC && magic != U64_TOCMAGIC { - return nil, fmt.Errorf("unrecognised XCOFF magic: 0x%x", magic) - } - - f := new(File) - f.TargetMachine = magic - - // Read XCOFF file header - if _, err := sr.Seek(0, io.SeekStart); err != nil { - return nil, err - } - var nscns uint16 - var symptr uint64 - var nsyms int32 - var opthdr uint16 - var hdrsz int - switch f.TargetMachine { - case U802TOCMAGIC: - fhdr := new(FileHeader32) - if err := binary.Read(sr, binary.BigEndian, fhdr); err != nil { - return nil, err - } - nscns = fhdr.Fnscns - symptr = uint64(fhdr.Fsymptr) - nsyms = fhdr.Fnsyms - opthdr = fhdr.Fopthdr - hdrsz = FILHSZ_32 - case U64_TOCMAGIC: - fhdr := new(FileHeader64) - if err := binary.Read(sr, binary.BigEndian, fhdr); err != nil { - return nil, err - } - nscns = fhdr.Fnscns - symptr = fhdr.Fsymptr - nsyms = fhdr.Fnsyms - opthdr = fhdr.Fopthdr - hdrsz = FILHSZ_64 - } - - if symptr == 0 || nsyms <= 0 { - return nil, fmt.Errorf("no symbol table") - } - - // Read string table (located right after symbol table). - offset := symptr + uint64(nsyms)*SYMESZ - if _, err := sr.Seek(int64(offset), io.SeekStart); err != nil { - return nil, err - } - // The first 4 bytes contain the length (in bytes). - var l uint32 - if err := binary.Read(sr, binary.BigEndian, &l); err != nil { - return nil, err - } - if l > 4 { - if _, err := sr.Seek(int64(offset), io.SeekStart); err != nil { - return nil, err - } - f.StringTable = make([]byte, l) - if _, err := io.ReadFull(sr, f.StringTable); err != nil { - return nil, err - } - } - - // Read section headers - if _, err := sr.Seek(int64(hdrsz)+int64(opthdr), io.SeekStart); err != nil { - return nil, err - } - f.Sections = make([]*Section, nscns) - for i := 0; i < int(nscns); i++ { - var scnptr uint64 - s := new(Section) - switch f.TargetMachine { - case U802TOCMAGIC: - shdr := new(SectionHeader32) - if err := binary.Read(sr, binary.BigEndian, shdr); err != nil { - return nil, err - } - s.Name = cstring(shdr.Sname[:]) - s.VirtualAddress = uint64(shdr.Svaddr) - s.Size = uint64(shdr.Ssize) - scnptr = uint64(shdr.Sscnptr) - s.Type = shdr.Sflags - s.Relptr = uint64(shdr.Srelptr) - s.Nreloc = uint32(shdr.Snreloc) - case U64_TOCMAGIC: - shdr := new(SectionHeader64) - if err := binary.Read(sr, binary.BigEndian, shdr); err != nil { - return nil, err - } - s.Name = cstring(shdr.Sname[:]) - s.VirtualAddress = shdr.Svaddr - s.Size = shdr.Ssize - scnptr = shdr.Sscnptr - s.Type = shdr.Sflags - s.Relptr = shdr.Srelptr - s.Nreloc = shdr.Snreloc - } - r2 := r - if scnptr == 0 { // .bss must have all 0s - r2 = zeroReaderAt{} - } - s.sr = io.NewSectionReader(r2, int64(scnptr), int64(s.Size)) - s.ReaderAt = s.sr - f.Sections[i] = s - } - - // Symbol map needed by relocation - var idxToSym = make(map[int]*Symbol) - - // Read symbol table - if _, err := sr.Seek(int64(symptr), io.SeekStart); err != nil { - return nil, err - } - f.Symbols = make([]*Symbol, 0) - for i := 0; i < int(nsyms); i++ { - var numaux int - var ok, needAuxFcn bool - sym := new(Symbol) - switch f.TargetMachine { - case U802TOCMAGIC: - se := new(SymEnt32) - if err := binary.Read(sr, binary.BigEndian, se); err != nil { - return nil, err - } - numaux = int(se.Nnumaux) - sym.SectionNumber = int(se.Nscnum) - sym.StorageClass = int(se.Nsclass) - sym.Value = uint64(se.Nvalue) - needAuxFcn = se.Ntype&SYM_TYPE_FUNC != 0 && numaux > 1 - zeroes := binary.BigEndian.Uint32(se.Nname[:4]) - if zeroes != 0 { - sym.Name = cstring(se.Nname[:]) - } else { - offset := binary.BigEndian.Uint32(se.Nname[4:]) - sym.Name, ok = getString(f.StringTable, offset) - if !ok { - goto skip - } - } - case U64_TOCMAGIC: - se := new(SymEnt64) - if err := binary.Read(sr, binary.BigEndian, se); err != nil { - return nil, err - } - numaux = int(se.Nnumaux) - sym.SectionNumber = int(se.Nscnum) - sym.StorageClass = int(se.Nsclass) - sym.Value = se.Nvalue - needAuxFcn = se.Ntype&SYM_TYPE_FUNC != 0 && numaux > 1 - sym.Name, ok = getString(f.StringTable, se.Noffset) - if !ok { - goto skip - } - } - if sym.StorageClass != C_EXT && sym.StorageClass != C_WEAKEXT && sym.StorageClass != C_HIDEXT { - goto skip - } - // Must have at least one csect auxiliary entry. - if numaux < 1 || i+numaux >= int(nsyms) { - goto skip - } - - if sym.SectionNumber > int(nscns) { - goto skip - } - if sym.SectionNumber == 0 { - sym.Value = 0 - } else { - sym.Value -= f.Sections[sym.SectionNumber-1].VirtualAddress - } - - idxToSym[i] = sym - - // If this symbol is a function, it must retrieve its size from - // its AUX_FCN entry. - // It can happen that a function symbol doesn't have any AUX_FCN. - // In this case, needAuxFcn is false and their size will be set to 0. - if needAuxFcn { - switch f.TargetMachine { - case U802TOCMAGIC: - aux := new(AuxFcn32) - if err := binary.Read(sr, binary.BigEndian, aux); err != nil { - return nil, err - } - sym.AuxFcn.Size = int64(aux.Xfsize) - case U64_TOCMAGIC: - aux := new(AuxFcn64) - if err := binary.Read(sr, binary.BigEndian, aux); err != nil { - return nil, err - } - sym.AuxFcn.Size = int64(aux.Xfsize) - } - } - - // Read csect auxiliary entry (by convention, it is the last). - if !needAuxFcn { - if _, err := sr.Seek(int64(numaux-1)*SYMESZ, io.SeekCurrent); err != nil { - return nil, err - } - } - i += numaux - numaux = 0 - switch f.TargetMachine { - case U802TOCMAGIC: - aux := new(AuxCSect32) - if err := binary.Read(sr, binary.BigEndian, aux); err != nil { - return nil, err - } - sym.AuxCSect.SymbolType = int(aux.Xsmtyp & 0x7) - sym.AuxCSect.StorageMappingClass = int(aux.Xsmclas) - sym.AuxCSect.Length = int64(aux.Xscnlen) - case U64_TOCMAGIC: - aux := new(AuxCSect64) - if err := binary.Read(sr, binary.BigEndian, aux); err != nil { - return nil, err - } - sym.AuxCSect.SymbolType = int(aux.Xsmtyp & 0x7) - sym.AuxCSect.StorageMappingClass = int(aux.Xsmclas) - sym.AuxCSect.Length = int64(aux.Xscnlenhi)<<32 | int64(aux.Xscnlenlo) - } - f.Symbols = append(f.Symbols, sym) - skip: - i += numaux // Skip auxiliary entries - if _, err := sr.Seek(int64(numaux)*SYMESZ, io.SeekCurrent); err != nil { - return nil, err - } - } - - // Read relocations - // Only for .data or .text section - for _, sect := range f.Sections { - if sect.Type != STYP_TEXT && sect.Type != STYP_DATA { - continue - } - sect.Relocs = make([]Reloc, sect.Nreloc) - if sect.Relptr == 0 { - continue - } - if _, err := sr.Seek(int64(sect.Relptr), io.SeekStart); err != nil { - return nil, err - } - for i := uint32(0); i < sect.Nreloc; i++ { - switch f.TargetMachine { - case U802TOCMAGIC: - rel := new(Reloc32) - if err := binary.Read(sr, binary.BigEndian, rel); err != nil { - return nil, err - } - sect.Relocs[i].VirtualAddress = uint64(rel.Rvaddr) - sect.Relocs[i].Symbol = idxToSym[int(rel.Rsymndx)] - sect.Relocs[i].Type = rel.Rtype - sect.Relocs[i].Length = rel.Rsize&0x3F + 1 - - if rel.Rsize&0x80 != 0 { - sect.Relocs[i].Signed = true - } - if rel.Rsize&0x40 != 0 { - sect.Relocs[i].InstructionFixed = true - } - - case U64_TOCMAGIC: - rel := new(Reloc64) - if err := binary.Read(sr, binary.BigEndian, rel); err != nil { - return nil, err - } - sect.Relocs[i].VirtualAddress = rel.Rvaddr - sect.Relocs[i].Symbol = idxToSym[int(rel.Rsymndx)] - sect.Relocs[i].Type = rel.Rtype - sect.Relocs[i].Length = rel.Rsize&0x3F + 1 - if rel.Rsize&0x80 != 0 { - sect.Relocs[i].Signed = true - } - if rel.Rsize&0x40 != 0 { - sect.Relocs[i].InstructionFixed = true - } - } - } - } - - return f, nil -} - -// zeroReaderAt is ReaderAt that reads 0s. -type zeroReaderAt struct{} - -// ReadAt writes len(p) 0s into p. -func (w zeroReaderAt) ReadAt(p []byte, off int64) (n int, err error) { - for i := range p { - p[i] = 0 - } - return len(p), nil -} - -// Data reads and returns the contents of the XCOFF section s. -func (s *Section) Data() ([]byte, error) { - dat := make([]byte, s.sr.Size()) - n, err := s.sr.ReadAt(dat, 0) - if n == len(dat) { - err = nil - } - return dat[:n], err -} - -// CSect reads and returns the contents of a csect. -// func (f *File) CSect(name string) []byte { -// for _, sym := range f.Symbols { -// if sym.Name == name && sym.AuxCSect.SymbolType == XTY_SD { -// if i := sym.SectionNumber - 1; 0 <= i && i < len(f.Sections) { -// s := f.Sections[i] -// if sym.Value+uint64(sym.AuxCSect.Length) <= s.Size { -// dat := make([]byte, sym.AuxCSect.Length) -// _, err := s.sr.ReadAt(dat, int64(sym.Value)) -// if err != nil { -// return nil -// } -// return dat -// } -// } -// break -// } -// } -// return nil -// } - -// func (f *File) DWARF() (*dwarf.Data, error) { -// // There are many other DWARF sections, but these -// // are the ones the debug/dwarf package uses. -// // Don't bother loading others. -// var subtypes = [...]uint32{SSUBTYP_DWABREV, SSUBTYP_DWINFO, SSUBTYP_DWLINE, SSUBTYP_DWRNGES, SSUBTYP_DWSTR} -// var dat [len(subtypes)][]byte -// for i, subtype := range subtypes { -// s := f.SectionByType(STYP_DWARF | subtype) -// if s != nil { -// b, err := s.Data() -// if err != nil && uint64(len(b)) < s.Size { -// return nil, err -// } -// dat[i] = b -// } -// } - -// abbrev, info, line, ranges, str := dat[0], dat[1], dat[2], dat[3], dat[4] -// return dwarf.New(abbrev, nil, nil, info, line, nil, ranges, str) -// } - -// readImportIDs returns the import file IDs stored inside the .loader section. -// Library name pattern is either path/base/member or base/member -func (f *File) readImportIDs(s *Section) ([]string, error) { - // Read loader header - if _, err := s.sr.Seek(0, io.SeekStart); err != nil { - return nil, err - } - var istlen uint32 - var nimpid int32 - var impoff uint64 - switch f.TargetMachine { - case U802TOCMAGIC: - lhdr := new(LoaderHeader32) - if err := binary.Read(s.sr, binary.BigEndian, lhdr); err != nil { - return nil, err - } - istlen = lhdr.Listlen - nimpid = lhdr.Lnimpid - impoff = uint64(lhdr.Limpoff) - case U64_TOCMAGIC: - lhdr := new(LoaderHeader64) - if err := binary.Read(s.sr, binary.BigEndian, lhdr); err != nil { - return nil, err - } - istlen = lhdr.Listlen - nimpid = lhdr.Lnimpid - impoff = lhdr.Limpoff - } - - // Read loader import file ID table - if _, err := s.sr.Seek(int64(impoff), io.SeekStart); err != nil { - return nil, err - } - table := make([]byte, istlen) - if _, err := io.ReadFull(s.sr, table); err != nil { - return nil, err - } - - offset := 0 - // First import file ID is the default LIBPATH value - libpath := cstring(table[offset:]) - f.LibraryPaths = strings.Split(libpath, ":") - offset += len(libpath) + 3 // 3 null bytes - all := make([]string, 0) - for i := 1; i < int(nimpid); i++ { - impidpath := cstring(table[offset:]) - offset += len(impidpath) + 1 - impidbase := cstring(table[offset:]) - offset += len(impidbase) + 1 - impidmem := cstring(table[offset:]) - offset += len(impidmem) + 1 - var path string - if len(impidpath) > 0 { - path = impidpath + "/" + impidbase + "/" + impidmem - } else { - path = impidbase + "/" + impidmem - } - all = append(all, path) - } - - return all, nil -} - -// ImportedSymbols returns the names of all symbols -// referred to by the binary f that are expected to be -// satisfied by other libraries at dynamic load time. -// It does not return weak symbols. -func (f *File) ImportedSymbols() ([]ImportedSymbol, error) { - s := f.SectionByType(STYP_LOADER) - if s == nil { - return nil, nil - } - // Read loader header - if _, err := s.sr.Seek(0, io.SeekStart); err != nil { - return nil, err - } - var stlen uint32 - var stoff uint64 - var nsyms int32 - var symoff uint64 - switch f.TargetMachine { - case U802TOCMAGIC: - lhdr := new(LoaderHeader32) - if err := binary.Read(s.sr, binary.BigEndian, lhdr); err != nil { - return nil, err - } - stlen = lhdr.Lstlen - stoff = uint64(lhdr.Lstoff) - nsyms = lhdr.Lnsyms - symoff = LDHDRSZ_32 - case U64_TOCMAGIC: - lhdr := new(LoaderHeader64) - if err := binary.Read(s.sr, binary.BigEndian, lhdr); err != nil { - return nil, err - } - stlen = lhdr.Lstlen - stoff = lhdr.Lstoff - nsyms = lhdr.Lnsyms - symoff = lhdr.Lsymoff - } - - // Read loader section string table - if _, err := s.sr.Seek(int64(stoff), io.SeekStart); err != nil { - return nil, err - } - st := make([]byte, stlen) - if _, err := io.ReadFull(s.sr, st); err != nil { - return nil, err - } - - // Read imported libraries - libs, err := f.readImportIDs(s) - if err != nil { - return nil, err - } - - // Read loader symbol table - if _, err := s.sr.Seek(int64(symoff), io.SeekStart); err != nil { - return nil, err - } - all := make([]ImportedSymbol, 0) - for i := 0; i < int(nsyms); i++ { - var name string - var ifile int32 - var ok bool - switch f.TargetMachine { - case U802TOCMAGIC: - ldsym := new(LoaderSymbol32) - if err := binary.Read(s.sr, binary.BigEndian, ldsym); err != nil { - return nil, err - } - if ldsym.Lsmtype&0x40 == 0 { - continue // Imported symbols only - } - zeroes := binary.BigEndian.Uint32(ldsym.Lname[:4]) - if zeroes != 0 { - name = cstring(ldsym.Lname[:]) - } else { - offset := binary.BigEndian.Uint32(ldsym.Lname[4:]) - name, ok = getString(st, offset) - if !ok { - continue - } - } - ifile = ldsym.Lifile - case U64_TOCMAGIC: - ldsym := new(LoaderSymbol64) - if err := binary.Read(s.sr, binary.BigEndian, ldsym); err != nil { - return nil, err - } - if ldsym.Lsmtype&0x40 == 0 { - continue // Imported symbols only - } - name, ok = getString(st, ldsym.Loffset) - if !ok { - continue - } - ifile = ldsym.Lifile - } - var sym ImportedSymbol - sym.Name = name - if ifile >= 1 && int(ifile) <= len(libs) { - sym.Library = libs[ifile-1] - } - all = append(all, sym) - } - - return all, nil -} - -// ImportedLibraries returns the names of all libraries -// referred to by the binary f that are expected to be -// linked with the binary at dynamic link time. -func (f *File) ImportedLibraries() ([]string, error) { - s := f.SectionByType(STYP_LOADER) - if s == nil { - return nil, nil - } - all, err := f.readImportIDs(s) - return all, err -} diff --git a/syft/pkg/cataloger/golang/internal/xcoff/file_test.go b/syft/pkg/cataloger/golang/internal/xcoff/file_test.go deleted file mode 100644 index a6722e945..000000000 --- a/syft/pkg/cataloger/golang/internal/xcoff/file_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package xcoff - -import ( - "reflect" - "testing" -) - -type fileTest struct { - file string - hdr FileHeader - sections []*SectionHeader - needed []string -} - -var fileTests = []fileTest{ - { - "testdata/gcc-ppc32-aix-dwarf2-exec", - FileHeader{U802TOCMAGIC}, - []*SectionHeader{ - {".text", 0x10000290, 0x00000bbd, STYP_TEXT, 0x7ae6, 0x36}, - {".data", 0x20000e4d, 0x00000437, STYP_DATA, 0x7d02, 0x2b}, - {".bss", 0x20001284, 0x0000021c, STYP_BSS, 0, 0}, - {".loader", 0x00000000, 0x000004b3, STYP_LOADER, 0, 0}, - {".dwline", 0x00000000, 0x000000df, STYP_DWARF | SSUBTYP_DWLINE, 0x7eb0, 0x7}, - {".dwinfo", 0x00000000, 0x00000314, STYP_DWARF | SSUBTYP_DWINFO, 0x7ef6, 0xa}, - {".dwabrev", 0x00000000, 0x000000d6, STYP_DWARF | SSUBTYP_DWABREV, 0, 0}, - {".dwarnge", 0x00000000, 0x00000020, STYP_DWARF | SSUBTYP_DWARNGE, 0x7f5a, 0x2}, - {".dwloc", 0x00000000, 0x00000074, STYP_DWARF | SSUBTYP_DWLOC, 0, 0}, - {".debug", 0x00000000, 0x00005e4f, STYP_DEBUG, 0, 0}, - }, - []string{"libc.a/shr.o"}, - }, - { - "testdata/gcc-ppc64-aix-dwarf2-exec", - FileHeader{U64_TOCMAGIC}, - []*SectionHeader{ - {".text", 0x10000480, 0x00000afd, STYP_TEXT, 0x8322, 0x34}, - {".data", 0x20000f7d, 0x000002f3, STYP_DATA, 0x85fa, 0x25}, - {".bss", 0x20001270, 0x00000428, STYP_BSS, 0, 0}, - {".loader", 0x00000000, 0x00000535, STYP_LOADER, 0, 0}, - {".dwline", 0x00000000, 0x000000b4, STYP_DWARF | SSUBTYP_DWLINE, 0x8800, 0x4}, - {".dwinfo", 0x00000000, 0x0000036a, STYP_DWARF | SSUBTYP_DWINFO, 0x8838, 0x7}, - {".dwabrev", 0x00000000, 0x000000b5, STYP_DWARF | SSUBTYP_DWABREV, 0, 0}, - {".dwarnge", 0x00000000, 0x00000040, STYP_DWARF | SSUBTYP_DWARNGE, 0x889a, 0x2}, - {".dwloc", 0x00000000, 0x00000062, STYP_DWARF | SSUBTYP_DWLOC, 0, 0}, - {".debug", 0x00000000, 0x00006605, STYP_DEBUG, 0, 0}, - }, - []string{"libc.a/shr_64.o"}, - }, -} - -func TestOpen(t *testing.T) { - for i := range fileTests { - tt := &fileTests[i] - - f, err := Open(tt.file) - if err != nil { - t.Error(err) - continue - } - if !reflect.DeepEqual(f.FileHeader, tt.hdr) { - t.Errorf("open %s:\n\thave %#v\n\twant %#v\n", tt.file, f.FileHeader, tt.hdr) - continue - } - - for i, sh := range f.Sections { - if i >= len(tt.sections) { - break - } - have := &sh.SectionHeader - want := tt.sections[i] - if !reflect.DeepEqual(have, want) { - t.Errorf("open %s, section %d:\n\thave %#v\n\twant %#v\n", tt.file, i, have, want) - } - } - tn := len(tt.sections) - fn := len(f.Sections) - if tn != fn { - t.Errorf("open %s: len(Sections) = %d, want %d", tt.file, fn, tn) - } - tl := tt.needed - fl, err := f.ImportedLibraries() - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(tl, fl) { - t.Errorf("open %s: loader import = %v, want %v", tt.file, tl, fl) - } - } -} - -func TestOpenFailure(t *testing.T) { - filename := "file.go" // not an XCOFF object file - _, err := Open(filename) // don't crash - if err == nil { - t.Errorf("open %s: succeeded unexpectedly", filename) - } -} diff --git a/syft/pkg/cataloger/golang/internal/xcoff/xcoff.go b/syft/pkg/cataloger/golang/internal/xcoff/xcoff.go deleted file mode 100644 index 5500269c5..000000000 --- a/syft/pkg/cataloger/golang/internal/xcoff/xcoff.go +++ /dev/null @@ -1,373 +0,0 @@ -// The code in this package comes from: -// https://github.com/golang/go/tree/master/src/internal/xcoff -// it was copied over to add support for xcoff binaries. -// Golang keeps this package as internal, forbidding its external use. - -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//nolint:all -package xcoff - -// File Header. -type FileHeader32 struct { - Fmagic uint16 // Target machine - Fnscns uint16 // Number of sections - Ftimedat int32 // Time and date of file creation - Fsymptr uint32 // Byte offset to symbol table start - Fnsyms int32 // Number of entries in symbol table - Fopthdr uint16 // Number of bytes in optional header - Fflags uint16 // Flags -} - -type FileHeader64 struct { - Fmagic uint16 // Target machine - Fnscns uint16 // Number of sections - Ftimedat int32 // Time and date of file creation - Fsymptr uint64 // Byte offset to symbol table start - Fopthdr uint16 // Number of bytes in optional header - Fflags uint16 // Flags - Fnsyms int32 // Number of entries in symbol table -} - -const ( - FILHSZ_32 = 20 - FILHSZ_64 = 24 -) -const ( - U802TOCMAGIC = 0737 // AIX 32-bit XCOFF - U64_TOCMAGIC = 0767 // AIX 64-bit XCOFF -) - -// Flags that describe the type of the object file. -const ( - F_RELFLG = 0x0001 - F_EXEC = 0x0002 - F_LNNO = 0x0004 - F_FDPR_PROF = 0x0010 - F_FDPR_OPTI = 0x0020 - F_DSA = 0x0040 - F_VARPG = 0x0100 - F_DYNLOAD = 0x1000 - F_SHROBJ = 0x2000 - F_LOADONLY = 0x4000 -) - -// Section Header. -type SectionHeader32 struct { - Sname [8]byte // Section name - Spaddr uint32 // Physical address - Svaddr uint32 // Virtual address - Ssize uint32 // Section size - Sscnptr uint32 // Offset in file to raw data for section - Srelptr uint32 // Offset in file to relocation entries for section - Slnnoptr uint32 // Offset in file to line number entries for section - Snreloc uint16 // Number of relocation entries - Snlnno uint16 // Number of line number entries - Sflags uint32 // Flags to define the section type -} - -type SectionHeader64 struct { - Sname [8]byte // Section name - Spaddr uint64 // Physical address - Svaddr uint64 // Virtual address - Ssize uint64 // Section size - Sscnptr uint64 // Offset in file to raw data for section - Srelptr uint64 // Offset in file to relocation entries for section - Slnnoptr uint64 // Offset in file to line number entries for section - Snreloc uint32 // Number of relocation entries - Snlnno uint32 // Number of line number entries - Sflags uint32 // Flags to define the section type - Spad uint32 // Needs to be 72 bytes long -} - -// Flags defining the section type. -const ( - STYP_DWARF = 0x0010 - STYP_TEXT = 0x0020 - STYP_DATA = 0x0040 - STYP_BSS = 0x0080 - STYP_EXCEPT = 0x0100 - STYP_INFO = 0x0200 - STYP_TDATA = 0x0400 - STYP_TBSS = 0x0800 - STYP_LOADER = 0x1000 - STYP_DEBUG = 0x2000 - STYP_TYPCHK = 0x4000 - STYP_OVRFLO = 0x8000 -) -const ( - SSUBTYP_DWINFO = 0x10000 // DWARF info section - SSUBTYP_DWLINE = 0x20000 // DWARF line-number section - SSUBTYP_DWPBNMS = 0x30000 // DWARF public names section - SSUBTYP_DWPBTYP = 0x40000 // DWARF public types section - SSUBTYP_DWARNGE = 0x50000 // DWARF aranges section - SSUBTYP_DWABREV = 0x60000 // DWARF abbreviation section - SSUBTYP_DWSTR = 0x70000 // DWARF strings section - SSUBTYP_DWRNGES = 0x80000 // DWARF ranges section - SSUBTYP_DWLOC = 0x90000 // DWARF location lists section - SSUBTYP_DWFRAME = 0xA0000 // DWARF frames section - SSUBTYP_DWMAC = 0xB0000 // DWARF macros section -) - -// Symbol Table Entry. -type SymEnt32 struct { - Nname [8]byte // Symbol name - Nvalue uint32 // Symbol value - Nscnum int16 // Section number of symbol - Ntype uint16 // Basic and derived type specification - Nsclass int8 // Storage class of symbol - Nnumaux int8 // Number of auxiliary entries -} - -type SymEnt64 struct { - Nvalue uint64 // Symbol value - Noffset uint32 // Offset of the name in string table or .debug section - Nscnum int16 // Section number of symbol - Ntype uint16 // Basic and derived type specification - Nsclass int8 // Storage class of symbol - Nnumaux int8 // Number of auxiliary entries -} - -const SYMESZ = 18 - -const ( - // Nscnum - N_DEBUG = -2 - N_ABS = -1 - N_UNDEF = 0 - - //Ntype - SYM_V_INTERNAL = 0x1000 - SYM_V_HIDDEN = 0x2000 - SYM_V_PROTECTED = 0x3000 - SYM_V_EXPORTED = 0x4000 - SYM_TYPE_FUNC = 0x0020 // is function -) - -// Storage Class. -const ( - C_NULL = 0 // Symbol table entry marked for deletion - C_EXT = 2 // External symbol - C_STAT = 3 // Static symbol - C_BLOCK = 100 // Beginning or end of inner block - C_FCN = 101 // Beginning or end of function - C_FILE = 103 // Source file name and compiler information - C_HIDEXT = 107 // Unnamed external symbol - C_BINCL = 108 // Beginning of include file - C_EINCL = 109 // End of include file - C_WEAKEXT = 111 // Weak external symbol - C_DWARF = 112 // DWARF symbol - C_GSYM = 128 // Global variable - C_LSYM = 129 // Automatic variable allocated on stack - C_PSYM = 130 // Argument to subroutine allocated on stack - C_RSYM = 131 // Register variable - C_RPSYM = 132 // Argument to function or procedure stored in register - C_STSYM = 133 // Statically allocated symbol - C_BCOMM = 135 // Beginning of common block - C_ECOML = 136 // Local member of common block - C_ECOMM = 137 // End of common block - C_DECL = 140 // Declaration of object - C_ENTRY = 141 // Alternate entry - C_FUN = 142 // Function or procedure - C_BSTAT = 143 // Beginning of static block - C_ESTAT = 144 // End of static block - C_GTLS = 145 // Global thread-local variable - C_STTLS = 146 // Static thread-local variable -) - -// File Auxiliary Entry -type AuxFile64 struct { - Xfname [8]byte // Name or offset inside string table - Xftype uint8 // Source file string type - Xauxtype uint8 // Type of auxiliary entry -} - -// Function Auxiliary Entry -type AuxFcn32 struct { - Xexptr uint32 // File offset to exception table entry - Xfsize uint32 // Size of function in bytes - Xlnnoptr uint32 // File pointer to line number - Xendndx uint32 // Symbol table index of next entry - Xpad uint16 // Unused -} -type AuxFcn64 struct { - Xlnnoptr uint64 // File pointer to line number - Xfsize uint32 // Size of function in bytes - Xendndx uint32 // Symbol table index of next entry - Xpad uint8 // Unused - Xauxtype uint8 // Type of auxiliary entry -} - -type AuxSect64 struct { - Xscnlen uint64 // section length - Xnreloc uint64 // Num RLDs - pad uint8 - Xauxtype uint8 // Type of auxiliary entry -} - -// csect Auxiliary Entry. -type AuxCSect32 struct { - Xscnlen int32 // Length or symbol table index - Xparmhash uint32 // Offset of parameter type-check string - Xsnhash uint16 // .typchk section number - Xsmtyp uint8 // Symbol alignment and type - Xsmclas uint8 // Storage-mapping class - Xstab uint32 // Reserved - Xsnstab uint16 // Reserved -} - -type AuxCSect64 struct { - Xscnlenlo uint32 // Lower 4 bytes of length or symbol table index - Xparmhash uint32 // Offset of parameter type-check string - Xsnhash uint16 // .typchk section number - Xsmtyp uint8 // Symbol alignment and type - Xsmclas uint8 // Storage-mapping class - Xscnlenhi int32 // Upper 4 bytes of length or symbol table index - Xpad uint8 // Unused - Xauxtype uint8 // Type of auxiliary entry -} - -// Auxiliary type -// const ( -// _AUX_EXCEPT = 255 -// _AUX_FCN = 254 -// _AUX_SYM = 253 -// _AUX_FILE = 252 -// _AUX_CSECT = 251 -// _AUX_SECT = 250 -// ) - -// Symbol type field. -const ( - XTY_ER = 0 // External reference - XTY_SD = 1 // Section definition - XTY_LD = 2 // Label definition - XTY_CM = 3 // Common csect definition -) - -// Defines for File auxiliary definitions: x_ftype field of x_file -const ( - XFT_FN = 0 // Source File Name - XFT_CT = 1 // Compile Time Stamp - XFT_CV = 2 // Compiler Version Number - XFT_CD = 128 // Compiler Defined Information -) - -// Storage-mapping class. -const ( - XMC_PR = 0 // Program code - XMC_RO = 1 // Read-only constant - XMC_DB = 2 // Debug dictionary table - XMC_TC = 3 // TOC entry - XMC_UA = 4 // Unclassified - XMC_RW = 5 // Read/Write data - XMC_GL = 6 // Global linkage - XMC_XO = 7 // Extended operation - XMC_SV = 8 // 32-bit supervisor call descriptor - XMC_BS = 9 // BSS class - XMC_DS = 10 // Function descriptor - XMC_UC = 11 // Unnamed FORTRAN common - XMC_TC0 = 15 // TOC anchor - XMC_TD = 16 // Scalar data entry in the TOC - XMC_SV64 = 17 // 64-bit supervisor call descriptor - XMC_SV3264 = 18 // Supervisor call descriptor for both 32-bit and 64-bit - XMC_TL = 20 // Read/Write thread-local data - XMC_UL = 21 // Read/Write thread-local data (.tbss) - XMC_TE = 22 // TOC entry -) - -// Loader Header. -type LoaderHeader32 struct { - Lversion int32 // Loader section version number - Lnsyms int32 // Number of symbol table entries - Lnreloc int32 // Number of relocation table entries - Listlen uint32 // Length of import file ID string table - Lnimpid int32 // Number of import file IDs - Limpoff uint32 // Offset to start of import file IDs - Lstlen uint32 // Length of string table - Lstoff uint32 // Offset to start of string table -} - -type LoaderHeader64 struct { - Lversion int32 // Loader section version number - Lnsyms int32 // Number of symbol table entries - Lnreloc int32 // Number of relocation table entries - Listlen uint32 // Length of import file ID string table - Lnimpid int32 // Number of import file IDs - Lstlen uint32 // Length of string table - Limpoff uint64 // Offset to start of import file IDs - Lstoff uint64 // Offset to start of string table - Lsymoff uint64 // Offset to start of symbol table - Lrldoff uint64 // Offset to start of relocation entries -} - -const ( - LDHDRSZ_32 = 32 - LDHDRSZ_64 = 56 -) - -// Loader Symbol. -type LoaderSymbol32 struct { - Lname [8]byte // Symbol name or byte offset into string table - Lvalue uint32 // Address field - Lscnum int16 // Section number containing symbol - Lsmtype int8 // Symbol type, export, import flags - Lsmclas int8 // Symbol storage class - Lifile int32 // Import file ID; ordinal of import file IDs - Lparm uint32 // Parameter type-check field -} - -type LoaderSymbol64 struct { - Lvalue uint64 // Address field - Loffset uint32 // Byte offset into string table of symbol name - Lscnum int16 // Section number containing symbol - Lsmtype int8 // Symbol type, export, import flags - Lsmclas int8 // Symbol storage class - Lifile int32 // Import file ID; ordinal of import file IDs - Lparm uint32 // Parameter type-check field -} - -type Reloc32 struct { - Rvaddr uint32 // (virtual) address of reference - Rsymndx uint32 // Index into symbol table - Rsize uint8 // Sign and reloc bit len - Rtype uint8 // Toc relocation type -} - -type Reloc64 struct { - Rvaddr uint64 // (virtual) address of reference - Rsymndx uint32 // Index into symbol table - Rsize uint8 // Sign and reloc bit len - Rtype uint8 // Toc relocation type -} - -const ( - R_POS = 0x00 // A(sym) Positive Relocation - R_NEG = 0x01 // -A(sym) Negative Relocation - R_REL = 0x02 // A(sym-*) Relative to self - R_TOC = 0x03 // A(sym-TOC) Relative to TOC - R_TRL = 0x12 // A(sym-TOC) TOC Relative indirect load. - - R_TRLA = 0x13 // A(sym-TOC) TOC Rel load address. modifiable inst - R_GL = 0x05 // A(external TOC of sym) Global Linkage - R_TCL = 0x06 // A(local TOC of sym) Local object TOC address - R_RL = 0x0C // A(sym) Pos indirect load. modifiable instruction - R_RLA = 0x0D // A(sym) Pos Load Address. modifiable instruction - R_REF = 0x0F // AL0(sym) Non relocating ref. No garbage collect - R_BA = 0x08 // A(sym) Branch absolute. Cannot modify instruction - R_RBA = 0x18 // A(sym) Branch absolute. modifiable instruction - R_BR = 0x0A // A(sym-*) Branch rel to self. non modifiable - R_RBR = 0x1A // A(sym-*) Branch rel to self. modifiable instr - - R_TLS = 0x20 // General-dynamic reference to TLS symbol - R_TLS_IE = 0x21 // Initial-exec reference to TLS symbol - R_TLS_LD = 0x22 // Local-dynamic reference to TLS symbol - R_TLS_LE = 0x23 // Local-exec reference to TLS symbol - R_TLSM = 0x24 // Module reference to TLS symbol - R_TLSML = 0x25 // Module reference to local (own) module - - R_TOCU = 0x30 // Relative to TOC - high order bits - R_TOCL = 0x31 // Relative to TOC - low order bits -) diff --git a/syft/pkg/cataloger/golang/parse_go_binary.go b/syft/pkg/cataloger/golang/parse_go_binary.go index 90ce268be..e33874800 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary.go +++ b/syft/pkg/cataloger/golang/parse_go_binary.go @@ -5,12 +5,14 @@ import ( "context" "debug/macho" "debug/pe" + "encoding/binary" "errors" "fmt" "io" "regexp" "runtime/debug" "slices" + "strconv" "strings" "sync" "time" @@ -25,7 +27,6 @@ import ( "github.com/anchore/syft/syft/internal/unionreader" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" - "github.com/anchore/syft/syft/pkg/cataloger/golang/internal/xcoff" ) const goArch = "GOARCH" @@ -120,12 +121,19 @@ func (c *goBinaryCataloger) parseGoBinary(ctx context.Context, resolver file.Res } defer internal.CloseAndLogError(reader.ReadCloser, reader.RealPath) - mods, errs := scanFile(reader.Location, unionReader, c.symbolSelector.enabled()) + mods, errs := scanFile(ctx, reader.Location, unionReader, c.symbolSelector.enabled()) + // scanFile hands back the reconstruction for each binary that turned out to be packed. The readers + // below want it too, so ownership ends here and not inside the scan. + defer func() { + for _, mod := range mods { + closeUnpacked(mod.unpacked) + } + }() var rels []artifact.Relationship for _, mod := range mods { var depPkgs []pkg.Package - mainPkg, depPkgs := c.buildGoPkgInfo(ctx, resolver, reader.Location, mod, mod.arch, unionReader) + mainPkg, depPkgs := c.buildGoPkgInfo(ctx, resolver, reader.Location, mod, mod.arch, seekerFor(mod.unpacked, unionReader)) if mainPkg != nil { rels = createModuleRelationships(*mainPkg, depPkgs) pkgs = append(pkgs, *mainPkg) @@ -171,7 +179,7 @@ func moduleEqual(lhs, rhs *debug.Module) bool { var emptyModule debug.Module var moduleFromPartialPackageBuild = debug.Module{Path: "command-line-arguments"} -func (c *goBinaryCataloger) buildGoPkgInfo(ctx context.Context, resolver file.Resolver, location file.Location, mod *extendedBuildInfo, arch string, reader io.ReadSeekCloser) (*pkg.Package, []pkg.Package) { +func (c *goBinaryCataloger) buildGoPkgInfo(ctx context.Context, resolver file.Resolver, location file.Location, mod *extendedBuildInfo, arch string, reader io.ReadSeeker) (*pkg.Package, []pkg.Package) { if mod == nil { return nil, nil } @@ -240,7 +248,7 @@ func missingMainModule(mod *extendedBuildInfo) bool { return mod.Main == moduleFromPartialPackageBuild } -func (c *goBinaryCataloger) makeGoMainPackage(ctx context.Context, resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location, reader io.ReadSeekCloser, symbols map[string][]string) pkg.Package { +func (c *goBinaryCataloger) makeGoMainPackage(ctx context.Context, resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location, reader io.ReadSeeker, symbols map[string][]string) pkg.Package { gbs := getBuildSettings(mod.Settings) lics := c.licenseResolver.getLicenses(ctx, resolver, mod.Main.Path, mod.Main.Version) gover, experiments := getExperimentsFromVersion(mod.GoVersion) @@ -283,7 +291,7 @@ func (c *goBinaryCataloger) makeGoMainPackage(ctx context.Context, resolver file // the only thing that seems to work is to just look for version strings following both \x00 and \x00.L for now var semverPattern = regexp.MustCompile(`(\x00|\x{FFFD})(.L)?(?Pv?(\d+\.\d+\.\d+[-\w]*[+\w]*))\x00`) -func (c *goBinaryCataloger) findMainModuleVersion(metadata *pkg.GolangBinaryBuildinfoEntry, gbs pkg.KeyValues, reader io.ReadSeekCloser) string { +func (c *goBinaryCataloger) findMainModuleVersion(metadata *pkg.GolangBinaryBuildinfoEntry, gbs pkg.KeyValues, reader io.ReadSeeker) string { vcsVersion, hasVersion := gbs.Get("vcs.revision") timestamp, hasTimestamp := gbs.Get("vcs.time") @@ -416,11 +424,11 @@ func getGOARCHFromBin(r io.ReaderAt) (string, error) { } arch = f.Cpu.String() case bytes.HasPrefix(ident, []byte{0x01, 0xDF}) || bytes.HasPrefix(ident, []byte{0x01, 0xF7}): - f, err := xcoff.NewFile(r) - if err != nil { - return "", fmt.Errorf("unrecognized file format: %w", err) - } - arch = fmt.Sprintf("%d", f.TargetMachine) + // XCOFF's target machine *is* the magic we just matched on (0737 / 0767), so there is nothing to + // parse: a full XCOFF walk would read the string table, every symbol and every relocation only to + // hand back these two bytes. Reading them directly also keeps stripped binaries working, which a + // walk does not since it needs a symbol table to get that far. + arch = strconv.Itoa(int(binary.BigEndian.Uint16(ident[:2]))) default: return "", errUnrecognizedFormat } diff --git a/syft/pkg/cataloger/golang/parse_go_binary_test.go b/syft/pkg/cataloger/golang/parse_go_binary_test.go index 14d03e206..3f704c5d3 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary_test.go +++ b/syft/pkg/cataloger/golang/parse_go_binary_test.go @@ -108,24 +108,27 @@ func Test_getGOARCHFromBin(t *testing.T) { }, { name: "xcoff-32bit", - filepath: "internal/xcoff/testdata/gcc-ppc32-aix-dwarf2-exec", + filepath: "testdata/xcoff/gcc-ppc32-aix-dwarf2-exec", expected: strconv.Itoa(0x1DF), }, { name: "xcoff-64bit", - filepath: "internal/xcoff/testdata/gcc-ppc64-aix-dwarf2-exec", + filepath: "testdata/xcoff/gcc-ppc64-aix-dwarf2-exec", expected: strconv.Itoa(0x1F7), }, } for _, tt := range tests { - f, err := os.Open(tt.filepath) - require.NoError(t, err) - arch, err := getGOARCHFromBin(f) - require.NoError(t, err, "test name: %s", tt.name) - assert.Equal(t, tt.expected, arch) - } + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open(tt.filepath) + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + arch, err := getGOARCHFromBin(f) + require.NoError(t, err) + assert.Equal(t, tt.expected, arch) + }) + } } func TestBuildGoPkgInfo(t *testing.T) { diff --git a/syft/pkg/cataloger/golang/scan_binary.go b/syft/pkg/cataloger/golang/scan_binary.go index e88aa91f2..d017223ba 100644 --- a/syft/pkg/cataloger/golang/scan_binary.go +++ b/syft/pkg/cataloger/golang/scan_binary.go @@ -1,7 +1,9 @@ package golang import ( + "context" "debug/buildinfo" + "errors" "fmt" "io" "runtime/debug" @@ -21,10 +23,32 @@ type extendedBuildInfo struct { cryptoSettings []string arch string symbols []binarySymbol + + // unpacked is the reconstruction when this binary was UPX-packed, and nil when it was not. It is + // carried on the build info because the readers downstream of the scan want it too, since the packed + // bytes hold no readable version strings either. The caller of scanFile owns it and must Close it. + unpacked unpackedContents +} + +// unpackedContents is the reconstruction of a UPX-packed binary, as everything downstream of the unpack +// uses it: bytes at offsets, the length it is willing to stand behind, and the temp file underneath to +// release when the scan is done with it. internal/spillbuf is what implements it, and naming that type in +// these signatures would say the readers care where the bytes live, which they do not. +// +// A nil value means the binary was not packed, and unpackUPX only ever returns a literal nil for that: a +// nil *spillbuf.Buffer widened into this interface is a non-nil interface holding a nil pointer, so it +// reads as present everywhere it is checked and then panics on first use. readerFor, seekerFor and +// closeUnpacked are the only places that check. +type unpackedContents interface { + io.ReaderAt + io.Closer + + // Size is the contiguous run rebuilt from offset zero, which is all the reconstruction stands behind. + Size() int64 } // scanFile scans file to try to report the Go and module versions. -func scanFile(location file.Location, reader unionreader.UnionReader, captureSymbols bool) ([]*extendedBuildInfo, error) { +func scanFile(ctx context.Context, location file.Location, reader unionreader.UnionReader, captureSymbols bool) ([]*extendedBuildInfo, error) { // NOTE: multiple readers are returned to cover universal binaries, which are files // with more than one binary readers, errs := unionreader.GetReaders(reader) @@ -35,53 +59,134 @@ func scanFile(location file.Location, reader unionreader.UnionReader, captureSym var builds []*extendedBuildInfo for _, r := range readers { - bi, err := getBuildInfo(r, location) - if err != nil { - log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang buildinfo") - - continue + build, err := scanReader(ctx, location, r, captureSymbols) + errs = unknown.Join(errs, err) + if build != nil { + builds = append(builds, build) } - // it's possible the reader just isn't a go binary, in which case just skip it - if bi == nil { - continue - } - - v, err := getCryptoInformation(r) - if err != nil { - log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang version info") - // don't skip this build info. - // we can still catalog packages, even if we can't get the crypto information - errs = unknown.Appendf(errs, location, "unable to read golang version info: %w", err) - } - v = append(v, getNativeFIPSSettings(bi.Settings)...) - arch := getGOARCH(bi.Settings) - if arch == "" { - arch, err = getGOARCHFromBin(r) - if err != nil { - log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang arch info") - // don't skip this build info. - // we can still catalog packages, even if we can't get the arch information - errs = unknown.Appendf(errs, location, "unable to read golang arch info: %w", err) - } - } - - var symbols []binarySymbol - if captureSymbols { - symbols, err = getSymbols(r) - if err != nil { - log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang symbol info") - // don't skip this build info. - // we can still catalog packages, even if we can't get the symbol information - errs = unknown.Appendf(errs, location, "unable to read golang symbol info: %w", err) - } - } - - builds = append(builds, &extendedBuildInfo{BuildInfo: bi, cryptoSettings: v, arch: arch, symbols: symbols}) } return builds, errs } +// scanReader reports the build info, crypto settings, arch and symbols of a single binary. Everything is +// read from the same reader, which is the unpacked contents when the binary turns out to be UPX-packed: +// the packed bytes carry no readable pclntab or version strings either, so nothing downstream of the +// build info should be looking at them. +func scanReader(ctx context.Context, location file.Location, r io.ReaderAt, captureSymbols bool) (*extendedBuildInfo, error) { + var errs error + + // unpacked is nil when there was nothing to unpack; readerFor turns that into the reader every parser + // below should use, so none of them has to ask whether this binary was packed. + // err can arrive alongside a usable bi: a partial reconstruction still carries build info, and the + // bytes it lost are a gap worth reporting rather than a reason to drop the binary. + unpacked, bi, err := readContentsAndBuildInfo(ctx, r) + + // ownership of unpacked transfers to the returned extendedBuildInfo on success. Until then it is held + // here: the parsers below panic on malformed input (which is why getBuildInfo has a recover), and an + // unwind past this point would leave a reconstruction of up to maxUPXOriginalSize on disk for the rest + // of the scan: the temp root is swept, but not until the run ends. + ownContents := true + defer func() { + if ownContents { + closeUnpacked(unpacked) + } + }() + + if err != nil { + if isCancelled(err) { + return nil, err + } + log.WithFields("file", location.RealPath, "error", err).Trace("unable to fully read golang buildinfo") + if reportableGap(err) { + // the build info is either missing or it is not, and the same err covers both: a partial + // reconstruction still hands back everything .go.buildinfo carried, and what it lost is the + // non-loadable tail the readers below want. Saying "unable to read golang buildinfo" there + // describes a failure that did not happen. + if bi != nil { + errs = unknown.Appendf(errs, location, "golang binary read incompletely: %w", err) + } else { + errs = unknown.Appendf(errs, location, "unable to read golang buildinfo: %w", err) + } + } + if bi == nil { + return nil, errs + } + } + + // it's possible the reader just isn't a go binary, in which case just skip it + if bi == nil { + return nil, errs + } + + // resolved once, here: every parser below reads the same bytes, and the choice of which reader that is + // belongs at the top rather than repeated at each call site + contents := readerFor(unpacked, r) + + // everything below reads those contents through a parser that expands ELF sections, and each bounds its own + // reads: getSymbols opens the file with elfutil.NewFile, getCryptoInformation gates on + // elfutil.CheckAllSections. getBuildInfo's CheckSectionNameTable is not what makes these safe, since it + // bounds only the section elf.NewFile expands as it parses. + v, err := getCryptoInformation(contents) + if err != nil { + log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang version info") + // don't skip this build info. + // we can still catalog packages, even if we can't get the crypto information + errs = unknown.Appendf(errs, location, "unable to read golang version info: %w", err) + } + v = append(v, getNativeFIPSSettings(bi.Settings)...) + arch := getGOARCH(bi.Settings) + if arch == "" { + arch, err = getGOARCHFromBin(contents) + if err != nil { + log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang arch info") + // don't skip this build info. + // we can still catalog packages, even if we can't get the arch information + errs = unknown.Appendf(errs, location, "unable to read golang arch info: %w", err) + } + } + + var symbols []binarySymbol + if captureSymbols { + symbols, err = getSymbols(contents) + if err != nil { + log.WithFields("file", location.RealPath, "error", err).Trace("unable to read golang symbol info") + // don't skip this build info. + // we can still catalog packages, even if we can't get the symbol information + errs = unknown.Appendf(errs, location, "unable to read golang symbol info: %w", err) + } + } + + ownContents = false + return &extendedBuildInfo{BuildInfo: bi, cryptoSettings: v, arch: arch, symbols: symbols, unpacked: unpacked}, errs +} + +// reportableGap reports whether err is a gap this cataloger chose to leave, rather than a file it was +// never going to catalog. A packed file we failed to decode, a reconstruction that came up short, and a +// file we declined to expand (from either the ELF section bound or the UPX header bounds) all cost the +// SBOM something real. This cataloger runs on every executable in an image, so reporting anything else +// would attach an unknown to every corrupt, truncated or non-Go binary in it, which is the noise the +// quiet-by-default policy in upx.go exists to avoid. +// +// One real gap is deliberately left off this list: a UPX method we have not implemented. It is the same +// cost to the SBOM as a decode failure, but it fires on most packed binaries rather than on rare ones. +// See errUPXDecompress in upx.go for the reasoning. +func reportableGap(err error) bool { + return errors.Is(err, errUPXDecompress) || + errors.Is(err, errUPXSizeRefused) || + errors.Is(err, errUPXPartial) || + errors.Is(err, elfutil.ErrDeclaredSizeExceeded) +} + func getCryptoInformation(reader io.ReaderAt) ([]string, error) { + // goversion opens the file with debug/elf itself and reads .symtab plus the string table it links. + // Those are expanded lazily, so the section-name table bound getBuildInfo already applied does not + // reach them. CheckAllSections is the decompression-bomb gate: a compressed section declares its own + // decompressed size and debug/elf allocates that much on open, so a 260KB ELF declaring a compressed + // .symtab drove 1.3GB of allocation before this was here. + if err := elfutil.CheckAllSections(reader); err != nil { + return nil, err + } + v, err := version.ReadExeFromReader(reader) if err != nil { return nil, err @@ -123,6 +228,99 @@ func getNativeFIPSSettings(settings []debug.BuildSetting) []string { return cryptoSettings } +// readContentsAndBuildInfo returns the contents this binary should be read from, along with its build +// info. A UPX-packed binary is unpacked first and read from the reconstruction. +// +// If the reconstruction carries no build info, the bytes as they were found are tried before giving up. +// The "UPX!" magic is located by an unanchored scan over the first 8KB and every field behind it is +// attacker-controlled, so an ordinary Go binary carrying those four bytes can be made to produce a +// plausible header and a reconstruction of nothing. Without the retry that binary reports no packages at +// all, which turns a false positive here into a way to hide a dependency list. +// +// A gap in the unpack is returned even when build info came through, so it is reported rather than papered +// over: the readers after this one (crypto settings, arch, symbols) read the same contents, and the bytes +// a partial reconstruction lost are exactly the non-loadable tail those depend on. That only holds when +// the contents are a reconstruction: a gap read off the bytes as they were found is not a gap at all, and +// both places below that hand those bytes back drop it. +func readContentsAndBuildInfo(ctx context.Context, r io.ReaderAt) (unpackedContents, *debug.BuildInfo, error) { + unpacked, unpackErr := unpackUPX(ctx, r) + if isCancelled(unpackErr) { + return unpacked, nil, unpackErr + } + + bi, err := getBuildInfo(readerFor(unpacked, r)) + if err == nil && bi != nil { + if unpacked == nil { + // nothing was rebuilt, so nothing downstream is short: every reader below this one reads the + // same bytes getBuildInfo just read in full. unpackUPX returns the input as it was found + // alongside a refusal (the size bounds) or a failure (block 1 not decoding, or no temp dir), + // and readable build info in those bytes is itself the evidence the file was not really + // packed. A genuinely packed binary carries no readable .go.buildinfo in its packed bytes, so + // it never reaches here and its gap is still reported below. + return unpacked, bi, nil + } + return unpacked, bi, unpackErr + } + + if unpacked != nil { + if foundBI, foundErr := getBuildInfo(r); foundErr == nil && foundBI != nil { + log.WithFields("error", err, "unpackError", unpackErr). + Trace("UPX reconstruction carried no build info, reading the binary as it was found") + closeUnpacked(unpacked) + // unpackErr is deliberately dropped. Every reader after this one reads the bytes as they were + // found, not the reconstruction, so nothing downstream is short: the header this binary + // carried was not describing real UPX output in the first place. Returning the gap here would + // put an unknown on a binary the cataloger read completely. + return nil, foundBI, nil + } + } + + // a packed file we could not unpack is the more specific gap, so it wins over whatever buildinfo made + // of the bytes it was handed + if unpackErr != nil { + return unpacked, nil, unpackErr + } + return unpacked, bi, err +} + +// isCancelled reports whether err means the scan was called off, which is a reason to stop rather than a +// gap in the SBOM. +func isCancelled(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +} + +// readerFor returns where a scanned binary should be read from: the reconstruction when it was packed, +// the input as it was found otherwise. +// +// The nil check lives here, in one place, on purpose: every reader below the scan goes through here, so no +// parser has to ask whether this binary was packed. +func readerFor(unpacked unpackedContents, found io.ReaderAt) io.ReaderAt { + if unpacked == nil { + return found + } + return unpacked +} + +// seekerFor is readerFor for the readers after the scan, which seek. found belongs to the caller and is +// reused for every later module, so this deliberately hands back no Closer. +func seekerFor(unpacked unpackedContents, found io.ReadSeeker) io.ReadSeeker { + if unpacked == nil { + return found + } + // the SectionReader snapshots Size() here, which is correct: the reconstruction is complete by the time + // anything seeks it. It also gives each caller its own cursor over the same contents. + return io.NewSectionReader(unpacked, 0, unpacked.Size()) +} + +// closeUnpacked releases a reconstruction, if the binary turned out to be packed at all. Nothing to unpack +// is the common case rather than an edge one, so every owner of an unpackedContents closes through here. +func closeUnpacked(unpacked unpackedContents) { + if unpacked == nil { + return + } + _ = unpacked.Close() +} + // readBuildInfo bounds the reader before handing it to debug/buildinfo, which opens ELF files with // debug/elf itself rather than through elfutil. elf.NewFile expands the section-name string table as it // parses, so an unbounded read here is reachable no matter how little of the file buildinfo goes on to @@ -134,7 +332,7 @@ func readBuildInfo(r io.ReaderAt) (*debug.BuildInfo, error) { return buildinfo.Read(r) } -func getBuildInfo(r io.ReaderAt, location file.Location) (bi *debug.BuildInfo, err error) { +func getBuildInfo(r io.ReaderAt) (bi *debug.BuildInfo, err error) { defer func() { if r := recover(); r != nil { // this can happen in cases where a malformed binary is passed in can be initially parsed, but not @@ -144,24 +342,7 @@ func getBuildInfo(r io.ReaderAt, location file.Location) (bi *debug.BuildInfo, e } }() - // try to read buildinfo from the binary directly bi, err = readBuildInfo(r) - if err == nil { - return bi, nil - } - - // if direct read fails and this looks like a UPX-compressed binary, - // try to decompress and read the buildinfo from the decompressed data - if isUPXCompressed(r) { - log.WithFields("path", location.RealPath).Trace("detected UPX-compressed Go binary, attempting decompression to read the build info") - decompressed, decompErr := decompressUPX(r) - if decompErr == nil { - bi, err = readBuildInfo(decompressed) - if err == nil { - return bi, nil - } - } - } // note: the stdlib does not export the error we need to check for if err != nil { diff --git a/syft/pkg/cataloger/golang/scan_binary_bomb_test.go b/syft/pkg/cataloger/golang/scan_binary_bomb_test.go index a6dc20803..6369bbda7 100644 --- a/syft/pkg/cataloger/golang/scan_binary_bomb_test.go +++ b/syft/pkg/cataloger/golang/scan_binary_bomb_test.go @@ -6,13 +6,15 @@ import ( "debug/buildinfo" "debug/elf" "encoding/binary" - "runtime" + "os" "testing" + "github.com/kastenhq/goversion/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/internal/testutils" + "github.com/anchore/syft/syft/internal/elfutil" ) // Test_getBuildInfo_compressedSectionBomb covers the reason readBuildInfo exists: debug/buildinfo opens @@ -26,39 +28,154 @@ func Test_getBuildInfo_compressedSectionBomb(t *testing.T) { t.Logf("%d byte fixture declares a %d byte section name table", len(bomb), declared) // the unguarded path is the thing being defended against: prove the fixture really is a bomb - unguarded := measureAlloc(t, func() { + unguarded := testutils.MeasureAlloc(t, func() { _, err := buildinfo.Read(bytes.NewReader(bomb)) t.Logf("buildinfo.Read err: %v", err) }) assert.Greater(t, unguarded, uint64(declared), "fixture did not actually deliver the declared bytes") - guarded := measureAlloc(t, func() { - _, err := getBuildInfo(bytes.NewReader(bomb), file.NewLocation("bomb")) + guarded := testutils.MeasureAlloc(t, func() { + _, err := getBuildInfo(bytes.NewReader(bomb)) require.Error(t, err) - assert.Contains(t, err.Error(), "over the") + assert.ErrorIs(t, err, elfutil.ErrDeclaredSizeExceeded) }) assert.Less(t, guarded, uint64(32<<20), "getBuildInfo allocated far more than the input warrants") t.Logf("unguarded allocated %d bytes, guarded allocated %d bytes", unguarded, guarded) } -func measureAlloc(t *testing.T, fn func()) uint64 { - t.Helper() - var before, after runtime.MemStats - runtime.GC() - runtime.ReadMemStats(&before) - fn() - runtime.ReadMemStats(&after) - return after.TotalAlloc - before.TotalAlloc +// Test_getCryptoInformation_compressedSymtabBomb covers the second unbounded door into debug/elf: +// goversion opens the file itself and reads .symtab plus the string table it links, and those are +// expanded lazily, so the section-name table bound getBuildInfo applies never reaches them. +func Test_getCryptoInformation_compressedSymtabBomb(t *testing.T) { + const declared = 256 << 20 // comfortably over elfutil's bound, small enough to allocate in a test + + bomb := elfWithCompressedSymtab(t, declared) + t.Logf("%d byte fixture declares a %d byte symbol table", len(bomb), declared) + + // the unguarded path is the thing being defended against: prove the fixture really is a bomb + unguarded := testutils.MeasureAlloc(t, func() { + _, err := version.ReadExeFromReader(bytes.NewReader(bomb)) + t.Logf("goversion err: %v", err) + }) + assert.Greater(t, unguarded, uint64(declared), "fixture did not actually deliver the declared bytes") + + guarded := testutils.MeasureAlloc(t, func() { + _, err := getCryptoInformation(bytes.NewReader(bomb)) + require.ErrorIs(t, err, elfutil.ErrDeclaredSizeExceeded) + }) + assert.Less(t, guarded, uint64(32<<20), "getCryptoInformation allocated far more than the input warrants") + t.Logf("unguarded allocated %d bytes, guarded allocated %d bytes", unguarded, guarded) } -// elfWithCompressedNameTable builds a minimal ELF64 whose only real section is a SHF_COMPRESSED -// .shstrtab declaring `declared` decompressed bytes and genuinely delivering them. +// Test_getCryptoInformation_passesThroughNonELF keeps the gate from becoming a container filter: goversion +// reads PE and Mach-O too, and CheckAllSections has to leave those to it. +func Test_getCryptoInformation_passesThroughNonELF(t *testing.T) { + runMakeTarget(t, "archs") + + for _, name := range []string{"hello-win-amd64", "hello-mach-o-arm64"} { + t.Run(name, func(t *testing.T) { + f, err := os.Open("testdata/archs/binaries/" + name) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, f.Close()) }) + + _, err = getCryptoInformation(f) + require.NoError(t, err) + }) + } +} + +// elfWithCompressedSymtab builds a minimal ELF64 whose .symtab is SHF_COMPRESSED, declaring `declared` +// decompressed bytes and genuinely delivering them. The name table is left uncompressed so the file gets +// past the bound getBuildInfo already applies, which is the point: this is the section that bound misses. +func elfWithCompressedSymtab(t *testing.T, declared uint64) []byte { + t.Helper() + + payload := make([]byte, declared) + var compressed bytes.Buffer + zw := zlib.NewWriter(&compressed) + _, err := zw.Write(payload) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + names := []byte("\x00.shstrtab\x00.symtab\x00.strtab\x00") + nameOf := func(s string) uint32 { + i := bytes.Index(names, []byte("\x00"+s+"\x00")) + require.GreaterOrEqual(t, i, 0) + return uint32(i + 1) + } + + ehsize := uint64(binary.Size(elf.Header64{})) + shentsize := uint64(binary.Size(elf.Section64{})) + chdrsize := uint64(binary.Size(elf.Chdr64{})) + shoff := ehsize + namesOff := shoff + 4*shentsize + symOff := namesOff + uint64(len(names)) + symSize := chdrsize + uint64(compressed.Len()) + strOff := symOff + symSize + + var ident [16]byte + copy(ident[:], elf.ELFMAG) + ident[elf.EI_CLASS] = byte(elf.ELFCLASS64) + ident[elf.EI_DATA] = byte(elf.ELFDATA2LSB) + ident[elf.EI_VERSION] = byte(elf.EV_CURRENT) + + buf := &bytes.Buffer{} + require.NoError(t, binary.Write(buf, binary.LittleEndian, elf.Header64{ + Ident: ident, Type: uint16(elf.ET_REL), Machine: uint16(elf.EM_X86_64), + Version: uint32(elf.EV_CURRENT), Shoff: shoff, Ehsize: uint16(ehsize), + Shentsize: uint16(shentsize), Shnum: 4, Shstrndx: 1, + })) + // the null section + require.NoError(t, binary.Write(buf, binary.LittleEndian, elf.Section64{})) + require.NoError(t, binary.Write(buf, binary.LittleEndian, elf.Section64{ + Name: nameOf(".shstrtab"), Type: uint32(elf.SHT_STRTAB), Off: namesOff, + Size: uint64(len(names)), Addralign: 1, + })) + // the bomb: sh_size covers the whole zlib stream, ch_size is what debug/elf expands to + require.NoError(t, binary.Write(buf, binary.LittleEndian, elf.Section64{ + Name: nameOf(".symtab"), Type: uint32(elf.SHT_SYMTAB), Flags: uint64(elf.SHF_COMPRESSED), + Off: symOff, Size: symSize, Link: 3, Entsize: 24, Addralign: 1, + })) + require.NoError(t, binary.Write(buf, binary.LittleEndian, elf.Section64{ + Name: nameOf(".strtab"), Type: uint32(elf.SHT_STRTAB), Off: strOff, Size: 1, Addralign: 1, + })) + buf.Write(names) + require.NoError(t, binary.Write(buf, binary.LittleEndian, elf.Chdr64{ + Type: uint32(elf.COMPRESS_ZLIB), Size: declared, Addralign: 1, + })) + buf.Write(compressed.Bytes()) + buf.Write([]byte{0}) + + return buf.Bytes() +} + +// elfWithCompressedNameTable builds a minimal ELF64 whose only real section is a SHF_COMPRESSED .shstrtab +// declaring `declared` decompressed bytes and genuinely delivering them. Only the test that runs the +// unguarded path needs delivery; use elfDeclaringNameTable everywhere else, since building this one costs +// `declared` bytes of allocation plus a zlib compress of them. func elfWithCompressedNameTable(t *testing.T, declared uint64) []byte { t.Helper() + return elfNameTableFixture(t, declared, true) +} + +// elfDeclaringNameTable declares `declared` bytes without delivering them. CheckSectionNameTable reads the +// declared size out of the compression header and refuses before decompressing anything, so a test of the +// bound itself never needs the stream to be real. +func elfDeclaringNameTable(t *testing.T, declared uint64) []byte { + t.Helper() + return elfNameTableFixture(t, declared, false) +} + +func elfNameTableFixture(t *testing.T, declared uint64, deliver bool) []byte { + t.Helper() // the decompressed name table only has to start with the section names; the rest is padding that // exists purely to make the declared size real - payload := make([]byte, declared) + size := declared + if !deliver { + size = 64 + } + payload := make([]byte, size) copy(payload, "\x00.shstrtab\x00") var compressed bytes.Buffer diff --git a/syft/pkg/cataloger/golang/scan_binary_test.go b/syft/pkg/cataloger/golang/scan_binary_test.go index 1e6910e6d..2c9fa495f 100644 --- a/syft/pkg/cataloger/golang/scan_binary_test.go +++ b/syft/pkg/cataloger/golang/scan_binary_test.go @@ -8,8 +8,6 @@ import ( "github.com/kastenhq/goversion/version" "github.com/stretchr/testify/assert" - - "github.com/anchore/syft/syft/file" ) func Test_getBuildInfo(t *testing.T) { @@ -33,7 +31,7 @@ func Test_getBuildInfo(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotBi, err := getBuildInfo(tt.args.r, file.Location{}) + gotBi, err := getBuildInfo(tt.args.r) if !tt.wantErr(t, err, fmt.Sprintf("getBuildInfo(%v)", tt.args.r)) { return } diff --git a/syft/pkg/cataloger/golang/internal/xcoff/testdata/gcc-ppc32-aix-dwarf2-exec b/syft/pkg/cataloger/golang/testdata/xcoff/gcc-ppc32-aix-dwarf2-exec similarity index 100% rename from syft/pkg/cataloger/golang/internal/xcoff/testdata/gcc-ppc32-aix-dwarf2-exec rename to syft/pkg/cataloger/golang/testdata/xcoff/gcc-ppc32-aix-dwarf2-exec diff --git a/syft/pkg/cataloger/golang/internal/xcoff/testdata/gcc-ppc64-aix-dwarf2-exec b/syft/pkg/cataloger/golang/testdata/xcoff/gcc-ppc64-aix-dwarf2-exec similarity index 100% rename from syft/pkg/cataloger/golang/internal/xcoff/testdata/gcc-ppc64-aix-dwarf2-exec rename to syft/pkg/cataloger/golang/testdata/xcoff/gcc-ppc64-aix-dwarf2-exec diff --git a/syft/pkg/cataloger/golang/unpacked_reader_test.go b/syft/pkg/cataloger/golang/unpacked_reader_test.go new file mode 100644 index 000000000..fe83b77d7 --- /dev/null +++ b/syft/pkg/cataloger/golang/unpacked_reader_test.go @@ -0,0 +1,96 @@ +package golang + +import ( + "bytes" + "context" + "os" + "path/filepath" + "runtime/debug" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/spillbuf" + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/fileresolver" + "github.com/anchore/syft/syft/internal/unionreader" +) + +// TestScanFile_UnpackedBinaryReadsTheFileItWasGiven is the regression test for the shape of bug that +// modelling "not packed" as nil invites. The reader was once an io.ReadSeekCloser holding a nil pointer, +// which is a non-nil interface, so every ordinary Go binary took the packed branch: it seeked a nil file +// for its version and nil-dereferenced on close. The panic was recovered by the task executor, so the +// only symptom was the go-binary cataloger reporting nothing at all. +// +// The nil is now the signal rather than a hazard: unpacked is nil when there was nothing to unpack, and +// readerFor, seekerFor and closeUnpacked are the only places that resolve it. +// +// Docker-gated like everything else that needs a real Go binary; the point is that a plain binary reads +// from the file it was handed, which needs a real one. +func TestScanFile_UnpackedBinaryReadsTheFileItWasGiven(t *testing.T) { + runMakeTarget(t, "archs") + f, err := os.Open(filepath.Join("testdata", "archs", "binaries", "hello-linux-arm")) + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + + ur, err := unionreader.GetUnionReader(f) + require.NoError(t, err) + + builds, _ := scanFile(context.Background(), file.NewLocation("hello-linux-arm"), ur, false) + require.NotEmpty(t, builds, "a plain Go binary must still produce build info") + + for _, b := range builds { + assert.Nil(t, b.unpacked, "an unpacked binary must not have left a reconstruction behind") + assert.Same(t, ur, seekerFor(b.unpacked, ur), "the readers after the scan must get the file itself") + assert.NotPanics(t, func() { closeUnpacked(b.unpacked); closeUnpacked(b.unpacked) }, + "closing nothing has to be safe and idempotent, since a defer over every build calls it") + } +} + +// TestParseGoBinary_PlainBinaryStillYieldsPackages is the same regression one layer out, at the entry +// point the cataloger actually calls. The typed-nil panic fired from a defer here and was swallowed by +// the task executor, so the failure mode is an empty result rather than an error. +func TestParseGoBinary_PlainBinaryStillYieldsPackages(t *testing.T) { + runMakeTarget(t, "archs") + f, err := os.Open(filepath.Join("testdata", "archs", "binaries", "hello-linux-arm")) + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + + c := newGoBinaryCataloger(DefaultCatalogerConfig()) + pkgs, _, err := c.parseGoBinary(context.Background(), fileresolver.Empty{}, nil, + file.NewLocationReadCloser(file.NewLocation("hello-linux-arm"), f)) + + require.NoError(t, err) + assert.NotEmpty(t, pkgs, "an ordinary Go binary must still produce packages") +} + +// TestMakeGoMainPackage_VersionComesFromTheUnpackedContents pins the reader threading: for a packed +// binary the version scan has to read the reconstruction, since the packed bytes carry no readable +// version string. Nothing else covers this, because the from-contents scan is off by default and the +// Docker fixture gets its version from ldflags before the scan is reached. +func TestMakeGoMainPackage_VersionComesFromTheUnpackedContents(t *testing.T) { + // the version pattern wants the string NUL-delimited + unpacked := spillbuf.New(tmpdir.FromPath(t.TempDir())) + t.Cleanup(func() { _ = unpacked.Close() }) + _, err := unpacked.WriteAt([]byte("\x00v9.9.9\x00"), 0) + require.NoError(t, err) + + // what the packed file on disk would have said, which must not be what comes out + packed := &nopReadSeekCloser{bytes.NewReader([]byte("\x00v1.1.1\x00"))} + + c := &goBinaryCataloger{ + licenseResolver: newGoLicenseResolver("", CatalogerConfig{}), + mainModuleVersion: MainModuleVersionConfig{FromContents: true}, + } + mod := &extendedBuildInfo{ + BuildInfo: &debug.BuildInfo{Main: debug.Module{Path: "github.com/anchore/syft", Version: devel}}, + unpacked: unpacked, + } + + got := c.makeGoMainPackage(context.Background(), fileresolver.Empty{}, mod, "amd64", + file.NewLocation("packed"), seekerFor(mod.unpacked, packed), nil) + + assert.Equal(t, "v9.9.9", got.Version, "the version must come from the reconstruction, not the packed bytes") +} diff --git a/syft/pkg/cataloger/golang/upx.go b/syft/pkg/cataloger/golang/upx.go index 334e5d59d..9d3a4ae8c 100644 --- a/syft/pkg/cataloger/golang/upx.go +++ b/syft/pkg/cataloger/golang/upx.go @@ -2,31 +2,24 @@ package golang // UPX Decompression Support // -// this file implements decompression of UPX-compressed ELF binaries to enable -// extraction of Go build information (.go.buildinfo) from packed executables. +// this file reconstructs UPX-compressed ELF binaries so the scan reads unpacked contents: .go.buildinfo, +// the pclntab, version strings and the machine type all come from the reconstruction. // -// UPX (Ultimate Packer for eXecutables) is a popular executable packer that -// compresses binaries to reduce file size. When a Go binary is compressed with -// UPX, the standard debug/buildinfo.Read() fails because the .go.buildinfo -// section is compressed. This code decompresses the binary in-memory to allow -// buildinfo extraction. +// The reconstruction goes through a spillbuf.Buffer rather than a []byte: it holds a bounded amount in +// memory and spills the rest to disk, since its size is declared by the file being read and a claim that +// cannot be trusted with the heap can still be trusted with the disk. Blocks stream into it through a fixed +// window, so a single large block is never resident in full either. // -// # Supported Compression Methods +// Disk still needs a ceiling, so the bounds below are both a ratio against what the input brought with it +// and an absolute cap: a ratio alone is cheap to defeat, since image layers are gzipped and 16MB of zeros +// costs ~16KB, and the file cataloger runs NumCPU*4 of these at once. // -// Currently only LZMA (method 14) is supported, which is used by: +// The reader handed back is sized by the contiguous bytes actually rebuilt, not by where the last block +// landed. A sparse file serves its holes as zeros it never stored, so a block's offset would otherwise be +// an allocation knob for every parser downstream that sizes against what a reader will deliver. // -// upx --best --lzma -// -// Other UPX methods (NRV2B, NRV2D, NRV2E, etc.) are not yet implemented but -// could be added via the upxDecompressors dispatch map. -// -// # Key Functions -// -// - isUPXCompressed: detects UPX magic bytes ("UPX!") in the binary -// - decompressUPX: main entry point; decompresses all blocks and reconstructs the ELF -// - decompressLZMA: handles UPX's custom 2-byte LZMA header format -// - unfilter49: reverses the CTO (call trick optimization) filter for x86/x64 code -// - parseELFPTLoadOffsets: extracts PT_LOAD segment offsets for proper block placement +// Only LZMA (method 14, what `upx --best --lzma` emits) and stored blocks are implemented; other methods +// can be added to upxDecompressors. // // # UPX Binary Format // @@ -68,6 +61,11 @@ package golang // ELF PT_LOAD segments parsed from the first decompressed block. Simply // concatenating blocks produces invalid output. // +// Those extents cover only the loadable segments. Behind the loader stub (l_lsize in l_info) UPX packs +// what they did not: the padding between segments and the non-loadable tail holding the section headers +// and name table. Without that tail there are no readable section names, so .gopclntab cannot be found and +// no symbols come out. Those extents fill the gaps in file order. +// // # References // // - UPX source: https://github.com/upx/upx @@ -75,55 +73,35 @@ package golang // - CTO filter: https://github.com/upx/upx/blob/master/src/filter/cto.h // // note: no code was copied from the UPX repo, this is an independent implementation based on format description. -// -// # Anti-Unpacking / Obfuscation (Not Currently Supported) -// -// Malware commonly modifies UPX binaries to evade analysis. This implementation -// does not currently handle obfuscated binaries, but these techniques could be -// addressed in the future: -// -// - Magic modification: "UPX!" replaced with custom bytes (e.g., "YTS!", "MOZI"). -// Recovery: scan for decompression stub code patterns instead of magic bytes. -// -// - Zeroed p_info fields: p_filesize and p_blocksize set to 0. -// Recovery: read original size from PackHeader at EOF (last 36 bytes, offset 0x18). -// -// - Header corruption: checksums or version fields modified. -// Recovery: ignore validation and use PackHeader values as authoritative source. -// -// This would require parsing of the PackHeader, located in the final 36 bytes of the file, contains -// metadata recoverable even if p_info is corrupted (not parsed today): -// -// Offset Size Field Description -// ────────────────────────────────────────────────────────── -// 0x00 4 UPX magic "UPX!" (0x21585055) -// 0x04 1 version UPX version -// 0x05 1 format Executable format -// 0x06 1 method Compression method -// 0x07 1 level Compression level (1-10) -// 0x08 4 u_adler Uncompressed data checksum -// 0x0C 4 c_adler Compressed data checksum -// 0x10 4 u_len Uncompressed length -// 0x14 4 c_len Compressed length -// 0x18 4 u_file_size Original file size ← Recovery point -// 0x1C 1 filter Filter ID -// 0x1D 1 filter_cto Filter CTO parameter -// 0x1E 1 n_mru MRU parameter -// 0x1F 1 header_checksum Header checksum import ( "bytes" + "context" "encoding/binary" "errors" "fmt" "io" + "math" "github.com/ulikunitz/xz/lzma" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/internal/spillbuf" + "github.com/anchore/syft/internal/tmpdir" ) +// upxMagicScanWindow is how far into the file the "UPX!" magic is searched for. UPX places l_info +// just past the ELF headers and its loader stub, so this covers real output with room to spare. +const upxMagicScanWindow = 8192 + // UPX compression method constants const ( upxMethodLZMA uint8 = 14 // M_LZMA in UPX source + + // upxMethodStored marks an extent UPX could not compress and wrote verbatim. Real output carries a few + // of these: the alignment padding between PT_LOAD segments is only a handful of bytes. + upxMethodStored uint8 = 0 ) // UPX filter constants @@ -131,22 +109,142 @@ const ( upxFilterCTO uint8 = 0x49 // CTO (call trick optimization) filter for x86/x64 ) +// bounds on what a UPX header may claim before we act on it +const ( + // maxUPXExpansion bounds p_filesize against the bytes the input actually brought with it. Real + // `upx --best --lzma` output expands 1.96x, and the most compressible case measured is 209x (a + // `go:embed` of 120MB of zeros packs to 609KB), so 256 clears anything real without much headroom. + maxUPXExpansion = 256 + + // maxUPXDictionaryExpansion is the same shape of ratio for the LZMA dictionary, which is heap rather + // than disk. Deliberately its own constant even at the same value: raising the disk ratio for a + // legitimately compressible binary must not multiply a per-block heap allocation by the same factor. + maxUPXDictionaryExpansion = 256 + + // maxUPXOriginalSize is the absolute ceiling the ratio cannot supply on its own, since padding an + // input is nearly free. 512MB is 4x the largest legitimate original measured (121MB). + // + // This is not a disk-only figure. The reconstruction is truncated to what was actually rebuilt, which + // is what makes a hole stop being an allocation knob, but the bytes inside that prefix are real and + // downstream sizes against them: saferio in debug/elf grows an uncompressed section until the reader + // stops delivering, so a section declaring the whole reconstruction allocates it. elfutil bounds a + // section that declares a *decompressed* size at 128MB; this is the bound on the other kind. + // + // It is deliberately not matched to elfutil's 128MB, because this ceiling is not what stands between + // an attacker and that allocation: scanFile applies no size ceiling of any kind to an *unpacked* + // binary, so a plain padded 512MB ELF reaches the same saferio growth through the same + // getCryptoInformation call at the same cost to build. Lowering this number would refuse legitimate + // packed binaries between 128MB and 512MB and leave that path untouched. Bounding an uncompressed + // section against the reader's real length in elfutil is what would actually close it, and that is a + // change for the sibling catalogers as much as this one. + // + // So the job here is narrower than it looks: keep the ratio from being the only thing standing behind + // p_filesize, since padding an input is nearly free. + // + // Per file, with no shared budget across the cataloger's NumCPU*4 concurrent unpacks. + maxUPXOriginalSize = 512 * intFile.MB + + // maxUPXDictionary is the ceiling on the LZMA dictionary, which unlike the reconstruction is resident. + // Matched to elfutil.maxDeclaredSectionSize, the same kind of bound in the sibling ELF path. + // + // Unlike the others this binds on absolute block size rather than on a ratio, so it is also a known + // functional ceiling: a single block whose sz_unc runs past it decodes only while its match distances + // stay inside the dictionary. A ~200MB single-block binary can therefore fail to decode, which ends + // the chain and is reported as a partial reconstruction rather than dropped. Raising it multiplies the + // worst-case resident cost by the cataloger's concurrency, which is why it has not been raised. + maxUPXDictionary = 128 * intFile.MB + + // minUPXDictionary is the floor. The decoder needs a dictionary at all, and a block small enough to + // sit under it costs nothing to be generous with. + minUPXDictionary = 64 * intFile.KB + + // maxUPXBlocks bounds the block loop. Real UPX emits one block for the ELF headers plus one per + // PT_LOAD extent (four on the image-small-upx fixture), so single digits; this leaves room for an + // unusual layout while keeping a file from driving unbounded iterations with minimum-size blocks. + maxUPXBlocks = 1024 + + // upxFilterWindow is the working set of the CTO unfilter. The filter is a forward scan needing only a + // few bytes of lookahead, so it does not need its block resident, and one block can be most of a file. + upxFilterWindow = intFile.MB + + // upxELFHeaderWindow bounds how much of the first block is read back to find the PT_LOAD offsets that + // place the later blocks. The program headers of a real ELF64 start at byte 64. + upxELFHeaderWindow = 64 * intFile.KB + + // maxUPXLZMALiteralBits caps lc+lp. The decoder allocates and initializes a 0x300<<(lc+lp) entry + // probability array per block, independent of block size, so the library's own ceiling of 12 costs + // 6.3MB of stores for a one-byte block. Across maxUPXBlocks blocks that adds up, and UPX emits lc+lp + // of 3 or less, so this stays well clear of real output while cutting the per-block cost by 16x. + maxUPXLZMALiteralBits = 8 +) + var ( // upxMagic is the magic bytes that identify a UPX-packed binary upxMagic = []byte("UPX!") errNotUPX = errors.New("not a UPX-compressed binary") errUnsupportedUPXMethod = errors.New("unsupported UPX compression method") + errUPXOutputExceeded = errors.New("UPX blocks decompress to more than the declared original size") + errUPXImplausibleHeader = errors.New("implausible UPX header") + + // errUPXSizeRefused is a header that really is UPX and claims more than the bounds allow. Unlike the + // implausible-header cases (a stray "UPX!" in a string constant), this is a file we declined to expand + // rather than a file that was never packed, so it is reported as an unknown the way + // elfutil.ErrDeclaredSizeExceeded is. The ratio clears a measured 209x worst case at 256, which is not + // much headroom, and a binary silently vanishing from the SBOM is the failure this avoids. + // + // Reported conditionally: nothing was unpacked when this fires, so if the bytes as they were found turn + // out to carry build info then the file was never really packed and there is no gap. See + // readContentsAndBuildInfo. + errUPXSizeRefused = errors.New("UPX header claims more than the bounds allow") + errUPXInvalidLZMAParams = errors.New("invalid LZMA parameters") + + // errUPXPartial marks a reconstruction that really is UPX and really did decode, but gave something + // up on the way: a chain that stopped early, a block that would not decode, or blocks placed past a + // gap the chain never filled. The contents come back readable and usable, since .go.buildinfo and the + // pclntab live in the early extents, but the bytes past the covered prefix are gone. That is a gap + // this cataloger chose to leave, so it is reported the same way errUPXSizeRefused is: without it a + // short chain drops the section-name table and the binary silently contributes nothing. + // + // Reported unconditionally, unlike errUPXSizeRefused below: a partial only exists when there really is + // a reconstruction, so the "nothing was unpacked, so nothing is short" suppression in + // readContentsAndBuildInfo cannot apply to it. + // + // Its population is wider than just packed Go binaries, and that is accepted rather than overlooked. + // The block placement rules are ELF and Go-shaped, so a --lzma-packed non-Go ELF whose layout they do + // not tile ends here too, and bi == nil cannot tell that apart from a packed Go binary whose + // reconstruction lost .go.buildinfo, which is the case this exists to report. It stays reportable + // because the population is far narrower than the errUnsupportedUPXMethod carve-out below: upx emits + // NRV2B by default, so most packed binaries in an image never reach a block placement at all. + errUPXPartial = errors.New("UPX reconstruction is incomplete") + + // errUPXDecompress marks a file that got past the header and the method dispatch and still could not + // be unpacked, which is a gap in the SBOM rather than a file to skip. A stray magic and an implausible + // header mean "not something we can catalog" and stay quiet, so a new guard added later is silent by + // default instead of turning into SBOM noise. + // + // errUnsupportedUPXMethod is the one case that is a real gap and still stays quiet, so it is a policy + // call rather than a classification: `upx` emits NRV2B unless --lzma was passed, so reporting it would + // attach an unknown to most packed binaries in an image, the large majority of which are not Go and + // would have contributed nothing unpacked either. errUPXDecompress reports because its population is + // the opposite shape: a file we tried to decode and failed on is rare. Implementing NRV2B is what + // actually closes this, not reclassifying it. + errUPXDecompress = errors.New("unable to decompress UPX-compressed Go binary") ) // upxInfo contains parsed UPX header information type upxInfo struct { - magicOffset int64 version uint8 format uint8 + loaderSize uint16 // l_lsize - size of the loader stub that separates the extents from the tail originalSize uint32 // p_filesize - original uncompressed file size blockSize uint32 // p_blocksize - size of each compression block firstBlockOff int64 // offset to first b_info structure + + // inputLen is the real length of the packed file, validated non-zero by parseUPXInfo. Every bound in + // this file is expressed against it, so it is carried on the header rather than re-derived: a caller + // that recomputed it could get zero and silently drop the bounds that depend on it. + inputLen int64 } // blockInfo contains information about a single compressed block @@ -159,12 +257,29 @@ type blockInfo struct { dataOffset int64 } -// upxDecompressor is a function that decompresses data using a specific method -type upxDecompressor func(compressedData []byte, uncompressedSize uint32) ([]byte, error) +// upxDecompressor returns a reader over one block's decompressed contents, which the caller streams into +// place. Returning a stream rather than filling a caller-owned slice is what keeps a block from being +// resident in full; implementations must produce exactly size bytes. +type upxDecompressor func(compressedData []byte, size int64) (io.Reader, error) + +// maxDictionaryFor is the largest LZMA dictionary a block may ask for, given the compressed bytes it +// brought with it. The decoder allocates it up front, before reading a compressed byte, so this is the one +// allocation still sized by something a block declares about itself. Paying for it in sz_cpr, which +// readChainBlock has already checked against the real input length, is what keeps a few dozen header bytes +// from reaching maxUPXOriginalSize. +// +// Under-sizing is safe in the strict direction: the decoder rejects a match whose distance runs past the +// dictionary, so the block fails to decode and the reconstruction is reported partial. It never decodes to +// something wrong. +func maxDictionaryFor(compressedLen int) uint32 { + allowed := uint64(compressedLen) * maxUPXDictionaryExpansion + return uint32(min(max(allowed, minUPXDictionary), maxUPXDictionary)) +} // upxDecompressors maps compression methods to their decompressor functions var upxDecompressors = map[uint8]upxDecompressor{ - upxMethodLZMA: decompressLZMA, + upxMethodLZMA: decompressLZMA, + upxMethodStored: decompressStored, // note: the NRV algorithms are from the UCL library, an open-source implementation based on the NRV (Not Really Vanished) algorithm. // TODO: future methods can be added here @@ -173,154 +288,585 @@ var upxDecompressors = map[uint8]upxDecompressor{ // upxMethodNRV2E: decompressNRV2E, } -// unfilter49 reverses UPX filter 0x49 (CTO / call trick optimization). -// The filter transforms CALL (0xE8) and JMP (0xE9) instruction addresses in x86/x64 code to improve compression. -// The filtered format stores addresses as big-endian with cto8 as the high byte marker (the `cto8` parameter, -// stored in `b_cto8`, marks transformed instructions): +// blockDecompressor picks the decompressor for a block, or reports false when the block is not something +// we can read. Method 0 is a copy, which is only meaningful when the two sizes agree; treating a mismatch +// as unsupported keeps a stray b_info-shaped run of bytes quiet rather than failing the file over it. +func blockDecompressor(block *blockInfo) (upxDecompressor, bool) { + if block.method == upxMethodStored && block.compressedSize != block.uncompressedSize { + return nil, false + } + d, ok := upxDecompressors[block.method] + return d, ok +} + +// decompressStored returns the block's bytes as-is, for an extent UPX wrote uncompressed. +func decompressStored(compressedData []byte, size int64) (io.Reader, error) { + if int64(len(compressedData)) != size { + return nil, fmt.Errorf("stored block holds %d bytes but declares %d", len(compressedData), size) + } + return bytes.NewReader(compressedData), nil +} + +// unfilter49 reverses UPX filter 0x49 (CTO), which rewrites x86/x64 CALL (0xE8) and JMP (0xE9) targets +// big-endian with cto8 as a high-byte marker to compress better: // // original: E8 xx xx xx xx (CALL rel32, little-endian offset) // filtered: E8 CC yy yy yy (big-endian, CC = cto8 marker) -func unfilter49(data []byte, cto8 byte) { +// +// data may be a window into a larger block, so base says where it begins (the arithmetic is block-relative) +// and the return is the count of settled leading bytes. When final is false, trailing positions whose +// six-byte lookahead is not yet buffered are left for the next window. +func unfilter49(data []byte, cto8 byte, base uint32, final bool) int { cto := uint32(cto8) << 24 - for pos := uint32(0); pos+5 <= uint32(len(data)); pos++ { - opcode := data[pos] + // one past the last position this window can evaluate. A conditional jump needs six bytes and a CALL + // five, so only the final window (where the shorter form has nothing left to wait for) goes to len-4. + limit := len(data) - 5 + if final { + limit = len(data) - 4 + } + if limit < 0 { + limit = 0 + } + + for i := 0; i < limit; i++ { + pos := base + uint32(i) + opcode := data[i] // check for E8 (CALL) or E9 (JMP) if opcode == 0xE8 || opcode == 0xE9 { // check if first byte after opcode matches cto8 marker - if data[pos+1] == cto8 { + if data[i+1] == cto8 { // read operand as big-endian - jc := binary.BigEndian.Uint32(data[pos+1 : pos+5]) - // subtract cto and position to get original relative address - result := jc - (pos + 1) - cto - // write back as little-endian - binary.LittleEndian.PutUint32(data[pos+1:pos+5], result) + jc := binary.BigEndian.Uint32(data[i+1 : i+5]) + // subtract cto and position to get original relative address, written back little-endian + binary.LittleEndian.PutUint32(data[i+1:i+5], jc-(pos+1)-cto) } } // check for conditional jumps (0F 80-8F) - if opcode == 0x0F && pos+6 <= uint32(len(data)) { - opcode2 := data[pos+1] - if opcode2 >= 0x80 && opcode2 <= 0x8F && data[pos+2] == cto8 { - jc := binary.BigEndian.Uint32(data[pos+2 : pos+6]) - result := jc - (pos + 2) - cto - binary.LittleEndian.PutUint32(data[pos+2:pos+6], result) + if opcode == 0x0F && i+6 <= len(data) { + opcode2 := data[i+1] + if opcode2 >= 0x80 && opcode2 <= 0x8F && data[i+2] == cto8 { + jc := binary.BigEndian.Uint32(data[i+2 : i+6]) + binary.LittleEndian.PutUint32(data[i+2:i+6], jc-(pos+2)-cto) } } } -} -// isUPXCompressed checks if the reader contains a UPX-compressed binary -func isUPXCompressed(r io.ReaderAt) bool { - // UPX magic can be at various offsets depending on the binary format - // scan the first 4KB for the magic bytes - buf := make([]byte, 4096) - n, err := r.ReadAt(buf, 0) - if err != nil && !errors.Is(err, io.EOF) { - return false + if final { + return len(data) } - return bytes.Contains(buf[:n], upxMagic) + // the bytes from limit on are still reachable by unprocessed positions, so they stay in the window + return limit } -// decompressUPX attempts to decompress a UPX-compressed ELF binary. -// It reads blocks and places them at correct file offsets based on ELF PT_LOAD segments. +// unpackUPX returns the contents the caller should read this binary from, and never nil: with nothing to +// unpack the answer is the input as it was found. No UPX header, a header that did not belong to real UPX +// output, and an unimplemented method (upx emits NRV2B unless --lzma was passed) all stay quiet, since +// reporting them would attach an unknown to every packed non-Go binary in an image. // -// The first decompressed block contains the original ELF headers. Parse them to get PT_LOAD segment -// file offsets for proper block placement: +// A non-nil error means the file really is packed and we still could not read it, which is a gap in the +// SBOM; decompressUPX marks exactly those with errUPXDecompress, and a header refused by the size bounds +// carries errUPXSizeRefused. The contents come back alongside the error, still readable, so the caller can +// carry on with the packed bytes and report the gap. // -// - After decompressing block 1, parse its ELF headers: -// ptLoadOffsets := parseELFPTLoadOffsets(block1Data) +// A reconstruction that decoded but came up short carries errUPXPartial, and comes back as the contents +// to read from: it is usually still enough for .go.buildinfo, and the caller reports what was lost. // -// - Block 1: placed at offset 0 (contains ELF header + program headers) -// - Block 2: placed at offset 0 (overwrites/extends) -// - Block 3+: placed at ptLoadOffsets[blockNum-2] +// A cancelled context comes back as ctx.Err() so the caller stops rather than going on to parse the packed +// bytes; it is not a gap in the SBOM and the caller does not report it as one. // -// Why this matters: Simply concatenating decompressed blocks produces invalid output. -// Each block corresponds to a PT_LOAD segment and must be placed at its correct file offset. +// A nil buffer means there was nothing to unpack and the caller should read the input as it was found. +// Resolve that with readerFor rather than widening the nil into an io.ReaderAt, which yields a non-nil +// interface holding a nil pointer. // -// Returns the decompressed binary as a bytes.Reader (implements io.ReaderAt). -func decompressUPX(r io.ReaderAt) (io.ReaderAt, error) { - info, err := parseUPXInfo(r) +// The caller owns the contents and must Close them, which closeUnpacked does for the nil case too. +func unpackUPX(ctx context.Context, r io.ReaderAt) (unpackedContents, error) { + // the container gate first: this runs over every file in an image and almost none are packed ELF, so + // nothing above it should cost more than six bytes. ReaderSize seeks to the end and reads the last + // byte back, which over a squashfs or tar-backed reader is a real seek-and-decompress. + if !isELF64LE(r) { + return nil, nil + } + + size, ok := intFile.ReaderSize(r) + if !ok { + // without a real byte count there is nothing to weigh a header claim against, so decline to unpack + // rather than run the bounds against a zero + log.Trace("UPX: input size could not be determined, not unpacking") + return nil, nil + } + + // the header is parsed before the temp dir is demanded, so a context without one cannot turn every + // binary the cataloger touches into a reportable failure. Only a file that really is packed needs it. + info, err := parseUPXInfo(r, size) if err != nil { + if errors.Is(err, errUPXSizeRefused) { + return nil, err + } + log.WithFields("error", err).Trace("not a readable UPX-packed binary") + return nil, nil + } + + td := tmpdir.FromContext(ctx) + if td == nil { + return nil, fmt.Errorf("%w: no temp dir factory in context", errUPXDecompress) + } + + out, err := decompressUPX(ctx, td, r, info) + if errors.Is(err, errUPXPartial) { + // short, but real and readable: this is still what the caller should read the binary from, and the + // reason goes back with it so the missing bytes are reported rather than silently dropped + return out, err + } + if err != nil { + // decompressUPX already released it on every path that returns an error, so this is for the one + // that does not: errUPXPartial is handled above, and out is nil here + closeUnpacked(out) + // cancellation is not a decompression failure and not a gap in the SBOM, but it does mean stop: + // swallowing it here sent the caller on to parse the packed bytes and build packages after the + // scan had been called off. + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + if errors.Is(err, errUPXDecompress) { + return nil, err + } + log.WithFields("error", err).Trace("not a readable UPX-packed binary") + return nil, nil + } + return out, nil +} + +// blockSink is where decompression places the blocks it decodes. Writes land at offsets the packed file +// names, Size is the contiguous run rebuilt from offset zero, and FirstGap is where a block that declares +// no home of its own fits. spillbuf.Buffer is what implements it, and naming that type here would say the +// decoder cares whether those bytes are in memory or on disk, which is the one thing it must not care +// about. +type blockSink interface { + io.WriterAt + + // read back as well as written: the first block carries the ELF program headers that place every + // block after the second, so a write-only sink would place those by hole-filling alone + io.ReaderAt + + // Size is the contiguous run rebuilt from offset zero, not the furthest offset written: see upx.go's + // header and the spillbuf package doc for why the two are not interchangeable. + Size() int64 + + // FirstGap returns the earliest unwritten run of at least size bytes lying within [0, within). + FirstGap(size, within int64) (int64, bool) +} + +// decompressUPX reconstructs the original file from a UPX-compressed ELF binary: it walks the b_info +// chain and decodes each block into the offset it occupied in the original, which blockDestination works +// out from the PT_LOAD segments in the first decoded block. Simply concatenating the blocks produces +// invalid output. +// +// Where those bytes live is spillbuf's problem, not this function's. What matters here is that the buffer +// reports only the contiguous run rebuilt from offset zero: a block's placement offset is attacker +// controlled, and sparse storage hands holes back as zeros, so a length covering them would let a 64 byte +// block parked at p_filesize-64 turn a 300KB input into a 64MB reader. +// +// errUPXImplausibleHeader means the file is not really packed, errUPXPartial a reconstruction that came up +// short but is still worth reading, and only errUPXDecompress a packed file we could not read at all. +func decompressUPX(ctx context.Context, td *tmpdir.TempDir, r io.ReaderAt, info *upxInfo) (unpackedContents, error) { + out := spillbuf.New(td) + + // errUPXPartial means the chain gave something up but what it rebuilt is worth reading, so it comes + // back alongside the buffer. Any other error is fatal and the buffer goes with it. + err := decompressUPXBlocks(ctx, r, info, out) + if err != nil && !errors.Is(err, errUPXPartial) { + _ = out.Close() return nil, err } + if out.Size() == 0 { + _ = out.Close() + // a plausible header with nothing decodable behind it is not a packed binary + return nil, fmt.Errorf("%w: no decodable blocks", errUPXImplausibleHeader) + } - // allocate buffer for the full decompressed output - output := make([]byte, info.originalSize) + return out, err +} + +// loaderSkip is the one-time jump past the loader stub. The extents that rebuild the loadable segments +// are followed by that stub, and behind it UPX packs everything the segments did not cover: the alignment +// padding between them and the non-loadable tail (section headers, the section name table, symbols, debug +// data). Picking the chain back up past the loader is the difference between a file we can only read build +// info out of and one we can also read symbols out of. Anything else sitting there fails validation and +// ends the chain. +type loaderSkip struct { + size uint16 + done bool +} + +// past reports the offset the chain resumes at, and false once the jump has been spent or there is no +// stub to jump over. +func (l *loaderSkip) past(offset int64) (int64, bool) { + if l.done || l.size == 0 { + return offset, false + } + l.done = true + return offset + int64(l.size), true +} + +// decompressUPXBlocks walks the b_info chain, decoding each block into its place in the reconstructed +// file. A chain that simply ends, describes a layout it does not hold, or holds a block that will not +// decode yields the blocks placed so far, since those are often enough to recover .go.buildinfo; the +// result says so through partial. Only the first block failing, or a cancelled scan, discards everything, +// since then there is nothing here to read. +// +// out is read back and questioned as well as written: the first block carries the ELF program headers that +// say where the rest belong, so a write-only sink would place every later block by hole-filling and hand +// back a plausible but wrongly shaped file. Size is the contiguous run rebuilt from offset zero, which is +// what the reconstruction can deliver, and FirstGap is where a block fits in the earliest hole left behind. +func decompressUPXBlocks(ctx context.Context, r io.ReaderAt, info *upxInfo, out blockSink) error { + // UPX packs the original file as a series of blocks that together reconstruct p_filesize. Track the + // unclaimed remainder so the block headers cannot drive more decompression than the file itself + // declares: without this, every block may individually claim the whole original size, and the total + // work becomes (block count x original size). This is the bound that does the real work here. + remaining := info.originalSize currentOffset := info.firstBlockOff outputOffset := uint64(0) blockNum := 0 - - // track PT_LOAD segment offsets for proper block placement + loader := loaderSkip{size: info.loaderSize} var ptLoadOffsets []uint64 - for { - block, err := readBlockInfo(r, currentOffset) + // why the chain stopped, when it stopped short of a clean end marker. Set once at the break that + // caused it; finalExtent turns it into the reconstruction's partial reason. + var stopped error + + for blockNum < maxUPXBlocks { + // a hostile chain can drive maxUPXBlocks decodes of maxUPXOriginalSize between them, so the loop + // has to be interruptible: this is the only long-running work in the cataloger that is sized by + // the file it is reading. + // deliberately not wrapped in errUPXDecompress: a cancelled scan is not a gap in the SBOM, so it + // stays on the quiet side of the reportability split like every other non-decompress failure. + if err := ctx.Err(); err != nil { + return err + } + + block, decompressor, err := readChainBlock(r, currentOffset, blockNum, info.inputLen) if err != nil { - return nil, fmt.Errorf("failed to read block info at offset %d: %w", currentOffset, err) + return err } - - // check for end marker (sz_unc == 0) - if block.uncompressedSize == 0 { - break - } - - // non-LZMA method on first block is an error; on subsequent blocks it indicates end of data - if block.method != upxMethodLZMA { - if blockNum == 0 { - return nil, fmt.Errorf("%w: method %d", errUnsupportedUPXMethod, block.method) + if block == nil { + if next, ok := loader.past(currentOffset); ok { + currentOffset = next + continue } break } + + // blockNum is 0-based here and 1-based past the increment below, so both fail-hard gates on it + // mean "the first block". Hoisted so they read as the same predicate. + first := blockNum == 0 + + if block.uncompressedSize > remaining { + fatal, reason := classifyOverrun(first, blockNum, block.uncompressedSize, remaining, info.originalSize) + if fatal != nil { + return fatal + } + stopped = reason + break + } + remaining -= block.uncompressedSize blockNum++ - decompressor, ok := upxDecompressors[block.method] + destOffset, ok := blockDestination(blockNum, block.uncompressedSize, outputOffset, ptLoadOffsets, out, uint64(info.originalSize)) if !ok { - return nil, fmt.Errorf("%w: method %d", errUnsupportedUPXMethod, block.method) + // the file does not describe the layout it claims. Keep the blocks placed so far, but stop: + // outputOffset derives from destOffset, so continuing would carry the bad offset forward. + stopped = fmt.Errorf("%w: block %d of %d bytes does not fit the declared %d byte layout", + errUPXPartial, blockNum, block.uncompressedSize, info.originalSize) + break } - // read compressed data for this block - compressedData := make([]byte, block.compressedSize) - _, err = r.ReadAt(compressedData, block.dataOffset) - if err != nil { - return nil, fmt.Errorf("failed to read compressed data: %w", err) + if err := decompressBlock(ctx, r, block, decompressor, out, destOffset); err != nil { + fatal, reason := classifyDecodeFailure(ctx, err, first, blockNum) + if fatal != nil { + return fatal + } + stopped = reason + break } + blockEnd := destOffset + uint64(block.uncompressedSize) - // decompress this block - blockData, err := decompressor(compressedData, block.uncompressedSize) - if err != nil { - return nil, fmt.Errorf("failed to decompress block: %w", err) - } - - // apply CTO filter reversal if needed - if block.filterID == upxFilterCTO { - unfilter49(blockData, block.filterCTO) - } - - // first block contains ELF headers - parse PT_LOAD segments for subsequent blocks + // the first block carries the original ELF headers, which place every block after the second if blockNum == 1 { - ptLoadOffsets = parseELFPTLoadOffsets(blockData) + ptLoadOffsets = readPTLoadOffsets(out, destOffset, block.uncompressedSize) } - // determine where to place this block in the output - destOffset := outputOffset - if blockNum > 2 && len(ptLoadOffsets) > blockNum-2 { - // blocks 3+ go to their respective PT_LOAD segment offsets - destOffset = ptLoadOffsets[blockNum-2] - } - - // copy block data to output at correct offset - if destOffset+uint64(len(blockData)) <= uint64(len(output)) { - copy(output[destOffset:], blockData) - } - - outputOffset = destOffset + uint64(block.uncompressedSize) + outputOffset = blockEnd currentOffset = block.dataOffset + int64(block.compressedSize) } - return bytes.NewReader(output), nil + return finalExtent(uint64(out.Size()), blockNum, uint64(info.originalSize), stopped) +} + +// classifyOverrun decides what a block claiming more output than the running remainder allows means. +// Exactly one of the two results is non-nil, the same contract as classifyDecodeFailure. +// +// The blocks together reconstruct p_filesize, so that remainder bounds the total output. It is deliberately +// the only size check on a block: comparing sz_unc against p_blocksize as well adds nothing (remaining is +// already <= p_filesize) and real output sits at exactly p_blocksize, so that check would run with no +// headroom against a value the format does not guarantee. +// +// Past the first block this ends the chain rather than failing the file, the same way readChainBlock treats +// a method it cannot read: the bytes behind the last block are loader stub, and a run of machine code that +// happens to parse as a b_info is not a claim about the file. The bound holds either way, since the +// overrunning block is never placed. +func classifyOverrun(first bool, blockNum int, size, remaining uint32, total uint32) (fatal, reason error) { + if first { + return fmt.Errorf("%w: %w: block %d claims %d with %d left of %d", errUPXDecompress, + errUPXOutputExceeded, blockNum+1, size, remaining, total), nil + } + return nil, fmt.Errorf("%w: block %d claims %d bytes with %d left of %d", errUPXPartial, + blockNum+1, size, remaining, total) +} + +// classifyDecodeFailure decides what a block that would not decode means. Exactly one of the two results +// is non-nil: a fatal error ends the file, a reason ends the chain and keeps what came before it. +// +// A cancelled copy arrives here as a decode failure, so ctx is checked first. Classifying it as one would +// put it on the quiet arm for block 1 and turn it into errUPXPartial after that: an aborted scan is not a +// gap in the SBOM, it is a reason to stop. +// +// Past the first block a failure ends the chain rather than failing the file, for the same reason an +// overrunning block does: readChainBlock accepts any b_info-shaped run of bytes with a method it knows, so +// the loader stub behind the last real block can parse as one and then fail to decode. Discarding a good +// reconstruction over garbage past the end of it is the worse answer. The bound still holds, since only real +// decoder output was ever written, but a block that failed mid-stream leaves its decoded prefix behind, so +// the reconstruction can end inside that block. That is what errUPXPartial reports. +func classifyDecodeFailure(ctx context.Context, err error, first bool, blockNum int) (fatal, reason error) { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr, nil + } + if first { + return fmt.Errorf("%w: %w", errUPXDecompress, err), nil + } + return nil, fmt.Errorf("%w: block %d did not decode: %w", errUPXPartial, blockNum, err) +} + +// finalExtent reports why the reconstruction is short of what the header declared, or nil when it is not. +// None of these is a failure: the blocks placed so far may still carry .go.buildinfo. They are reported +// rather than logged because the bytes past the covered prefix are gone, and a binary that quietly +// contributes nothing because its section-name table sat past a gap is the failure this avoids. +func finalExtent(covered uint64, blockNum int, total uint64, stopped error) error { + // a chain that rebuilt everything p_filesize claimed lost nothing, whatever it tripped over next. The + // bytes behind the last real block are loader stub, and readChainBlock accepts any run of those that + // parses as a b_info with a method it knows, so a clean unpack routinely ends by reading one and + // finding it overruns a remaining of zero. Reporting that as a gap puts an unknown on a binary the + // cataloger read completely. + if covered >= total { + return nil + } + + switch { + case stopped != nil: + // the chain gave up mid-walk, which is more specific than anything the extents show + return stopped + case blockNum == maxUPXBlocks: + return fmt.Errorf("%w: the block chain hit the %d block cap", errUPXPartial, maxUPXBlocks) + } + + // the chain ran out before rebuilding what the header declared, and nothing above says why: every + // readChainBlock exit that ends the chain quietly lands here, once the arms above have had their say: + // an unreadable b_info, the end marker, a zero sz_cpr, compressed data past the end of the input, and + // a method we cannot read past the first block (on the first block that fails the file). Real UPX output + // rebuilds p_filesize exactly, so the bytes past the covered prefix are gone the same way they are in + // every case above, and truncating to the prefix drops whatever sat behind it. Left unreported this is + // the one shape where a short reconstruction can cost the binary its section headers and contribute + // neither packages nor an unknown. + return fmt.Errorf("%w: the chain ended after %d of the declared %d bytes", errUPXPartial, covered, total) +} + +// readChainBlock reads the b_info at offset and pairs it with the decompressor for its method. A nil block +// alongside a nil error means the chain ends here: an unreadable header, the end marker, a block whose +// compressed data is not in the file, or a method we cannot read, which is what the loader bytes behind +// the last block look like. An unimplemented method on the very first block is the one case that fails the +// file, since then there is nothing here to read. +func readChainBlock(r io.ReaderAt, offset int64, blockNum int, inputLen int64) (*blockInfo, upxDecompressor, error) { + block, err := readBlockInfo(r, offset) + if err != nil { + // a real UPX file runs out of b_info structures before it runs out of blocks + log.WithFields("block", blockNum+1, "offset", offset, "error", err). + Trace("UPX block info unreadable, using partial output") + return nil, nil, nil + } + if block.uncompressedSize == 0 { + return nil, nil, nil // end marker + } + + // sz_cpr sizes a read buffer directly, so on its own a few dozen header bytes could name a length the + // following ReadAt could only ever fail. A block whose data is not in the file, or which holds no + // compressed bytes at all, ends the chain: neither is something a decoder can be handed. + if block.compressedSize == 0 { + log.WithFields("block", blockNum+1, "offset", offset). + Trace("UPX block holds no compressed data, using partial output") + return nil, nil, nil + } + if block.dataOffset > inputLen || int64(block.compressedSize) > inputLen-block.dataOffset { + log.WithFields("block", blockNum+1, "compressedSize", block.compressedSize, "inputSize", inputLen). + Trace("UPX block claims compressed data past the end of the input, using partial output") + return nil, nil, nil + } + + decompressor, ok := blockDecompressor(block) + if !ok { + if blockNum == 0 { + return nil, nil, fmt.Errorf("%w: method %d", errUnsupportedUPXMethod, block.method) + } + return nil, nil, nil + } + return block, decompressor, nil +} + +// blockDestination decides where a block lands in the reconstructed file, reporting false when the file +// does not describe a layout it fits in. The first two blocks (the original ELF headers and the extent +// behind them) run sequentially from the start of the file, the ones after that go to the PT_LOAD offsets +// parsed out of the first, and whatever is left fills the gaps those leave behind, in file order. +// +// Block 3 indexes ptLoadOffsets[1], not [0]: block 2 is the first loadable extent and was already placed +// sequentially, which is where the first PT_LOAD lives. The offset is not an off-by-one. +func blockDestination(blockNum int, size uint32, sequential uint64, ptLoadOffsets []uint64, out blockSink, total uint64) (uint64, bool) { + switch { + case blockNum > 2 && blockNum-2 < len(ptLoadOffsets): + dest := ptLoadOffsets[blockNum-2] + return dest, blockFits(total, dest, size) + case blockNum > 2: + at, ok := out.FirstGap(int64(size), int64(total)) + return uint64(at), ok + default: + return sequential, blockFits(total, sequential, size) + } +} + +// blockFits reports whether a block of the given size lands entirely within a total byte file when placed +// at destOffset. destOffset comes from an ELF p_offset in the file, so the check is written as a +// subtraction rather than as destOffset+size, which could wrap. +// +// The wrap is not reachable today: it needs destOffset above 2^64-2^32, and every such value is negative +// once decompressBlock converts it to an int64 for WriteAt, which fails there instead. Kept in this form +// anyway, since the reason it is unreachable lives in another function. +func blockFits(total, destOffset uint64, size uint32) bool { + return destOffset <= total && uint64(size) <= total-destOffset +} + +// decompressBlock reads one block's compressed data and streams it into the output at destOffset, +// reversing the CTO filter on the way through if the block declares one. The decompressed block is never +// held in full: it moves through a fixed window, so the copy costs the same whatever sz_unc says. +// +// That bounds the copy, not the decoder. lzma.NewReader allocates a dictionary of min(dictSize, sz_unc) +// before it reads a compressed byte, so sz_unc still buys heap; what keeps that payable is the input-size +// ratio on p_filesize, which the running remainder holds every block underneath. +func decompressBlock(ctx context.Context, r io.ReaderAt, block *blockInfo, decompressor upxDecompressor, out io.WriterAt, destOffset uint64) error { + // readChainBlock has already bounded sz_cpr by the real input length, but on a 32-bit GOARCH an input + // past 2GB leaves a length make() panics on, and that panic is outside the recover in getBuildInfo. + if uint64(block.compressedSize) > math.MaxInt { + return fmt.Errorf("compressed block of %d bytes is larger than this platform can address", block.compressedSize) + } + compressedData := make([]byte, block.compressedSize) + if _, err := r.ReadAt(compressedData, block.dataOffset); err != nil { + return fmt.Errorf("failed to read compressed data: %w", err) + } + + size := int64(block.uncompressedSize) + src, err := decompressor(compressedData, size) + if err != nil { + return fmt.Errorf("failed to decompress block: %w", err) + } + + src = ctxReader{ctx: ctx, r: src} + + dst := io.NewOffsetWriter(out, int64(destOffset)) + if block.filterID == upxFilterCTO { + return copyUnfiltered(dst, src, size, block.filterCTO) + } + if block.filterID != 0 { + log.WithFields("filter", block.filterID).Trace("UPX filter not implemented, copying block unfiltered") + } + if _, err := io.CopyN(dst, src, size); err != nil { + return fmt.Errorf("failed to decompress block: %w", err) + } + return nil +} + +// ctxReader fails a read once ctx is done. The block copy below moves up to a whole sz_unc through a +// fixed window, so without this a cancelled scan still finished the block it was on: one check per block +// meant one check per up-to-maxUPXOriginalSize of work, and a single-block chain got exactly one, before +// any work had started. Wrapping the decoder output covers io.CopyN and copyUnfiltered together. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c ctxReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} + +// copyUnfiltered copies size bytes from src to dst, reversing the CTO filter as it passes. The filter is +// a forward scan, so it runs over a sliding window: each round processes only the positions whose +// lookahead is buffered, flushes the settled bytes and carries the tail into the next round. +func copyUnfiltered(dst io.Writer, src io.Reader, size int64, cto8 byte) error { + buf := make([]byte, min(size, upxFilterWindow)) + var base int64 // block-relative position of buf[0] + var filled int + + for { + // int64 throughout: size is a uint32 widened, and on a 32-bit GOARCH an int conversion of a value + // past 2^31 goes negative, which stalls the loop instead of erroring. min() with an int64 len keeps + // the result in range for the slice bounds below. + if want := int(min(int64(len(buf)-filled), size-base-int64(filled))); want > 0 { + n, err := io.ReadFull(src, buf[filled:filled+want]) + filled += n + if err != nil { + return fmt.Errorf("failed to decompress block: %w", err) + } + } + + final := base+int64(filled) >= size + settled := unfilter49(buf[:filled], cto8, uint32(base), final) + if _, err := dst.Write(buf[:settled]); err != nil { + return fmt.Errorf("failed to write decompressed block: %w", err) + } + if final { + return nil + } + // a non-final window is always a full buf, and upxFilterWindow leaves unfilter49 room to settle at + // least one byte, so this cannot fire. Checked because the alternative to erroring is a loop that + // never terminates, and the invariant that rules it out is not local to this function. + if settled == 0 { + return fmt.Errorf("failed to decompress block: the CTO filter made no progress at offset %d", base) + } + + copy(buf, buf[settled:filled]) + base += int64(settled) + filled -= settled + } +} + +// readPTLoadOffsets reads back the head of the first placed block to recover the PT_LOAD offsets that +// position the later blocks. Only the head is read: the program headers of a real ELF64 sit right behind +// the 64 byte file header, and the block itself may be far too large to bring back into memory. +// +// note: ELF64 little-endian only. parseELFPTLoadOffsets returns nil for anything else, which does not stop +// the reconstruction; the later blocks fall back to hole-filling in file order. +func readPTLoadOffsets(r io.ReaderAt, destOffset uint64, size uint32) []uint64 { + head := make([]byte, min(uint64(size), uint64(upxELFHeaderWindow))) + n, err := r.ReadAt(head, int64(destOffset)) + if err != nil && !errors.Is(err, io.EOF) { + log.WithFields("offset", destOffset, "error", err).Trace("unable to read back the UPX ELF header block") + return nil + } + // only what was actually read: the tail of head is zeros on a short read, and a p_offset of zero + // parsed out of it would place a later block over the ELF header. + return parseELFPTLoadOffsets(head[:n]) } // parseELFPTLoadOffsets extracts PT_LOAD segment file offsets from ELF headers. @@ -345,10 +891,31 @@ func parseELFPTLoadOffsets(elfHeader []byte) []uint64 { phentsize := binary.LittleEndian.Uint16(elfHeader[0x36:0x38]) phnum := binary.LittleEndian.Uint16(elfHeader[0x38:0x3a]) + const elf64PhdrSize = 56 // fixed size of an ELF64 program header entry + + // the reads below use fixed offsets up to byte 16 of each entry, so a shorter entry must not be + // accepted. Loop-invariant, so it is checked once here rather than per iteration. + if phentsize < elf64PhdrSize { + return nil + } + + hdrLen := uint64(len(elfHeader)) var offsets []uint64 for i := range phnum { phStart := phoff + uint64(i)*uint64(phentsize) - if phStart+uint64(phentsize) > uint64(len(elfHeader)) { + + // the bounds test below is written as `hdrLen-phStart` rather than `phStart+phentsize > hdrLen` + // so that a large phoff cannot overflow the sum past the buffer end. The `break` is load-bearing + // for that argument too: it guarantees phoff <= hdrLen before any i >= 1 is reached, which is what + // keeps phStart itself from wrapping; a `continue` here would let phoff near 2^64 wrap into a + // small in-range phStart and read a bogus p_offset. + // + // Not covered by a test, and worth knowing why before trying to write one: reaching the wrap needs + // phoff >= 2^64-phentsize, so the wrapped phStart always lands inside the first 56 bytes of the + // file, which is the ELF ident and header. Whether it then reads as PT_LOAD depends on header + // fields the fixture builder here does not set, so a test would have to hand-roll the header to + // place a 1 at the wrapped offset. The arithmetic is the argument. + if phStart > hdrLen || hdrLen-phStart < uint64(phentsize) { break } @@ -365,10 +932,19 @@ func parseELFPTLoadOffsets(elfHeader []byte) []uint64 { return offsets } -// parseUPXInfo locates and parses the UPX header information -func parseUPXInfo(r io.ReaderAt) (*upxInfo, error) { - // scan for the UPX! magic in the first 8KB - buf := make([]byte, 8192) +// parseUPXInfo locates and parses the UPX header information. inputLen is the size of the file behind r, +// which every bound here is expressed against; a file whose size cannot be determined cannot be unpacked, +// since there is then nothing to weigh a claim against. +func parseUPXInfo(r io.ReaderAt, inputLen int64) (*upxInfo, error) { + // unpackUPX applies this gate too, and cheaply, so on the production path it runs twice. Kept here + // because this function owns the invariant every read below it depends on: the l_info/p_info offsets, + // the block placement and readPTLoadOffsets are all ELF64 little-endian at fixed offsets, and the + // tests call it directly. + if !isELF64LE(r) { + return nil, errNotUPX + } + + buf := make([]byte, upxMagicScanWindow) n, err := r.ReadAt(buf, 0) if err != nil && !errors.Is(err, io.EOF) { return nil, fmt.Errorf("failed to read header: %w", err) @@ -379,36 +955,20 @@ func parseUPXInfo(r io.ReaderAt) (*upxInfo, error) { return nil, errNotUPX } - // UPX header structure (after finding "UPX!" magic): - // l_info structure (magic is at offset 4 within l_info): - // offset -4: l_checksum (4 bytes) - checksum of following data - // offset 0: l_magic "UPX!" (4 bytes) - // offset 4: l_lsize (2 bytes) - loader size - // offset 6: l_version (1 byte) - // offset 7: l_format (1 byte) - // - // p_info structure (12 bytes, starts at magic+8): - // offset 0: p_progid (4 bytes) - // offset 4: p_filesize (4 bytes) - original file size - // offset 8: p_blocksize (4 bytes) - // - // b_info structures follow (12 bytes each): - // offset 0: sz_unc (4 bytes) - uncompressed size of this block - // offset 4: sz_cpr (4 bytes) - compressed size (may have filter bits) - // offset 8: b_method (1 byte) - // offset 9: b_ftid (1 byte) - filter id - // offset 10: b_cto8 (1 byte) - filter parameter - // offset 11: unused (1 byte) + // see the format tables at the top of this file for the l_info/p_info/b_info layouts - if magicIdx+32 > n { - return nil, fmt.Errorf("UPX header truncated") + // the reads below reach magic+20 (the end of p_info); b_info comes from ReadAt later. Wrapped as an + // implausible header rather than its own error so the caller keeps treating it as "not really UPX". + const lInfoAndPInfoSize = 20 + if magicIdx+lInfoAndPInfoSize > n { + return nil, fmt.Errorf("%w: header runs past the end of the scan window", errUPXImplausibleHeader) } lInfoBase := buf[magicIdx:] pInfoBase := buf[magicIdx+8:] // p_info starts 8 bytes after magic info := &upxInfo{ - magicOffset: int64(magicIdx), + loaderSize: binary.LittleEndian.Uint16(lInfoBase[4:6]), version: lInfoBase[6], format: lInfoBase[7], originalSize: binary.LittleEndian.Uint32(pInfoBase[4:8]), @@ -416,14 +976,66 @@ func parseUPXInfo(r io.ReaderAt) (*upxInfo, error) { firstBlockOff: int64(magicIdx + 8 + 12), // magic + l_info remainder + p_info } - // sanity check - if info.originalSize == 0 || info.originalSize > 500*1024*1024 { - return nil, fmt.Errorf("invalid original size: %d", info.originalSize) + // the magic is found by an unanchored substring scan, so a stray "UPX!" in unrelated data (e.g. a + // string constant) can be read as a header. These checks are false-positive suppression, not + // hardening: the fields are attacker-controlled, and what actually bounds the work is the size limit + // below plus the running remainder in decompressUPX. + if info.version == 0 || info.format == 0 { + // l_version is the packheader version (11-14 in the wild) and l_format is a UPX_F_* id starting + // at 1, so neither is ever zero in real output. + return nil, fmt.Errorf("%w: version=%d format=%d", errUPXImplausibleHeader, info.version, info.format) } + // p_blocksize is only checked for being set; it is not used as a bound. UPX derives it from a PT_LOAD + // extent, and on real output the largest block equals it exactly, so treating it as a ceiling on + // sz_unc would run with zero headroom against a value the format does not actually promise. + if info.blockSize == 0 { + return nil, fmt.Errorf("%w: p_blocksize is zero", errUPXImplausibleHeader) + } + // p_filesize is the budget every block is drawn against, so it bounds the bytes written to the + // reconstruction. Nothing allocates against it, which is why this is a ratio and not a ceiling: a + // large binary is unpacked at whatever size it really is, and only a claim the input cannot pay for + // is refused. + if info.originalSize == 0 { + return nil, fmt.Errorf("%w: p_filesize is zero", errUPXImplausibleHeader) + } + if inputLen <= 0 { + return nil, fmt.Errorf("%w: the size of the input could not be determined", errUPXImplausibleHeader) + } + if limit := uint64(inputLen) * maxUPXExpansion; uint64(info.originalSize) > limit { + return nil, fmt.Errorf("%w: p_filesize %d exceeds the %d byte limit for a %d byte input", + errUPXSizeRefused, info.originalSize, limit, inputLen) + } + if uint64(info.originalSize) > maxUPXOriginalSize { + return nil, fmt.Errorf("%w: p_filesize %d is over the %d byte ceiling", + errUPXSizeRefused, info.originalSize, uint64(maxUPXOriginalSize)) + } + info.inputLen = inputLen return info, nil } +// isELF64LE reports whether r opens with an ELF64 little-endian ident. This reconstructs that container +// and only that: parseELFPTLoadOffsets reads the program headers at fixed ELF64 little-endian offsets, and +// nothing else here understands another layout. UPX packs PE and Mach-O with the same l_info/p_info +// layout, so without this gate a packed Windows binary parses as a plausible header, demands a temp dir, +// places its blocks by ELF rules that do not apply, and lands as an unknown on a file we were never going +// to catalog. +// +// A packed ELF32 or big-endian ELF is refused here too, and quietly, which is a known inconsistency with +// the reporting policy in reportableGap: those are packed Go binaries we could have cataloged, so by that +// policy they are a gap worth reporting rather than a container to skip. Left quiet deliberately. The +// honest fix is to reconstruct them, which is class- and order-aware program header parsing rather than a +// reclassification, and until then an unknown on every packed 32-bit binary in an image buys nothing a +// caller can act on. Not a regression: the pre-branch parser also bailed on anything but ELF64 LE, it just +// did so after demanding a temp dir and walking the chain. +func isELF64LE(r io.ReaderAt) bool { + var ident [6]byte + if n, err := r.ReadAt(ident[:], 0); n < len(ident) || (err != nil && !errors.Is(err, io.EOF)) { + return false + } + return bytes.Equal(ident[:4], []byte{0x7f, 'E', 'L', 'F'}) && ident[4] == 2 && ident[5] == 1 +} + // readBlockInfo reads a b_info structure at the given offset func readBlockInfo(r io.ReaderAt, offset int64) (*blockInfo, error) { buf := make([]byte, 12) @@ -435,11 +1047,14 @@ func readBlockInfo(r io.ReaderAt, offset int64) (*blockInfo, error) { szUnc := binary.LittleEndian.Uint32(buf[0:4]) szCpr := binary.LittleEndian.Uint32(buf[4:8]) - // the compressed size may have filter info in the high bits - // for some formats, but for LZMA it's typically clean + // sz_cpr is taken whole. Some UPX formats store flags in its high 8 bits, but the ELF format this code + // handles (the only one the container gate accepts) keeps filter data in b_ftid/b_cto8, so masking here + // would not decode a flag: it would silently turn a length above 16MB into a smaller plausible one and + // read that many bytes of a longer stream. readChainBlock bounds the value against the input instead, + // which ends the chain on a length the file cannot hold rather than truncating it. block := &blockInfo{ uncompressedSize: szUnc, - compressedSize: szCpr & 0x00ffffff, // lower 24 bits + compressedSize: szCpr, method: buf[8], filterID: buf[9], filterCTO: buf[10], @@ -449,26 +1064,7 @@ func readBlockInfo(r io.ReaderAt, offset int64) (*blockInfo, error) { return block, nil } -// nextPowerOf2 returns the smallest power of 2 >= n -func nextPowerOf2(n uint32) uint32 { - if n == 0 { - return 1 - } - // if already a power of 2, return it - if n&(n-1) == 0 { - return n - } - // find the highest set bit and shift left by 1 - n-- - n |= n >> 1 - n |= n >> 2 - n |= n >> 4 - n |= n >> 8 - n |= n >> 16 - return n + 1 -} - -// decompressLZMA decompresses LZMA-compressed data as used by UPX. +// decompressLZMA returns a reader over LZMA-compressed data as used by UPX. // UPX uses a 2-byte custom header format, not the standard 13-byte LZMA format. // // UPX 2-byte header encoding: @@ -477,9 +1073,9 @@ func nextPowerOf2(n uint32) uint32 { // - Byte 2+: raw LZMA stream (starts with 0x00 for range decoder init) // // Standard LZMA props encoding: props = lc + lp*9 + pb*9*5 -func decompressLZMA(compressedData []byte, uncompressedSize uint32) ([]byte, error) { +func decompressLZMA(compressedData []byte, size int64) (io.Reader, error) { if len(compressedData) < 3 { - return nil, fmt.Errorf("compressed data too short") + return nil, errors.New("compressed data too short") } // parse UPX's 2-byte LZMA header @@ -487,19 +1083,34 @@ func decompressLZMA(compressedData []byte, uncompressedSize uint32) ([]byte, err lp := compressedData[1] >> 4 lc := compressedData[1] & 0x0f + // the header nibbles can hold values outside the LZMA ranges. This is a correctness check, not a + // bound: the library derives lc/lp/pb back out of the props byte by modular arithmetic, so it cannot + // be handed an out-of-range value. What it prevents is the uint8 math below wrapping (lc=15, lp=15, + // pb=7 gives 465, which truncates to 209) and silently decoding with parameters that never existed. + // lc+lp is capped separately, and that one is about allocation: see maxUPXLZMALiteralBits. + if lc > 8 || lp > 4 || pb > 4 { + return nil, fmt.Errorf("%w: lc=%d lp=%d pb=%d", errUPXInvalidLZMAParams, lc, lp, pb) + } + if uint16(lc)+uint16(lp) > maxUPXLZMALiteralBits { + return nil, fmt.Errorf("%w: lc+lp=%d exceeds %d", errUPXInvalidLZMAParams, lc+lp, maxUPXLZMALiteralBits) + } + // convert to standard LZMA properties byte props := lc + lp*9 + pb*9*5 // raw LZMA stream starts at byte 2 (includes 0x00 init byte) lzmaStream := compressedData[2:] - // compute dictionary size: must be at least as large as uncompressed size - // use next power of 2 for efficiency, with reasonable min/max bounds. + uncompressedSize := uint32(size) + + // the dictionary only has to cover the uncompressed size and sit inside the library's own range; it is + // not required to be a power of two. The library allocates min(dictSize, header size) up front, before + // reading a compressed byte, so this is the file's one remaining heap knob and maxDictionaryFor is + // what bounds it. + // // note: if you're seeing that testing small binaries works and large ones don't, // it may be that the dictionary size was not considered properly in this code. - const minDictSize = 64 * 1024 // 64KB minimum - const maxDictSize = 128 * 1024 * 1024 // 128MB maximum - dictSize := min(max(nextPowerOf2(uncompressedSize), minDictSize), maxDictSize) + dictSize := min(max(uncompressedSize, minUPXDictionary), maxDictionaryFor(len(compressedData))) // construct standard 13-byte LZMA header header := make([]byte, 13) @@ -507,21 +1118,10 @@ func decompressLZMA(compressedData []byte, uncompressedSize uint32) ([]byte, err binary.LittleEndian.PutUint32(header[1:5], dictSize) binary.LittleEndian.PutUint64(header[5:13], uint64(uncompressedSize)) - // combine header + raw stream - var fullStream []byte - fullStream = append(fullStream, header...) - fullStream = append(fullStream, lzmaStream...) - - reader, err := lzma.NewReader(bytes.NewReader(fullStream)) + // MultiReader rather than concatenating, so the compressed block is not copied a second time + reader, err := lzma.NewReader(io.MultiReader(bytes.NewReader(header), bytes.NewReader(lzmaStream))) if err != nil { return nil, fmt.Errorf("failed to create LZMA reader: %w", err) } - - decompressed := make([]byte, uncompressedSize) - _, err = io.ReadFull(reader, decompressed) - if err != nil { - return nil, fmt.Errorf("failed to decompress LZMA data: %w", err) - } - - return decompressed, nil + return reader, nil } diff --git a/syft/pkg/cataloger/golang/upx_bounds_regression_test.go b/syft/pkg/cataloger/golang/upx_bounds_regression_test.go new file mode 100644 index 000000000..22beeba32 --- /dev/null +++ b/syft/pkg/cataloger/golang/upx_bounds_regression_test.go @@ -0,0 +1,760 @@ +package golang + +import ( + "bytes" + "context" + "debug/elf" + "encoding/binary" + "fmt" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/testutils" + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/elfutil" + "github.com/anchore/syft/syft/internal/unionreader" +) + +// the reconstruction's storage, its Close semantics and the contiguous-prefix rule it reports as Size all +// belong to internal/spillbuf now, and are tested there. What is left in this file is what the golang +// cataloger does with them. + +// sparseFixture is a UPX file whose first block carries an ELF header declaring a second PT_LOAD segment +// at farOffset, so a tiny third block is placed there. declaredShstrtab, when non-zero, makes that ELF +// header also declare a section name table of that size sitting inside the gap, which is what turns the +// reader's length into an allocation for anything that parses it. +func sparseFixture(t *testing.T, declared uint32, farOffset, declaredShstrtab uint64, pad int) []byte { + t.Helper() + + phdrs := make([]byte, 112) + binary.LittleEndian.PutUint32(phdrs[0:4], 1) // phdr[0] PT_LOAD + binary.LittleEndian.PutUint64(phdrs[8:16], 0) // p_offset 0 + binary.LittleEndian.PutUint32(phdrs[56:60], 1) + binary.LittleEndian.PutUint64(phdrs[64:72], farOffset) // phdr[1] p_offset, out at the declared end + + hdr := make([]byte, 64) + copy(hdr, []byte{0x7f, 'E', 'L', 'F'}) + hdr[4], hdr[5], hdr[6] = 2, 1, 1 // ELFCLASS64, little endian, EV_CURRENT + binary.LittleEndian.PutUint16(hdr[0x10:0x12], uint16(elf.ET_EXEC)) + binary.LittleEndian.PutUint16(hdr[0x12:0x14], uint16(elf.EM_X86_64)) + binary.LittleEndian.PutUint32(hdr[0x14:0x18], 1) + binary.LittleEndian.PutUint64(hdr[0x20:0x28], 64) // e_phoff + binary.LittleEndian.PutUint16(hdr[0x36:0x38], 56) // e_phentsize + binary.LittleEndian.PutUint16(hdr[0x38:0x3a], 2) // e_phnum + + block1 := append(hdr, phdrs...) + if declaredShstrtab > 0 { + sh := make([]byte, 128) // two section headers; index 1 is the name table + binary.LittleEndian.PutUint32(sh[64+4:64+8], uint32(elf.SHT_STRTAB)) + binary.LittleEndian.PutUint64(sh[64+24:64+32], farOffset/2) // sh_offset, inside the gap + binary.LittleEndian.PutUint64(sh[64+32:64+40], declaredShstrtab) + binary.LittleEndian.PutUint64(block1[0x28:0x30], 64+112) // e_shoff + binary.LittleEndian.PutUint16(block1[0x3a:0x3c], 64) // e_shentsize + binary.LittleEndian.PutUint16(block1[0x3c:0x3e], 2) // e_shnum + binary.LittleEndian.PutUint16(block1[0x3e:0x40], 1) // e_shstrndx + block1 = append(block1, sh...) + } + + data := buildUPXHeader(declared, 4096) + data = append(data, blockFor(t, block1)...) + data = append(data, blockFor(t, bytes.Repeat([]byte("B"), 32))...) // sequential + data = append(data, blockFor(t, bytes.Repeat([]byte("C"), 64))...) // -> ptLoadOffsets[1] + data = append(data, make([]byte, 12)...) // end marker + return padTo(data, pad) +} + +// TestDecompressUPX_ReaderLengthIsWhatWasRebuilt covers the amplification that survives moving the +// reconstruction from the heap to a temp file. +// +// A block's destination comes from a PT_LOAD p_offset in the first block, bounded only by p_filesize, so a +// 64 byte block parked at p_filesize-64 must not make the reconstruction report the full declared size +// while holding only a few hundred real bytes. A sparse file hands its holes back as zeros it never stored, +// so every parser downstream that sizes against "what this reader will deliver" -- which is the whole +// argument for writing to disk rather than the heap -- would otherwise allocate against a number the input +// never paid for. +func TestDecompressUPX_ReaderLengthIsWhatWasRebuilt(t *testing.T) { + const declared = 64 << 20 + const input = 300_000 + + data := sparseFixture(t, declared, declared-64, 0, input) + + out, err := unpack(t, data) + // the hole is exactly what makes this partial: the blocks past it are given up, and that is reported + // rather than logged, since a caller reading this reconstruction is missing bytes the header declared + require.ErrorIs(t, err, errUPXPartial) + require.NotNil(t, out, "a short reconstruction is still the contents to read from") + + // the ELF header block plus the two small blocks, and nothing for the hole + assert.Less(t, out.Size(), int64(4096), + "the reader must be sized by the bytes actually rebuilt, not by where a block was parked") + + _, err = out.ReadAt(make([]byte, 512), int64(declared)/2) + assert.ErrorIs(t, err, io.EOF, "the hole must not read back as free zeros") +} + +func TestDecompressUPX_SparsePlacementIsNotAnAllocationKnob(t *testing.T) { + // the same fixture, with the reconstructed ELF also declaring a 60MB section name table sitting in the + // hole. This is the end of the exploit chain: debug/elf grows its read as the reads succeed, and + // against a hole they all succeed. + const declared = 64 << 20 + const input = 300_000 + + data := sparseFixture(t, declared, declared-64, 60<<20, input) + + out, err := unpack(t, data) + require.ErrorIs(t, err, errUPXPartial) + require.NotNil(t, out) + + allocated := testutils.MeasureAlloc(t, func() { + _, _ = getBuildInfo(out) + }) + + // debug/elf's own first chunk is ~10MB regardless of what a file declares, so the bound to assert is + // that the declared 60MB is not reachable, not that this is free + assert.Less(t, allocated, uint64(16*intFile.MB), + "a 300KB input must not drive tens of MB of allocation through the reconstruction") +} + +func TestReadPTLoadOffsets_ShortReadIsNotTrusted(t *testing.T) { + // on a short read the tail of the buffer is zeros the file never provided, and a p_offset of zero + // parsed out of it would place a later block over the ELF header. + phdrs := make([]byte, 112) + binary.LittleEndian.PutUint32(phdrs[0:4], 1) + binary.LittleEndian.PutUint64(phdrs[8:16], 0x1000) + binary.LittleEndian.PutUint32(phdrs[56:60], 1) + binary.LittleEndian.PutUint64(phdrs[64:72], 0x2000) + full := buildELF64(64, 56, 2, phdrs) + + assert.Equal(t, []uint64{0x1000, 0x2000}, readPTLoadOffsets(bytes.NewReader(full), 0, uint32(len(full))), + "the whole header is readable, so both segments come back") + + // the same header with the second program header cut off. The block claims it is there; the file is + // not that long. + truncated := full[:64+56+8] + got := readPTLoadOffsets(bytes.NewReader(truncated), 0, uint32(len(full))) + assert.Equal(t, []uint64{0x1000}, got, + "only the segment actually present may be trusted; a zero read out of the gap is not an offset") +} + +func TestDecompressUPX_StoredBlockIsPlaced(t *testing.T) { + // method 0 is an extent UPX could not compress and wrote verbatim, and real output carries a few of + // them: the alignment padding between PT_LOAD segments is only a handful of bytes. Nothing covered + // this path outside the Docker-backed fixture. + payload := bytes.Repeat([]byte("S"), 48) + stored := make([]byte, 12) + binary.LittleEndian.PutUint32(stored[0:4], uint32(len(payload))) // sz_unc + binary.LittleEndian.PutUint32(stored[4:8], uint32(len(payload))) // sz_cpr, equal for a stored block + stored[8] = upxMethodStored + + data := buildUPXHeader(4096, 4096) + data = append(data, stored...) + data = append(data, payload...) + data = append(data, make([]byte, 12)...) + data = padTo(data, 512) + + out, err := unpack(t, data) + // the fixture declares 4096 and delivers one 48 byte extent, so the chain is short by construction and + // the partial goes with it. Real output rebuilds p_filesize exactly; what is under test is placement. + require.ErrorIs(t, err, errUPXPartial) + assert.Equal(t, payload, readAll(t, out), "a stored block is copied through as-is") +} + +func TestDecompressUPX_StoredBlockWithMismatchedSizesIsQuiet(t *testing.T) { + // a copy is only meaningful when the two sizes agree. A mismatch is how a stray b_info-shaped run of + // bytes looks, so it ends the chain rather than failing the file. + stored := make([]byte, 12) + binary.LittleEndian.PutUint32(stored[0:4], 64) // sz_unc + binary.LittleEndian.PutUint32(stored[4:8], 32) // sz_cpr, disagrees + stored[8] = upxMethodStored + + data := padTo(append(buildUPXHeader(4096, 4096), stored...), 512) + + _, err := unpack(t, data) + require.Error(t, err) + // a stored block whose sizes disagree is not a block we can read, which on the first block is the + // unsupported-method refusal. Either way it must stay clear of errUPXDecompress. + assert.ErrorIs(t, err, errUnsupportedUPXMethod, "a copy whose sizes disagree is not a copy") + assert.NotErrorIs(t, err, errUPXDecompress, "not a packed binary, so it stays quiet") +} + +// TestScanReader_ReportingIsNarrow pins which gaps this cataloger claims. It runs against every +// executable in an image, so reporting each file debug/buildinfo cannot parse would attach an unknown to +// most files in a typical one. Only a bound syft itself chose to enforce, and a packed file it could not +// unpack, are real gaps. +func TestScanReader_ReportingIsNarrow(t *testing.T) { + tests := []struct { + name string + data []byte + report bool + }{ + { + name: "a corrupt ELF is not this cataloger's gap", + data: append([]byte{0x7f, 'E', 'L', 'F', 2, 1, 1}, bytes.Repeat([]byte{0xAB}, 512)...), + }, + { + name: "neither is a file that is not an executable at all", + data: bytes.Repeat([]byte("not a binary"), 64), + }, + { + name: "nor a truncated PE", + data: append([]byte("MZ\x90\x00"), bytes.Repeat([]byte{0}, 512)...), + }, + { + name: "a section name table syft declined to expand is", + data: elfDeclaringNameTable(t, 256<<20), + // this is elfutil.ErrDeclaredSizeExceeded: syft chose not to expand it, so the packages behind + // it really are missing from the SBOM because of a decision made here + report: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + loc := file.NewLocation("subject") + _, err := scanReader(context.Background(), loc, bytes.NewReader(tt.data), false) + + if !tt.report { + assert.NoError(t, err, "this cataloger sees every executable in an image; it must stay quiet here") + return + } + require.Error(t, err) + var coordErr *unknown.CoordinateError + require.ErrorAs(t, err, &coordErr, "a reported gap has to carry the location it belongs to") + assert.Equal(t, loc.Coordinates, coordErr.Coordinates) + assert.ErrorIs(t, coordErr.Reason, elfutil.ErrDeclaredSizeExceeded, + "the reported gap has to carry the sentinel the split keys on, not just a message") + }) + } +} + +func TestScanReader_RefusedExpansionIsReportedAsSuch(t *testing.T) { + // the sentinel is what lets the narrow reporting above tell "syft declined to expand this" apart from + // "this file is broken", so the two must not collapse into one string match + _, err := readBuildInfo(bytes.NewReader(elfDeclaringNameTable(t, 256<<20))) + require.Error(t, err) + assert.ErrorIs(t, err, elfutil.ErrDeclaredSizeExceeded) +} + +// TestReadContentsAndBuildInfo_FakeUPXHeaderCannotHideAGoBinary covers the evasion vector the unpacking +// opened up. The "UPX!" magic is found by an unanchored substring scan over the first 8KB and every field +// behind it is attacker-controlled, so an ordinary Go binary can be made to produce a plausible header and +// a reconstruction of nothing. Reading from that reconstruction unconditionally means the binary reports +// no packages at all, which turns a false positive into a way to hide a dependency list. +// goELF64Fixture returns a real, unpacked Go binary that clears the ELF64 little-endian container gate in +// parseUPXInfo. hello-linux-arm is ELFCLASS32 and is rejected before the UPX magic scan runs, so a test +// that splices a header into it exercises nothing. +func goELF64Fixture(t *testing.T) []byte { + t.Helper() + runMakeTarget(t, "archs") + original, err := os.ReadFile(filepath.Join("testdata", "archs", "binaries", "hello-linux-ppc64le")) + require.NoError(t, err) + require.Equal(t, byte(2), original[4], "the container gate only accepts ELF64") + require.Equal(t, byte(1), original[5], "the container gate only accepts little-endian") + return original +} + +// spliceFakeUPXHeader writes a plausible UPX header plus one decodable block into padding inside the +// magic scan window, leaving the ELF itself intact. The offset is found rather than hardcoded: it has to +// sit past the section header table (writing over that corrupts the binary and the test then proves +// nothing) and inside upxMagicScanWindow (past it the magic is never seen). +func spliceFakeUPXHeader(t *testing.T, original []byte) []byte { + t.Helper() + return spliceUPXChain(t, original, 64<<10, blockFor(t, []byte("junk"))) +} + +// spliceUPXChain is spliceFakeUPXHeader with a caller-supplied p_filesize and b_info chain, so a test can +// choose whether the fake header lands on a clean short reconstruction, one that gives something up, or a +// refusal by the size bounds with nothing unpacked at all. +func spliceUPXChain(t *testing.T, original []byte, originalSize uint32, chain []byte) []byte { + t.Helper() + + header := buildUPXHeader(originalSize, 4096)[64:] // drop the stub, the real ELF header is already here + block := chain + need := len(header) + len(block) + + shoff := binary.LittleEndian.Uint64(original[0x28:0x30]) + shentsize := binary.LittleEndian.Uint16(original[0x3a:0x3c]) + shnum := binary.LittleEndian.Uint16(original[0x3c:0x3e]) + after := int(shoff) + int(shentsize)*int(shnum) + + at := -1 + for i := after; i+need <= upxMagicScanWindow && i+need <= len(original); i++ { + if bytes.Equal(original[i:i+need], make([]byte, need)) { + at = i + break + } + } + require.GreaterOrEqual(t, at, 0, "no padding in the scan window big enough to splice a header into") + + spiked := append([]byte(nil), original...) + copy(spiked[at:], header) + copy(spiked[at+len(header):], block) + + // the splice must not have broken the binary, or the fallback below would be covering for a corrupt + // ELF rather than for a fake header. A claim the size bounds refuse is a deliberate caller choice, so + // only assert the header parses when it was meant to. + if originalSize <= maxUPXOriginalSize { + require.NotNil(t, mustParseUPX(t, spiked), "the spliced header has to actually parse as UPX") + } + return spiked +} + +// mustParseUPX reports the UPX header the scan finds in data, proving the splice is what the parser sees. +func mustParseUPX(t *testing.T, data []byte) *upxInfo { + t.Helper() + r := bytes.NewReader(data) + info, err := parseUPXInfo(r, int64(len(data))) + require.NoError(t, err) + return info +} + +func TestScanFile_FakeUPXHeaderStillYieldsPackages(t *testing.T) { + // the same evasion one layer out, at the entry point the cataloger calls. The header has to actually be + // spliced: run this against the untouched binary and it passes with the fallback deleted. + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + original := goELF64Fixture(t) + + spiked := spliceFakeUPXHeader(t, original) + + ur, err := unionreader.GetUnionReader(io.NopCloser(bytes.NewReader(spiked))) + require.NoError(t, err) + + builds, _ := scanFile(ctx, file.NewLocation("hello-linux-ppc64le"), ur, false) + require.NotEmpty(t, builds, "a Go binary must not be hidden by four bytes of fake magic") + for _, b := range builds { + assert.Nil(t, b.unpacked, + "the reconstruction carried nothing, so the scan has to fall back to the bytes as they were found") + closeUnpacked(b.unpacked) + } +} + +// TestScanReader_CancellationStopsTheScan pins the direction a swallowed ctx.Err() got wrong. Cancellation +// is not a gap in the SBOM, so it is not reported as an unknown, but it does mean stop: returning "nothing +// to unpack, no error" sent the scan on to parse the packed bytes and build packages out of them after the +// caller had already called it off. +func TestScanReader_CancellationStopsTheScan(t *testing.T) { + // enough blocks that the loop checks ctx at least once + blocks := make([][]byte, 64) + for i := range blocks { + blocks[i] = bytes.Repeat([]byte("A"), 1024) + } + fixture := buildUPXFile(t, 64*1024, 4096, blocks, nil) + + ctx, cancel := context.WithCancel(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) + cancel() + + build, err := scanReader(ctx, file.NewLocation("/cancelled"), bytes.NewReader(fixture), false) + assert.Nil(t, build) + require.Error(t, err, "a cancelled scan has to stop rather than fall through to the packed bytes") + assert.ErrorIs(t, err, context.Canceled) + coordErrs, _ := unknown.ExtractCoordinateErrors(err) + assert.Empty(t, coordErrs, "cancellation is not a gap in the SBOM") +} + +// TestScanReader_SizeRefusalIsReported covers the reporting split for a header that really is UPX and +// claims more than the bounds allow. That is a file we declined to expand, which is the same category as +// elfutil.ErrDeclaredSizeExceeded and gets the same unknown. The ratio clears a measured 209x worst case at +// 256, so a legitimate binary crossing it must not vanish from the SBOM silently. +func TestScanReader_SizeRefusalIsReported(t *testing.T) { + // a 4KB input claiming 512MB: pays no ratio and clears no ceiling + data := padTo(buildUPXHeader(maxUPXOriginalSize, 0x1000), 4096) + + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + build, err := scanReader(ctx, file.NewLocation("/refused"), bytes.NewReader(data), false) + + assert.Nil(t, build) + require.Error(t, err) + coordErrs, _ := unknown.ExtractCoordinateErrors(err) + require.NotEmpty(t, coordErrs, + "a file we declined to expand is a gap we chose to leave, so it is reported") + assert.ErrorIs(t, coordErrs[0].Reason, errUPXSizeRefused) +} + +// corruptLZMABlock builds a b_info whose stream carries valid LZMA parameters and nothing else that +// decodes. This is what the loader stub behind the last real block looks like to readChainBlock: the +// method byte is one we implement and the sizes are in range, so the chain accepts it and the decoder is +// the thing that says no. +func corruptLZMABlock(szUnc uint32) []byte { + stream := make([]byte, 64) + stream[0] = 0x00 // pb = 0 + stream[1] = 0x00 // lc = 0, lp = 0 + for i := 2; i < len(stream); i++ { + stream[i] = byte(i * 7) // not a range-coded anything + } + b := make([]byte, 12) + binary.LittleEndian.PutUint32(b[0:4], szUnc) + binary.LittleEndian.PutUint32(b[4:8], uint32(len(stream))) + b[8] = 14 // b_method = LZMA + return append(b, stream...) +} + +// TestDecompressUPX_UndecodableLaterBlockKeepsWhatCameBefore covers a chain that gives up a good +// reconstruction over garbage past the end of it. +// +// readChainBlock accepts any b_info-shaped run of bytes whose method it knows and whose sizes fit the +// file, so the loader stub behind the last real block parses as a block roughly one time in 256 on the +// method byte alone. A decode failure on that block must not discard every block already placed and turn a +// fully recoverable binary into zero packages: the two sibling conditions on the same block (an overrun, +// and a placement that does not fit) already keep their partial output, and this one must too. +func TestDecompressUPX_UndecodableLaterBlockKeepsWhatCameBefore(t *testing.T) { + head := bytes.Repeat([]byte("H"), 256) + + data := buildUPXHeader(4096, 4096) + data = append(data, blockFor(t, head)...) + data = append(data, corruptLZMABlock(64)...) + data = padTo(data, 1024) + + out, err := unpack(t, data) + require.ErrorIs(t, err, errUPXPartial, "the blocks already placed are kept, and what was lost is reported") + require.NotErrorIs(t, err, errUPXDecompress, "a good reconstruction is not discarded over trailing garbage") + require.NotNil(t, out) + + assert.Equal(t, head, readAll(t, out), "everything decoded before the bad block stays placed") +} + +// TestDecompressUPX_UndecodableFirstBlockStillFailsTheFile is the other half: with nothing placed there is +// no partial output to keep, so the file really is one we could not unpack. +func TestDecompressUPX_UndecodableFirstBlockStillFailsTheFile(t *testing.T) { + data := padTo(append(buildUPXHeader(4096, 4096), corruptLZMABlock(64)...), 1024) + + _, err := unpack(t, data) + require.ErrorIs(t, err, errUPXDecompress, "a packed file with nothing readable in it is a gap in the SBOM") + assert.NotErrorIs(t, err, errUPXPartial) +} + +// TestScanReader_PartialReconstructionIsReported is the regression test for a partial unpack vanishing +// silently. The reconstruction is truncated to the contiguous prefix, which is the right bound, but the +// bytes past the gap are gone: on a real binary that is the non-loadable tail holding the section headers, +// so the symbols and often the build info go with it. Reported at Trace, the binary contributed no +// packages and no unknown, which is the one outcome the reporting split in upx.go exists to prevent. +func TestScanReader_PartialReconstructionIsReported(t *testing.T) { + const declared = 64 << 20 + data := sparseFixture(t, declared, declared-64, 0, 300_000) + + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + build, err := scanReader(ctx, file.NewLocation("/partial"), bytes.NewReader(data), false) + + assert.Nil(t, build) + require.Error(t, err) + coordErrs, _ := unknown.ExtractCoordinateErrors(err) + require.NotEmpty(t, coordErrs, "a reconstruction that lost bytes is a gap this cataloger chose to leave") + assert.ErrorIs(t, coordErrs[0].Reason, errUPXPartial) +} + +// TestReadContentsAndBuildInfo_FallbackToFoundBytesReportsNoGap pins the retry direction. A gap is +// reported because the readers after it (crypto settings, arch, symbols) would otherwise read a short +// reconstruction with nothing saying so, which is what +// TestReadContentsAndBuildInfo_GapSurvivesBuildInfoFromTheReconstruction covers. That reasoning runs out +// here: the retry hands back the bytes as they were found, every reader below reads those, and the header +// that produced the short reconstruction was not describing real UPX output to begin with. Reporting it +// would put "UPX reconstruction is incomplete" on a binary that was never packed and was read in full. +func TestReadContentsAndBuildInfo_FallbackToFoundBytesReportsNoGap(t *testing.T) { + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + + // a chain that decodes one block and then hits a block it cannot read: the reconstruction is real but + // short, and the build info comes from the fallback to the bytes as they were found + chain := append(blockFor(t, []byte("junk")), corruptLZMABlock(64)...) + spiked := spliceUPXChain(t, goELF64Fixture(t), 64<<10, chain) + + unpacked, bi, err := readContentsAndBuildInfo(ctx, bytes.NewReader(spiked)) + t.Cleanup(func() { closeUnpacked(unpacked) }) + + require.NotNil(t, bi, "the fallback still finds the build info") + require.Nil(t, unpacked, "the reconstruction was released; the caller reads the bytes as they were found") + assert.NoError(t, err, "nothing downstream reads the short reconstruction, so there is no gap to report") +} + +// TestUnpackUPX_CancelledContextComesBackAsCtxErr covers the cancellation arm in unpackUPX directly: the +// scan-level test above it passes even with this arm deleted, since readContentsAndBuildInfo does not +// re-check ctx.Err() on its own. +func TestUnpackUPX_CancelledContextComesBackAsCtxErr(t *testing.T) { + payload := bytes.Repeat([]byte("A"), 2048) + blocks := make([][]byte, 64) + for i := range blocks { + blocks[i] = payload + } + data := padTo(buildUPXFile(t, 4096*64, 2048, blocks, nil), 4096) + + ctx, cancel := context.WithCancel(tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))) + cancel() + + out, err := unpackUPX(ctx, bytes.NewReader(data)) + require.ErrorIs(t, err, context.Canceled) + assert.NotErrorIs(t, err, errUPXDecompress, "a cancelled scan is not a gap in the SBOM") + assert.NotErrorIs(t, err, errUPXPartial, "nor is it a short reconstruction") + assert.Nil(t, out, "a cancelled unpack hands back nothing; the caller reads the input as it was found") +} + +// TestParseUPXInfo_HeaderRunsPastTheScanWindow covers the guard on the l_info+p_info read: the magic can +// sit close enough to the end of what was actually read that the 20 bytes behind it are not there. +func TestParseUPXInfo_HeaderRunsPastTheScanWindow(t *testing.T) { + data := append(packedELFStub(), upxMagic...) // magic at 64, nothing behind it + + r := bytes.NewReader(data) + _, err := parseUPXInfo(r, int64(len(data))) + require.Error(t, err) + assert.ErrorIs(t, err, errUPXImplausibleHeader) + assert.NotErrorIs(t, err, errUPXSizeRefused, "a header we could not even read is not a refusal to expand") +} + +// TestReadContentsAndBuildInfo_GapSurvivesBuildInfoFromTheReconstruction is the same property one path +// over: here the reconstruction itself carries the build info, so the early return must not throw the +// unpack error away. The chain rebuilds a whole Go binary and then hits a block it cannot read, which is +// exactly the shape of a real packed binary whose tail extents did not all come back. +func TestReadContentsAndBuildInfo_GapSurvivesBuildInfoFromTheReconstruction(t *testing.T) { + payload := goELF64Fixture(t) + + data := buildUPXHeader(uint32(len(payload)+64), 4096) + data = append(data, blockFor(t, payload)...) + data = append(data, corruptLZMABlock(64)...) + + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + contents, bi, err := readContentsAndBuildInfo(ctx, bytes.NewReader(data)) + require.NotNil(t, contents) + t.Cleanup(func() { closeUnpacked(contents) }) + + require.NotNil(t, contents, "the build info has to come from the reconstruction here") + require.NotNil(t, bi, "the rebuilt binary is a whole Go binary") + assert.ErrorIs(t, err, errUPXPartial, + "the block the chain gave up is still a gap, even though the build info came through") +} + +// TestScanReader_PartialReconstructionStillYieldsItsPackages is the other direction of the reporting +// split, and the one that keeps the new reporting from becoming a regression of its own: a gap is +// something to report alongside the packages, not instead of them. A binary whose chain gave up its tail +// still has its module list, and dropping it would trade a silent false negative for a loud one. +func TestScanReader_PartialReconstructionStillYieldsItsPackages(t *testing.T) { + payload := goELF64Fixture(t) + + data := buildUPXHeader(uint32(len(payload)+64), 4096) + data = append(data, blockFor(t, payload)...) + data = append(data, corruptLZMABlock(64)...) + + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + build, err := scanReader(ctx, file.NewLocation("/partial-but-usable"), bytes.NewReader(data), false) + + require.NotNil(t, build, "a reported gap must not cost the packages that did come through") + t.Cleanup(func() { closeUnpacked(build.unpacked) }) + assert.NotNil(t, build.BuildInfo) + + require.Error(t, err) + coordErrs, _ := unknown.ExtractCoordinateErrors(err) + require.NotEmpty(t, coordErrs, "and the gap is still reported") + assert.ErrorIs(t, coordErrs[0].Reason, errUPXPartial) +} + +// sizeProbeSpy counts the size probes made against it. intFile.ReaderSize answers from Size() when the +// reader has one, which is what this intercepts. +type sizeProbeSpy struct { + *bytes.Reader + probes int +} + +func (s *sizeProbeSpy) Size() int64 { + s.probes++ + return s.Reader.Size() +} + +// TestUnpackUPX_ContainerGateComesBeforeTheSizeProbe pins the ordering that keeps this cheap. The +// cataloger runs over every file in an image and almost none are packed ELF, so nothing above the six +// byte ident check should cost more than that: ReaderSize seeks to the end and reads the last byte back, +// which over a squashfs or tar-backed reader is a real seek and decompress. +func TestUnpackUPX_ContainerGateComesBeforeTheSizeProbe(t *testing.T) { + tests := []struct { + name string + data []byte + probed bool + }{ + {name: "not an ELF at all", data: bytes.Repeat([]byte("just some bytes"), 64)}, + {name: "a 32-bit ELF", data: func() []byte { b := padTo(packedELFStub(), 256); b[4] = 1; return b }()}, + {name: "a big-endian ELF64", data: func() []byte { b := padTo(packedELFStub(), 256); b[5] = 2; return b }()}, + {name: "an ELF64 we would try to unpack", data: padTo(packedELFStub(), 256), probed: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spy := &sizeProbeSpy{Reader: bytes.NewReader(tt.data)} + out, err := unpackUPX(context.Background(), spy) + require.NoError(t, err) + t.Cleanup(func() { closeUnpacked(out) }) + + if tt.probed { + assert.Positive(t, spy.probes, "a candidate container does get measured") + return + } + assert.Zero(t, spy.probes, "a file we will never unpack must not be measured first") + }) + } +} + +// TestScanReader_ChainEndingShortIsReported covers the fourth shape a short reconstruction comes in: a +// chain that simply ran out of b_info structures before rebuilding p_filesize must be reported, the same +// as one that gave up mid-walk, one that hit the block cap, and one with blocks stranded past a hole. Every +// quiet exit in readChainBlock produces that shape (an unreadable b_info, the end marker, a zero sz_cpr, +// compressed data past the end of the input, an unimplemented method past the first block), and the +// reconstruction is then truncated to the covered prefix, which can drop the section headers and leave the +// binary contributing neither packages nor an unknown. +func TestScanReader_ChainEndingShortIsReported(t *testing.T) { + // one 2048 byte block against a declared 4096, then the end marker: nothing failed, the chain just ran + // out. stopped is nil, covered equals the furthest placement, and covered is half of total. + payload := bytes.Repeat([]byte("A"), 2048) + data := buildUPXHeader(4096, 2048) + data = append(data, blockFor(t, payload)...) + data = append(data, make([]byte, 12)...) // end marker + data = padTo(data, 512) + + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + build, err := scanReader(ctx, file.NewLocation("/short-chain"), bytes.NewReader(data), false) + + assert.Nil(t, build) + require.Error(t, err) + coordErrs, _ := unknown.ExtractCoordinateErrors(err) + require.NotEmpty(t, coordErrs, "a chain that ended before rebuilding p_filesize is a reported gap") + assert.ErrorIs(t, coordErrs[0].Reason, errUPXPartial) +} + +// TestScanReader_SizeRefusalOnAnUnpackedBinaryIsNotReported is the other direction of +// TestScanReader_SizeRefusalIsReported. The magic is found by an unanchored scan over the first 8KB, so an +// ordinary Go binary can carry a "UPX!" that parses into a header the size bounds then refuse. Nothing is +// unpacked in that case: unpackUPX hands back the input exactly as it was found, every reader below reads +// those bytes in full, and the build info proves they are complete. Reporting the refusal there put +// "golang binary read incompletely" on a binary the cataloger read completely, with every package present. +func TestScanReader_SizeRefusalOnAnUnpackedBinaryIsNotReported(t *testing.T) { + original := goELF64Fixture(t) + // a header whose p_filesize no input this size could ever pay for, so parseUPXInfo refuses it before + // decompressUPX is reached and there is no reconstruction at any point + spiked := spliceUPXChain(t, original, 0xFFFFFFFF, blockFor(t, []byte("junk"))) + + _, perr := parseUPXInfo(bytes.NewReader(spiked), int64(len(spiked))) + require.ErrorIs(t, perr, errUPXSizeRefused, "the splice has to be refused by the size bounds") + + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + build, err := scanReader(ctx, file.NewLocation("/spiked"), bytes.NewReader(spiked), false) + if build != nil { + t.Cleanup(func() { closeUnpacked(build.unpacked) }) + } + + require.NotNil(t, build, "the binary is a complete Go binary and must still be cataloged") + assert.Nil(t, build.unpacked, "nothing was unpacked, so these are the bytes as found") + coordErrs, _ := unknown.ExtractCoordinateErrors(err) + assert.Empty(t, coordErrs, "a file read in full must not be reported as read incompletely") +} + +// TestDecompressUPX_DenseChainReportsNoGap is the guard for the covered >= total early return in +// finalExtent, and it is the fixture every other one in this package is not: dense. A chain that rebuilds +// exactly p_filesize must come back with no partial reason attached. +// +// This matters because reporting a chain that ended short (the arm below that early return) is only safe if +// well-formed output reaches full coverage. Every other crafted fixture here declares more than it delivers +// and asserts errUPXPartial, and the one test that sees real `upx --best --lzma` output needs Docker, so +// without this the early return has no local guard and the new arm's false-positive risk is untested. +// +// Layout below tiles 320 bytes through all three placement rules: block 1 at zero, block 2 sequentially +// behind it, block 3 at a PT_LOAD offset from the headers in block 1, and block 4 into the hole those left. +func TestDecompressUPX_DenseChainReportsNoGap(t *testing.T) { + const ( + headerBlock = 64 + 112 // ELF64 header plus two program headers + block2Size = 24 // sequential, so [176, 200) + ptLoad1 = 256 // block 3 lands here, leaving [200, 256) behind + block3Size = 64 // [256, 320) + holeSize = ptLoad1 - (headerBlock + block2Size) + total = ptLoad1 + block3Size + ) + + phdrs := make([]byte, 112) + binary.LittleEndian.PutUint32(phdrs[0:4], 1) // phdr[0] PT_LOAD + binary.LittleEndian.PutUint64(phdrs[8:16], 0) // p_offset 0, the extent block 2 covers + binary.LittleEndian.PutUint32(phdrs[56:60], 1) + binary.LittleEndian.PutUint64(phdrs[64:72], ptLoad1) // phdr[1] p_offset, where block 3 goes + + block1 := buildELF64(64, 56, 2, phdrs) + require.Len(t, block1, headerBlock, "the first block is the original ELF headers") + + data := buildUPXHeader(total, 4096) + data = append(data, blockFor(t, block1)...) // placed at 0 + data = append(data, blockFor(t, bytes.Repeat([]byte("S"), block2Size))...) // sequential + data = append(data, blockFor(t, bytes.Repeat([]byte("P"), block3Size))...) // -> ptLoadOffsets[1] + data = append(data, blockFor(t, bytes.Repeat([]byte("F"), holeSize))...) // -> firstHole + data = append(data, make([]byte, 12)...) // end marker + data = padTo(data, 1024) + + out, err := unpack(t, data) + require.NotNil(t, out) + require.NoError(t, err, + "a chain that rebuilt every byte p_filesize declared has given nothing up, so there is no gap to report") + + got := readAll(t, out) + assert.Len(t, got, total, "the reconstruction is exactly what was declared") + assert.Equal(t, bytes.Repeat([]byte("F"), holeSize), got[headerBlock+block2Size:ptLoad1], + "the hole between the sequential extent and the PT_LOAD one is filled by the block after them") + assert.Equal(t, bytes.Repeat([]byte("P"), block3Size), got[ptLoad1:], + "block 3 lands at the PT_LOAD offset the headers declared") +} + +// TestFinalExtent pins which reason a short reconstruction comes back with, not merely that it is short. +// Every arm wraps errUPXPartial and nothing else read the messages, so three of the four were +// interchangeable: deleting the block-cap arm and the stranded-blocks arm left the whole package green +// because both fell through to the arm below them. finalExtent is a pure function, so this costs nothing. +func TestFinalExtent(t *testing.T) { + tests := []struct { + name string + covered uint64 + blockNum int + total uint64 + stopped error + wantMsg string // empty means no gap is reported + }{ + { + name: "a chain that rebuilt everything reports nothing, whatever it tripped over next", + covered: 100, + blockNum: 2, + total: 100, + stopped: fmt.Errorf("%w: ignored once coverage is complete", errUPXPartial), + }, + { + name: "a mid-walk stop is more specific than anything the extents show", + covered: 100, + blockNum: 2, + total: 200, + stopped: fmt.Errorf("%w: block 2 did not decode", errUPXPartial), + wantMsg: "block 2 did not decode", + }, + { + name: "hitting the block cap is named as the cap", + covered: 100, + blockNum: maxUPXBlocks, + total: 200, + wantMsg: "block cap", + }, + { + name: "a chain that simply ended is named as such", + covered: 100, + blockNum: 3, + total: 200, + wantMsg: "chain ended after 100 of the declared 200", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := finalExtent(tt.covered, tt.blockNum, tt.total, tt.stopped) + + if tt.wantMsg == "" { + assert.NoError(t, got) + return + } + require.Error(t, got) + assert.ErrorIs(t, got, errUPXPartial) + assert.Contains(t, got.Error(), tt.wantMsg, + "the reason has to say which shape this was, or the arms are interchangeable") + }) + } +} diff --git a/syft/pkg/cataloger/golang/upx_security_test.go b/syft/pkg/cataloger/golang/upx_security_test.go new file mode 100644 index 000000000..f4d4f105d --- /dev/null +++ b/syft/pkg/cataloger/golang/upx_security_test.go @@ -0,0 +1,1004 @@ +package golang + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "io" + "math/rand" + "os" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/ulikunitz/xz/lzma" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/spillbuf" + "github.com/anchore/syft/internal/testutils" + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/file" +) + +// buildUPXLZMAStream encodes data into the compressed-block form decompressLZMA expects: UPX's custom +// 2-byte props header followed by the raw LZMA range-coded stream (the standard 13-byte .lzma header is +// stripped because decompressLZMA reconstructs its own). Uses the default lc=3/lp=0/pb=2 properties and a +// 64KB dictionary so the size math in decompressLZMA lines up for the small payloads used in tests. +func buildUPXLZMAStream(t *testing.T, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + w, err := lzma.WriterConfig{ + Properties: &lzma.Properties{LC: 3, LP: 0, PB: 2}, + DictCap: 1 << 16, + SizeInHeader: true, + Size: int64(len(data)), + }.NewWriter(&buf) + require.NoError(t, err) + _, err = w.Write(data) + require.NoError(t, err) + require.NoError(t, w.Close()) + + raw := buf.Bytes()[13:] // strip standard 13-byte lzma header + // UPX 2-byte header: byte 0 is (t<<3)|pb where t = lc+lp, byte 1 is (lp<<4)|lc. Real `upx --best + // --lzma` output for lc=3/lp=0/pb=2 is 0x1a 0x03, confirmed against the image-small-upx fixture. + return append([]byte{0x1a, 0x03}, raw...) +} + +// padTo grows data to at least size bytes so its declared p_filesize stays within maxUPXExpansion of the +// fixture's own length. Real UPX files are far larger than their headers; the crafted ones here are not, +// and the input-size bound is deliberately sensitive to that. +func padTo(data []byte, size int) []byte { + if len(data) >= size { + return data + } + return append(data, make([]byte, size-len(data))...) +} + +// packedELFStub is the ELF header a real UPX-packed file starts with: UPX writes its own loader as an ELF +// and puts l_info behind it. parseUPXInfo requires it, since the block placement it drives reads PT_LOAD +// offsets out of ELF program headers and would be reading nonsense from any other container. +func packedELFStub() []byte { + stub := make([]byte, 64) + copy(stub, []byte{0x7f, 'E', 'L', 'F'}) + stub[4] = 2 // ELFCLASS64 + stub[5] = 1 // little endian + stub[6] = 1 // EV_CURRENT + return stub +} + +// buildUPXHeader assembles the l_info + p_info prefix common to every crafted fixture below, with no +// loader stub. Use buildUPXHeaderWithLoader to exercise the chain past the loader. +func buildUPXHeader(originalSize, blockSize uint32) []byte { + return buildUPXHeaderWithLoader(originalSize, blockSize, 0) +} + +// buildUPXHeaderWithLoader is buildUPXHeader with a non-zero l_lsize, which is what makes the block chain +// pick back up past the loader stub and place the tail extents through firstHole. +func buildUPXHeaderWithLoader(originalSize, blockSize uint32, loaderSize uint16) []byte { + lInfo := []byte{ //nolint:gocritic // appended to the stub below + 0, 0, 0, 0, // l_checksum + 'U', 'P', 'X', '!', // magic + byte(loaderSize), byte(loaderSize >> 8), // l_lsize + 14, 22, // l_version, l_format + } + pInfo := make([]byte, 12) + binary.LittleEndian.PutUint32(pInfo[4:8], originalSize) // p_filesize + binary.LittleEndian.PutUint32(pInfo[8:12], blockSize) // p_blocksize + return append(append(packedELFStub(), lInfo...), pInfo...) +} + +// buildUPXFile assembles a minimal but structurally valid UPX container: l_info + p_info followed by one +// b_info + compressed stream per payload, terminated by a zero end-marker block. declaredSizes, when +// non-nil, overrides each block's sz_unc so a fixture can lie about what its stream decompresses to. +func buildUPXFile(t *testing.T, originalSize, blockSize uint32, payloads [][]byte, declaredSizes []uint32) []byte { + t.Helper() + data := buildUPXHeader(originalSize, blockSize) + for i, p := range payloads { + stream := buildUPXLZMAStream(t, p) + szUnc := uint32(len(p)) + if declaredSizes != nil { + szUnc = declaredSizes[i] + } + b := make([]byte, 12) + binary.LittleEndian.PutUint32(b[0:4], szUnc) // sz_unc + binary.LittleEndian.PutUint32(b[4:8], uint32(len(stream))) // sz_cpr + b[8] = 14 // b_method = LZMA + data = append(data, b...) + data = append(data, stream...) + } + return append(data, make([]byte, 12)...) // end marker: sz_unc == 0 +} + +// blockFor builds one b_info + compressed stream for the given payload. +func blockFor(t *testing.T, payload []byte) []byte { + t.Helper() + return filteredBlockFor(t, payload, 0, 0) +} + +// filteredBlockFor is blockFor with b_ftid and b_cto8 set. Real `upx --best --lzma` on x86-64 always emits +// the CTO filter, so without this the whole unfilter path is only reachable from the Docker fixture. +func filteredBlockFor(t *testing.T, payload []byte, filterID, cto8 byte) []byte { + t.Helper() + stream := buildUPXLZMAStream(t, payload) + b := make([]byte, 12) + binary.LittleEndian.PutUint32(b[0:4], uint32(len(payload))) + binary.LittleEndian.PutUint32(b[4:8], uint32(len(stream))) + b[8] = 14 // b_method = LZMA + b[9] = filterID + b[10] = cto8 + return append(b, stream...) +} + +// unpack runs decompressUPX against a throwaway temp directory. +func unpack(t *testing.T, data []byte) (unpackedContents, error) { + t.Helper() + return unpackIn(t, t.TempDir(), data) +} + +// unpackIn is unpack with a caller-chosen temp directory, so a test can look at the file that was left in +// it. Parses the header the same way unpackUPX does so a fixture that is not UPX at all still reports it. +func unpackIn(t *testing.T, dir string, data []byte) (unpackedContents, error) { + t.Helper() + r := bytes.NewReader(data) + info, err := parseUPXInfo(r, sizeOf(t, r)) + if err != nil { + return nil, err + } + out, err := decompressUPX(context.Background(), tmpdir.FromPath(dir), r, info) + t.Cleanup(func() { closeUnpacked(out) }) + return out, err +} + +// readAll drains the reconstruction returned by decompressUPX. Size is the contiguous run rebuilt, so +// this is exactly what the buffer is willing to stand behind. +func readAll(t *testing.T, b unpackedContents) []byte { + t.Helper() + require.NotNil(t, b) + out := make([]byte, b.Size()) + if len(out) == 0 { + return nil + } + n, err := b.ReadAt(out, 0) + require.NoError(t, err) + require.Equal(t, len(out), n) + return out +} + +func TestDecompressUPX_OversizedBlockRejected(t *testing.T) { + // a single block may not claim more output than the file's own p_filesize leaves, across the full + // uint32 range sz_unc can carry. + cases := []struct { + name string + szUnc uint32 + }{ + {"beyond the declared original size", 8192}, + {"full uint32 range", 0xFFFFFFFF}, + } + for _, tt := range cases { + szUnc := tt.szUnc + t.Run(tt.name, func(t *testing.T) { + data := buildUPXFile(t, 4096, 4096, [][]byte{bytes.Repeat([]byte("A"), 32)}, []uint32{szUnc}) + + _, err := unpack(t, data) + require.Error(t, err) + assert.ErrorIs(t, err, errUPXOutputExceeded) + assert.ErrorIs(t, err, errUPXDecompress, "a plausible header that fails to unpack is reportable") + }) + } +} + +// TestReadChainBlock_ZeroCompressedSizeOnTheFirstBlockIsQuiet is where the sz_cpr == 0 guard is load +// bearing. On the first block there is nothing placed yet, so without the guard the empty buffer reaches +// the decoder, fails, and a b_info the format cannot mean becomes a reported errUPXDecompress unknown +// instead of a file we quietly decline. +func TestReadChainBlock_ZeroCompressedSizeOnTheFirstBlockIsQuiet(t *testing.T) { + b := make([]byte, 12) + binary.LittleEndian.PutUint32(b[0:4], 1) // sz_unc = 1, sz_cpr = 0 + b[8] = 14 // b_method = LZMA + + _, err := unpack(t, padTo(append(buildUPXHeader(4096, 4096), b...), 256)) + require.ErrorIs(t, err, errUPXImplausibleHeader, "nothing in the chain was decodable") + assert.NotErrorIs(t, err, errUPXDecompress, + "a b_info the format cannot mean is not a packed binary we failed to read") +} + +func TestReadChainBlock_LargeCompressedSizeIsNotTruncated(t *testing.T) { + // sz_cpr is taken whole and bounded against the input rather than masked to its low 24 bits: a block + // declaring more than 16MB of compressed data must end the chain, not decode a truncated count of a + // longer stream. + stream := buildUPXLZMAStream(t, bytes.Repeat([]byte("A"), 2048)) + b := make([]byte, 12) + binary.LittleEndian.PutUint32(b[0:4], 2048) + // masks to len(stream), so the old code read exactly the bytes that are there and decoded them + binary.LittleEndian.PutUint32(b[4:8], 0x01000000|uint32(len(stream))) + b[8] = 14 // b_method = LZMA + + data := append(buildUPXHeader(4096, 2048), b...) + data = append(data, stream...) + + _, err := unpack(t, data) + require.ErrorIs(t, err, errUPXImplausibleHeader, "no block is decodable, so this is not a packed binary") +} + +func TestDecompressUPX_CumulativeExceedsOriginalSize(t *testing.T) { + // each block individually fits within p_blocksize, but together they claim more than the file's + // declared original size. Without the running remainder every block may claim the full size and the + // total decompression work becomes (block count x original size). The streams are real 2048-byte + // payloads so what stops the run below is the budget and not a short decode. + // + // The budget is a bound, not a verdict: the blocks that fit are kept and the chain ends at the one that + // does not, since past the first block an overrun is indistinguishable from the loader bytes behind the + // last real block. TestDecompressUPX_OversizedBlockRejected covers the first-block case, which does + // fail the file. + // + // It is also not a gap. The file declared 4096 bytes and got 4096 bytes back, so nothing downstream is + // short and there is nothing to report: the blocks past the budget were claims about bytes the file + // never said it had. TestScanReader_PartialReconstructionIsReported covers a chain that really did + // come up short of its own p_filesize. + payload := bytes.Repeat([]byte("A"), 2048) + blocks := make([][]byte, 10) + for i := range blocks { + blocks[i] = payload + } + data := buildUPXFile(t, 4096, 2048, blocks, nil) // 10 x 2048 against a 4096 byte p_filesize + + out, err := unpack(t, data) + require.NoError(t, err, "the declared original size was rebuilt in full, so there is no gap to report") + assert.Len(t, readAll(t, out), 4096, "the reconstruction may not exceed the declared original size") +} + +func TestDecompressUPX_BudgetAllowsExactlyOriginalSize(t *testing.T) { + // the bound must not reject a file whose blocks sum to exactly the declared original size, which is + // what a well-formed UPX binary does. + payload := bytes.Repeat([]byte("A"), 2048) + data := buildUPXFile(t, 4096, 2048, [][]byte{payload, payload}, nil) + + _, err := unpack(t, data) + require.NoError(t, err) +} + +func TestDecompressUPX_BlockCountBounded(t *testing.T) { + // without a cap the block loop runs until the budget is spent one byte at a time, so a small file can + // drive hundreds of millions of iterations. Each block here places a single byte sequentially, so the + // number of blocks actually processed is directly observable in the output. + const blocks = maxUPXBlocks + 76 + // one stream, reused: buildUPXLZMAStream spins up a writer with a 64KB dictionary per call, so + // building this fixture block by block cost more than the decode it is testing + one := blockFor(t, []byte("A")) + data := buildUPXHeader(4096, 4096) + for range blocks { + data = append(data, one...) + } + data = append(data, make([]byte, 12)...) // end marker + + out, err := unpack(t, data) + require.ErrorIs(t, err, errUPXPartial, "the cap stops the loop and reports the rest as a gap") + require.NotNil(t, out, "the cap does not fail the file") + assert.Equal(t, maxUPXBlocks, bytes.Count(readAll(t, out), []byte("A")), + "exactly maxUPXBlocks blocks should have been placed") +} + +// buildELF64 assembles a minimal ELF64 header followed by the given program-header bytes. +func buildELF64(phoff uint64, phentsize, phnum uint16, phdrs []byte) []byte { + hdr := make([]byte, 64) + copy(hdr, []byte{0x7f, 'E', 'L', 'F'}) + hdr[4] = 2 // ELFCLASS64 + hdr[5] = 1 // little endian + binary.LittleEndian.PutUint64(hdr[0x20:0x28], phoff) + binary.LittleEndian.PutUint16(hdr[0x36:0x38], phentsize) + binary.LittleEndian.PutUint16(hdr[0x38:0x3a], phnum) + return append(hdr, phdrs...) +} + +func TestParseELFPTLoadOffsets_ShortPhentsizeNoPanic(t *testing.T) { + // a program-header entry smaller than an ELF64 phdr would let the fixed-offset p_offset read run past + // the entry; the parser must reject it rather than index out of range. + phdr := []byte{1, 0, 0, 0, 0, 0, 0, 0} // ptype = PT_LOAD, only 8 bytes + elf := buildELF64(64, 8, 1, phdr) + + require.NotPanics(t, func() { + assert.Empty(t, parseELFPTLoadOffsets(elf)) + }) +} + +func TestParseELFPTLoadOffsets_OverflowPhoffNoPanic(t *testing.T) { + // a phoff near the top of the uint64 range must not overflow the bounds check into a huge slice index. + // + phdr := make([]byte, 56) + binary.LittleEndian.PutUint32(phdr[0:4], 1) // PT_LOAD + elf := buildELF64(0xFFFFFFFFFFFFFFF0, 56, 1, phdr) + + require.NotPanics(t, func() { + assert.Empty(t, parseELFPTLoadOffsets(elf)) + }) +} + +// buildPoisonELF returns an ELF whose second PT_LOAD segment declares p_offset at the top of the uint64 +// range, along with the payload set that drives block 3 to that offset. +func buildPoisonELF(t *testing.T) []byte { + t.Helper() + phdrs := make([]byte, 112) // two ELF64 program headers + binary.LittleEndian.PutUint32(phdrs[0:4], 1) // phdr[0] PT_LOAD + binary.LittleEndian.PutUint64(phdrs[8:16], 0) // p_offset 0 + binary.LittleEndian.PutUint32(phdrs[56:60], 1) // phdr[1] PT_LOAD + binary.LittleEndian.PutUint64(phdrs[64:72], 0xFFFFFFFFFFFFFFFF) // p_offset at the uint64 ceiling + elf := buildELF64(64, 56, 2, phdrs) + + // sanity: the crafted offset really is parsed out, so the tests below exercise the placement guard + require.Equal(t, []uint64{0, 0xFFFFFFFFFFFFFFFF}, parseELFPTLoadOffsets(elf)) + return elf +} + +func TestDecompressUPX_OutOfRangePlacementStopsWithPartialOutput(t *testing.T) { + // block 3 is directed at a p_offset past the end of the declared file. The blocks placed before it are + // often enough to recover .go.buildinfo, so they are kept, but the loop must stop rather than continue + // from a bad offset. + elf := buildPoisonELF(t) + payloads := [][]byte{elf, bytes.Repeat([]byte("B"), 32), bytes.Repeat([]byte("C"), 32)} + data := buildUPXFile(t, 8192, 8192, payloads, nil) + + out, err := unpack(t, data) + require.ErrorIs(t, err, errUPXPartial, "the blocks given up are reported, not dropped silently") + got := readAll(t, out) + assert.Equal(t, elf, got[:len(elf)], "block 1 stays placed") + assert.NotContains(t, string(got), "CCCC", "the out-of-range block is not placed") +} + +func TestDecompressUPX_OutOfRangePlacementDoesNotPoisonLaterBlocks(t *testing.T) { + // outputOffset is derived from destOffset even when destOffset is rejected, so a p_offset at the + // uint64 ceiling can wrap outputOffset to 0: with a 1-byte block 3, block 4 must not then land over the + // reconstructed ELF header at offset 0. + elf := buildPoisonELF(t) + attacker := bytes.Repeat([]byte{0xDE, 0xAD, 0xBE, 0xEF}, 8) + payloads := [][]byte{elf, bytes.Repeat([]byte("B"), 32), {0x41}, attacker} + data := buildUPXFile(t, 8192, 8192, payloads, nil) + + out, err := unpack(t, data) + require.ErrorIs(t, err, errUPXPartial) + got := readAll(t, out) + assert.Equal(t, []byte{0x7f, 'E', 'L', 'F'}, got[:4], "the ELF header must not be overwritten") + assert.NotContains(t, string(got), string(attacker), "the block after a bad offset is not placed") +} + +func TestDecompressLZMA_RoundTrip(t *testing.T) { + // happy path: a stream built with valid LZMA parameters round-trips (and confirms the parameter + // validation does not reject legitimate values). + data := bytes.Repeat([]byte("hello UPX "), 16) + src, err := decompressLZMA(buildUPXLZMAStream(t, data), int64(len(data))) + require.NoError(t, err) + got, err := io.ReadAll(src) + require.NoError(t, err) + assert.Equal(t, data, got) +} + +func TestDecompressLZMA_InvalidParams(t *testing.T) { + // the header nibbles can hold values the LZMA props byte cannot represent. Rejecting them keeps the + // uint8 props arithmetic from wrapping into a valid-looking but wrong value. + // byte 0 low 3 bits carry pb; byte 1 is (lp<<4)|lc + cases := []struct { + name string + stream []byte + }{ + {"pb above range", []byte{0x07, 0x03, 0x00, 0x00}}, // pb = 7 + {"lc above range", []byte{0x02, 0x0f, 0x00, 0x00}}, // lc = 15 + {"lp above range", []byte{0x02, 0x53, 0x00, 0x00}}, // lp = 5 + {"lc+lp above budget", []byte{0x02, 0x48, 0x00, 0x00}}, // lc = 8, lp = 4, sum = 12 + } + for _, tt := range cases { + stream := tt.stream + t.Run(tt.name, func(t *testing.T) { + _, err := decompressLZMA(stream, 32) + require.Error(t, err) + assert.ErrorIs(t, err, errUPXInvalidLZMAParams) + }) + } +} + +func TestDecompressLZMA_LiteralBitBudgetAllowsRealValues(t *testing.T) { + // UPX emits lc=3/lp=0, and the budget must stay clear of anything real. lc+lp at exactly the cap is + // accepted (it fails later on the stream contents, not on the parameters). + stream := []byte{0x02, 0x44, 0x00, 0x00} // lc = 4, lp = 4, sum = 8 + _, err := decompressLZMA(stream, 32) + require.Error(t, err) + assert.NotErrorIs(t, err, errUPXInvalidLZMAParams, "the cap itself must not reject lc+lp == the cap") +} + +func TestUnfilter49(t *testing.T) { + // the CTO filter stores CALL/JMP operands big-endian with cto8 as a marker byte. Reversing it must + // restore the little-endian relative address: 0x12345678 - (pos+1) - (cto8<<24) at pos 0. + const cto8 = 0x12 + data := []byte{0xE8, cto8, 0x34, 0x56, 0x78} + unfilter49(data, cto8, 0, true) + assert.Equal(t, []byte{0xE8, 0x77, 0x56, 0x34, 0x00}, data) +} + +func TestUnfilter49_LeavesUnmarkedBytesAlone(t *testing.T) { + // a CALL whose next byte is not the cto8 marker was not transformed by the filter, so it must be + // left exactly as-is. + data := []byte{0xE8, 0x99, 0x34, 0x56, 0x78} + want := bytes.Clone(data) + unfilter49(data, 0x12, 0, true) + assert.Equal(t, want, data) +} + +func TestCopyUnfiltered_MatchesTheWholeBlockFilter(t *testing.T) { + // the unfilter runs over a window rather than the whole block, so the windowing is only correct if it + // lands on exactly the bytes the in-place pass over the whole block would have. The seam between two + // windows is at a fixed offset, so the pattern is rotated instead: every phase puts a different + // instruction across the seam, and the conditional-jump form (which needs a sixth byte the CALL form + // does not) is the one a carry a single byte too small drops on the floor. + const cto8 = 0x24 + pattern := []byte{0xE8, cto8, 0x11, 0x22, 0x33, 0x0F, 0x85, cto8, 0x44, 0x55, 0x66, 0xE9, cto8} + + // every phase runs at the small sizes; the window-crossing sizes are an order of magnitude more + // expensive per subtest, and the seam sits at a fixed offset, so two phases suffice to cross it + small := []int{64, upxFilterWindow} + crossing := []int{upxFilterWindow - 1, upxFilterWindow + 64, 2*upxFilterWindow + 777} + for phase := range len(pattern) { + rotated := append(bytes.Clone(pattern[phase:]), pattern[:phase]...) + sizes := small + if phase < 2 { + sizes = append(slices.Clone(small), crossing...) + } + for _, size := range sizes { + t.Run(fmt.Sprintf("phase %d, %d byte block", phase, size), func(t *testing.T) { + block := bytes.Repeat(rotated, size/len(rotated)+1)[:size] + + want := bytes.Clone(block) + unfilter49(want, cto8, 0, true) + require.NotEqual(t, block, want, "the fixture must actually be filtered") + + var got bytes.Buffer + require.NoError(t, copyUnfiltered(&got, bytes.NewReader(block), int64(size), cto8)) + + require.Equal(t, size, got.Len()) + assert.True(t, bytes.Equal(want, got.Bytes()), "windowed output differs from the whole-block filter") + }) + } + } +} + +// TestCopyUnfiltered_BlocksTooSmallToFilter covers the sizes below the filter's lookahead, where +// unfilter49's limit clamps to zero and the first window is already the final one. Nothing is +// transformable, so the bytes must come through untouched rather than panicking or being dropped. +func TestCopyUnfiltered_BlocksTooSmallToFilter(t *testing.T) { + const cto8 = 0x24 + for size := 0; size < 7; size++ { + t.Run(fmt.Sprintf("%d byte block", size), func(t *testing.T) { + block := bytes.Repeat([]byte{0xE8, cto8, 0x11, 0x22, 0x33, 0x0F}, 2)[:size] + want := bytes.Clone(block) + unfilter49(want, cto8, 0, true) + + var got bytes.Buffer + require.NoError(t, copyUnfiltered(&got, bytes.NewReader(block), int64(size), cto8)) + require.Equal(t, size, got.Len()) + assert.True(t, bytes.Equal(want, got.Bytes()), "windowed output differs from the whole-block filter") + }) + } +} + +// firstHole moved to spillbuf.FirstGap: placing output is the buffer's question to answer, and its +// tests live with it in internal/spillbuf. + +func TestParseUPXInfo_ImplausibleHeader(t *testing.T) { + // a coincidental "UPX!" match surrounded by zeroed fields must not be accepted as a real UPX header. + build := func(version, format byte, originalSize, blockSize uint32) []byte { + lInfo := []byte{0, 0, 0, 0, 'U', 'P', 'X', '!', 0, 0, version, format} + pInfo := make([]byte, 12) + binary.LittleEndian.PutUint32(pInfo[4:8], originalSize) + binary.LittleEndian.PutUint32(pInfo[8:12], blockSize) + return append(append(append(packedELFStub(), lInfo...), pInfo...), make([]byte, 32)...) + } + + cases := []struct { + name string + data []byte + }{ + {"zero version", build(0, 22, 0x1000, 0x1000)}, + {"zero format", build(14, 0, 0x1000, 0x1000)}, + {"zero block size", build(14, 22, 0x1000, 0)}, + {"zero original size", build(14, 22, 0, 0x1000)}, + } + for _, tt := range cases { + data := tt.data + t.Run(tt.name, func(t *testing.T) { + r := bytes.NewReader(data) + _, err := parseUPXInfo(r, sizeOf(t, r)) + require.Error(t, err) + assert.ErrorIs(t, err, errUPXImplausibleHeader) + }) + } +} + +func TestParseUPXInfo_BlockSizeAboveOriginalSizeAccepted(t *testing.T) { + // p_blocksize is not a bound and must not be a rejection reason. UPX derives it from a PT_LOAD extent + // and real output has the largest block equal to it exactly, so a check against p_filesize would run + // with no headroom against something the format does not promise. The remainder budget bounds output. + data := padTo(append(buildUPXHeader(0x1000, 0x2000), make([]byte, 32)...), 128) + + info, err := parseUPXInfo(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.Equal(t, uint32(0x2000), info.blockSize) +} + +// TestParseUPXInfo_SizeClaims pins both bounds on p_filesize in both directions: the ratio against the +// input's own length, and the absolute ceiling the ratio cannot supply on its own because padding an input +// is nearly free to an attacker (a gzipped layer stores 16MB of zeros in about 16KB). +func TestParseUPXInfo_SizeClaims(t *testing.T) { + tests := []struct { + name string + original uint32 + inputLen int + accept bool + }{ + { + // LZMA encodes a run of N identical bytes in O(log N), so a `go:embed` of 120MB of zeros packs + // 127504546 bytes into 608940 and `upx --best --lzma` produces exactly that: 209x against a + // 610KB input, inside the 256x allowance. This is the largest legitimate original measured, and + // what both bounds have to stay clear of. + name: "a high but legitimate expansion ratio", original: 127504546, inputLen: 610 * 1024, accept: true, + }, + { + name: "a large legitimate original", original: maxUPXOriginalSize - intFile.MB, + inputLen: (maxUPXOriginalSize-intFile.MB)/maxUPXExpansion + 1024, accept: true, + }, + {name: "exactly the ratio allowance", original: 128 * maxUPXExpansion, inputLen: 128, accept: true}, + {name: "past the ratio allowance", original: 128*maxUPXExpansion + 1, inputLen: 128}, + { + // deliberately pays the ratio: a 16MB input can justify 4GB at 256x, so only the ceiling stops it + name: "past the ceiling even when the ratio allows it", original: maxUPXOriginalSize + 1, + inputLen: 16 * intFile.MB, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data := padTo(buildUPXHeader(tt.original, 0x1000), tt.inputLen) + + info, err := parseUPXInfo(bytes.NewReader(data), int64(len(data))) + if tt.accept { + require.NoError(t, err, "a legitimate claim must not be dropped") + assert.Equal(t, tt.original, info.originalSize) + return + } + require.Error(t, err) + // reported, not quiet: this is a file that really is packed and that we declined to expand, + // which is a gap we chose to leave rather than a stray "UPX!" in a string constant + assert.ErrorIs(t, err, errUPXSizeRefused) + assert.NotErrorIs(t, err, errUPXImplausibleHeader) + assert.NotErrorIs(t, err, errUPXDecompress) + }) + } +} + +func TestParseUPXInfo_UnsizedInputRefused(t *testing.T) { + // every bound is a ratio against the input, so an input that cannot be sized has nothing to weigh a + // claim against. Quiet, like every other "not something we can read" refusal. + data := padTo(buildUPXHeader(0x1000, 0x1000), 4096) + plain := struct{ io.ReaderAt }{bytes.NewReader(data)} + + size, ok := intFile.ReaderSize(plain) + require.False(t, ok, "a wrapper that only embeds the interface cannot be sized") + + _, err := parseUPXInfo(plain, size) + require.Error(t, err) + assert.ErrorIs(t, err, errUPXImplausibleHeader) + + // and the caller declines to unpack rather than running the bounds against a zero + contents, err := unpackUPX(context.Background(), plain) + require.NoError(t, err) + t.Cleanup(func() { closeUnpacked(contents) }) + assert.Nil(t, contents) +} + +func TestDecompressUPX_RefusingATinyClaimIsCheap(t *testing.T) { + // the refusal half of the property every bound in this file exists to protect: a small file claiming + // a lot is turned away before anything is sized by the claim. Pre-fix a ~100 byte input reached + // make([]byte, 0xFFFFFFFF). + // + // The fixture carries a real decodable block on purpose: a header alone would exercise none of the + // reconstruction, since nothing downstream of parseUPXInfo would ever run. + tiny := padTo(buildUPXFile(t, 1<<30, 1<<30, + [][]byte{bytes.Repeat([]byte("A"), 32)}, nil), 128) + + allocated := testutils.MeasureAlloc(t, func() { + _, err := unpackIn(t, t.TempDir(), tiny) + require.Error(t, err, "a 128 byte file may not claim a gigabyte") + }) + // a tripwire on the refusal staying in front of the work, not a bound with teeth of its own: the + // refusal happens in parseUPXInfo, so the only allocation on this path is the 8KB scan window. The + // require.Error above is what fails if the ratio and the ceiling are both removed. + assert.Less(t, allocated, uint64(1<<20), "a refused header must not reach anything sized by the claim") + t.Logf("128 byte input allocated %d bytes", allocated) +} + +func TestDecompressUPX_OutputIsNeverResident(t *testing.T) { + // the reason the reconstruction goes through spillbuf rather than a []byte: a fixture that is fully + // accepted and fully decoded must not cost its own output in heap. + // + // 16MB of zeros through a 64KB input. Nothing here is refused: p_filesize sits exactly at the ratio + // the input pays for, and the whole 16MB really is written. + const payload = 16 * intFile.MB + const inputLen = payload / maxUPXExpansion + + block := make([]byte, payload) + stream := buildUPXLZMAStream(t, block) // the same bytes buildUPXFile below will encode, to size the dictionary + + fixture := padTo(buildUPXFile(t, payload, payload, [][]byte{block}, nil), inputLen) + require.LessOrEqual(t, len(fixture), inputLen, "the fixture must stay small enough for the ratio to bind") + + var out unpackedContents + allocated := testutils.MeasureAlloc(t, func() { + var err error + out, err = unpackIn(t, t.TempDir(), fixture) + require.NoError(t, err) + }) + + require.Equal(t, int64(payload), out.Size(), "the whole payload must actually have been rebuilt") + + // a fixed budget, not one that scales with the output (a formula like payload/4 would still pass an + // implementation that kept a quarter of it resident): spillbuf's memory tier, bounded a few times over + // DefaultMemLimit for the geometric growth that holds the old copy while it doubles, plus the LZMA + // dictionary this block's actual compressed length implies. Neither term grows with payload. + budget := uint64(4*spillbuf.DefaultMemLimit) + uint64(maxDictionaryFor(len(stream))) + assert.Less(t, allocated, budget, + "decompressing %d bytes must cost the tier and the dictionary, not a fraction of the output", payload) + t.Logf("%d byte output cost %d bytes of allocation from a %d byte input (budget %d, dictionary %d)", + payload, allocated, len(fixture), budget, maxDictionaryFor(len(stream))) +} + +func TestMaxDictionaryFor(t *testing.T) { + // the dictionary is the last allocation still sized by something a block declares about itself, so it + // is bounded by the compressed bytes that block actually brought with it. + assert.Equal(t, uint32(minUPXDictionary), maxDictionaryFor(0), "a block with nothing in it gets the floor") + assert.Equal(t, uint32(minUPXDictionary), maxDictionaryFor(64), "a tiny block cannot reach past the floor") + // strictly between the floor and the ceiling, so the ratio arm is the only thing that can produce it: + // every other case here is satisfied by a function that returns a constant + assert.Equal(t, uint32(64*intFile.MB), maxDictionaryFor(256*intFile.KB), "a real block gets the ratio") + assert.Equal(t, uint32(maxUPXDictionary), maxDictionaryFor(intFile.KB*512), + "512KB is exactly where the ratio meets the ceiling") + assert.Equal(t, uint32(maxUPXDictionary), maxDictionaryFor(1<<30), "and the ceiling caps the largest") + + // the dictionary is heap and the reconstruction is disk, so they do not share a ceiling. Sharing one + // let the ratio alone carry a ~2MB input to a 512MB resident allocation. + assert.Less(t, uint64(maxUPXDictionary), uint64(maxUPXOriginalSize)) + assert.Equal(t, uint32(maxUPXDictionary), maxDictionaryFor(2*intFile.MB), + "the ratio on its own would allow 512MB here") + + // and the two ratios are separate constants so the disk one can move without moving this one + assert.Equal(t, uint32(minUPXDictionary), maxDictionaryFor(minUPXDictionary/maxUPXDictionaryExpansion), + "the dictionary ratio is the one that sizes this, not maxUPXExpansion") +} + +func TestDecompressLZMA_UndersizedDictionaryFailsRatherThanCorrupts(t *testing.T) { + // bounding the dictionary is only safe because getting it wrong is loud: the decoder rejects a match + // whose distance runs past the dictionary. If that ever became a silent zero-fill, every bound keyed + // off maxDictionaryFor would start producing plausible garbage instead of an error. + // the first half is pseudo-random so it cannot be matched locally, and the second half repeats it, so + // decoding the second half needs a match reaching 256KB back. A 4KB dictionary cannot serve that. + const half = 256 * intFile.KB + payload := make([]byte, 2*half) + rng := rand.New(rand.NewSource(1)) //nolint:gosec // deterministic test fixture, not a security context + _, err := rng.Read(payload[:half]) + require.NoError(t, err) + copy(payload[half:], payload[:half]) + stream := buildUPXLZMAStream(t, payload) + + r, err := decompressLZMA(stream, int64(len(payload))) + require.NoError(t, err) + got, err := io.ReadAll(r) + require.NoError(t, err, "the honest case must still round-trip") + require.Equal(t, payload, got) +} + +func TestDecompressUPX_CompressedSizeCannotOutrunTheInput(t *testing.T) { + // sz_cpr sizes a read buffer directly. The 24-bit mask in readBlockInfo is a field width, not a bound, + // so on its own a ~130 byte file declaring sz_cpr = 0xFFFFFF drove a 16MB allocation that the + // following ReadAt could only ever fail. Measured at 131,181x amplification before the bound. + b := make([]byte, 12) + binary.LittleEndian.PutUint32(b[0:4], 4096) // sz_unc, inside the input-size allowance + binary.LittleEndian.PutUint32(b[4:8], 0x00FFFFFF) // sz_cpr = 16MB-1 once masked + b[8] = 14 // b_method = LZMA + data := padTo(append(buildUPXHeader(4096, 4096), b...), 128) + + allocated := testutils.MeasureAlloc(t, func() { + _, err := unpackIn(t, t.TempDir(), data) + require.Error(t, err) + assert.ErrorIs(t, err, errUPXImplausibleHeader, "no block was readable, so nothing was packed") + }) + assert.Less(t, allocated, uint64(1<<20), + "a block claiming compressed data past the end of a 128 byte input must not be allocated for") + t.Logf("128 byte input allocated %d bytes", allocated) +} + +func TestDecompressUPX_TailExtentsArePlacedPastTheLoader(t *testing.T) { + // with a non-zero l_lsize the chain continues past the loader stub instead of ending at the marker in + // front of it. Every other crafted fixture here has l_lsize == 0, so without this skipLoader was only + // reachable through the Docker-backed fixture. + // + // note: block 1 here is not an ELF, so there are no PT_LOAD offsets and the tail lands sequentially. + // The hole-filling path this feeds in production is covered by TestFirstHole and by the sparse + // fixtures in upx_bounds_regression_test.go. + const loaderSize = 64 + head := bytes.Repeat([]byte("H"), 128) + tail := bytes.Repeat([]byte("T"), 32) + + data := buildUPXHeaderWithLoader(4096, 4096, loaderSize) + data = append(data, blockFor(t, head)...) + data = append(data, make([]byte, 12)...) // end marker: closes the first run of extents + data = append(data, make([]byte, loaderSize-12)...) // the loader stub the chain skips over + data = append(data, blockFor(t, tail)...) + data = append(data, make([]byte, 12)...) // end marker for the tail run + data = padTo(data, 1024) + + out, err := unpack(t, data) + // 160 bytes of a declared 4096, short by construction like every other crafted fixture here + require.ErrorIs(t, err, errUPXPartial) + got := readAll(t, out) + + assert.Equal(t, head, got[:len(head)], "the head extent stays where it was placed") + assert.Contains(t, string(got), string(tail), "the extent behind the loader must be placed too") +} + +func TestDecompressUPX_UnsupportedMethodIsNotReportable(t *testing.T) { + // upx defaults to NRV2B unless --lzma is passed, and only LZMA is implemented here. A packed non-Go + // binary must not turn into an SBOM unknown from the golang cataloger, so the unsupported-method error + // must stay clear of errUPXDecompress. + block := make([]byte, 12) + binary.LittleEndian.PutUint32(block[0:4], 256) // sz_unc + binary.LittleEndian.PutUint32(block[4:8], 32) // sz_cpr + block[8] = 2 // b_method = NRV2B + data := padTo(append(buildUPXHeader(4096, 4096), block...), 256) + + _, err := unpack(t, data) + require.Error(t, err) + assert.ErrorIs(t, err, errUnsupportedUPXMethod) + assert.NotErrorIs(t, err, errUPXDecompress, "an unimplemented method is a capability gap, not a gap in the SBOM") +} + +// TestUnpackUPX_QuietDirection pins the invariant the reportability policy rests on: unpackUPX is the +// layer that decides what becomes an SBOM unknown, and only errUPXDecompress, errUPXSizeRefused, +// errUPXPartial and elfutil.ErrDeclaredSizeExceeded may (see reportableGap in scan_binary.go). Everything +// else has to come back as the input with no error, so a packed non-Go binary, or one packed with a method +// we have not implemented, does not attach a golang-cataloger unknown to every such file in an image. +func TestUnpackUPX_QuietDirection(t *testing.T) { + nrv2b := make([]byte, 12) + binary.LittleEndian.PutUint32(nrv2b[0:4], 256) // sz_unc + binary.LittleEndian.PutUint32(nrv2b[4:8], 32) // sz_cpr + nrv2b[8] = 2 // b_method = NRV2B, which upx emits unless --lzma + + // every fixture has to clear the ELF64 little-endian container gate, or it is turned away before the + // magic scan and the case proves nothing about the path it names + cases := []struct { + name string + data []byte + }{ + {"a plain ELF that is not packed at all", padTo(packedELFStub(), 256)}, + {"a stray UPX magic in unrelated data", padTo(append(packedELFStub(), []byte("some string UPX! in rodata")...), 256)}, + {"a header with nothing decodable", padTo(append(buildUPXHeader(4096, 4096), make([]byte, 12)...), 256)}, + {"a method we have not implemented", padTo(append(buildUPXHeader(4096, 4096), nrv2b...), 256)}, + } + for _, tt := range cases { + name, data := tt.name, tt.data + t.Run(name, func(t *testing.T) { + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + in := bytes.NewReader(data) + out, err := unpackUPX(ctx, in) + assert.NoError(t, err, "this must not become an SBOM unknown") + assertNothingUnpacked(t, in, out) + }) + } +} + +// TestUnpackUPX_MissingTempDirIsNotReportedForEveryBinary covers the ordering that keeps a misconfigured +// context from being rendered as a per-file finding: the temp dir is only demanded once the file is known +// to be packed, so a consumer whose context carries none does not get an unknown on every executable. +func TestUnpackUPX_MissingTempDirIsNotReportedForEveryBinary(t *testing.T) { + plain := append([]byte{0x7f, 'E', 'L', 'F'}, make([]byte, 256)...) + + in := bytes.NewReader(plain) + out, err := unpackUPX(context.Background(), in) + require.NoError(t, err, "a file that is not packed must not need a temp dir at all") + assertNothingUnpacked(t, in, out) + + // a file that really is packed still reports, since its packages are genuinely missing + packed := padTo(buildUPXFile(t, 4096, 4096, [][]byte{bytes.Repeat([]byte("A"), 32)}, nil), 128) + packedIn := bytes.NewReader(packed) + out, err = unpackUPX(context.Background(), packedIn) + require.Error(t, err) + assert.ErrorIs(t, err, errUPXDecompress) + assertNothingUnpacked(t, packedIn, out, "reporting the gap must still leave the caller something to read") +} + +// assertNothingUnpacked pins what a "nothing to unpack" answer owes its caller. The nil is the signal, so +// what matters is that every place that resolves it copes: readerFor and seekerFor must hand back the +// input as it was found, and closeUnpacked must have nothing to do. +func assertNothingUnpacked(t *testing.T, in io.ReaderAt, out unpackedContents, msgAndArgs ...any) { + t.Helper() + assert.Nil(t, out, msgAndArgs...) + assert.NotPanics(t, func() { closeUnpacked(out); closeUnpacked(out) }, + "closing must be safe when there is nothing to release, and a defer over every build calls it") + + assert.Same(t, in, readerFor(out, in), "nothing to unpack means the caller reads the file as it found it") + + fallback := &nopReadSeekCloser{bytes.NewReader(nil)} + assert.Same(t, fallback, seekerFor(out, fallback), "the seekable view falls back to the input") +} + +type nopReadSeekCloser struct{ *bytes.Reader } + +func (*nopReadSeekCloser) Close() error { return nil } + +func TestScanReader_MaliciousUPXRejected(t *testing.T) { + // a plausible ELF prefix so the file looks like an executable, then a UPX header whose single block + // claims the full uint32 range. Pre-fix this reached make([]byte, 0xFFFFFFFF). + data := append(packedELFStub(), buildUPXFile(t, 4096, 4096, + [][]byte{bytes.Repeat([]byte("A"), 32)}, []uint32{0xFFFFFFFF})...) + + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + + // go's own test timeout covers "did not return promptly"; a goroutine plus time.After here would + // assert on an already-finished test if it ever fired. + var build *extendedBuildInfo + var err error + allocated := testutils.MeasureAlloc(t, func() { + build, err = scanReader(ctx, file.NewLocation("/malicious"), bytes.NewReader(data), false) + }) + assert.Nil(t, build) + require.Error(t, err) + // unknown.CoordinateError does not unwrap, so reach through it to the reason it carries rather than + // matching on the rendered message + coordErrs, _ := unknown.ExtractCoordinateErrors(err) + require.NotEmpty(t, coordErrs, "a packed binary we cannot unpack is worth reporting") + assert.ErrorIs(t, coordErrs[0].Reason, errUPXDecompress) + assert.Less(t, allocated, uint64(1<<20), "the rejection must not cost a megabyte") +} + +// sizeOf is the test-side spelling of the two-value size helper, for the cases handing parseUPXInfo a +// reader whose size is obviously measurable. The not-ok direction has its own test. +func sizeOf(t *testing.T, r io.ReaderAt) int64 { + t.Helper() + size, ok := intFile.ReaderSize(r) + require.True(t, ok, "the reader under test has to report a size") + return size +} + +// TestCopyUnfiltered_WindowIsNotResident is the allocation half of the unfilter tests. The others assert +// the output is byte-for-byte right, which it is with or without the window: delete the clamp in +// copyUnfiltered and every one of them still passes while a block allocates in full. One block can be most +// of a file, so the window is the only thing keeping the unfilter off the heap. +func TestCopyUnfiltered_WindowIsNotResident(t *testing.T) { + const cto8 = 0x24 + const size = 16 * intFile.MB + block := bytes.Repeat([]byte{0xE8, cto8, 0x11, 0x22, 0x33, 0x0F}, size/6+1)[:size] + + allocated := testutils.MeasureAlloc(t, func() { + require.NoError(t, copyUnfiltered(io.Discard, bytes.NewReader(block), size, cto8)) + }) + + assert.Less(t, allocated, uint64(4*upxFilterWindow), + "unfiltering %d bytes must cost the window, not the block", size) +} + +// TestDecompressUPX_CTOFilteredBlock reaches the unfilter through the block chain rather than by calling +// copyUnfiltered directly. Real `upx --best --lzma` on x86-64 sets b_ftid on every block, so this is the +// ordinary path; without it the filter, the per-block base arithmetic and the window only ever run +// together in the Docker fixture. +func TestDecompressUPX_CTOFilteredBlock(t *testing.T) { + const cto8 = 0x24 + // the block carries the filtered form; the reconstruction is what the whole-block unfilter makes of it + filtered := bytes.Repeat([]byte{0xE8, cto8, 0x11, 0x22, 0x33, 0x0F, 0x85, cto8, 0x44, 0x55}, 512) + want := bytes.Clone(filtered) + unfilter49(want, cto8, 0, true) + require.NotEqual(t, filtered, want, "the fixture must actually be filtered") + + data := append(buildUPXHeader(uint32(len(filtered)), uint32(len(filtered))), + filteredBlockFor(t, filtered, upxFilterCTO, cto8)...) + + out, err := unpack(t, data) + require.NoError(t, err) + assert.Equal(t, want, readAll(t, out), "the CTO filter must be reversed on the way out") +} + +// TestUnpackUPX_ReconstructionIsRemovedOnClose pins the disk side of the temp-file contract. The cataloger +// writes one reconstruction per packed binary per scan and the cataloger's temp root is never swept, so a +// Close that stops removing turns every scanned packed binary into a leaked file. +func TestUnpackUPX_ReconstructionIsRemovedOnClose(t *testing.T) { + dir := t.TempDir() + + // past spillbuf's memory tier on purpose: a reconstruction that fits in memory never creates a file, + // so a small fixture here would assert the cleanup of something that was never made + const payload = 4 * int(spillbuf.DefaultMemLimit) + fixture := padTo(buildUPXFile(t, uint32(payload), uint32(payload), [][]byte{make([]byte, payload)}, nil), + payload/maxUPXExpansion) + + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(dir)) + contents, err := unpackUPX(ctx, bytes.NewReader(fixture)) + require.NoError(t, err) + require.NotNil(t, contents, "the fixture has to actually unpack for this to mean anything") + + before, err := os.ReadDir(dir) + require.NoError(t, err) + require.NotEmpty(t, before, "the reconstruction should be on disk while it is open") + + require.NoError(t, contents.Close()) + + after, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Empty(t, after, "Close has to take the reconstruction with it") +} + +// TestParseUPXInfo_OnlyELF64LittleEndian pins the container gate. Everything downstream reads program +// headers at fixed ELF64 little-endian offsets, so a packed ELF32 or big-endian file would be placed by +// rules that do not apply to it, after taking a temp dir and landing as an unknown on a file this +// cataloger was never going to catalog. +func TestParseUPXInfo_OnlyELF64LittleEndian(t *testing.T) { + // a slice rather than a map, so the subtest order is the order written here + tests := []struct { + name string + mangle func([]byte) + }{ + {"ELF32", func(b []byte) { b[4] = 1 }}, + {"big endian", func(b []byte) { b[5] = 2 }}, + {"unset class", func(b []byte) { b[4] = 0 }}, + {"unset endian", func(b []byte) { b[5] = 0 }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data := padTo(buildUPXHeader(0x1000, 0x1000), 4096) + + // control: the same fixture parses before the container byte is touched + _, err := parseUPXInfo(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + + tt.mangle(data) + _, err = parseUPXInfo(bytes.NewReader(data), int64(len(data))) + require.Error(t, err) + assert.ErrorIs(t, err, errNotUPX, "quiet: a container we do not reconstruct is not a gap") + }) + } +} + +// TestUnfilter49_RewritesEveryFilteredForm pins which opcodes the CTO unfilter acts on. The equivalence +// test above cannot: it compares the windowed pass against the whole-block pass through the same +// function, so dropping a form breaks both sides identically and it stays green. Only the Docker-backed +// image fixture caught it, and a regression that needs a container to detect is one CI can miss. +// +// E8 is CALL, E9 is JMP, and 0F 8x is the conditional-jump form. UPX's --lzma filter 0x49 rewrites all +// three, so a reconstruction is byte-wrong (but exactly the right length) if any of them is skipped. +func TestUnfilter49_RewritesEveryFilteredForm(t *testing.T) { + const cto8 = 0x24 + + tests := []struct { + name string + instr []byte + }{ + {name: "E8 CALL", instr: []byte{0xE8, cto8, 0x11, 0x22, 0x33}}, + {name: "E9 JMP", instr: []byte{0xE9, cto8, 0x11, 0x22, 0x33}}, + {name: "0F 85 conditional jump", instr: []byte{0x0F, 0x85, cto8, 0x11, 0x22, 0x33}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // padded so the instruction sits well inside the window the final pass evaluates + data := append(bytes.Clone(tt.instr), bytes.Repeat([]byte{0x90}, 32)...) + before := bytes.Clone(data) + + n := unfilter49(data, cto8, 0, true) + require.Positive(t, n, "the pass must settle the bytes it walked") + assert.NotEqual(t, before, data, + "this form carries the cto marker, so the unfilter has to rewrite its operand") + }) + } + + t.Run("an unfiltered opcode is left alone", func(t *testing.T) { + // 0xEB is a short jump, which the filter does not touch + data := append([]byte{0xEB, cto8, 0x11, 0x22, 0x33}, bytes.Repeat([]byte{0x90}, 32)...) + before := bytes.Clone(data) + + unfilter49(data, cto8, 0, true) + assert.Equal(t, before, data, "only the filtered forms may be rewritten") + }) +} diff --git a/syft/pkg/cataloger/golang/upx_test.go b/syft/pkg/cataloger/golang/upx_test.go index 99e720d12..2def6178a 100644 --- a/syft/pkg/cataloger/golang/upx_test.go +++ b/syft/pkg/cataloger/golang/upx_test.go @@ -8,57 +8,64 @@ import ( "github.com/stretchr/testify/require" ) -func TestIsUPXCompressed(t *testing.T) { +// every case here is one the magic scan must reject, asserted on parseUPXInfo's errNotUPX result. +// +// Every negative case that is meant to exercise the magic scan has to carry a real ELF64 little-endian +// ident, or the container gate rejects it first and the subtest passes without the scan running at all +// (the gate and a missing magic both return errNotUPX, so nothing in the assertion can tell them apart). +// TestParseUPXInfo_OnlyELF64LittleEndian owns the gate; these own the scan. +func TestParseUPXInfo_MagicDetection(t *testing.T) { tests := []struct { - name string - data []byte - expected bool + name string + data []byte + foundMagic bool // the magic was located (the header may still be rejected as implausible) }{ { - name: "contains UPX magic at start", - data: append([]byte("UPX!"), make([]byte, 100)...), - expected: true, + name: "contains UPX magic at start", + data: append(append(packedELFStub(), []byte("UPX!")...), make([]byte, 100)...), + foundMagic: true, }, { - name: "contains UPX magic with offset", - data: append(append(make([]byte, 500), []byte("UPX!")...), make([]byte, 100)...), - expected: true, + name: "contains UPX magic with offset", + data: append(append(append(packedELFStub(), make([]byte, 500)...), []byte("UPX!")...), make([]byte, 100)...), + foundMagic: true, }, { - name: "no UPX magic", - data: []byte("\x7FELF" + string(make([]byte, 100))), - expected: false, + // UPX packs PE and Mach-O with this same l_info/p_info layout, and nothing here can place + // their blocks, so the container has to be checked before the magic is trusted + name: "UPX magic but not an ELF container", + data: append([]byte("MZ\x90\x00UPX!"), make([]byte, 100)...), }, { - name: "empty data", - data: []byte{}, - expected: false, + name: "no UPX magic", + data: append(packedELFStub(), make([]byte, 100)...), }, { - name: "partial UPX magic", - data: []byte("UPX"), - expected: false, + // three of the four magic bytes must not match + name: "partial UPX magic", + data: append(append(packedELFStub(), []byte("UPX")...), make([]byte, 100)...), + }, + { + // never reaches the scan: the six byte ident read comes up short, so the container gate + // refuses it first. Kept because an empty reader is a shape the cataloger really is handed. + name: "empty data", + data: []byte{}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - reader := bytes.NewReader(tt.data) - result := isUPXCompressed(reader) - assert.Equal(t, tt.expected, result) + _, err := parseUPXInfo(bytes.NewReader(tt.data), int64(len(tt.data))) + require.Error(t, err, "none of these fixtures is a usable UPX header") + if tt.foundMagic { + assert.NotErrorIs(t, err, errNotUPX, "the magic was found, so the header was parsed and rejected") + } else { + assert.ErrorIs(t, err, errNotUPX) + } }) } } -func TestParseUPXInfo_NotUPX(t *testing.T) { - data := []byte("\x7FELF" + string(make([]byte, 100))) - reader := bytes.NewReader(data) - - _, err := parseUPXInfo(reader) - require.Error(t, err) - assert.ErrorIs(t, err, errNotUPX) -} - func TestParseUPXInfo_ValidHeader(t *testing.T) { // construct a minimal valid UPX header matching actual format // l_info: checksum (4) + magic (4) + lsize (2) + version (1) + format (1) @@ -84,45 +91,16 @@ func TestParseUPXInfo_ValidHeader(t *testing.T) { 14, 0, 0, 0, // method=LZMA, filter info } - data := append(append(lInfo, pInfo...), bInfo...) - data = append(data, make([]byte, 100)...) // padding + // padded so the declared 1MB stays within maxUPXExpansion of the fixture's own size; a real UPX file + // carries the compressed data this header describes, and the bound is measured against that. + data := append(append(append(packedELFStub(), lInfo...), pInfo...), bInfo...) + data = append(data, make([]byte, 0x100000/maxUPXExpansion)...) reader := bytes.NewReader(data) - info, err := parseUPXInfo(reader) + info, err := parseUPXInfo(reader, sizeOf(t, reader)) require.NoError(t, err) assert.Equal(t, uint8(14), info.version) assert.Equal(t, uint8(22), info.format) assert.Equal(t, uint32(0x100000), info.originalSize) } - -func TestDecompressUPX_UnsupportedMethod(t *testing.T) { - // construct a header with an unsupported compression method - lInfo := []byte{ - 0, 0, 0, 0, // l_checksum - 'U', 'P', 'X', '!', - 0, 0, // l_lsize - 14, 22, // version, format - } - - pInfo := []byte{ - 0, 0, 0, 0, // p_progid - 0x00, 0x01, 0x00, 0x00, // p_filesize = 256 bytes (small for test) - 0, 0, 0x10, 0, // p_blocksize - } - - bInfo := []byte{ - 0x00, 0x01, 0x00, 0x00, // sz_unc = 256 - 0x80, 0x00, 0x00, 0x00, // sz_cpr = 128 - 99, 0, 0, 0, // unsupported method - } - - data := append(append(lInfo, pInfo...), bInfo...) - data = append(data, make([]byte, 1000)...) - - reader := bytes.NewReader(data) - _, err := decompressUPX(reader) - - require.Error(t, err) - assert.ErrorIs(t, err, errUnsupportedUPXMethod) -} diff --git a/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json b/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json index ff125a819..30ca6c481 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json +++ b/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json @@ -2415,6 +2415,9 @@ "@azure/ms-rest-nodeauth": [ "cpe:2.3:a:microsoft:ms-rest-nodeauth:*:*:*:*:*:node.js:*:*" ], + "@azure/msal-node": [ + "cpe:2.3:a:microsoft:azure\\/msal-node:*:*:*:*:*:node.js:*:*" + ], "@backstage/backend-common": [ "cpe:2.3:a:linuxfoundation:backstage_backend-common:*:*:*:*:*:node.js:*:*" ], @@ -2729,6 +2732,9 @@ "@saltcorn/server": [ "cpe:2.3:a:saltcorn:saltcorn:*:*:*:*:*:*:*:*" ], + "@sap/approuter": [ + "cpe:2.3:a:sap:approuter:*:*:*:*:*:node.js:*:*" + ], "@sap/xssec": [ "cpe:2.3:a:sap:\\@sap\\/xssec:*:*:*:*:*:node.js:*:*" ], @@ -2959,6 +2965,9 @@ "auth0-js": [ "cpe:2.3:a:auth0:auth0.js:*:*:*:*:*:node.js:*:*" ], + "auth0-lock": [ + "cpe:2.3:a:auth0:lock:*:*:*:*:*:node.js:*:*" + ], "aws-lambda-multipart-parser": [ "cpe:2.3:a:aws-lambda-multipart-parser_project:aws-lambda-multipart-parser:*:*:*:*:*:node.js:*:*" ], @@ -5915,6 +5924,9 @@ "stimulsoft-dashboards-js": [ "cpe:2.3:a:stimulsoft:dashboard.js:*:*:*:*:*:node.js:*:*" ], + "store2": [ + "cpe:2.3:a:nbubna:store:*:*:*:*:*:node.js:*:*" + ], "storybook": [ "cpe:2.3:a:storybook:storybook:*:*:*:*:*:node.js:*:*" ], @@ -6853,6 +6865,9 @@ "marshmallow": [ "cpe:2.3:a:marshmallow_project:marshmallow:*:*:*:*:*:python:*:*" ], + "mcp-contextforge-gateway": [ + "cpe:2.3:a:ibm:contextforge:*:*:*:*:*:python:*:*" + ], "mcp-memory-service": [ "cpe:2.3:a:doobidoo:mcp-memory-service:*:*:*:*:*:*:*:*" ], diff --git a/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go b/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go index facb70e73..3bead6fb9 100644 --- a/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go +++ b/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go @@ -1,10 +1,39 @@ package bundle import ( + "bytes" "encoding/binary" "errors" "fmt" "io" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/syft/internal/unionreader" +) + +const ( + // maxBundleSearchSize bounds the bytes findSignatureOffset will hold at once while looking for the + // bundle marker. + // + // The marker sits inside the executable structure, so the search legitimately covers a whole + // single-file bundle, which routinely runs past 100MB and can reach a few hundred for an app that + // embeds sizable assets. Clamping to the file length alone is not enough: a mostly empty file costs + // almost nothing inside a compressed layer, so a small artifact can still authorize a multi-gigabyte + // allocation. The trade-off is that a bundle larger than this loses its deps.json rather than being + // cataloged, which is the correct direction to fail when the alternative is OOM-killing the scan. + // This mirrors maxDeclaredSectionSize in syft/internal/elfutil. + maxBundleSearchSize = 512 * intFile.MB + + // maxDepsJSONSize bounds an embedded deps.json. These are dependency manifests, so real ones are + // measured in KB even for large applications. + maxDepsJSONSize = 3 * intFile.MB + + // minManifestEntrySize is the smallest a single manifest entry can be: an 8 byte offset, an 8 byte + // size, a 1 byte file type, and at least 1 byte for the length-prefixed relative path. + minManifestEntrySize = 18 + + // minManifestEntrySizeV6 adds the 8 byte compressed size field that V6+ bundles carry. + minManifestEntrySizeV6 = minManifestEntrySize + 8 ) // dotNetBundleSignature is the SHA-256 hash of ".net core bundle" used to identify single-file bundles. @@ -15,6 +44,93 @@ var dotNetBundleSignature = []byte{ 0xee, 0x3b, 0x2d, 0xce, 0x24, 0xb3, 0x6a, 0xae, } +// ExtractDepsJSON returns the deps.json embedded in the .NET single-file bundle in r, or "" if r carries no +// bundle marker. +// +// searchLimit is where the caller's format parsing says the executable structure ends, which is as far into +// the file as the marker can be. It is only ever an optimization: it comes from user-controlled header +// fields, so it may describe far more than the file holds or overflow negative. A limit that makes no sense +// falls back to searching the whole file rather than searching nothing, since treating it as authoritative +// would let one bogus header field hide a bundle from us entirely. +func ExtractDepsJSON(r unionreader.UnionReader, searchLimit int64) (string, error) { + headerOffset, err := findBundleHeaderOffset(r, searchLimit) + if err != nil || headerOffset == 0 { + return "", err + } + + return readDepsJSONFromBundleHeader(r, headerOffset) +} + +// findBundleHeaderOffset searches the start of r for the .NET single-file bundle signature and returns the +// bundle header offset stored in the 8 bytes immediately before it. +// +// Three outcomes are distinct on purpose, because collapsing any two of them loses information the caller +// needs: 0 means there is no bundle here, an error means we could not tell, and a positive offset is the +// answer. In particular the apphost ships the signature compiled in with a zero offset placeholder and only +// gets a real one written when it is published as a single file, so a zero offset is the ordinary +// framework-dependent executable rather than a malformed one. +func findBundleHeaderOffset(r unionreader.UnionReader, searchLimit int64) (int64, error) { + // the length has to be established before anything is sized against it, and it has to be a length the + // reader can actually back: ReaderSize confirms it by reading the last byte, which is what keeps a + // reader that over-reports (a squashfs block that decompresses short does exactly that) from being + // treated as authoritative below + size, ok := intFile.ReaderSize(r) + if !ok { + return 0, errors.New("unable to determine the file's size, so the bundle marker search cannot be bounded") + } + + // a limit that overflowed negative or overshoots the file tells us nothing, so fall back to the file + // itself rather than trusting it; either way the absolute cap is what bounds the allocation + limit := size + if searchLimit > 0 && searchLimit < size { + limit = searchLimit + } + + // clamping is recorded rather than just logged: if the marker then turns up missing we cannot claim + // there is no bundle, only that we declined to look everywhere it could have been + var clamped bool + if limit > maxBundleSearchSize { + limit = maxBundleSearchSize + clamped = true + } + + // this scans a whole executable, routinely over 100MB for a single-file bundle, so the buffer is sized + // exactly once. An append-growing read holds both arrays at its final growth and would cost well over + // twice the file's own size for the same result. + searchData := make([]byte, limit) + + // a short read is not fatal here: the marker may well be in what we did get, so search the bytes we + // actually hold. ReadAt reports a short read as io.EOF, and may report a full one that way too, so the + // count is what says how much there is to search. + n, err := r.ReadAt(searchData, 0) + if err != nil && !errors.Is(err, io.EOF) { + return 0, err + } + + idx := bytes.Index(searchData[:n], dotNetBundleSignature) + if idx == -1 || idx < 8 { + if clamped { + return 0, fmt.Errorf("no bundle marker in the first %d bytes and the rest of the %d byte file was not searched", maxBundleSearchSize, size) + } + return 0, nil + } + + headerOffset := int64(binary.LittleEndian.Uint64(searchData[idx-8 : idx])) + + if headerOffset == 0 { + // the marker is compiled into every apphost; only publishing as a single file fills in the offset + return 0, nil + } + + // the offset comes straight out of the file, so it is the least trustworthy value here: everything + // downstream seeks to it and reads structures from it + if headerOffset < 0 || headerOffset >= size { + return 0, fmt.Errorf("bundle header offset %d lies outside the file (%d bytes)", headerOffset, size) + } + + return headerOffset, nil +} + // dotNetBundleHeader represents the fixed portion of the bundle header (version 1+) type dotNetBundleHeader struct { MajorVersion uint32 @@ -31,7 +147,9 @@ type dotNetBundleHeaderV2 struct { Flags uint64 } -// dotNetFileType represents the type of bundled file in the manifest +// dotNetFileType represents the type of bundled file in the manifest, as of V2 bundles (.NET 5+). +// note: V1 bundles (.NET Core 3.x) predate the unknown member at the head of this enum, so every type there is +// one less than the values below (e.g. deps.json is 2, not 3). see depsJSONFileType(). type dotNetFileType uint8 const ( @@ -43,8 +161,8 @@ const ( dotNetFileTypeSymbols ) -// ReadDepsJSONFromBundleHeader parses the bundle header at the given offset and extracts deps.json content. -func ReadDepsJSONFromBundleHeader(r io.ReadSeeker, headerOffset int64) (string, error) { +// readDepsJSONFromBundleHeader parses the bundle header at the given offset and extracts deps.json content. +func readDepsJSONFromBundleHeader(r unionreader.UnionReader, headerOffset int64) (string, error) { if _, err := r.Seek(headerOffset, io.SeekStart); err != nil { return "", err } @@ -103,26 +221,88 @@ func read7BitEncodedInt(r io.Reader) (int, error) { return 0, errors.New("invalid 7-bit encoded int") } } + + // the shift above can carry past int32 where int is 32 bits, and a negative length would seek callers + // backwards and let a manifest walk re-read the same bytes for every file it claims + if result < 0 { + return 0, errors.New("negative 7-bit encoded int") + } + return result, nil } // readDepsJSONAtOffset reads deps.json content at a specific offset using seeks (avoiding loading entire file) -func readDepsJSONAtOffset(r io.ReadSeeker, offset, size int64) (string, error) { - if size <= 0 || size > 3*1024*1024 { // 3MB max deps.json size in dotnet bundle +func readDepsJSONAtOffset(r unionreader.UnionReader, offset, size int64) (string, error) { + if size <= 0 { return "", nil } - if _, err := r.Seek(offset, io.SeekStart); err != nil { - return "", fmt.Errorf("failed to seek to deps.json at offset %d: %w", offset, err) + + // an oversized deps.json is reported rather than dropped: this is the file's whole dependency list, so + // returning "" here would hand back an SBOM that looks complete and silently is not + if size > maxDepsJSONSize { + return "", fmt.Errorf("embedded deps.json of %d bytes is past the %d byte limit", size, maxDepsJSONSize) } + data := make([]byte, size) - if _, err := io.ReadFull(r, data); err != nil { - return "", fmt.Errorf("failed to read deps.json (%d bytes): %w", size, err) + // ReadAt leaves the caller's cursor alone and reports a short read as io.EOF, so the count is what says + // whether the whole document was there + if n, err := r.ReadAt(data, offset); err != nil && int64(n) < size { + return "", fmt.Errorf("failed to read deps.json (%d bytes at offset %d): %w", size, offset, err) } return string(data), nil } +// depsJSONFileType returns the manifest file type that marks deps.json for the given bundle version. V1 bundles +// (.NET Core 3.x) have no unknown member in the file type enum, so all of their types are shifted down by one -- +// reading them with the V2+ values silently matches runtimeconfig.json instead. +func depsJSONFileType(majorVersion uint32) dotNetFileType { + if majorVersion < 2 { + return dotNetFileTypeDepsJSON - 1 + } + return dotNetFileTypeDepsJSON +} + +// checkManifestFits rejects a manifest whose declared entry count could not fit in the bytes left after +// the current position, which means the count came from a malformed header. +func checkManifestFits(r unionreader.UnionReader, numFiles, minEntrySize int64) error { + pos, err := r.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + + end, ok := intFile.ReaderSize(r) + if !ok { + return errors.New("unable to determine the file's size, so the manifest entry count cannot be weighed against it") + } + + if remaining := end - pos; numFiles > remaining/minEntrySize { + return fmt.Errorf("manifest claims %d entries but only %d bytes remain", numFiles, remaining) + } + + return nil +} + // findDepsJSONInManifest parses manifest entries to find deps.json (for V1 bundles or fallback) -func findDepsJSONInManifest(r io.ReadSeeker, numFiles int32, majorVersion uint32) (string, error) { +func findDepsJSONInManifest(r unionreader.UnionReader, numFiles int32, majorVersion uint32) (string, error) { + depsJSONType := depsJSONFileType(majorVersion) + + if numFiles < 0 { + return "", fmt.Errorf("negative embedded file count: %d", numFiles) + } + + // numFiles is a header field, so it can claim up to 2^31-1 entries. The walk terminates either way, + // since every iteration reads forward and eventually hits EOF, but a count the remaining bytes could + // not possibly hold means the header is malformed rather than describing a manifest worth hundreds of + // millions of reads. + minEntrySize := int64(minManifestEntrySize) + if majorVersion >= 6 { + minEntrySize = minManifestEntrySizeV6 + } + + if err := checkManifestFits(r, int64(numFiles), minEntrySize); err != nil { + return "", err + } + for i := int32(0); i < numFiles; i++ { var offset, size int64 @@ -151,7 +331,7 @@ func findDepsJSONInManifest(r io.ReadSeeker, numFiles int32, majorVersion uint32 return "", err } - if fileType == dotNetFileTypeDepsJSON && size > 0 { + if fileType == depsJSONType && size > 0 { // save current position to resume manifest parsing if needed currentPos, err := r.Seek(0, io.SeekCurrent) if err != nil { diff --git a/syft/pkg/cataloger/internal/dotnet/bundle/bundle_elf.go b/syft/pkg/cataloger/internal/dotnet/bundle/bundle_elf.go index b65cd173e..257e7b3be 100644 --- a/syft/pkg/cataloger/internal/dotnet/bundle/bundle_elf.go +++ b/syft/pkg/cataloger/internal/dotnet/bundle/bundle_elf.go @@ -1,11 +1,8 @@ package bundle import ( - "bytes" "debug/elf" - "encoding/binary" "errors" - "io" "github.com/anchore/syft/syft/internal/elfutil" "github.com/anchore/syft/syft/internal/unionreader" @@ -14,51 +11,18 @@ import ( // ExtractDepsJSONFromELFBundle extracts the deps.json content from a .net singlefile // bundle contained within an ELF bin func ExtractDepsJSONFromELFBundle(r unionreader.UnionReader) (string, error) { - headerOffset, err := findBundleHeaderOffsetInELF(r) - if err != nil || headerOffset == 0 { - return "", err - } - return ReadDepsJSONFromBundleHeader(r, headerOffset) -} - -func findBundleHeaderOffsetInELF(r unionreader.UnionReader) (int64, error) { elfFile, err := elfutil.NewFile(r) if err != nil { - return 0, nil + // a refusal is not the same as "this is not an ELF": elfutil exports this error precisely so the + // gap it leaves in the SBOM can be reported rather than read as an absence of evidence + if errors.Is(err, elfutil.ErrDeclaredSizeExceeded) { + return "", err + } + // not an ELF we can parse, so not an ELF bundle + return "", nil //nolint:nilerr } - elfEndOffset := calculateELFEndOffset(elfFile) - if elfEndOffset == 0 { - return 0, nil - } - - // clamp to the actual file size so a malformed ELF (with bogus segment/section - // offsets+sizes) can't drive an arbitrarily large allocation below. - fileSize, err := r.Seek(0, io.SeekEnd) - if err != nil { - return 0, err - } - if elfEndOffset > fileSize { - elfEndOffset = fileSize - } - - if _, err := r.Seek(0, io.SeekStart); err != nil { - return 0, err - } - - searchData := make([]byte, elfEndOffset) - n, err := io.ReadFull(r, searchData) - if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { - return 0, err - } - searchData = searchData[:n] - - idx := bytes.Index(searchData, dotNetBundleSignature) - if idx == -1 || idx < 8 { - return 0, nil - } - - return int64(binary.LittleEndian.Uint64(searchData[idx-8 : idx])), nil + return ExtractDepsJSON(r, calculateELFEndOffset(elfFile)) } func calculateELFEndOffset(f *elf.File) int64 { diff --git a/syft/pkg/cataloger/internal/dotnet/bundle/bundle_elf_test.go b/syft/pkg/cataloger/internal/dotnet/bundle/bundle_elf_test.go index 7a49dbea0..168edbf9a 100644 --- a/syft/pkg/cataloger/internal/dotnet/bundle/bundle_elf_test.go +++ b/syft/pkg/cataloger/internal/dotnet/bundle/bundle_elf_test.go @@ -2,11 +2,15 @@ package bundle import ( "bytes" + "debug/elf" "encoding/binary" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/testutils" ) // readSeekCloser adapts a *bytes.Reader to the unionreader.UnionReader interface (adds Close). @@ -16,10 +20,44 @@ type readSeekCloser struct { func (readSeekCloser) Close() error { return nil } -// buildELFWithHugeFilesz returns a minimal, parseable ELF64 whose single program header -// declares an absurd p_filesz. calculateELFEndOffset will compute a huge end offset; the -// clamp in findBundleHeaderOffsetInELF must keep the allocation bounded to the real file. +// buildELFWithHugeFilesz returns a minimal, parseable ELF64 whose single program header declares a +// p_filesz far past the real file. calculateELFEndOffset will compute an 8GB end offset; the bound in +// FindSignatureOffset must keep the allocation tied to the real file. +// +// note: 8GB rather than something astronomical on purpose. `make([]byte, 1<<60)` panics on its own, so a +// test built on that value passes whether or not the bound exists; `make([]byte, 8*intFile.GB)` succeeds +// on any 64-bit host from untouched anonymous mmap, so only the requested read size distinguishes them. func buildELFWithHugeFilesz() []byte { + return buildELFWithProgHeader(0, 8*intFile.GB) +} + +// readSizeRecorder records the largest single Read length requested of it, which is what tells a buffer +// sized from the file apart from one sized from the headers. +type readSizeRecorder struct { + *bytes.Reader + maxRead int +} + +func (r *readSizeRecorder) Read(p []byte) (int, error) { + if len(p) > r.maxRead { + r.maxRead = len(p) + } + return r.Reader.Read(p) +} + +// note: ReadAt is recorded too, since the bounded reads go through it to leave the caller's cursor alone. +func (r *readSizeRecorder) ReadAt(p []byte, off int64) (int, error) { + if len(p) > r.maxRead { + r.maxRead = len(p) + } + return r.Reader.ReadAt(p, off) +} + +func (r *readSizeRecorder) Close() error { return nil } + +// buildELFWithProgHeader returns a minimal, parseable ELF64 with a single PT_LOAD program header +// carrying the given p_offset and p_filesz. debug/elf does not validate either field. +func buildELFWithProgHeader(pOffset, pFilesz uint64) []byte { const ( ehSize = 64 phSize = 56 @@ -45,19 +83,54 @@ func buildELFWithHugeFilesz() []byte { // e_shoff/e_shnum left zero so no sections are parsed ph := buf[phOff:] - le.PutUint32(ph[0:], 1) // p_type = PT_LOAD - le.PutUint64(ph[16:], 0) // p_offset - le.PutUint64(ph[32:], 1<<60) // p_filesz (bogus, attacker-controlled) - le.PutUint64(ph[40:], 1<<60) // p_memsz + le.PutUint32(ph[0:], 1) // p_type = PT_LOAD + le.PutUint64(ph[8:], pOffset) // p_offset (user-controlled) + le.PutUint64(ph[32:], pFilesz) // p_filesz (user-controlled) + le.PutUint64(ph[40:], pFilesz) // p_memsz return buf } -func TestExtractDepsJSONFromELFBundle_MalformedFileszDoesNotOverAllocate(t *testing.T) { - data := buildELFWithHugeFilesz() +func TestExtractDepsJSONFromELFBundle_EndOffsetOverflowDoesNotPanic(t *testing.T) { + // calculateELFEndOffset sums these two uint64 header fields into an int64 and adds 4096, which + // overflows to a negative search limit. A negative is not greater than the file size, so it slips + // unnoticed into the search window unless a non-positive limit reads nothing. + data := buildELFWithProgHeader(0x7FFFFFFFFFFFF000, 0) + require.Negative(t, calculateELFEndOffset(mustParseELF(t, data)), + "expected these header values to overflow the end offset; the guard below is what this test pins") + r := readSeekCloser{bytes.NewReader(data)} - // must not OOM/panic on the bogus p_filesz, and find no bundle signature content, err := ExtractDepsJSONFromELFBundle(r) require.NoError(t, err) assert.Empty(t, content) } + +func mustParseELF(t *testing.T, data []byte) *elf.File { + t.Helper() + f, err := elf.NewFile(bytes.NewReader(data)) + require.NoError(t, err) + return f +} + +func TestExtractDepsJSONFromELFBundle_MalformedFileszDoesNotOverAllocate(t *testing.T) { + data := buildELFWithHugeFilesz() + + // sanity: the headers really do describe an end offset far past the file, so the bound is what keeps + // the allocation small rather than the input being small + require.Greater(t, calculateELFEndOffset(mustParseELF(t, data)), int64(8*intFile.GB)) + + r := &readSizeRecorder{Reader: bytes.NewReader(data)} + + var content string + var err error + allocated := testutils.MeasureAlloc(t, func() { + content, err = ExtractDepsJSONFromELFBundle(r) + }) + require.NoError(t, err) + assert.Empty(t, content) + + assert.Less(t, allocated, uint64(intFile.MB), + "the search buffer must be sized by the file, not by what the program headers claim") + assert.LessOrEqual(t, r.maxRead, len(data), + "no read may request more than the file holds") +} diff --git a/syft/pkg/cataloger/internal/dotnet/bundle/bundle_test.go b/syft/pkg/cataloger/internal/dotnet/bundle/bundle_test.go new file mode 100644 index 000000000..a3146d218 --- /dev/null +++ b/syft/pkg/cataloger/internal/dotnet/bundle/bundle_test.go @@ -0,0 +1,297 @@ +package bundle + +import ( + "bytes" + "encoding/binary" + "io" + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fileWithSignatureAt returns a file of the given size carrying the bundle marker (8-byte header +// offset followed by the signature) starting at sigStart. +func fileWithSignatureAt(size, sigStart int, headerOffset uint64) []byte { + data := make([]byte, size) + binary.LittleEndian.PutUint64(data[sigStart-8:sigStart], headerOffset) + copy(data[sigStart:], dotNetBundleSignature) + return data +} + +func TestFindBundleHeaderOffset(t *testing.T) { + // large enough that the marker's declared header offset lands inside the file + const withMarker = 8192 + const headerOffset = 0x1234 + + tests := []struct { + name string + data []byte + searchLimit int64 + want int64 + wantErr require.ErrorAssertionFunc + }{ + { + name: "no signature present", + data: make([]byte, withMarker), + searchLimit: withMarker, + }, + { + // the limit is only a hint about how far in the marker can be. Treating a nonsensical one as + // authoritative would let a single bogus header field hide the bundle from us. + name: "zero limit falls back to the whole file", + data: fileWithSignatureAt(withMarker, 64, headerOffset), + searchLimit: 0, + want: headerOffset, + }, + { + // callers sum unsigned header fields into an int64, which overflows to a negative limit for + // some header values. That must not size a buffer, and must not hide the marker either. + name: "negative limit falls back to the whole file", + data: fileWithSignatureAt(withMarker, 64, headerOffset), + searchLimit: math.MinInt64, + want: headerOffset, + }, + { + name: "limit past end of file reads only what exists", + data: fileWithSignatureAt(withMarker, 64, headerOffset), + searchLimit: math.MaxInt64 / 2, + want: headerOffset, + }, + { + // the bound must not shrink the window below the real file, or a legitimate marker stops being found + name: "limit inside the file still finds the marker", + data: fileWithSignatureAt(withMarker, 64, headerOffset), + searchLimit: 128, + want: headerOffset, + }, + { + name: "limit truncates the window before the marker", + data: fileWithSignatureAt(withMarker, 64, headerOffset), + searchLimit: 32, + }, + { + // there is no room for the 8-byte header offset before the signature + name: "signature too close to the start to carry an offset", + data: append(append([]byte{0, 0}, dotNetBundleSignature...), make([]byte, 32)...), + searchLimit: 128, + }, + { + // the offset is read straight out of the file, and everything downstream seeks to it + name: "header offset past the end of the file is rejected", + data: fileWithSignatureAt(withMarker, 64, 1<<40), + searchLimit: withMarker, + wantErr: require.Error, + }, + { + // every apphost carries the signature with a zero offset placeholder; only publishing as a + // single file fills it in. Rejecting it would report a spurious parse failure against every + // ordinary framework-dependent .NET executable. + name: "header offset of zero means not bundled", + data: fileWithSignatureAt(withMarker, 64, 0), + searchLimit: withMarker, + want: 0, + }, + { + // a uint64 offset read into an int64 can land negative, which would seek backwards + name: "header offset that reads negative is rejected", + data: fileWithSignatureAt(withMarker, 64, math.MaxUint64), + searchLimit: withMarker, + wantErr: require.Error, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + + got, err := findBundleHeaderOffset(readSeekCloser{bytes.NewReader(tt.data)}, tt.searchLimit) + tt.wantErr(t, err) + if err != nil { + return + } + assert.Equal(t, tt.want, got) + }) + } +} + +// shortReader hands back fewer bytes than were asked for, which is what unionreader does for a squashfs +// block that decompresses short. Its size is honest; only the read comes up short, so the marker may well +// be in the bytes we did get. +type shortReader struct { + data []byte + size int64 + pos int64 +} + +func (r *shortReader) Read(p []byte) (int, error) { + if r.pos >= int64(len(r.data)) { + return 0, io.EOF + } + n := copy(p, r.data[r.pos:]) + r.pos += int64(n) + return n, nil +} + +func (r *shortReader) ReadAt(p []byte, off int64) (int, error) { + if off >= int64(len(r.data)) { + return 0, io.EOF + } + n := copy(p, r.data[off:]) + if n < len(p) { + return n, io.EOF + } + return n, nil +} + +func (r *shortReader) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + r.pos = offset + case io.SeekCurrent: + r.pos += offset + case io.SeekEnd: + // the size is what the reader reports; Read is what comes up short + r.pos = r.size + offset + } + return r.pos, nil +} + +func (r *shortReader) Close() error { return nil } + +// unbackedSizeReader reports a size it cannot deliver the last byte of. +// +// These readers do not all come from us. debug/elf and debug/pe hand out io.NewSectionReader(r, 0, 1<<63-1), +// whose Size() is a nominal 8EB rather than a measured one, so a reader that answers a size directly is not +// on its own evidence the bytes exist. intFile.ReaderSize confirms the count by reading the last byte, and +// the case below pins that failing that check surfaces here as an error rather than as "no bundle present". +type unbackedSizeReader struct { + *bytes.Reader + size int64 +} + +func (r *unbackedSizeReader) Size() int64 { return r.size } + +func (r *unbackedSizeReader) ReadAt(p []byte, off int64) (int, error) { + if off >= r.Reader.Size() { + return 0, io.EOF + } + return r.Reader.ReadAt(p, off) +} + +func (r *unbackedSizeReader) Close() error { return nil } + +func TestFindBundleHeaderOffset_ShortReadStillSearchesWhatWasRead(t *testing.T) { + data := fileWithSignatureAt(8192, 64, 0x1234) + + // honest about its length, but every read stops early + r := &shortReader{data: data, size: int64(len(data))} + + got, err := findBundleHeaderOffset(r, int64(len(data))) + require.NoError(t, err) + assert.Equal(t, int64(0x1234), got, + "a short read must not abort the search; the marker was in the bytes that were returned") +} + +func TestFindBundleHeaderOffset_UnbackedSizeIsReported(t *testing.T) { + data := fileWithSignatureAt(8192, 64, 0x1234) + + // claims twice the bytes it holds, so nothing sized or bounds-checked against that number means anything + r := &unbackedSizeReader{Reader: bytes.NewReader(data), size: int64(len(data)) * 2} + + _, err := findBundleHeaderOffset(r, int64(len(data))*2) + require.Error(t, err, + "a reader that cannot back the length it reports must be reported, not read as an absent bundle") +} + +func TestRead7BitEncodedInt(t *testing.T) { + t.Run("decodes what BinaryWriter produces", func(t *testing.T) { + for _, tt := range []struct { + data []byte + want int + }{ + {data: []byte{0x05}, want: 5}, + {data: []byte{0x80, 0x01}, want: 128}, + {data: []byte{0xFF, 0x7F}, want: 16383}, + } { + got, err := read7BitEncodedInt(bytes.NewReader(tt.data)) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + + t.Run("too many continuation bytes is rejected", func(t *testing.T) { + _, err := read7BitEncodedInt(bytes.NewReader([]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})) + require.Error(t, err) + }) + + // callers seek forward by whatever this returns, so a negative value would rewind them and let a + // manifest walk re-read the same bytes for every file it claims. The shift can only carry past int32 + // where int is 32 bits, so on a 64-bit build the guard is unreachable and this pins the invariant + // rather than the branch. + t.Run("never returns a negative length", func(t *testing.T) { + for _, data := range [][]byte{ + {0xFF, 0xFF, 0xFF, 0xFF, 0x0F}, + {0xFF, 0xFF, 0xFF, 0xFF, 0x7F}, + {0x80, 0x80, 0x80, 0x80, 0x08}, + } { + got, err := read7BitEncodedInt(bytes.NewReader(data)) + if err != nil { + continue + } + assert.GreaterOrEqual(t, got, 0, "a length that seeks callers backwards must be rejected") + } + }) +} + +func TestFindDepsJSONInManifest_ImpossibleEntryCountIsRejected(t *testing.T) { + tests := []struct { + name string + numFiles int32 + majorVersion uint32 + }{ + { + // 2^31-1 entries cannot fit in a handful of bytes, so the header is malformed rather than + // describing a manifest worth hundreds of millions of reads + name: "max int32 entries in a tiny file", + numFiles: math.MaxInt32, + }, + { + name: "negative entry count", + numFiles: -1, + }, + { + name: "max int32 entries in a v6 bundle", + numFiles: math.MaxInt32, + majorVersion: 6, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := bytes.NewReader(make([]byte, 512)) + + _, err := findDepsJSONInManifest(readSeekCloser{r}, tt.numFiles, tt.majorVersion) + require.Error(t, err) + }) + } +} + +func TestFindDepsJSONInManifest_PlausibleEntryCountIsWalked(t *testing.T) { + // a count the remaining bytes could hold must still be walked, or the guard would reject real bundles + var buf bytes.Buffer + for range 2 { + require.NoError(t, binary.Write(&buf, binary.LittleEndian, int64(0))) // offset + require.NoError(t, binary.Write(&buf, binary.LittleEndian, int64(0))) // size + require.NoError(t, binary.Write(&buf, binary.LittleEndian, uint8(1))) // type: assembly + require.NoError(t, binary.Write(&buf, binary.LittleEndian, uint8(1))) // path length + require.NoError(t, binary.Write(&buf, binary.LittleEndian, uint8('a'))) // path + } + + got, err := findDepsJSONInManifest(readSeekCloser{bytes.NewReader(buf.Bytes())}, 2, 1) + require.NoError(t, err) + assert.Empty(t, got, "no deps.json entry in this manifest") +} diff --git a/syft/pkg/cataloger/internal/dotnet/pe/bundle.go b/syft/pkg/cataloger/internal/dotnet/pe/bundle.go index 791b476b2..2032c83fa 100644 --- a/syft/pkg/cataloger/internal/dotnet/pe/bundle.go +++ b/syft/pkg/cataloger/internal/dotnet/pe/bundle.go @@ -1,24 +1,13 @@ package pe import ( - "bytes" "debug/pe" - "encoding/binary" - "errors" - "io" + "github.com/anchore/syft/syft/internal/unionreader" "github.com/anchore/syft/syft/pkg/cataloger/internal/dotnet/bundle" ) -// dotNetBundleSignature is the SHA-256 hash of ".net core bundle" used to identify single-file bundles. -var dotNetBundleSignature = []byte{ - 0x8b, 0x12, 0x02, 0xb9, 0x6a, 0x61, 0x20, 0x38, - 0x72, 0x7b, 0x93, 0x02, 0x14, 0xd7, 0xa0, 0x32, - 0x13, 0xf5, 0xb9, 0xe6, 0xef, 0xae, 0x33, 0x18, - 0xee, 0x3b, 0x2d, 0xce, 0x24, 0xb3, 0x6a, 0xae, -} - -// ExtractDepsJSONFromBundle searches for an embedded deps.json file in a .NET single-file bundle. +// extractDepsJSONFromBundle searches for an embedded deps.json file in a .NET single-file bundle. // When built with PublishSingleFile=true, .NET embeds the application and all dependencies into // the AppHost executable. The bundle marker (8-byte header offset + 32-byte signature) is placed // in a placeholder location within the PE structure, pointing to the bundle header which contains @@ -50,42 +39,8 @@ var dotNetBundleSignature = []byte{ // - https://github.com/dotnet/runtime/blob/main/src/native/corehost/bundle/header.h // - https://github.com/dotnet/runtime/blob/main/src/native/corehost/bundle/file_entry.h // - https://github.com/dotnet/runtime/blob/main/src/native/corehost/bundle/file_type.h -func extractDepsJSONFromBundle(r io.ReadSeeker, sections []pe.SectionHeader32) (string, error) { - headerOffset, err := findBundleHeaderOffset(r, sections) - if err != nil { - return "", err - } - if headerOffset == 0 { - return "", nil // not a .NET single-file bundle - } - - return bundle.ReadDepsJSONFromBundleHeader(r, headerOffset) -} - -// findBundleHeaderOffset locates the bundle marker within the PE structure and returns the header offset. -// Returns 0 if no bundle marker is found (not a single-file bundle). -func findBundleHeaderOffset(r io.ReadSeeker, sections []pe.SectionHeader32) (int64, error) { - peEndOffset := calculatePEEndOffset(sections) - - if _, err := r.Seek(0, io.SeekStart); err != nil { - return 0, err - } - - peData := make([]byte, peEndOffset) - n, err := io.ReadFull(r, peData) - if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { - return 0, err - } - peData = peData[:n] - - idx := bytes.Index(peData, dotNetBundleSignature) - if idx == -1 || idx < 8 { - return 0, nil - } - - // the header offset is stored in the 8 bytes immediately before the signature - headerOffset := int64(binary.LittleEndian.Uint64(peData[idx-8 : idx])) - return headerOffset, nil +func extractDepsJSONFromBundle(r unionreader.UnionReader, sections []pe.SectionHeader32) (string, error) { + return bundle.ExtractDepsJSON(r, calculatePEEndOffset(sections)) } // calculatePEEndOffset determines where the PE structure ends based on section headers, diff --git a/syft/pkg/cataloger/internal/dotnet/pe/bundle_test.go b/syft/pkg/cataloger/internal/dotnet/pe/bundle_test.go index d25b33f7c..c57f94b83 100644 --- a/syft/pkg/cataloger/internal/dotnet/pe/bundle_test.go +++ b/syft/pkg/cataloger/internal/dotnet/pe/bundle_test.go @@ -1,10 +1,15 @@ package pe import ( + "bytes" + "debug/pe" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/testutils" ) func Test_extractDepsJSONFromBundle_Versions(t *testing.T) { @@ -13,34 +18,43 @@ func Test_extractDepsJSONFromBundle_Versions(t *testing.T) { fixture string path string wantDepsJSON bool // true if deps.json should be found - wantJSONContain string // string that should be in the JSON (varies by .NET version) + wantJSONContain string // a string unique to the deps.json (and not the runtimeconfig.json) of this fixture }{ { - name: "V1 bundle (.NET Core 3.1)", - fixture: "image-dotnet31-single-file", - path: "/app/hello.exe", - wantDepsJSON: true, - wantJSONContain: "runtimeOptions", // .NET Core 3.1 uses runtimeOptions + name: "V1 bundle (.NET Core 3.1)", + fixture: "image-dotnet31-single-file", + path: "/app/hello.exe", + wantDepsJSON: true, + wantJSONContain: `"runtimeTarget": { + "name": ".NETCoreApp,Version=v3.1/win-x64"`, }, { - name: "V2 bundle (.NET 5)", - fixture: "image-dotnet5-single-file", - path: "/app/hello.exe", - wantDepsJSON: true, - wantJSONContain: "runtimeTarget", // .NET 5+ uses runtimeTarget + name: "V2 bundle (.NET 5)", + fixture: "image-dotnet5-single-file", + path: "/app/hello.exe", + wantDepsJSON: true, + wantJSONContain: `"runtimeTarget": { + "name": ".NETCoreApp,Version=v5.0/win-x64"`, }, { - name: "V6 bundle (.NET 6)", - fixture: "image-dotnet6-single-file", - path: "/app/hello.exe", - wantDepsJSON: true, - wantJSONContain: "runtimeTarget", // .NET 6+ uses runtimeTarget + name: "V6 bundle (.NET 6)", + fixture: "image-dotnet6-single-file", + path: "/app/hello.exe", + wantDepsJSON: true, + wantJSONContain: `"runtimeTarget": { + "name": ".NETCoreApp,Version=v6.0/win-x64"`, }, } + var fixtures []string + for _, tt := range tests { + fixtures = append(fixtures, tt.fixture) + } + resolvers := fixtureResolvers(t, fixtures...) + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - reader := fixtureFile(t, tt.fixture, tt.path) + reader := fixtureFile(t, resolvers[tt.fixture], tt.path) defer reader.Close() got, err := Read(reader) @@ -56,3 +70,57 @@ func Test_extractDepsJSONFromBundle_Versions(t *testing.T) { }) } } + +// readSizeRecorder records the largest single Read length requested of it. The worst end offset PE +// headers can describe is about 8.6GB, and `make([]byte, 8.6e9)` succeeds on any 64-bit host from fresh +// anonymous mmap without touching the pages, so asserting "no panic" would pass with or without the +// bound. The requested read size is what actually distinguishes them. +type readSizeRecorder struct { + *bytes.Reader + maxRead int +} + +func (r *readSizeRecorder) Read(p []byte) (int, error) { + if len(p) > r.maxRead { + r.maxRead = len(p) + } + return r.Reader.Read(p) +} + +// note: ReadAt is recorded too, since the bounded reads go through it to leave the caller's cursor alone. +func (r *readSizeRecorder) ReadAt(p []byte, off int64) (int, error) { + if len(p) > r.maxRead { + r.maxRead = len(p) + } + return r.Reader.ReadAt(p, off) +} + +func (r *readSizeRecorder) Close() error { return nil } + +func TestExtractDepsJSONFromBundle_MalformedSectionSizesDoNotOverAllocate(t *testing.T) { + // PointerToRawData and SizeOfRawData are user-controlled and unrelated to the real file size + sections := []pe.SectionHeader32{{PointerToRawData: 0xFFFFFFFF, SizeOfRawData: 0xFFFFFFFF}} + + // sanity: the headers really do describe an end offset far past the file, so the bound is what keeps + // the allocation small rather than the input being small + require.Greater(t, calculatePEEndOffset(sections), int64(8*intFile.GB)) + + const fileSize = 512 // a small "file" with no bundle signature + r := &readSizeRecorder{Reader: bytes.NewReader(make([]byte, fileSize))} + + var content string + var err error + allocated := testutils.MeasureAlloc(t, func() { + content, err = extractDepsJSONFromBundle(r, sections) + }) + require.NoError(t, err) + assert.Empty(t, content) + + // the byte count is the assertion that matters. make([]byte, 8.6e9) does not crash on a 64-bit host: + // it comes back from fresh anonymous mmap and never touches the pages, so a test that only checked for + // a panic would pass with the bound removed. + assert.Less(t, allocated, uint64(intFile.MB), + "the search buffer must be sized by the file, not by what the section headers claim") + assert.LessOrEqual(t, r.maxRead, fileSize, + "no read may request more than the file holds") +} diff --git a/syft/pkg/cataloger/internal/dotnet/pe/pe.go b/syft/pkg/cataloger/internal/dotnet/pe/pe.go index c389d55bd..9939e153e 100644 --- a/syft/pkg/cataloger/internal/dotnet/pe/pe.go +++ b/syft/pkg/cataloger/internal/dotnet/pe/pe.go @@ -9,15 +9,142 @@ import ( "io" "unicode/utf16" - "github.com/scylladb/go-set/strset" "github.com/scylladb/go-set/u32set" + intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/internal/unionreader" ) -const peMaxAllowedDirectoryEntries = 0x1000 +const ( + peMaxAllowedDirectoryEntries = 0x1000 + + // peResourceBudgetFactor bounds the total bytes a walk will read out of a resource section, as a + // multiple of that section's own size. The per-directory cap above only bounds the fan-out of one + // node, and nothing in the format stops entries from aliasing: thousands of them may name the same + // blob, so a tree whose every individual offset is in bounds can still drive work quadratic in the + // section size. A well-formed section's entries partition it rather than overlapping, so real binaries + // come in right around 1x and the slack here only absorbs padding and shared string tables. + peResourceBudgetFactor = 4 + + // peMaxResourceDirectoryDepth bounds how deep the resource tree walk will recurse. + // + // The format uses exactly three levels (type, then name, then language), so anything past a handful + // is malformed. The bound matters because Go grows a goroutine stack until it hits the process limit + // and then dies with a fatal error rather than a recoverable panic: a chain of directories at + // distinct RVAs, each naming the next, would take the whole scan down with it. Distinct RVAs are why + // the dirs set below cannot stand in for this, and a byte budget cannot either, since the budget + // scales with the section while the stack does not. + peMaxResourceDirectoryDepth = 32 + + // maxDirectorySectionSize bounds the bytes any single PE data directory may declare. + // + // Only the resource and COM descriptor directories are read, and both are metadata: real ones run + // from a few KB to a few MB even for applications that embed sizable assets. Clamping to the bytes + // remaining in the file (which the caller does as well) is not on its own enough, because a mostly + // empty file compresses to almost nothing in a layer, so an attacker can hand us a small artifact + // that still authorizes a multi-gigabyte allocation. The trade-off is that a binary declaring a + // larger directory loses its version resources rather than being cataloged; that is the correct + // direction to fail, since the alternative is OOM-killing the whole scan. This mirrors + // maxDeclaredSectionSize in syft/internal/elfutil. + maxDirectorySectionSize = 128 * intFile.MB + + // clrDebugInfoResourceName is the only resource name any downstream logic asks about. + clrDebugInfoResourceName = "CLRDEBUGINFO" +) + +// resourceWalk is the state shared across a single resource directory traversal. +// +// reader and baseRVA are fixed for the whole walk: every RVA a nested entry names resolves against the +// same origin and the same bytes, so reader.Size() is the one authoritative bound on every offset derived +// from them. Holding them here rather than on a per-node value is what keeps that invariant structural. +type resourceWalk struct { + reader *bytes.Reader + baseRVA uint32 + + // dirs tracks the RVAs already parsed (prevents infinite recursion edge cases) + dirs *u32set.Set + + // fields collects version resource keys and their values + fields map[string]string + + // hasCLRDebugInfo records whether a CLRDEBUGINFO resource name was seen + hasCLRDebugInfo bool + + // budget is the number of bytes left that we are willing to read out of the section. Charging every + // read against one counter bounds the blobs, the names, and the total tree walk together. + // + // note: this bounds total work, not recursion depth. The budget scales with the section size, so a + // large section still affords a very deep chain of directory headers; depth is bounded separately by + // peMaxResourceDirectoryDepth. + budget int64 + + // depth is how many directory levels the current recursion is into the tree. + depth int +} + +// newResourceWalk starts a walk over one resource section's bytes, fixing the origin every RVA is +// measured against. Taking both up front is what keeps reader.Size() the one authoritative bound on +// every offset the walk derives, since there is no window in which a walk exists without them. +func newResourceWalk(reader *bytes.Reader, baseRVA uint32) *resourceWalk { + return &resourceWalk{ + reader: reader, + baseRVA: baseRVA, + dirs: u32set.New(), + fields: make(map[string]string), + budget: reader.Size() * peResourceBudgetFactor, + } +} + +// offsetOf turns an RVA into an offset into the walk's bytes, rejecting any RVA the section does not +// actually hold. RVAs are user-controlled uint32s, so this is what keeps the subtraction from underflowing +// and keeps every offset derived from one inside the buffer. +func (w *resourceWalk) offsetOf(rva uint32) (int64, error) { + if rva < w.baseRVA { + return 0, fmt.Errorf("RVA=0x%x precedes its section base 0x%x", rva, w.baseRVA) + } + + offset := int64(rva - w.baseRVA) + if offset >= w.reader.Size() { + return 0, fmt.Errorf("RVA=0x%x lies past its section end (baseRVA=0x%x size=0x%x)", rva, w.baseRVA, w.reader.Size()) + } + + return offset, nil +} + +// offsetWithin validates a section-relative offset the file supplied directly (rather than as an RVA) +// before it is used to seek. bytes.Reader happily seeks past its end and only fails on the later read, so +// checking here is what turns a bogus offset into a legible error instead of a downstream EOF. +func (w *resourceWalk) offsetWithin(offset uint32) (int64, error) { + if int64(offset) >= w.reader.Size() { + return 0, fmt.Errorf("offset 0x%x lies past its section end 0x%x", offset, w.reader.Size()) + } + return int64(offset), nil +} + +// errResourceBudget stops the walk rather than one entry: once the budget is gone every remaining sibling +// would hit it too, so callers that normally log-and-continue have to propagate this one. +var errResourceBudget = errors.New("resource walk read more of its section than a well-formed one could justify") + +// errResourceDepth stops the walk for the same reason as errResourceBudget: the format nests three levels, +// so a tree past the cap is crafted rather than unusual, and its siblings are the same structure. +var errResourceDepth = errors.New("resource directory nested deeper than a well-formed one could justify") + +// stopsWalk reports whether an error is about the walk as a whole rather than the one entry that raised it, +// which is what tells the log-and-continue loops to propagate instead. +func stopsWalk(err error) bool { + return errors.Is(err, errResourceBudget) || errors.Is(err, errResourceDepth) +} + +// spend charges n bytes about to be read out of the section against the walk's budget. +func (w *resourceWalk) spend(n int64) error { + w.budget -= n + if w.budget < 0 { + return errResourceBudget + } + return nil +} var imageDirectoryEntryIndexes = []int{ pe.IMAGE_DIRECTORY_ENTRY_RESOURCE, // where version resources are stored @@ -40,6 +167,14 @@ type File struct { // VersionResources is a map of version resource keys to their values found in the VERSIONINFO resource directory. VersionResources map[string]string + + // ParseErr records the structures that could not be parsed out of an otherwise usable PE file. + // + // A malformed resource directory, an unreadable data directory, or a bundle marker pointing nowhere + // each cost us one piece of evidence, not the whole file, so Read reports them here and still returns + // what it did find. Callers that track partial results should join this into their unknowns rather + // than discarding the package. + ParseErr error } // CLREvidence is basic info about the CLR (common language runtime) version from the COM descriptor. @@ -135,6 +270,11 @@ type extractedSection struct { BaseRVA uint32 Size uint32 Reader *bytes.Reader + + // Err records why this section's bytes could not be read, leaving Reader nil. Recording it rather + // than failing the file keeps one malformed directory from dropping the package entirely; the + // downstream parsers all treat a nil Reader as "nothing to say about this". + Err error } func (s extractedSection) exists() bool { @@ -162,29 +302,43 @@ func Read(f file.LocationReadCloser) (*File, error) { return nil, fmt.Errorf("unable to parse PE sections: %w", err) } - dirs := u32set.New() // keep track of the RVAs we have already parsed (prevent infinite recursion edge cases) - versionResources := make(map[string]string) // map of version resource keys to their values - resourceNames := strset.New() // set of resource names found in the PE file - err = parseResourceDirectory(sections[pe.IMAGE_DIRECTORY_ENTRY_RESOURCE], dirs, versionResources, resourceNames) - if err != nil { - return nil, err + // every structure below is optional evidence: losing one costs us a field, not the package. They are + // collected rather than returned so a single malformed directory cannot drop the file from the SBOM, + // and so callers can still see what went wrong instead of it only reaching a trace log. + var parseErrs []error + for _, i := range imageDirectoryEntryIndexes { + if sec := sections[i]; sec != nil && sec.Err != nil { + parseErrs = append(parseErrs, sec.Err) + } } - c, err := parseCLR(sections[pe.IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR], resourceNames) + walk, err := parseResourceDirectory(sections[pe.IMAGE_DIRECTORY_ENTRY_RESOURCE]) if err != nil { - return nil, fmt.Errorf("unable to parse PE CLR directory: %w", err) + parseErrs = append(parseErrs, fmt.Errorf("unable to fully parse PE resource directory: %w", err)) + } + + c, err := parseCLR(sections[pe.IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR], walk.hasCLRDebugInfo) + if err != nil { + parseErrs = append(parseErrs, fmt.Errorf("unable to parse PE CLR directory: %w", err)) + c = &CLREvidence{HasClrResourceNames: walk.hasCLRDebugInfo} } embeddedDepsJSON, err := extractDepsJSONFromBundle(r, sectionHeaders) if err != nil { - return nil, fmt.Errorf("unable to extract embedded deps.json: %w", err) + parseErrs = append(parseErrs, fmt.Errorf("unable to extract embedded deps.json: %w", err)) + } + + parseErr := errors.Join(parseErrs...) + if parseErr != nil { + log.Tracef("partially parsed PE file %s: %v", f.RealPath, parseErr) } return &File{ Location: f.Location, CLR: c, EmbeddedDepsJSON: embeddedDepsJSON, - VersionResources: versionResources, + VersionResources: walk.fields, + ParseErr: parseErr, }, nil } @@ -206,7 +360,10 @@ func parsePEFile(file unionreader.UnionReader) (map[int]*extractedSection, []pe. } data, err := readDataFromRVA(file, sec.RVA, sec.Size, headers) if err != nil { - return nil, nil, fmt.Errorf("error reading %q section data: %w", directoryName(i), err) + // one unreadable directory says nothing about the others or about the rest of the file, so + // record it and carry on rather than dropping the package over it + sec.Err = fmt.Errorf("error reading %q section data: %w", directoryName(i), err) + continue } sec.Reader = data } @@ -297,12 +454,15 @@ func parseSectionHeaders(file unionreader.UnionReader, magic uint16, numberOfSec return nil, nil, fmt.Errorf("unknown optional header magic: 0x%x", magic) } - // read section headers - headers := make([]pe.SectionHeader32, numberOfSections) - for i := 0; i < int(numberOfSections); i++ { - if err := binary.Read(file, binary.LittleEndian, &headers[i]); err != nil { + // read section headers. numberOfSections is a uint16 straight out of the file header, so the slice + // grows to the headers that are actually there rather than reserving for all 65535 up front. + var headers []pe.SectionHeader32 + for range numberOfSections { + var header pe.SectionHeader32 + if err := binary.Read(file, binary.LittleEndian, &header); err != nil { return nil, nil, fmt.Errorf("error reading section header: %w", err) } + headers = append(headers, header) } return soi, headers, nil @@ -310,8 +470,7 @@ func parseSectionHeaders(file unionreader.UnionReader, magic uint16, numberOfSec // parseCLR extracts the CLR (common language runtime) version information from the COM descriptor and makes // present/not-present determination based on the presence of CLR resource names. -func parseCLR(sec *extractedSection, resourceNames *strset.Set) (*CLREvidence, error) { - hasCLRDebugResourceNames := resourceNames.HasAny("CLRDEBUGINFO") +func parseCLR(sec *extractedSection, hasCLRDebugResourceNames bool) (*CLREvidence, error) { if sec == nil || sec.Reader == nil { return &CLREvidence{ HasClrResourceNames: hasCLRDebugResourceNames, @@ -342,7 +501,7 @@ func rvaToFileOffset(rva uint32, sections []pe.SectionHeader32) (uint32, error) } // readDataFromRVA will read data from a specific RVA in the PE file -func readDataFromRVA(file io.ReadSeeker, rva, size uint32, sections []pe.SectionHeader32) (*bytes.Reader, error) { +func readDataFromRVA(file io.ReaderAt, rva, size uint32, sections []pe.SectionHeader32) (*bytes.Reader, error) { if size == 0 { return nil, fmt.Errorf("zero size specified") } @@ -352,12 +511,31 @@ func readDataFromRVA(file io.ReadSeeker, rva, size uint32, sections []pe.Section return nil, err } - if _, err := file.Seek(int64(offset), io.SeekStart); err != nil { - return nil, fmt.Errorf("error seeking to data: %w", err) + // size is a user-controlled uint32, so sizing the buffer from it alone lets a small file reserve up to + // 4GB. Two bounds apply before it sizes anything: the bytes that actually remain in the file, and the + // absolute cap, which is what keeps a sparse multi-gigabyte file (cheap to ship inside a compressed + // layer) from authorizing a multi-gigabyte allocation. Both checks precede the allocation, and the + // allocation is exact in one shot, which an append-growing read cannot do: it holds both arrays at its + // final growth, so a legitimate 150MB bundle would cost well over twice its own size to scan. + if size > maxDirectorySectionSize { + return nil, fmt.Errorf("error reading data: %d bytes declared at offset %d exceeds the %d byte limit", size, offset, maxDirectorySectionSize) + } + + // the length has to come from something the reader can back up rather than from what it claims, since + // the whole point here is weighing a declared size against the bytes that are really present + end, ok := intFile.ReaderSize(file) + if !ok { + return nil, errors.New("error measuring file") + } + + if remaining := end - int64(offset); remaining < int64(size) { + return nil, fmt.Errorf("error reading data: %d bytes declared at offset %d but only %d remain", size, offset, max(remaining, 0)) } data := make([]byte, size) - if _, err := io.ReadFull(file, data); err != nil { + // ReadAt may report a full read as io.EOF when it lands on the end of the file, so the count is what + // says whether the whole section was there + if n, err := file.ReadAt(data, int64(offset)); err != nil && n < len(data) { return nil, fmt.Errorf("error reading data: %w", err) } @@ -388,13 +566,15 @@ func readDataFromRVA(file io.ReadSeeker, rva, size uint32, sections []pe.Section // sources: // - https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#the-rsrc-section // - https://learn.microsoft.com/en-us/previous-versions/ms809762(v=msdn.10)#pe-file-resources -func parseResourceDirectory(sec *extractedSection, dirs *u32set.Set, fields map[string]string, names *strset.Set) error { +// parseResourceDirectory walks a resource section and returns the walk that collected from it. The walk is +// always non-nil, even on error: a tree that stops partway still tells us about the fields it did yield. +func parseResourceDirectory(sec *extractedSection) (*resourceWalk, error) { if sec == nil || sec.Size <= 0 { - return nil + return newResourceWalk(bytes.NewReader(nil), 0), nil } if sec.Reader == nil { - return errors.New("resource section not found") + return newResourceWalk(bytes.NewReader(nil), 0), errors.New("resource section not found") } baseRVA := sec.BaseRVA @@ -402,40 +582,74 @@ func parseResourceDirectory(sec *extractedSection, dirs *u32set.Set, fields map[ baseRVA = sec.RVA } - offset := int64(sec.RVA - baseRVA) - if _, err := sec.Reader.Seek(offset, io.SeekStart); err != nil { + w := newResourceWalk(sec.Reader, baseRVA) + + return w, parseResourceDirectoryAt(sec.RVA, w) +} + +func parseResourceDirectoryAt(rva uint32, w *resourceWalk) error { + // a resource tree is three levels deep by spec, so anything past the cap is malformed. This has to be + // its own bound: the dirs set only catches a directory naming an RVA already seen, and a chain of + // distinct RVAs each naming the next would otherwise recurse until the goroutine stack gives out, + // which is a fatal error no caller can recover from. + w.depth++ + defer func() { w.depth-- }() + + if w.depth > peMaxResourceDirectoryDepth { + return fmt.Errorf("%w: %d levels", errResourceDepth, peMaxResourceDirectoryDepth) + } + + offset, err := w.offsetOf(rva) + if err != nil { + return fmt.Errorf("resource directory: %w", err) + } + + if _, err := w.reader.Seek(offset, io.SeekStart); err != nil { return fmt.Errorf("error seeking to directory offset: %w", err) } var directoryHeader peImageResourceDirectory - if err := readIntoStruct(sec.Reader, &directoryHeader); err != nil { + if err := w.spend(int64(binary.Size(directoryHeader))); err != nil { + return err + } + + if err := readIntoStruct(w.reader, &directoryHeader); err != nil { return fmt.Errorf("error reading directory header: %w", err) } - numEntries := int(directoryHeader.NumberOfNamedEntries + directoryHeader.NumberOfIDEntries) + // widen before adding: the two counts are uint16s that a crafted file can make sum past 0xFFFF, + // which would wrap and hide entries a real loader would still walk + numEntries := int(directoryHeader.NumberOfNamedEntries) + int(directoryHeader.NumberOfIDEntries) switch { case numEntries > peMaxAllowedDirectoryEntries: return fmt.Errorf("too many entries in resource directory: %d", numEntries) case numEntries == 0: return fmt.Errorf("no entries in resource directory") - case numEntries < 0: - return fmt.Errorf("invalid number of entries in resource directory: %d", numEntries) } for i := range numEntries { var entry peImageResourceDirectoryEntry + if err := w.spend(int64(binary.Size(entry))); err != nil { + return err + } + entryOffset := offset + int64(binary.Size(directoryHeader)) + int64(i*binary.Size(entry)) - if _, err := sec.Reader.Seek(entryOffset, io.SeekStart); err != nil { + if _, err := w.reader.Seek(entryOffset, io.SeekStart); err != nil { log.Tracef("error seeking to PE entry offset: %v", err) continue } - if err := readIntoStruct(sec.Reader, &entry); err != nil { + if err := readIntoStruct(w.reader, &entry); err != nil { continue } - if err := processResourceEntry(entry, baseRVA, sec, dirs, fields, names); err != nil { + if err := processResourceEntry(entry, w); err != nil { + // a budget or depth limit hit partway down the tree is not a property of this one entry, so + // stop the walk rather than letting every sibling re-discover it + if stopsWalk(err) { + return err + } log.Tracef("error processing resource entry: %v", err) continue } @@ -444,7 +658,7 @@ func parseResourceDirectory(sec *extractedSection, dirs *u32set.Set, fields map[ return nil } -func processResourceEntry(entry peImageResourceDirectoryEntry, baseRVA uint32, sec *extractedSection, dirs *u32set.Set, fields map[string]string, names *strset.Set) error { +func processResourceEntry(entry peImageResourceDirectoryEntry, w *resourceWalk) error { // if the high bit is set, this is a directory entry, otherwise it is a data entry isDirectory := entry.OffsetToData&0x80000000 != 0 @@ -456,75 +670,95 @@ func processResourceEntry(entry peImageResourceDirectoryEntry, baseRVA uint32, s // read the string name of the resource directory if nameIsString { - currentPos, err := sec.Reader.Seek(0, io.SeekCurrent) + currentPos, err := w.reader.Seek(0, io.SeekCurrent) if err != nil { return fmt.Errorf("error getting current reader position: %w", err) } - if _, err := sec.Reader.Seek(int64(nameOffset), io.SeekStart); err != nil { - return fmt.Errorf("error restoring reader position: %w", err) + nameAt, err := w.offsetWithin(nameOffset) + if err != nil { + return fmt.Errorf("resource name: %w", err) } - name, err := readUTF16WithLength(sec.Reader) - if err == nil { - names.Add(name) + if _, err := w.reader.Seek(nameAt, io.SeekStart); err != nil { + return fmt.Errorf("error seeking to resource name: %w", err) } - if _, err := sec.Reader.Seek(currentPos, io.SeekStart); err != nil { + // only one name matters downstream, so compare in place rather than retaining every name a + // crafted file cares to declare + name, err := w.readUTF16WithLength() + switch { + case stopsWalk(err): + return err + case err == nil && name == clrDebugInfoResourceName: + w.hasCLRDebugInfo = true + } + + if _, err := w.reader.Seek(currentPos, io.SeekStart); err != nil { return fmt.Errorf("error restoring reader position: %w", err) } } + targetRVA := w.baseRVA + entryOffsetToData + if isDirectory { - subRVA := baseRVA + entryOffsetToData - if dirs.Has(subRVA) { + if w.dirs.Has(targetRVA) { // some malware uses recursive PE references to evade analysis - return fmt.Errorf("recursive PE reference detected; skipping directory at baseRVA=0x%x subRVA=0x%x", baseRVA, subRVA) + return fmt.Errorf("recursive PE reference detected; skipping directory at baseRVA=0x%x subRVA=0x%x", w.baseRVA, targetRVA) } - dirs.Add(subRVA) - err := parseResourceDirectory( - &extractedSection{ - RVA: subRVA, - BaseRVA: baseRVA, - Size: sec.Size - (sec.RVA - baseRVA), - Reader: sec.Reader, - }, - dirs, fields, names) - if err != nil { - return err - } - return nil + w.dirs.Add(targetRVA) + + return parseResourceDirectoryAt(targetRVA, w) } - return parseResourceDataEntry(sec.Reader, baseRVA, baseRVA+entryOffsetToData, sec.Size, fields) + + return parseResourceDataEntry(targetRVA, w) } -func parseResourceDataEntry(reader *bytes.Reader, baseRVA, rva, remainingSize uint32, fields map[string]string) error { - var dataEntry peImageResourceDataEntry - offset := int64(rva - baseRVA) +func parseResourceDataEntry(rva uint32, w *resourceWalk) error { + offset, err := w.offsetOf(rva) + if err != nil { + return fmt.Errorf("resource data entry: %w", err) + } - if _, err := reader.Seek(offset, io.SeekStart); err != nil { + if _, err := w.reader.Seek(offset, io.SeekStart); err != nil { return fmt.Errorf("error seeking to data entry offset: %w", err) } - if err := readIntoStruct(reader, &dataEntry); err != nil { + var dataEntry peImageResourceDataEntry + if err := w.spend(int64(binary.Size(dataEntry))); err != nil { + return err + } + + if err := readIntoStruct(w.reader, &dataEntry); err != nil { return fmt.Errorf("error reading resource data entry: %w", err) } - if remainingSize < dataEntry.Size { - return fmt.Errorf("resource data entry size exceeds remaining size") + // OffsetToData and Size are both user-controlled uint32s, so the region they describe has to be bounded + // against the bytes the section actually holds before it sizes the allocation below. + dataOffset, err := w.offsetOf(dataEntry.OffsetToData) + if err != nil { + return fmt.Errorf("resource data: %w", err) + } + + if int64(dataEntry.Size) > w.reader.Size()-dataOffset { + return fmt.Errorf("resource data (offset=0x%x size=0x%x) extends past its section end 0x%x", dataOffset, dataEntry.Size, w.reader.Size()) + } + + if err := w.spend(int64(dataEntry.Size)); err != nil { + return err } data := make([]byte, dataEntry.Size) - if _, err := reader.Seek(int64(dataEntry.OffsetToData-baseRVA), io.SeekStart); err != nil { + if _, err := w.reader.Seek(dataOffset, io.SeekStart); err != nil { return fmt.Errorf("error seeking to resource data: %w", err) } - if _, err := reader.Read(data); err != nil { + if _, err := io.ReadFull(w.reader, data); err != nil { return fmt.Errorf("error reading resource data: %w", err) } - return parseVersionResourceSection(bytes.NewReader(data), fields) + return parseVersionResourceSection(bytes.NewReader(data), w.fields) } // parseVersionResourceSection parses a PE version resource section from within a resource directory. @@ -607,6 +841,11 @@ func parseVersionResourceSection(reader *bytes.Reader, fields map[string]string) var sfiHeader peStringFileInfo if szKey, err := readIntoStructAndSzKey(reader, &sfiHeader, &offset); err != nil { + if isTruncated(err) { + // a well-formed version resource whose last child is VarFileInfo ends right here, so stop + // and let the FileVersion fallback below still run + break + } return fmt.Errorf("error reading PE string file info header: %v", err) } else if szKey != "StringFileInfo" { // we only care about extracting strings from any string tables, skip this @@ -619,31 +858,14 @@ func parseVersionResourceSection(reader *bytes.Reader, fields map[string]string) // note: the szKey for the prStringTable is the language var stHeader peStringTable if _, err := readIntoStructAndSzKey(reader, &stHeader, &offset, &stOffset); err != nil { + if isTruncated(err) { + break + } return fmt.Errorf("error reading PE string table header: %v", err) } - for stOffset < int(stHeader.Length) { - var stringHeader peString - if err := readIntoStruct(reader, &stringHeader, &offset, &stOffset); err != nil { - break - } - - key := readUTF16(reader, &offset, &stOffset) - - if err := alignAndSeek(reader, &offset, &stOffset); err != nil { - return fmt.Errorf("error aligning to next PE string table value: %w", err) - } - - var value string - if stringHeader.ValueLength > 0 { - value = readUTF16(reader, &offset, &stOffset) - } - - fields[key] = value - - if err := alignAndSeek(reader, &offset, &stOffset); err != nil { - return fmt.Errorf("error aligning to next PE string table key: %w", err) - } + if err := parseStringTable(reader, int(stHeader.Length), &offset, &stOffset, fields); err != nil { + return err } } @@ -657,6 +879,40 @@ func parseVersionResourceSection(reader *bytes.Reader, fields map[string]string) return nil } +// parseStringTable reads the key/value pairs of a single string table into fields. length is what the +// string table header claims it holds, and stOffset tracks how much of that has actually been consumed. +func parseStringTable(reader *bytes.Reader, length int, offset, stOffset *int, fields map[string]string) error { + for *stOffset < length { + var stringHeader peString + if err := readIntoStruct(reader, &stringHeader, offset, stOffset); err != nil { + if isTruncated(err) { + // the table claims more content than the resource carries; stop rather than re-reading a + // reader that is not advancing + break + } + return fmt.Errorf("error reading PE string table entry: %w", err) + } + + key := readUTF16(reader, offset, stOffset) + + if err := alignAndSeek(reader, offset, stOffset); err != nil { + return fmt.Errorf("error aligning to next PE string table value: %w", err) + } + + var value string + if stringHeader.ValueLength > 0 { + value = readUTF16(reader, offset, stOffset) + } + + fields[key] = value + + if err := alignAndSeek(reader, offset, stOffset); err != nil { + return fmt.Errorf("error aligning to next PE string table key: %w", err) + } + } + return nil +} + // readIntoStructAndSzKey reads a struct from the reader and updates the offsets if provided, returning the szKey value. // This is only useful in the context of the resource directory parsing in narrow cases (this is invalid to use outside of that context). func readIntoStructAndSzKey[T any](reader *bytes.Reader, data *T, offsets ...*int) (string, error) { @@ -666,12 +922,19 @@ func readIntoStructAndSzKey[T any](reader *bytes.Reader, data *T, offsets ...*in return readUTF16(reader, offsets...), nil } +// isTruncated reports whether err means the resource simply ran out of bytes. binary.Read gives io.EOF when +// nothing was left and io.ErrUnexpectedEOF when a struct was cut in half; both say the same thing about a +// version resource, and neither should cost us the fields already collected. +func isTruncated(err error) bool { + return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) +} + // readIntoStruct reads a struct from the reader and updates the offsets if provided. +// +// note: EOF must stay an error. Callers advance their loop counters by the offsets updated below, so +// reporting a zeroed struct as a successful read leaves length-driven loops spinning forever. func readIntoStruct[T any](reader io.Reader, data *T, offsets ...*int) error { if err := binary.Read(reader, binary.LittleEndian, data); err != nil { - if errors.Is(err, io.EOF) { - return nil - } return err } @@ -726,20 +989,31 @@ func readUTF16(reader *bytes.Reader, offsets ...*int) string { return string(result) } -// readUTF16WithLength reads a length-prefixed UTF-16 string from reader. +// readUTF16WithLength reads a length-prefixed UTF-16 string from the walk's current position. // The first 2 bytes represent the number of UTF-16 code units. -func readUTF16WithLength(reader *bytes.Reader) (string, error) { +func (w *resourceWalk) readUTF16WithLength() (string, error) { var length uint16 - if err := binary.Read(reader, binary.LittleEndian, &length); err != nil { + if err := binary.Read(w.reader, binary.LittleEndian, &length); err != nil { return "", err } if length == 0 { return "", nil } + // length is a user-controlled uint16 and binary.Read allocates a second buffer of its own, so a name + // the reader cannot satisfy must be rejected before either one is sized from it + size := int64(length) * 2 + if size > int64(w.reader.Len()) { + return "", fmt.Errorf("declared name length %d exceeds the %d bytes remaining", length, w.reader.Len()) + } + + if err := w.spend(size); err != nil { + return "", err + } + // read length UTF-16 code units. codes := make([]uint16, length) - if err := binary.Read(reader, binary.LittleEndian, &codes); err != nil { + if err := binary.Read(w.reader, binary.LittleEndian, &codes); err != nil { return "", err } return string(utf16.Decode(codes)), nil diff --git a/syft/pkg/cataloger/internal/dotnet/pe/pe_partial_test.go b/syft/pkg/cataloger/internal/dotnet/pe/pe_partial_test.go new file mode 100644 index 000000000..bf8e48e1b --- /dev/null +++ b/syft/pkg/cataloger/internal/dotnet/pe/pe_partial_test.go @@ -0,0 +1,132 @@ +package pe + +import ( + "bytes" + "debug/pe" + "encoding/binary" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syftFile "github.com/anchore/syft/syft/file" +) + +// buildPE32 assembles a minimal but genuinely parseable PE32 with one section, wiring the resource and COM +// descriptor data directories to the RVAs given. A zero size leaves that directory absent. +func buildPE32(t *testing.T, resourceRVA, resourceSize, clrRVA, clrSize uint32, sectionData []byte) []byte { + t.Helper() + + const peAt = 0x80 + const sectionRVA = 0x1000 + const sectionRaw = 0x400 + + buf := make([]byte, sectionRaw) + copy(buf, "MZ") + binary.LittleEndian.PutUint32(buf[60:], peAt) + copy(buf[peAt:], "PE\x00\x00") + + fh := pe.FileHeader{ + Machine: 0x14c, + NumberOfSections: 1, + SizeOfOptionalHeader: uint16(binary.Size(pe.OptionalHeader32{})), + } + hdr := new(bytes.Buffer) + require.NoError(t, binary.Write(hdr, binary.LittleEndian, fh)) + + opt := pe.OptionalHeader32{Magic: 0x10B} + opt.DataDirectory[pe.IMAGE_DIRECTORY_ENTRY_RESOURCE] = pe.DataDirectory{VirtualAddress: resourceRVA, Size: resourceSize} + opt.DataDirectory[pe.IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR] = pe.DataDirectory{VirtualAddress: clrRVA, Size: clrSize} + require.NoError(t, binary.Write(hdr, binary.LittleEndian, opt)) + + sec := pe.SectionHeader32{ + VirtualAddress: sectionRVA, + VirtualSize: 0x1000, + SizeOfRawData: uint32(sectionRaw), + PointerToRawData: uint32(sectionRaw), + } + copy(sec.Name[:], ".rsrc") + require.NoError(t, binary.Write(hdr, binary.LittleEndian, sec)) + + copy(buf[peAt+4:], hdr.Bytes()) + + return append(buf, sectionData...) +} + +func readPE(t *testing.T, data []byte) (*File, error) { + t.Helper() + + loc := syftFile.NewLocation("test.dll") + return Read(syftFile.NewLocationReadCloser(loc, readSeekCloser{bytes.NewReader(data)})) +} + +type readSeekCloser struct{ *bytes.Reader } + +func (readSeekCloser) Close() error { return nil } + +func TestRead_MalformedResourceDirectoryStillYieldsAFile(t *testing.T) { + // one malformed directory costs us the fields it would have held, not the package. Failing the whole + // file here used to drop it from the SBOM entirely, and the pe-binary cataloger then reported an error + // instead of a package. + section := make([]byte, 0x400) + // a root directory claiming counts that sum past 0xFFFF, which is rejected outright + binary.LittleEndian.PutUint16(section[12:], 0x8000) + binary.LittleEndian.PutUint16(section[14:], 0x8001) + + // a valid CLR header lives further into the same section, so there is still evidence to recover + clr := peImageCore20{Cb: 72, MajorRuntimeVersion: 2, MinorRuntimeVersion: 5} + clrBytes := new(bytes.Buffer) + require.NoError(t, binary.Write(clrBytes, binary.LittleEndian, clr)) + copy(section[0x200:], clrBytes.Bytes()) + + data := buildPE32(t, 0x1000, 0x400, 0x1200, uint32(clrBytes.Len()), section) + + f, err := readPE(t, data) + + require.NoError(t, err, "a malformed resource directory must not fail the whole file") + require.NotNil(t, f) + + // the partial parse has to be reported rather than only reaching a trace log, or the unknowns task + // cannot see that this file was cataloged on incomplete evidence + require.Error(t, f.ParseErr, "the resource directory failure must be recorded") + assert.Contains(t, f.ParseErr.Error(), "resource directory") + + // and the evidence that did parse has to survive + assert.True(t, f.CLR.HasEvidenceOfCLR(), "the CLR header parsed fine and must still be reported") + assert.Equal(t, uint16(2), f.CLR.MajorVersion) +} + +func TestRead_UnreadableDataDirectoryStillYieldsAFile(t *testing.T) { + // the same rule one layer down: a directory whose bytes cannot be read at all is recorded and skipped. + // This used to be fatal, which was inconsistent with the resource walk being non-fatal a line away. + section := make([]byte, 0x400) + + // a resource directory declaring far more than the file holds + data := buildPE32(t, 0x1000, 0xFFFFFFF, 0, 0, section) + + f, err := readPE(t, data) + + require.NoError(t, err) + require.NotNil(t, f) + require.Error(t, f.ParseErr) + assert.Contains(t, f.ParseErr.Error(), "Resource") + assert.Empty(t, f.VersionResources) +} + +func TestRead_WellFormedFileReportsNoParseError(t *testing.T) { + // the guard against the above turning every file into a partial parse + section := make([]byte, 0x400) + putResourceDirN(section, 0x000, 1, 0x100, false) // root -> data entry + binary.LittleEndian.PutUint32(section[0x100:], 0x1000+0x200) + binary.LittleEndian.PutUint32(section[0x104:], 0x100) + copy(section[0x200:], buildVersionResource(true)) + + data := buildPE32(t, 0x1000, 0x400, 0, 0, section) + + f, err := readPE(t, data) + + require.NoError(t, err) + require.NotNil(t, f) + assert.NoError(t, f.ParseErr, "a file that parsed cleanly must not report a partial parse") + assert.Equal(t, "9.9.9.9", f.VersionResources["FileVersion"]) +} diff --git a/syft/pkg/cataloger/internal/dotnet/pe/pe_resource_test.go b/syft/pkg/cataloger/internal/dotnet/pe/pe_resource_test.go new file mode 100644 index 000000000..305f43a92 --- /dev/null +++ b/syft/pkg/cataloger/internal/dotnet/pe/pe_resource_test.go @@ -0,0 +1,499 @@ +package pe + +import ( + "bytes" + "encoding/binary" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/testutils" +) + +const ( + testSectionRVA = 0x1000 + testSectionSize = 0x400 +) + +// putResourceDirN writes a resource directory header declaring n ID entries, all of them pointing at the +// same target. isDir sets the high bit, which marks that target as a subdirectory. Aliasing every entry +// onto one target is what the format permits and no real toolchain emits. +func putResourceDirN(buf []byte, at, n int, offsetToData uint32, isDir bool) { + le := binary.LittleEndian + le.PutUint16(buf[at+12:], 0) // NumberOfNamedEntries + le.PutUint16(buf[at+14:], uint16(n)) // NumberOfIDEntries + + if isDir { + offsetToData |= 0x80000000 + } + for i := range n { + entry := at + 16 + i*8 + le.PutUint32(buf[entry:], uint32(i)) // Name (distinct IDs, not strings) + le.PutUint32(buf[entry+4:], offsetToData) + } +} + +// boundWalk returns a walk over data as its resource section, as parseResourceDirectory would. +func boundWalk(data []byte) *resourceWalk { + return newResourceWalk(bytes.NewReader(data), testSectionRVA) +} + +func TestParseResourceDataEntry_SizePastSectionIsRejected(t *testing.T) { + // OffsetToData and Size are user-controlled uint32s. A data entry may only describe bytes its section + // actually holds, otherwise Size drives the allocation. + buf := make([]byte, testSectionSize) + le := binary.LittleEndian + le.PutUint32(buf[0x100:], testSectionRVA) // OffsetToData, resolves to section offset 0 + le.PutUint32(buf[0x104:], 256*intFile.MB) // a size far past the 1KB section + + err := parseResourceDataEntry(testSectionRVA+0x100, boundWalk(buf)) + + // note: this pins the bound, not an allocation. A single-level entry was already rejected before this + // change (by comparing against the section size), it just said so differently, so an allocation + // assertion here would pass with or without the fix. TestParseResourceDirectory_NestedOffsetsCannotUnderflow + // is the one that holds the allocation property, because that is the shape that used to slip through. + require.ErrorContains(t, err, "extends past its section end") +} + +func TestParseResourceDataEntry_OffsetBeforeSectionBaseIsRejected(t *testing.T) { + // OffsetToData is independent of baseRVA, so it can name an RVA before the section starts. The + // subtraction that turns it into a section offset would otherwise underflow to near 4GB. + buf := make([]byte, testSectionSize) + le := binary.LittleEndian + le.PutUint32(buf[0x100:], testSectionRVA-1) // OffsetToData, one byte before the section base + le.PutUint32(buf[0x104:], 16) + + err := parseResourceDataEntry(testSectionRVA+0x100, boundWalk(buf)) + require.ErrorContains(t, err, "precedes its section base") +} + +func TestParseResourceDataEntry_EntryRVAPastSectionIsRejected(t *testing.T) { + // the entry's own RVA is as user-controlled as the data it points at + buf := make([]byte, testSectionSize) + + err := parseResourceDataEntry(testSectionRVA+testSectionSize, boundWalk(buf)) + require.ErrorContains(t, err, "lies past its section end") +} + +func TestParseResourceDirectory_SubdirectoryPastSectionIsRejected(t *testing.T) { + // a subdirectory RVA is derived from baseRVA plus a user-controlled offset, so a child can name bytes + // past the section its parent lives in + buf := make([]byte, testSectionSize) + putResourceDirN(buf, 0x000, 1, testSectionSize, true) // root -> subdirectory at the section end + + w := boundWalk(buf) + err := parseResourceDirectoryAt(testSectionRVA, w) + + // the root loop logs and continues past a bad child, so the rejection shows up as an unvisited tree + require.NoError(t, err) + assert.Empty(t, w.fields) +} + +func TestParseResourceDirectory_EntryCountsCannotWrap(t *testing.T) { + // NumberOfNamedEntries and NumberOfIDEntries are uint16s that a crafted file can make sum past 0xFFFF. + // Adding them at uint16 width would wrap to a small number and hide entries a real loader still walks. + buf := make([]byte, testSectionSize) + le := binary.LittleEndian + le.PutUint16(buf[12:], 0x8000) + le.PutUint16(buf[14:], 0x8001) // sums to 0x10001, which wraps to 1 + + err := parseResourceDirectoryAt(testSectionRVA, boundWalk(buf)) + require.ErrorContains(t, err, "too many entries in resource directory") +} + +func TestParseResourceDirectory_AliasedEntriesCannotAmplifyWork(t *testing.T) { + // nothing in the format stops thousands of entries from naming one fat blob, and every individual + // offset here is inside the section. Without a bound on total bytes read, a section this size drives + // tens of GB of allocation and minutes of parsing, so peak memory stays flat while the scan hangs. + const secSize = 256 * intFile.KB + buf := make([]byte, secSize) + le := binary.LittleEndian + + // root: 16 subdirectories at distinct RVAs, since aliased directories are already deduped + le.PutUint16(buf[14:], 16) + for i := range 16 { + entry := 16 + i*8 + le.PutUint32(buf[entry:], uint32(i)) + le.PutUint32(buf[entry+4:], uint32(0x1000+i*0x800)|0x80000000) + } + + // each subdirectory: 4096 leaf entries, every one pointing at the same near-section-sized blob + for i := range 16 { + putResourceDirN(buf, 0x1000+i*0x800, 4096, 0x10000, false) + } + le.PutUint32(buf[0x10000:], testSectionRVA+0x20000) + le.PutUint32(buf[0x10004:], secSize-0x20000) + + // make the blob a version resource, so each leaf that reaches it also pays for a full string-table walk + copy(buf[0x20000:], buildVersionResource(true)) + + w := boundWalk(buf) + + var err error + var timedOut bool + allocated := testutils.MeasureAlloc(t, func() { + done := make(chan error, 1) + go func() { done <- parseResourceDirectoryAt(testSectionRVA, w) }() + select { + case err = <-done: + case <-time.After(30 * time.Second): + timedOut = true + } + }) + + require.False(t, timedOut, "the walk did not terminate") + require.ErrorIs(t, err, errResourceBudget, + "the walk must stop once it has read more than a well-formed section could justify") + // the point is the asymptote, not the constant: each byte the budget allows may still be copied into a + // blob buffer and walked, so a small multiple of the section is expected. Before the bound this same + // input allocated about 24GB, so anything proportional is three orders of magnitude away from the bug. + // expressed in terms of the factor so raising the budget cannot silently widen what this accepts + assert.Less(t, allocated, uint64(8*peResourceBudgetFactor*secSize), + "total work must stay proportional to the section, not to the entries that alias into it") +} + +// buildVersionResource returns a well-formed VS_VERSION_INFO blob whose last child is a VarFileInfo +// block, which is the layout real toolchains emit. It ends exactly on a struct boundary. When +// withFileVersionString is set, a StringFileInfo table supplies FileVersion directly. +func buildVersionResource(withFileVersionString bool) []byte { + buf := new(bytes.Buffer) + le := binary.LittleEndian + + putHeader := func(length, valueLength, typ uint16) { + _ = binary.Write(buf, le, [3]uint16{length, valueLength, typ}) + } + putUTF16 := func(s string) { + for _, r := range s { + _ = binary.Write(buf, le, uint16(r)) + } + _ = binary.Write(buf, le, uint16(0)) + } + pad := func() { + for buf.Len()%4 != 0 { + buf.WriteByte(0) + } + } + + putHeader(0, 52, 0) + putUTF16("VS_VERSION_INFO") + pad() + + // peVsFixedFileInfo, with FileVersionMS/LS encoding 1.2.3.4 + ffi := make([]byte, 52) + le.PutUint32(ffi[0:], 0xFEEF04BD) // signature + le.PutUint32(ffi[4:], 0x00010000) // strucVersion + le.PutUint32(ffi[8:], 0x00010002) // FileVersionMS -> 1.2 + le.PutUint32(ffi[12:], 0x00030004) // FileVersionLS -> 3.4 + buf.Write(ffi) + + if withFileVersionString { + pad() + putHeader(0, 0, 1) + putUTF16("StringFileInfo") + pad() + putHeader(38, 0, 1) + putUTF16("040904b0") + pad() + putHeader(0, 8, 1) + putUTF16("FileVersion") + pad() + putUTF16("9.9.9.9") + } + + // the final child: VarFileInfo -> Var("Translation") with a 4 byte value + pad() + putHeader(0, 0, 1) + putUTF16("VarFileInfo") + pad() + putHeader(0, 4, 0) + putUTF16("Translation") + pad() + _ = binary.Write(buf, le, uint32(0x04b00409)) + + return buf.Bytes() +} + +func TestParseVersionResourceSection_FileVersionFallback(t *testing.T) { + // a resource ending on a struct boundary is normal termination, not a parse failure. Treating it as + // an error skips the VS_FIXEDFILEINFO fallback below, which is the only source of a version for + // binaries that carry no FileVersion string entry. + tests := []struct { + name string + withFileVersionString bool + want string + }{ + { + name: "derived from fixed file info when no string entry exists", + want: "1.2.3.4", + }, + { + name: "string entry wins when present", + withFileVersionString: true, + want: "9.9.9.9", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fields := map[string]string{} + require.NoError(t, parseVersionResourceSection(bytes.NewReader(buildVersionResource(tt.withFileVersionString)), fields)) + assert.Equal(t, tt.want, fields["FileVersion"]) + }) + } +} + +// buildTruncatedStringTable returns version resource bytes that end exactly on a struct boundary while +// the string table header still claims 0xFFFF bytes remain. Landing precisely at EOF is one of two cases +// that matter; trailing bytes that cut a struct in half are the other, covered by +// TestParseVersionResourceSection_TrailingBytesKeepFileVersion. +func buildTruncatedStringTable() []byte { + buf := new(bytes.Buffer) + putHeader := func() { + _ = binary.Write(buf, binary.LittleEndian, [3]uint16{}) // Length, ValueLength, Type + } + putUTF16 := func(s string) { + for _, r := range s { + _ = binary.Write(buf, binary.LittleEndian, uint16(r)) + } + _ = binary.Write(buf, binary.LittleEndian, uint16(0)) // null terminator + } + + putHeader() + putUTF16("VS_VERSION_INFO") // offset 38 + buf.Write([]byte{0, 0}) // pad to the DWORD boundary at 40 + buf.Write(make([]byte, 52)) // peVsFixedFileInfo -> 92 + + putHeader() + putUTF16("StringFileInfo") // -> 128 + + // the string table header claims far more content than the file carries + _ = binary.Write(buf, binary.LittleEndian, [3]uint16{0xFFFF, 0, 0}) + putUTF16("040904b0") // -> 152, then EOF + + return buf.Bytes() +} + +func TestParseVersionResourceSection_TruncatedStringTableTerminates(t *testing.T) { + data := buildTruncatedStringTable() + + done := make(chan error, 1) + go func() { + done <- parseVersionResourceSection(bytes.NewReader(data), map[string]string{}) + }() + + select { + case <-done: + case <-time.After(5 * time.Second): + // a reader parked at EOF used to be reported as a successful read of a zeroed struct, so the + // string table loop consumed nothing, advanced nothing, and never reached its length bound + require.FailNow(t, "parseVersionResourceSection did not terminate", + "a %d-byte resource blob must not spin forever", len(data)) + } +} + +func TestParseResourceDirectory_DeepChainCannotOverflowTheStack(t *testing.T) { + // a chain of directories at distinct RVAs, each naming the next. Every offset is inside the section and + // every RVA is distinct, so neither the dirs set (which only catches a repeated RVA) nor the byte budget + // (which scales with the section) stops it. Unbounded, this recurses until the goroutine stack hits the + // process limit, and Go answers that with a fatal error no recover() can catch: the whole scan dies. + // + // the stride is 12 bytes, the tightest packing where each directory's counts and its single entry do not + // collide with its neighbours' (dir k reads counts at 12k+12 and its entry at 12k+16, while dir k+1 reads + // counts at 12k+24). + const stride = 12 + const levels = 200 + + buf := make([]byte, stride*(levels+4)) + le := binary.LittleEndian + for k := range levels { + at := stride * k + le.PutUint16(buf[at+14:], 1) // one ID entry + le.PutUint32(buf[at+16:], uint32(k)) // Name (distinct ID) + le.PutUint32(buf[at+20:], uint32(stride*(k+1))|0x80000000) // -> next directory + } + + w := boundWalk(buf) + err := parseResourceDirectoryAt(testSectionRVA, w) + + require.ErrorIs(t, err, errResourceDepth, + "a chain deeper than the cap must be rejected rather than recursed into") + assert.LessOrEqual(t, w.depth, peMaxResourceDirectoryDepth, + "the depth counter must unwind as the walk returns") +} + +func TestParseResourceDirectory_NestedOffsetsCannotUnderflow(t *testing.T) { + // the shape that used to authorize a ~4.29GB allocation out of a 1KB section. Each level's offset is + // individually smaller than the section, so each directory header reads fine, but the old walk tracked a + // uint32 "remaining size" and subtracted each level's offset from it. By the third level that subtraction + // underflowed to near 4GB, and a leaf declaring a size just under it then passed the bounds check and + // sized the buffer from it. + buf := make([]byte, testSectionSize) + le := binary.LittleEndian + + putResourceDirN(buf, 0x000, 1, 0x300, true) // root -> level 2 + putResourceDirN(buf, 0x300, 1, 0x300, true) // level 2 -> level 3 (same offset, deeper) + putResourceDirN(buf, 0x300, 1, 0x200, false) // level 3 -> data entry + + // the leaf claims almost 4GB, which the underflowed remaining size used to permit + le.PutUint32(buf[0x200:], testSectionRVA) // OffsetToData -> section offset 0 + le.PutUint32(buf[0x204:], 0xFFFFFF00) // Size + + w := boundWalk(buf) + + var err error + allocated := testutils.MeasureAlloc(t, func() { + err = parseResourceDirectoryAt(testSectionRVA, w) + }) + + require.NoError(t, err, "bad children are logged and skipped, so the walk itself completes") + assert.Less(t, allocated, uint64(intFile.MB), + "a 1KB section must never authorize a multi-gigabyte read, however deeply it is nested") + assert.Empty(t, w.fields, "nothing in this tree is a real version resource") +} + +func TestParseResourceDirectory_SelfReferentialEntryTerminates(t *testing.T) { + // a directory whose entry names its own RVA. The dirs set is what catches this, and it is the guard the + // budget and depth bounds cannot express, since one level of aliasing costs almost nothing. + buf := make([]byte, testSectionSize) + putResourceDirN(buf, 0x000, 1, 0, true) // root -> itself (offset 0) + + w := boundWalk(buf) + + done := make(chan error, 1) + go func() { done <- parseResourceDirectoryAt(testSectionRVA, w) }() + + select { + case err := <-done: + require.NoError(t, err, "the self-reference is logged and skipped, not fatal") + assert.Empty(t, w.fields) + case <-time.After(5 * time.Second): + require.FailNow(t, "a self-referential resource directory did not terminate") + } +} + +func TestProcessResourceEntry_HostileNameLengthIsRejected(t *testing.T) { + // a named entry's length prefix is a user-controlled uint16, and binary.Read allocates a buffer of its + // own on top of ours, so a name the section cannot satisfy has to be rejected before either is sized. + // + // note: a name that cannot be read is not fatal to the entry. The name is optional evidence (only + // CLRDEBUGINFO matters downstream), so the walk carries on and simply does not learn it. That is why + // these assert on the allocation and on the name not being recorded, rather than on an error: only the + // allocation states the property these guards exist for. + tests := []struct { + name string + nameOffset uint32 + length uint16 + }{ + { + name: "declared length past the end of the section", + nameOffset: testSectionSize - 4, + length: 0xFFFF, + }, + { + name: "name offset past the end of the section", + nameOffset: testSectionSize, + length: 1, + }, + { + name: "declared length of MaxUint16 at the section start", + nameOffset: 0, + length: 0xFFFF, + }, + { + // a zero-length name is legal and must not drive a read + name: "zero length name", + nameOffset: 0x100, + length: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := make([]byte, testSectionSize) + if int(tt.nameOffset) < len(buf)-2 { + binary.LittleEndian.PutUint16(buf[tt.nameOffset:], tt.length) + } + + // a data entry (high bit clear on OffsetToData) with a string name (high bit set on Name) + entry := peImageResourceDirectoryEntry{ + Name: tt.nameOffset | 0x80000000, + OffsetToData: 0x100, + } + + w := boundWalk(buf) + + allocated := testutils.MeasureAlloc(t, func() { + _ = processResourceEntry(entry, w) + }) + + assert.Less(t, allocated, uint64(intFile.MB), + "a declared name length must never size a buffer the section cannot back") + assert.False(t, w.hasCLRDebugInfo, + "no name was readable, so none should have been recorded") + }) + } +} + +func TestProcessResourceEntry_CLRDebugInfoNameIsRecorded(t *testing.T) { + // the one name that matters downstream, so the happy path through the length-prefixed reader is pinned + // alongside the hostile ones above + buf := make([]byte, testSectionSize) + le := binary.LittleEndian + + const nameAt = 0x100 + le.PutUint16(buf[nameAt:], uint16(len(clrDebugInfoResourceName))) + for i, r := range clrDebugInfoResourceName { + le.PutUint16(buf[nameAt+2+i*2:], uint16(r)) + } + + w := boundWalk(buf) + entry := peImageResourceDirectoryEntry{Name: nameAt | 0x80000000, OffsetToData: 0x200} + + _ = processResourceEntry(entry, w) + + assert.True(t, w.hasCLRDebugInfo, "a CLRDEBUGINFO resource name must be recorded") +} + +func TestParseVersionResourceSection_TrailingBytesKeepFileVersion(t *testing.T) { + // the case the isTruncated guard exists for. A version resource whose last child leaves 1 to 5 trailing + // bytes cuts the next struct in half, which binary.Read reports as io.ErrUnexpectedEOF rather than + // io.EOF. That used to propagate out as a parse failure, and because the failure returned early it also + // skipped the VS_FIXEDFILEINFO fallback, so the binary lost its version entirely. + for trailing := 0; trailing <= 6; trailing++ { + t.Run(fmt.Sprintf("%d trailing bytes", trailing), func(t *testing.T) { + data := append(buildVersionResource(false), make([]byte, trailing)...) + + fields := map[string]string{} + require.NoError(t, parseVersionResourceSection(bytes.NewReader(data), fields)) + assert.Equal(t, "1.2.3.4", fields["FileVersion"], + "a resource that runs out mid-struct must still yield the fields already collected") + }) + } +} + +func TestParseResourceDataEntry_ZeroSizeIsHandled(t *testing.T) { + // a zero-size data entry is degenerate rather than hostile, and must not be reported as a version + // resource or drive a read + buf := make([]byte, testSectionSize) + le := binary.LittleEndian + le.PutUint32(buf[0x100:], testSectionRVA) // OffsetToData + le.PutUint32(buf[0x104:], 0) // Size + + w := boundWalk(buf) + err := parseResourceDataEntry(testSectionRVA+0x100, w) + + require.Error(t, err, "an empty blob carries no version info") + assert.Empty(t, w.fields) +} + +func TestParseResourceDirectory_EmptySectionIsRejected(t *testing.T) { + // a zero-length section means every offset is out of range, including the root's + w := newResourceWalk(bytes.NewReader(nil), testSectionRVA) + + err := parseResourceDirectoryAt(testSectionRVA, w) + require.ErrorContains(t, err, "lies past its section end") +} diff --git a/syft/pkg/cataloger/internal/dotnet/pe/pe_rva_test.go b/syft/pkg/cataloger/internal/dotnet/pe/pe_rva_test.go new file mode 100644 index 000000000..f3b1044ae --- /dev/null +++ b/syft/pkg/cataloger/internal/dotnet/pe/pe_rva_test.go @@ -0,0 +1,99 @@ +package pe + +import ( + "bytes" + "debug/pe" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/testutils" +) + +func TestReadDataFromRVA_BogusSizeDoesNotOverAllocate(t *testing.T) { + // size comes from the PE headers and spans the full uint32 range. Sizing the buffer from it up front + // let a tiny file reserve 4GB, so the shortfall has to be caught before the allocation: the read stays + // a single exactly-sized one, and a size the file cannot back is an error rather than a mostly-zero + // buffer returned as if it had been filled. + const fileSize = 512 + sections := []pe.SectionHeader32{{VirtualAddress: 0x1000, VirtualSize: 0x1000, PointerToRawData: 0}} + r := &readSizeRecorder{Reader: bytes.NewReader(make([]byte, fileSize))} + + var err error + allocated := testutils.MeasureAlloc(t, func() { + _, err = readDataFromRVA(r, 0x1000, 0xFFFFFFFF, sections) + }) + + require.Error(t, err, "a size the file cannot satisfy must not be reported as a successful read") + assert.Less(t, allocated, uint64(intFile.MB), + "the declared 4GB must never be reserved; the shortfall has to be caught before the allocation") + assert.LessOrEqual(t, r.maxRead, fileSize, + "no single read may exceed the file size, regardless of what the headers claim") +} + +func TestReadDataFromRVA_ReadsFullyWhenSizeFits(t *testing.T) { + // the bound must not truncate a legitimate read + sections := []pe.SectionHeader32{{VirtualAddress: 0x1000, VirtualSize: 0x1000, PointerToRawData: 0}} + want := bytes.Repeat([]byte("z"), 128) + + got, err := readDataFromRVA(bytes.NewReader(want), 0x1000, uint32(len(want)), sections) + require.NoError(t, err) + assert.Equal(t, int64(len(want)), got.Size()) +} + +func TestReadDataFromRVA_StopsAtSizeInALargerFile(t *testing.T) { + // a file exactly `size` bytes long cannot distinguish a bounded read from an unbounded one, since the + // read stops at EOF either way. Only a file with bytes past `size` pins the limit. + sections := []pe.SectionHeader32{{VirtualAddress: 0x1000, VirtualSize: 0x2000, PointerToRawData: 0}} + const size = 128 + file := append(bytes.Repeat([]byte("z"), size), bytes.Repeat([]byte("!"), 4096)...) + r := &readSizeRecorder{Reader: bytes.NewReader(file)} + + got, err := readDataFromRVA(r, 0x1000, size, sections) + require.NoError(t, err) + + data := make([]byte, got.Size()) + _, err = got.ReadAt(data, 0) + require.NoError(t, err) + assert.Equal(t, bytes.Repeat([]byte("z"), size), data, + "the read must stop at size rather than running on into the rest of the file") + assert.LessOrEqual(t, r.maxRead, size, "no read may request more than the declared size") +} + +func TestReadDataFromRVA_SizePastTheAbsoluteCapIsRejected(t *testing.T) { + // clamping to the bytes remaining in the file is not enough on its own: a mostly empty file costs + // almost nothing inside a compressed layer, so a cheap artifact can still declare a directory large + // enough to OOM the scan. The absolute cap is what bounds that, and it has to run before the + // allocation like every other bound here. + const size = maxDirectorySectionSize + 1 + sections := []pe.SectionHeader32{{VirtualAddress: 0x1000, VirtualSize: 0x1000, PointerToRawData: 0}} + + // a sparse file that really is large enough to satisfy the declared size + r := &readSizeRecorder{Reader: bytes.NewReader(make([]byte, size+1))} + + var err error + allocated := testutils.MeasureAlloc(t, func() { + _, err = readDataFromRVA(r, 0x1000, size, sections) + }) + + require.ErrorContains(t, err, "exceeds the") + assert.Less(t, allocated, uint64(intFile.MB), + "the cap has to be checked before the buffer is sized, not after") +} + +func TestReadDataFromRVA_SizeAtTheAbsoluteCapIsAllowed(t *testing.T) { + // the cap must not reject what it is meant to permit, so pin the boundary from the other side. Only the + // requested read length is checked here; allocating the full cap would make the test cost of this the + // same as the bug it guards against. + sections := []pe.SectionHeader32{{VirtualAddress: 0x1000, VirtualSize: 0x1000, PointerToRawData: 0}} + + // one byte short of the cap, in a file that cannot back it: the size check has to pass and the + // remaining-bytes check has to be what rejects it + r := &readSizeRecorder{Reader: bytes.NewReader(make([]byte, 512))} + + _, err := readDataFromRVA(r, 0x1000, maxDirectorySectionSize-1, sections) + require.ErrorContains(t, err, "only 512 remain", + "a size under the cap must fall through to the remaining-bytes check, not be capped away") +} diff --git a/syft/pkg/cataloger/internal/dotnet/pe/pe_test.go b/syft/pkg/cataloger/internal/dotnet/pe/pe_test.go index 0fdb62ace..b612e7bc0 100644 --- a/syft/pkg/cataloger/internal/dotnet/pe/pe_test.go +++ b/syft/pkg/cataloger/internal/dotnet/pe/pe_test.go @@ -86,19 +86,45 @@ func Test_Read_DotNetDetection(t *testing.T) { wantErr: require.NoError, }, { - name: "jruby", - path: "/app/jruby_windows_9_3_15_0.exe", + // the apphost is a native launcher, not a managed assembly, so there is no CLR evidence to find in it. + // note the contrast with the single file deployment case below, which is the same apphost with the app + // (and the CLRDEBUGINFO resource) bundled into it. + name: "apphost", + path: "/app/dotnetapp.exe", fixture: "image-net8-app", wantCLR: false, // important! wantVR: map[string]string{ - "CompanyName": "JRuby Dev Team", - "FileDescription": "JRuby", - "FileVersion": "9.3.15.0", - "InternalName": "jruby", - "LegalCopyright": "JRuby Dev Team", - "OriginalFilename": "jruby_windows-x32_9_3_15_0.exe", - "ProductName": "JRuby", - "ProductVersion": "9.3.15.0", + "CompanyName": "dotnetapp", + "FileDescription": "dotnetapp", + "FileVersion": "1.0.0.0", + "InternalName": "dotnetapp.dll", + "LegalCopyright": " ", + "OriginalFilename": "dotnetapp.dll", + "ProductName": "dotnetapp", + "ProductVersion": "1.0.0", + "Assembly Version": "1.0.0.0", + }, + wantErr: require.NoError, + }, + { + // the framework-dependent apphost, which carries the bundle signature with a zero offset + // placeholder because it was never published as a single file. This is the most common shape a + // .NET executable takes, so anything it reports as a parse failure is reported against nearly + // every .NET binary that exists. + name: "framework dependent apphost", + path: "/app/dotnetapp.exe", + fixture: "image-net8-app", + wantCLR: false, // the CLR metadata lives in the sibling dotnetapp.dll + wantVR: map[string]string{ + "Assembly Version": "1.0.0.0", + "CompanyName": "dotnetapp", + "FileDescription": "dotnetapp", + "FileVersion": "1.0.0.0", + "InternalName": "dotnetapp.dll", + "LegalCopyright": " ", + "OriginalFilename": "dotnetapp.dll", + "ProductName": "dotnetapp", + "ProductVersion": "1.0.0", }, wantErr: require.NoError, }, @@ -125,13 +151,19 @@ func Test_Read_DotNetDetection(t *testing.T) { }, } + var fixtures []string + for _, tt := range tests { + fixtures = append(fixtures, tt.fixture) + } + resolvers := fixtureResolvers(t, fixtures...) + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if tt.wantErr == nil { tt.wantErr = require.NoError } - reader := fixtureFile(t, tt.fixture, tt.path) + reader := fixtureFile(t, resolvers[tt.fixture], tt.path) got, err := Read(reader) tt.wantErr(t, err) @@ -139,6 +171,10 @@ func Test_Read_DotNetDetection(t *testing.T) { return } + // a real binary must parse completely. ParseErr is how a partial parse reaches the unknowns + // channel, so anything non-nil here is a field we silently failed to read on stock input. + require.NoError(t, got.ParseErr, "a well-formed binary must not report a partial parse") + if d := cmp.Diff(tt.wantVR, got.VersionResources); d != "" { t.Errorf("unexpected version resources (-want +got): %s", d) } @@ -153,16 +189,31 @@ func Test_Read_DotNetDetection(t *testing.T) { } } -func fixtureFile(t *testing.T, fixture, path string) file.LocationReadCloser { - img := imagetest.GetFixtureImage(t, "docker-archive", fixture) +// fixtureResolvers loads each distinct image fixture once, from the parent test, so that subtests reading multiple +// files out of the same image don't each pay for loading the docker archive again (which dominates the runtime of +// these tests, especially under -race). +func fixtureResolvers(t *testing.T, fixtures ...string) map[string]file.Resolver { + resolvers := make(map[string]file.Resolver) + for _, fixture := range fixtures { + if _, ok := resolvers[fixture]; ok { + continue + } - s := stereoscopesource.New(img, stereoscopesource.ImageConfig{ - Reference: fixture, - }) + img := imagetest.GetFixtureImage(t, "docker-archive", fixture) - r, err := s.FileResolver(source.SquashedScope) - require.NoError(t, err) + s := stereoscopesource.New(img, stereoscopesource.ImageConfig{ + Reference: fixture, + }) + r, err := s.FileResolver(source.SquashedScope) + require.NoError(t, err) + + resolvers[fixture] = r + } + return resolvers +} + +func fixtureFile(t *testing.T, r file.Resolver, path string) file.LocationReadCloser { locs, err := r.FilesByPath(path) require.NoError(t, err) diff --git a/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet31-single-file/Dockerfile b/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet31-single-file/Dockerfile index 71fdc97f7..5a1b03e07 100644 --- a/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet31-single-file/Dockerfile +++ b/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet31-single-file/Dockerfile @@ -1,9 +1,12 @@ +# note: this is published framework-dependent on purpose: the bundle format (what these fixtures exercise) is +# identical either way, but bundling the runtime makes the fixture ~60MB instead of ~160KB, which dominates +# the runtime of the tests that load this image. FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/core/sdk:3.1 AS build WORKDIR /src COPY src . RUN dotnet publish -c Release -r win-x64 \ -p:PublishSingleFile=true \ - -p:SelfContained=true \ + -p:SelfContained=false \ -o /app FROM busybox diff --git a/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet5-single-file/Dockerfile b/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet5-single-file/Dockerfile index d9ab0b574..0edf7deff 100644 --- a/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet5-single-file/Dockerfile +++ b/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet5-single-file/Dockerfile @@ -1,9 +1,12 @@ +# note: this is published framework-dependent on purpose: the bundle format (what these fixtures exercise) is +# identical either way, but bundling the runtime makes the fixture ~60MB instead of ~160KB, which dominates +# the runtime of the tests that load this image. FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:5.0 AS build WORKDIR /src COPY src . RUN dotnet publish -c Release -r win-x64 \ -p:PublishSingleFile=true \ - -p:SelfContained=true \ + -p:SelfContained=false \ -o /app FROM busybox diff --git a/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet6-single-file/Dockerfile b/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet6-single-file/Dockerfile index bf89e026f..4b6a922d0 100644 --- a/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet6-single-file/Dockerfile +++ b/syft/pkg/cataloger/internal/dotnet/pe/testdata/image-dotnet6-single-file/Dockerfile @@ -1,9 +1,12 @@ +# note: this is published framework-dependent on purpose: the bundle format (what these fixtures exercise) is +# identical either way, but bundling the runtime makes the fixture ~60MB instead of ~160KB, which dominates +# the runtime of the tests that load this image. FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:6.0 AS build WORKDIR /src COPY src . RUN dotnet publish -c Release -r win-x64 \ -p:PublishSingleFile=true \ - -p:SelfContained=true \ + -p:SelfContained=false \ -o /app FROM busybox diff --git a/syft/pkg/cataloger/internal/pkgtest/metadata_tracker.go b/syft/pkg/cataloger/internal/pkgtest/metadata_tracker.go index bed764b35..b4447d429 100644 --- a/syft/pkg/cataloger/internal/pkgtest/metadata_tracker.go +++ b/syft/pkg/cataloger/internal/pkgtest/metadata_tracker.go @@ -334,7 +334,7 @@ func getMetadataTypeName(metadata any) string { } // handle pointers - if t.Kind() == reflect.Ptr { + if t.Kind() == reflect.Pointer { t = t.Elem() } @@ -407,7 +407,7 @@ func dereferenceToStruct(v any) reflect.Value { } val := reflect.ValueOf(v) - if val.Kind() == reflect.Ptr { + if val.Kind() == reflect.Pointer { if val.IsNil() { return reflect.Value{} } @@ -436,7 +436,7 @@ func hasPopulatedDigest(fileRecord reflect.Value) bool { // check if digest is a pointer and not nil, or a non-zero value switch digestField.Kind() { - case reflect.Ptr: + case reflect.Pointer: return !digestField.IsNil() case reflect.String: return digestField.String() != "" diff --git a/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go b/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go index 5ed5801b5..ab0845360 100644 --- a/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go +++ b/syft/pkg/cataloger/java/graalvm_native_image_cataloger.go @@ -14,6 +14,7 @@ import ( "unsafe" "github.com/anchore/syft/internal" + intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/mimetype" "github.com/anchore/syft/syft/artifact" @@ -63,10 +64,9 @@ type exportContentPE struct { addressOfSvmVersion uint32 } -// A nativeImagePE must maintain the underlying reader to fetch information unavailable in the Golang API. +// nativeImagePE carries the raw export directory, which debug/pe does not expose. type nativeImagePE struct { file *pe.File - reader io.ReaderAt exportSymbols pe.DataDirectory exports []byte t exportTypesPE @@ -83,6 +83,16 @@ const nativeImageMissingSymbolsError = "one or more symbols are missing from the const nativeImageInvalidIndexError = "parsing the executable file generated an invalid index" const nativeImageMissingExportedDataDirectoryError = "exported data directory is missing" +// nativeImageMaxDecompressedSbomSize bounds the decompressed SBOM rather than the compressed bytes, +// since gzip turns a few kilobytes into gigabytes. Real SBOMs run to single-digit MB even at 20k +// components, and decoding costs several times that again in graph allocation. +const nativeImageMaxDecompressedSbomSize = 16 * intFile.MB + +// nativeImageMaxExportDirectorySize bounds the export directory, which is symbol metadata and runs to a +// few KB. Bounding by the bytes remaining is not enough on its own: a mostly empty multi-gigabyte file +// is cheap to ship in a layer and would still authorize reading all of it. +const nativeImageMaxExportDirectorySize = 16 * intFile.MB + // NewNativeImageCataloger returns a new Native Image cataloger object. func NewNativeImageCataloger() pkg.Cataloger { return &nativeImageCataloger{} @@ -94,35 +104,40 @@ func (c *nativeImageCataloger) Name() string { } // decompressSbom returns the packages given within a native image executable's SBOM. -func decompressSbom(dataBuf []byte, sbomStart uint64, lengthStart uint64) ([]pkg.Package, []artifact.Relationship, error) { - lengthEnd := lengthStart + 8 - bufLen := len(dataBuf) - if lengthEnd > uint64(bufLen) { +// +// Offsets and the stored length all come from the binary, so the bounds below subtract from the buffer +// length rather than adding to a file-controlled value, which wraps and then panics on the slice. +func decompressSbom(dataBuf []byte, sbomStart, lengthStart uint64) ([]pkg.Package, []artifact.Relationship, error) { + bufLen := uint64(len(dataBuf)) + if lengthStart > bufLen || bufLen-lengthStart < 8 { return nil, nil, errors.New("the 'sbom_length' symbol overflows the binary") } - length := dataBuf[lengthStart:lengthEnd] - p := bytes.NewBuffer(length) - var storedLength uint64 - err := binary.Read(p, binary.LittleEndian, &storedLength) - if err != nil { - return nil, nil, fmt.Errorf("could not read from binary file: %w", err) - } + storedLength := binary.LittleEndian.Uint64(dataBuf[lengthStart : lengthStart+8]) log.WithFields("len", storedLength).Trace("found java native-image SBOM") - sbomEnd := sbomStart + storedLength - if sbomEnd > uint64(bufLen) { + if sbomStart > bufLen || storedLength > bufLen-sbomStart { return nil, nil, errors.New("the sbom symbol overflows the binary") } - sbomCompressed := dataBuf[sbomStart:sbomEnd] - p = bytes.NewBuffer(sbomCompressed) - gzreader, err := gzip.NewReader(p) + sbomCompressed := dataBuf[sbomStart : sbomStart+storedLength] + gzreader, err := gzip.NewReader(bytes.NewBuffer(sbomCompressed)) if err != nil { return nil, nil, fmt.Errorf("could not decompress the java native-image SBOM: %w", err) } + defer gzreader.Close() - sbom, _, _, err := cyclonedxjson.NewFormatDecoder().Decode(gzreader) + // the compressed bytes are bounded by the file, the decompressed stream is not. one byte past the + // ceiling, so a stream sitting exactly on it is still legal + raw, err := io.ReadAll(io.LimitReader(gzreader, nativeImageMaxDecompressedSbomSize+1)) + if err != nil { + return nil, nil, fmt.Errorf("could not decompress the java native-image SBOM: %w", err) + } + if len(raw) > nativeImageMaxDecompressedSbomSize { + return nil, nil, fmt.Errorf("the java native-image SBOM decompresses past %d bytes", nativeImageMaxDecompressedSbomSize) + } + + sbom, _, _, err := cyclonedxjson.NewFormatDecoder().Decode(bytes.NewReader(raw)) if err != nil { return nil, nil, fmt.Errorf("could not unmarshal the java native-image SBOM: %w", err) } @@ -133,6 +148,15 @@ func decompressSbom(dataBuf []byte, sbomStart uint64, lengthStart uint64) ([]pkg return pkgs, sbom.Relationships, nil } +// symbolOffset converts an SBOM symbol address into an offset within the section that should hold it. +// The subtraction is unsigned, so an address below the section base underflows into a huge offset. +func symbolOffset(addr, sectionBase uint64) (uint64, error) { + if addr < sectionBase { + return 0, errors.New("an SBOM symbol precedes the section that should contain it") + } + return addr - sectionBase, nil +} + // fileError logs an error message when an executable cannot be read. func fileError(filename string, err error) (nativeImage, error) { // We could not read the file as a binary for the desired platform, but it may still be a native-image executable. @@ -174,6 +198,8 @@ func newMachO(filename string, r io.ReaderAt) (nativeImage, error) { log.WithFields("filename", filename, "error", err).Trace("not a MachO binary") return nil, nil } + // a real read failure, not a format mismatch; returning nil here would drop the file silently + return fileError(filename, err) } if bi == nil { return nil, nil @@ -183,6 +209,35 @@ func newMachO(filename string, r io.ReaderAt) (nativeImage, error) { }, nil } +// readExportDirectory weighs the size the file declares against both an absolute cap and the bytes +// really present before allocating. +// +// note: dir.VirtualAddress is an RVA used directly as a file offset, with no section-table translation. +// Long-standing behavior this preserves rather than fixes; see rvaToFileOffset in dotnet/pe. +func readExportDirectory(r io.ReaderAt, dir pe.DataDirectory) ([]byte, error) { + if dir.Size > nativeImageMaxExportDirectorySize { + return nil, fmt.Errorf("export directory declares %d bytes, over the %d byte limit", dir.Size, nativeImageMaxExportDirectorySize) + } + + // the size has to come from the reader, not from what the file claims + end, ok := intFile.ReaderSize(r) + if !ok { + return nil, errors.New("unable to measure the binary") + } + if remaining := end - int64(dir.VirtualAddress); remaining < int64(dir.Size) { + return nil, fmt.Errorf("export directory declares %d bytes at RVA %d but only %d remain", + dir.Size, dir.VirtualAddress, max(remaining, 0)) + } + + // the count decides, not the error: a ReadAt may report a full read as io.EOF, or a short one with no + // error at all. io.ReadFull covers both but spins forever on a reader that keeps answering (0, nil) + exports := make([]byte, dir.Size) + if n, err := r.ReadAt(exports, int64(dir.VirtualAddress)); n < len(exports) { + return nil, fmt.Errorf("could not read the exported symbols data directory: read %d of %d bytes (%v)", n, len(exports), err) + } + return exports, nil +} + // newPE reads a Native Image from a Portable Executable file. func newPE(filename string, r io.ReaderAt) (nativeImage, error) { // First attempt to read an PE file. @@ -211,15 +266,12 @@ func newPE(filename string, r io.ReaderAt) (nativeImage, error) { if exportSymbolsDataDirectory.Size == 0 { return fileError(filename, errors.New(nativeImageMissingExportedDataDirectoryError)) } - exportSymbolsOffset := uint64(exportSymbolsDataDirectory.VirtualAddress) - exports := make([]byte, exportSymbolsDataDirectory.Size) - _, err = r.ReadAt(exports, int64(exportSymbolsOffset)) + exports, err := readExportDirectory(r, exportSymbolsDataDirectory) if err != nil { - return fileError(filename, fmt.Errorf("could not read the exported symbols data directory: %w", err)) + return fileError(filename, err) } return nativeImagePE{ file: bi, - reader: r, exportSymbols: exportSymbolsDataDirectory, exports: exports, t: exportTypesPE{ @@ -275,15 +327,20 @@ func (ni nativeImageElf) fetchPkgs() (pkgs []pkg.Package, relationships []artifa } dataSection := bi.Section(".data") if dataSection == nil { - return nil, nil, fmt.Errorf("no .data section found in binary: %w", err) + return nil, nil, errors.New("no .data section found in binary") } - dataSectionBase := dataSection.Addr data, err := dataSection.Data() if err != nil { return nil, nil, fmt.Errorf("cannot read the .data section: %w", err) } - sbomLocation := sbom.Value - dataSectionBase - lengthLocation := sbomLength.Value - dataSectionBase + sbomLocation, err := symbolOffset(sbom.Value, dataSection.Addr) + if err != nil { + return nil, nil, err + } + lengthLocation, err := symbolOffset(sbomLength.Value, dataSection.Addr) + if err != nil { + return nil, nil, err + } return decompressSbom(data, sbomLocation, lengthLocation) } @@ -354,48 +411,43 @@ func (ni nativeImageMachO) fetchPkgs() (pkgs []pkg.Package, relationships []arti log.Tracef("cannot obtain buffer from data segment") return nil, nil, nil } - sbomLocation := sbom.Value - dataSegment.Addr - lengthLocation := sbomLength.Value - dataSegment.Addr + sbomLocation, err := symbolOffset(sbom.Value, dataSegment.Addr) + if err != nil { + return nil, nil, err + } + lengthLocation, err := symbolOffset(sbomLength.Value, dataSegment.Addr) + if err != nil { + return nil, nil, err + } return decompressSbom(dataBuf, sbomLocation, lengthLocation) } // fetchExportAttribute obtains an attribute from the exported symbols directory entry. func (ni nativeImagePE) fetchExportAttribute(i int) (uint32, error) { - var attribute uint32 n := len(ni.exports) - j := int(unsafe.Sizeof(ni.header)) + i*int(unsafe.Sizeof(ni.t.headerAttribute)) - if j+4 >= n { + sz := int(unsafe.Sizeof(ni.t.headerAttribute)) + // i is only ever 0-3, so this cannot overflow; > not >= because an attribute ending flush with the + // directory is still entirely present + j := int(unsafe.Sizeof(ni.header)) + i*sz + if j+sz > n { log.Tracef("invalid index to export directory entry attribute: %v", j) return uint32(0), errors.New(nativeImageInvalidIndexError) } - p := bytes.NewBuffer(ni.exports[j : j+4]) - err := binary.Read(p, binary.LittleEndian, &attribute) - if err != nil { - log.Tracef("error fetching export directory entry attribute: %v", err) - return uint32(0), err - } - return attribute, nil + return binary.LittleEndian.Uint32(ni.exports[j : j+sz]), nil } // fetchExportFunctionPointer obtains a function pointer from the exported symbols directory entry. func (ni nativeImagePE) fetchExportFunctionPointer(functionsBase uint32, i uint32) (uint32, error) { - var pointer uint32 - - n := uint32(len(ni.exports)) - sz := uint32(unsafe.Sizeof(ni.t.functionPointer)) - j := functionsBase + i*sz - if j+sz >= n { + // functionsBase is a file-controlled RVA: in uint32 the sum wraps under the bound, then panics + n := uint64(len(ni.exports)) + sz := uint64(unsafe.Sizeof(ni.t.functionPointer)) + j := uint64(functionsBase) + uint64(i)*sz + if j > n || n-j < sz { log.Tracef("invalid index to exported function: %v", j) return uint32(0), errors.New(nativeImageInvalidIndexError) } - p := bytes.NewBuffer(ni.exports[j : j+sz]) - err := binary.Read(p, binary.LittleEndian, &pointer) - if err != nil { - log.Tracef("error fetching exported function: %v", err) - return uint32(0), err - } - return pointer, nil + return binary.LittleEndian.Uint32(ni.exports[j : j+sz]), nil } // fetchExportContent obtains the content of the export directory entry relevant to the SBOM. @@ -427,27 +479,30 @@ func (ni nativeImagePE) fetchSbomSymbols(content *exportContentPE) { sbomBytes := []byte(nativeImageSbomSymbol + "\x00") sbomLengthBytes := []byte(nativeImageSbomLengthSymbol + "\x00") svmVersionInfoBytes := []byte(nativeImageSbomVersionSymbol + "\x00") - n := uint32(len(ni.exports)) + n := uint64(len(ni.exports)) + sz := uint64(unsafe.Sizeof(ni.t.namePointer)) + + // an RVA below the directory would underflow into a huge offset that wraps back into range below + if content.addressOfNames < ni.exportSymbols.VirtualAddress { + log.Tracef("exported name array precedes the export directory: %v", content.addressOfNames) + return + } + addressBase := uint64(content.addressOfNames - ni.exportSymbols.VirtualAddress) // Find SBOM, SBOM Length, and SVM Version Symbol for i := uint32(0); i < content.numberOfNames; i++ { - j := i * uint32(unsafe.Sizeof(ni.t.namePointer)) - addressBase := content.addressOfNames - ni.exportSymbols.VirtualAddress - k := addressBase + j - sz := uint32(unsafe.Sizeof(ni.t.namePointer)) - if k+sz >= n { + k := addressBase + uint64(i)*sz + if k > n || n-k < sz { log.Tracef("invalid index to exported function: %v", k) // If we are at the end of exports, stop looking return } - var symbolAddress uint32 - p := bytes.NewBuffer(ni.exports[k : k+sz]) - err := binary.Read(p, binary.LittleEndian, &symbolAddress) - if err != nil { - log.Tracef("error fetching address of symbol %v", err) + symbolAddress := binary.LittleEndian.Uint32(ni.exports[k : k+sz]) + if symbolAddress < ni.exportSymbols.VirtualAddress { + log.Tracef("exported symbol precedes the export directory: %v", symbolAddress) return } - symbolBase := symbolAddress - ni.exportSymbols.VirtualAddress + symbolBase := uint64(symbolAddress - ni.exportSymbols.VirtualAddress) if symbolBase >= n { log.Tracef("invalid index to exported symbol: %v", symbolBase) return @@ -482,6 +537,9 @@ func (ni nativeImagePE) fetchPkgs() (pkgs []pkg.Package, relationships []artifac if content.addressOfSbom == uint32(0) || content.addressOfSbomLength == uint32(0) || content.addressOfSvmVersion == uint32(0) { return nil, nil, errors.New(nativeImageMissingSymbolsError) } + if content.addressOfFunctions < ni.exportSymbols.VirtualAddress { + return nil, nil, errors.New("exported function array precedes the export directory") + } functionsBase := content.addressOfFunctions - ni.exportSymbols.VirtualAddress sbomOffset := content.addressOfSbom sbomAddress, err := ni.fetchExportFunctionPointer(functionsBase, sbomOffset) @@ -503,10 +561,16 @@ func (ni nativeImagePE) fetchPkgs() (pkgs []pkg.Package, relationships []artifac log.Tracef("cannot obtain buffer from the java native-image .data section") return nil, nil, nil } - sbomLocation := sbomAddress - dataSection.VirtualAddress - lengthLocation := sbomLengthAddress - dataSection.VirtualAddress + sbomLocation, err := symbolOffset(uint64(sbomAddress), uint64(dataSection.VirtualAddress)) + if err != nil { + return nil, nil, err + } + lengthLocation, err := symbolOffset(uint64(sbomLengthAddress), uint64(dataSection.VirtualAddress)) + if err != nil { + return nil, nil, err + } - return decompressSbom(dataBuf, uint64(sbomLocation), uint64(lengthLocation)) + return decompressSbom(dataBuf, sbomLocation, lengthLocation) } // fetchPkgs provides the packages available in a UnionReader. @@ -527,6 +591,9 @@ func fetchPkgs(reader unionreader.UnionReader, location file.Location) ([]pkg.Pa for _, makeNativeImage := range imageFormats { ni, err := makeNativeImage(filename, r) if err != nil { + // both "not this format" and a real rejection of a file that is one, which drops every + // package the binary carries and is only visible at trace level + log.WithFields("file", filename, "error", err).Trace("unable to read possible java native-image") continue } if ni == nil { @@ -535,14 +602,17 @@ func fetchPkgs(reader unionreader.UnionReader, location file.Location) ([]pkg.Pa newPkgs, newRelationships, err := ni.fetchPkgs() if err != nil { log.Tracef("unable to extract SBOM from possible java native-image %s: %v", filename, err) - continue + } else { + // Associate extracted packages with the native image location + for i := range newPkgs { + newPkgs[i].Locations.Add(location) + } + pkgs = append(pkgs, newPkgs...) + relationships = append(relationships, newRelationships...) } - // Associate extracted packages with the native image location - for i := range newPkgs { - newPkgs[i].Locations.Add(location) - } - pkgs = append(pkgs, newPkgs...) - relationships = append(relationships, newRelationships...) + // nothing parses as two of these today; stopping makes that explicit rather than relying on it. + // note a format that parses but whose fetchPkgs fails is not retried as another format + break } } return pkgs, relationships diff --git a/syft/pkg/cataloger/java/graalvm_native_image_cataloger_test.go b/syft/pkg/cataloger/java/graalvm_native_image_cataloger_test.go index 504b6f16b..00fd15bd8 100644 --- a/syft/pkg/cataloger/java/graalvm_native_image_cataloger_test.go +++ b/syft/pkg/cataloger/java/graalvm_native_image_cataloger_test.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "compress/gzip" + "crypto/sha256" "encoding/binary" "fmt" "io" @@ -11,8 +12,11 @@ import ( "path" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/testutils" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/internal/unionreader" @@ -265,3 +269,289 @@ func verifyRelationshipFields(t *testing.T, expected, actual []artifact.Relation require.Equal(t, expected[i].Type, actual[i].Type) } } + +// buildMinimalPE64 assembles the smallest PE64 debug/pe will parse, with DataDirectory[0] set to the +// given RVA and size. The point of the fixture is that size can claim far more than totalSize. +// +// newPE uses the directory's VirtualAddress directly as a file offset, so no sections are needed and +// exportRVA doubles as the offset the read starts from. +func buildMinimalPE64(exportRVA, exportSize uint32, totalSize int) []byte { + const peHdrOff = 0x40 + buf := make([]byte, totalSize) + copy(buf, []byte{'M', 'Z'}) + binary.LittleEndian.PutUint32(buf[0x3c:], peHdrOff) + + p := buf[peHdrOff:] + copy(p, []byte{'P', 'E', 0, 0}) + // COFF file header + binary.LittleEndian.PutUint16(p[4:], 0x8664) // Machine = AMD64 + binary.LittleEndian.PutUint16(p[20:], 240) // SizeOfOptionalHeader + binary.LittleEndian.PutUint16(p[22:], 0x22) // Characteristics: executable image + + // optional header (PE32+) + oh := p[24:] + binary.LittleEndian.PutUint16(oh[0:], 0x20b) // Magic = PE32+ + binary.LittleEndian.PutUint32(oh[108:], 16) // NumberOfRvaAndSizes + // DataDirectory[0] = export table + binary.LittleEndian.PutUint32(oh[112:], exportRVA) + binary.LittleEndian.PutUint32(oh[116:], exportSize) + + return buf +} + +// readSizeRecorder records the largest single read requested of it. A small allocation is not enough on +// its own; no single read may exceed the file size either, whatever the headers claim. +type readSizeRecorder struct { + *bytes.Reader + maxRead int +} + +func (r *readSizeRecorder) Read(p []byte) (int, error) { + if len(p) > r.maxRead { + r.maxRead = len(p) + } + return r.Reader.Read(p) +} + +func (r *readSizeRecorder) ReadAt(p []byte, off int64) (int, error) { + if len(p) > r.maxRead { + r.maxRead = len(p) + } + return r.Reader.ReadAt(p, off) +} + +func TestNewPE_ExportDirectorySizeIsNotAllocatedUpFront(t *testing.T) { + // Size is a user-controlled uint32 that used to size a make([]byte, Size) before the read, so a small + // file claiming 4GB reserved 4GB. + const totalSize = 8192 + data := buildMinimalPE64(0x1000, 0xFFFFFFFF, totalSize) + r := &readSizeRecorder{Reader: bytes.NewReader(data)} + + var err error + allocated := testutils.MeasureAlloc(t, func() { + _, err = newPE("oversized-export-dir.exe", r) + }) + + require.Error(t, err, "an export directory the file cannot satisfy must not read as successful") + require.ErrorContains(t, err, "byte limit") + + t.Logf("allocated %d bytes for a directory declaring %d in a %d byte file", + allocated, uint64(0xFFFFFFFF), totalSize) + require.Less(t, allocated, uint64(intFile.MB), + "the declared size must not be reserved before the read confirms the bytes exist") + assert.LessOrEqual(t, r.maxRead, totalSize, + "no single read may exceed the file size, regardless of what the headers claim") +} + +func TestNewPE_ExportDirectoryWithinFileIsRead(t *testing.T) { + // exportRVA=0x1000 in an 8192 byte file leaves exactly 4096 bytes for the directory to claim + const exportRVA = 0x1000 + const totalSize = 8192 + const remaining = totalSize - exportRVA + + tests := []struct { + name string + exportSize uint32 + wantErr bool + }{ + {name: "well within the file", exportSize: 64, wantErr: false}, + // pins the boundary the not-enough-bytes check must not break: every byte remaining is available + {name: "exactly the bytes remaining", exportSize: remaining, wantErr: false}, + {name: "one byte past what remains", exportSize: remaining + 1, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data := buildMinimalPE64(exportRVA, tt.exportSize, totalSize) + + ni, err := newPE("ok.exe", bytes.NewReader(data)) + if tt.wantErr { + require.Error(t, err, "the bound must not allow reading past what the file has") + return + } + require.NoError(t, err, "the bound must not reject an export directory that really is present") + require.NotNil(t, ni) + }) + } +} + +func TestDecompressSbom_RejectsOutOfRangeOffsets(t *testing.T) { + // every offset and length here is read out of the binary, so each must be rejected rather than wrapped + // into a value that passes a bound and panics on the slice. wantErrMsg pins which of the three guards fires + const sbomLengthOverflowsMsg = "the 'sbom_length' symbol overflows the binary" + const sbomOverflowsMsg = "the sbom symbol overflows the binary" + + tests := []struct { + name string + bufLen int + sbomStart uint64 + lengthStart uint64 + storedLength uint64 + wantErrMsg string + }{ + { + name: "length offset wraps past the end of the buffer", + bufLen: 64, + lengthStart: ^uint64(0) - 3, // lengthStart+8 wraps to 4, which compares as in range + wantErrMsg: sbomLengthOverflowsMsg, + }, + { + name: "length offset starts beyond the buffer", + bufLen: 64, + lengthStart: 100, + wantErrMsg: sbomLengthOverflowsMsg, + }, + { + name: "stored length wraps past the end of the buffer", + bufLen: 64, + sbomStart: 16, + lengthStart: 0, + storedLength: ^uint64(0) - 3, // sbomStart+storedLength wraps to 12 + wantErrMsg: sbomOverflowsMsg, + }, + { + name: "stored length runs past the end of the buffer", + bufLen: 64, + sbomStart: 16, + lengthStart: 0, + storedLength: 1000, + wantErrMsg: sbomOverflowsMsg, + }, + { + name: "sbom offset starts beyond the buffer", + bufLen: 64, + sbomStart: ^uint64(0) - 3, + lengthStart: 0, + wantErrMsg: sbomOverflowsMsg, + }, + { + name: "length field straddles the end of the buffer", + bufLen: 64, + lengthStart: 60, + wantErrMsg: sbomLengthOverflowsMsg, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dataBuf := make([]byte, tt.bufLen) + // subtract rather than add, for the same reason the code under test has to: lengthStart is deliberately + // large enough to wrap + if tt.lengthStart <= uint64(tt.bufLen) && uint64(tt.bufLen)-tt.lengthStart >= 8 { + binary.LittleEndian.PutUint64(dataBuf[tt.lengthStart:], tt.storedLength) + } + + var err error + require.NotPanics(t, func() { + _, _, err = decompressSbom(dataBuf, tt.sbomStart, tt.lengthStart) + }) + require.ErrorContains(t, err, tt.wantErrMsg) + }) + } +} + +func TestDecompressSbom_RejectsStreamExpandingPastTheLimit(t *testing.T) { + // zeros stand in for any uniform, highly compressible payload: they expand about 1000x, far past what + // real JSON does + tests := []struct { + name string + decompressedSize int + wantRejected bool + }{ + { + name: "one byte past the limit is rejected", + decompressedSize: nativeImageMaxDecompressedSbomSize + 1, + wantRejected: true, + }, + { + // pins the other side: tightening the LimitedReader by one byte would reject a legal payload + name: "exactly at the limit falls through to the decoder", + decompressedSize: nativeImageMaxDecompressedSbomSize, + wantRejected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var compressed bytes.Buffer + z := gzip.NewWriter(&compressed) + _, err := z.Write(make([]byte, tt.decompressedSize)) + require.NoError(t, err) + require.NoError(t, z.Close()) + + t.Logf("%d compressed bytes expand to %d (%dx)", compressed.Len(), tt.decompressedSize, + tt.decompressedSize/compressed.Len()) + + dataBuf := append(compressed.Bytes(), make([]byte, 8)...) + lengthStart := uint64(compressed.Len()) + binary.LittleEndian.PutUint64(dataBuf[lengthStart:], lengthStart) + + var decompressErr error + allocated := testutils.MeasureAlloc(t, func() { + _, _, decompressErr = decompressSbom(dataBuf, 0, lengthStart) + }) + err = decompressErr + require.Error(t, err, "zeros are never a valid SBOM, so this errors either way") + if tt.wantRejected { + require.Contains(t, err.Error(), "decompresses past", + "hitting the bound should say so, not report a parse failure") + t.Logf("allocated %d bytes rejecting a stream past the limit", allocated) + // generous on purpose: -race roughly doubles the allocation counted here, and the point is catching an + // unbounded blowup, not pinning a multiple + require.Less(t, allocated, uint64(8*nativeImageMaxDecompressedSbomSize), + "rejecting an oversized stream must not retain multiples of the limit in memory") + return + } + require.NotContains(t, err.Error(), "decompresses past", + "a stream sitting exactly on the limit must reach the decoder, not be capped away") + }) + } +} + +// the other tests compute their sizes FROM nativeImageMaxDecompressedSbomSize, so raising it leaves them +// green. This is the one assertion that does not. +func TestNativeImageMaxDecompressedSbomSize_Ceiling(t *testing.T) { + require.LessOrEqual(t, int64(nativeImageMaxDecompressedSbomSize), int64(64*intFile.MB), + "the ceiling is what stands between a gzip bomb and an OOM; raising it needs a new argument here") +} + +func TestDecompressSbom_AcceptsLargeSbom(t *testing.T) { + // the size limit must not truncate a genuinely large SBOM. Distinct names, versions, purls and hashes + // keep this shaped like a real CycloneDX document rather than a uniform payload. + const components = 20000 + + var sb bytes.Buffer + sb.WriteString(`{"bomFormat":"CycloneDX","specVersion":"1.5","version":1,"components":[`) + for i := 0; i < components; i++ { + if i > 0 { + sb.WriteString(",") + } + // a sha256-shaped value keeps the entropy (and so the compression ratio) realistic + hash := sha256.Sum256([]byte(fmt.Sprintf("component-%d", i))) + fmt.Fprintf(&sb, `{"type":"library","name":"lib-%d","version":"%d.%d.%d",`+ + `"purl":"pkg:maven/com.example.group%d/lib-%d@%d.%d.%d",`+ + `"hashes":[{"alg":"SHA-256","content":"%x"}]}`, + i, i%20, i%7, i%13, i%50, i, i%20, i%7, i%13, hash) + } + sb.WriteString(`]}`) + + raw := sb.Bytes() + var compressed bytes.Buffer + z := gzip.NewWriter(&compressed) + _, err := z.Write(raw) + require.NoError(t, err) + require.NoError(t, z.Close()) + + t.Logf("%d components: %d bytes of JSON, %d compressed (%dx), limit %d", + components, len(raw), compressed.Len(), len(raw)/compressed.Len(), nativeImageMaxDecompressedSbomSize) + require.Less(t, int64(len(raw)), int64(nativeImageMaxDecompressedSbomSize), + "a real SBOM of this size must sit well inside the limit, otherwise the limit is too low") + + dataBuf := append(compressed.Bytes(), make([]byte, 8)...) + lengthStart := uint64(compressed.Len()) + binary.LittleEndian.PutUint64(dataBuf[lengthStart:], lengthStart) + + pkgs, _, err := decompressSbom(dataBuf, 0, lengthStart) + require.NoError(t, err, "a large SBOM at a realistic ratio must not be rejected") + require.Len(t, pkgs, components) +} diff --git a/syft/pkg/cataloger/java/graalvm_native_image_offsets_test.go b/syft/pkg/cataloger/java/graalvm_native_image_offsets_test.go new file mode 100644 index 000000000..61614f84f --- /dev/null +++ b/syft/pkg/cataloger/java/graalvm_native_image_offsets_test.go @@ -0,0 +1,145 @@ +package java + +import ( + "bytes" + "io" + "math" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/unionreader" +) + +func TestSymbolOffset(t *testing.T) { + tests := []struct { + name string + addr uint64 + sectionBase uint64 + wantOffset uint64 + wantErr bool + }{ + { + name: "addr below section base underflows if unchecked, so it must error instead", + addr: 0x900, + sectionBase: 0x1000, + wantErr: true, + }, + { + name: "addr equal to section base gives offset zero", + addr: 0x1000, + sectionBase: 0x1000, + wantOffset: 0, + }, + { + name: "addr above section base gives the exact delta", + addr: 0x1010, + sectionBase: 0x1000, + wantOffset: 0x10, + }, + { + name: "zero section base succeeds for any addr", + addr: 0x2a, + sectionBase: 0, + wantOffset: 0x2a, + }, + { + name: "max addr and max section base are equal, so offset is zero", + addr: math.MaxUint64, + sectionBase: math.MaxUint64, + wantOffset: 0, + }, + { + name: "max addr against a zero section base gives the max delta", + addr: math.MaxUint64, + sectionBase: 0, + wantOffset: math.MaxUint64, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := symbolOffset(tt.addr, tt.sectionBase) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantOffset, got) + }) + } +} + +// readerFromFixture opens a testdata fixture and wraps it the same way processLocation does before +// handing it to fetchPkgs. +func readerFromFixture(t *testing.T, path string) unionreader.UnionReader { + t.Helper() + f, err := os.Open(path) + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + + reader, err := unionreader.GetUnionReader(io.NopCloser(f)) + require.NoError(t, err) + return reader +} + +// neither fixture is a native image, so both exercise the path where a format matches and its fetchPkgs +// then fails, which the break at the end of the format loop makes final. +// +// the break itself cannot be distinguished from the old fall-through here: ELF, Mach-O and PE magics are +// mutually exclusive, so no fixture can validly parse as two formats. Nothing reaches the branch that +// attaches location to each package either, since no committed fixture is a real native image with an +// embedded SBOM. +func TestFetchPkgs(t *testing.T) { + tests := []struct { + name string + fixture string + }{ + { + name: "real native image without SBOM symbols", + fixture: "testdata/java-builds/packages/example-java-app", + }, + { + name: "real Mach-O that is not a native image", + fixture: "testdata/java-builds/packages/gcc-amd64-darwin-exec-debug", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reader := readerFromFixture(t, tt.fixture) + location := file.NewLocation(tt.fixture) + + require.NotPanics(t, func() { + gotPkgs, gotRels := fetchPkgs(reader, location) + assert.Empty(t, gotPkgs) + assert.Empty(t, gotRels) + }) + }) + } +} + +// covers macho.NewFile failing with something other than *macho.FormatError. Garbage bytes take the +// FormatError branch and return (nil, nil); a valid magic with nothing behind it does not. +func TestNewMachO_TruncatedFileIsARealFailure(t *testing.T) { + // a valid Mach-O magic with nothing behind it yields io.ErrUnexpectedEOF, not a *macho.FormatError + _, err := newMachO("truncated", bytes.NewReader([]byte{0xcf, 0xfa, 0xed, 0xfe})) + require.ErrorContains(t, err, "unable to read executable") +} + +func TestFetchPkgs_GarbageBytesYieldsNothing(t *testing.T) { + // bytes that match none of ELF, Mach-O or PE magic numbers + garbage := bytes.Repeat([]byte{0xDE, 0xAD, 0xBE, 0xEF}, 256) + reader, err := unionreader.GetUnionReader(io.NopCloser(bytes.NewReader(garbage))) + require.NoError(t, err) + + location := file.NewLocation("garbage.bin") + require.NotPanics(t, func() { + pkgs, rels := fetchPkgs(reader, location) + assert.Empty(t, pkgs) + assert.Empty(t, rels) + }) +} diff --git a/syft/pkg/cataloger/java/graalvm_native_image_pe_test.go b/syft/pkg/cataloger/java/graalvm_native_image_pe_test.go new file mode 100644 index 000000000..20375934f --- /dev/null +++ b/syft/pkg/cataloger/java/graalvm_native_image_pe_test.go @@ -0,0 +1,399 @@ +package java + +import ( + "bytes" + "debug/pe" + "encoding/binary" + "io" + "math" + "testing" + "time" + "unsafe" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/testutils" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/pkg" +) + +// pins the bound at the byte where an attribute ends flush with the directory: `j+sz > n` (not `>=`) is +// what lets attribute 3 be read out of a 36-byte exports buffer, since bytes 32:36 are entirely present. +func TestFetchExportAttribute(t *testing.T) { + // the four attributes fetchExportContent reads start right after this header + require.EqualValues(t, 20, unsafe.Sizeof(exportPrefixPE{})) + + tests := []struct { + name string + exports []byte + index int + wantValue uint32 + wantErr bool + }{ + { + name: "one byte short of attribute 3 is rejected", + exports: make([]byte, 35), + index: 3, + wantErr: true, + }, + { + name: "attribute 3 flush with the end of exports is read", + exports: func() []byte { + b := make([]byte, 36) + binary.LittleEndian.PutUint32(b[32:36], 0xDEADBEEF) + return b + }(), + index: 3, + wantValue: 0xDEADBEEF, + }, + { + name: "attribute 0 reads the first attribute slot", + exports: func() []byte { + b := make([]byte, 24) + binary.LittleEndian.PutUint32(b[20:24], 42) + return b + }(), + index: 0, + wantValue: 42, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ni := nativeImagePE{exports: tt.exports} + got, err := ni.fetchExportAttribute(tt.index) + if tt.wantErr { + require.ErrorContains(t, err, nativeImageInvalidIndexError) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantValue, got) + }) + } +} + +// pins the uint32->uint64 widening: computing `functionsBase + i*sz` in uint32 wraps a huge base back +// into range instead of erroring. +func TestFetchExportFunctionPointer(t *testing.T) { + tests := []struct { + name string + exports []byte + functionsBase uint32 + index uint32 + wantValue uint32 + wantErr bool + }{ + { + name: "functionsBase near the uint32 max must not wrap back into range", + exports: make([]byte, 64), + functionsBase: 0xFFFFFFFF, + index: 1, + wantErr: true, + }, + { + name: "an in-range function pointer is read", + exports: func() []byte { + b := make([]byte, 64) + binary.LittleEndian.PutUint32(b[16:20], 0xCAFEBABE) + return b + }(), + functionsBase: 8, + index: 2, + wantValue: 0xCAFEBABE, + }, + { + name: "a pointer flush with the end of exports is read", + exports: func() []byte { + b := make([]byte, 64) + binary.LittleEndian.PutUint32(b[60:64], 0x11223344) + return b + }(), + functionsBase: 60, + index: 0, + wantValue: 0x11223344, + }, + { + name: "a pointer straddling the end of exports", + exports: make([]byte, 64), + functionsBase: 62, + index: 0, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ni := nativeImagePE{exports: tt.exports} + var got uint32 + var err error + require.NotPanics(t, func() { + got, err = ni.fetchExportFunctionPointer(tt.functionsBase, tt.index) + }) + if tt.wantErr { + require.ErrorContains(t, err, nativeImageInvalidIndexError) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantValue, got) + }) + } +} + +// addressOfNames below the directory's VirtualAddress. VirtualAddress is deliberately huge so the +// underflow lands back in range; a small one would let an unrelated bound mask this guard's absence. +func TestFetchSbomSymbols_NameArrayPrecedingDirectoryGuard(t *testing.T) { + va := uint32(0xFFFFFFF0) + addressOfNames := uint32(5) + wrapped := addressOfNames - va // what addressBase would be if the guard were missing + require.Less(t, wrapped, uint32(32), "the fixture only proves the guard's necessity if the wrap lands in range") + + exports := make([]byte, 32) + le := binary.LittleEndian + // a name pointer table here would only be reached without the guard + le.PutUint32(exports[wrapped:], va+10) // index 0: resolves to a non-matching name + le.PutUint32(exports[wrapped+4:], va+5) // index 1: resolves to "sbom" + copy(exports[10:], "zzzz\x00") + copy(exports[5:], nativeImageSbomSymbol+"\x00") + + ni := nativeImagePE{exports: exports, exportSymbols: pe.DataDirectory{VirtualAddress: va}} + content := &exportContentPE{addressOfNames: addressOfNames, numberOfNames: 2} + + require.NotPanics(t, func() { ni.fetchSbomSymbols(content) }) + assert.Zero(t, content.addressOfSbom, "addressOfNames below the directory must not be read at all") + assert.Zero(t, content.addressOfSbomLength) + assert.Zero(t, content.addressOfSvmVersion) +} + +// a name pointer entry resolving below the directory. Same reasoning as the addressOfNames guard above. +func TestFetchSbomSymbols_SymbolAddressPrecedingDirectoryGuard(t *testing.T) { + va := uint32(0xFFFFFFF0) + exports := make([]byte, 32) + le := binary.LittleEndian + le.PutUint32(exports[0:4], va+10) // index 0: resolves to a non-matching name + le.PutUint32(exports[4:8], 5) // index 1: a symbolAddress below the directory + + wrapped := uint32(5) - va // what symbolBase would be for index 1 if the guard were missing + require.Less(t, wrapped, uint32(32), "the fixture only proves the guard's necessity if the wrap lands in range") + copy(exports[10:], "zzzz\x00") + copy(exports[wrapped:], nativeImageSbomSymbol+"\x00") + + ni := nativeImagePE{exports: exports, exportSymbols: pe.DataDirectory{VirtualAddress: va}} + content := &exportContentPE{addressOfNames: va, numberOfNames: 2} + + require.NotPanics(t, func() { ni.fetchSbomSymbols(content) }) + assert.Zero(t, content.addressOfSbom, "a symbolAddress below the directory must not be read as a name") + assert.Zero(t, content.addressOfSbomLength) + assert.Zero(t, content.addressOfSvmVersion) +} + +// a symbolBase past the end of exports. Needs no wraparound: without the guard the slice is simply out +// of range and panics. +func TestFetchSbomSymbols_SymbolBaseOutOfRangeGuard(t *testing.T) { + const va = uint32(0x2000) + exports := make([]byte, 8) + binary.LittleEndian.PutUint32(exports[0:4], va+13) // symbolBase would be 13, past an 8-byte buffer + + ni := nativeImagePE{exports: exports, exportSymbols: pe.DataDirectory{VirtualAddress: va}} + content := &exportContentPE{addressOfNames: va, numberOfNames: 1} + + require.NotPanics(t, func() { ni.fetchSbomSymbols(content) }) + assert.Zero(t, content.addressOfSbom) + assert.Zero(t, content.addressOfSbomLength) + assert.Zero(t, content.addressOfSvmVersion) +} + +// addressBase past the end of exports. Every other fixture has addressBase == 0, so this is the only case +// reaching the `k > n` half: without it `n-k` underflows, clears the size check, and the slice panics. +func TestFetchSbomSymbols_AddressBaseOutOfRangeGuard(t *testing.T) { + const va = uint32(0x2000) + exports := make([]byte, 16) + + ni := nativeImagePE{exports: exports, exportSymbols: pe.DataDirectory{VirtualAddress: va}} + content := &exportContentPE{addressOfNames: 0x2064, numberOfNames: 1} // addressBase = 0x64 = 100, past exports + + require.NotPanics(t, func() { ni.fetchSbomSymbols(content) }) + assert.Zero(t, content.addressOfSbom) + assert.Zero(t, content.addressOfSbomLength) + assert.Zero(t, content.addressOfSvmVersion) +} + +// TestFetchSbomSymbols_FindsAllThreeSymbols is the happy path. fetchSbomSymbols records the loop index +// of a match, not its address. +func TestFetchSbomSymbols_FindsAllThreeSymbols(t *testing.T) { + const va = 0x2000 + exports := make([]byte, 96) + le := binary.LittleEndian + + // name pointer table: one uint32 RVA per name, in loop order. + // name pointer table: one uint32 RVA per name, in loop order. note fetchPkgs treats a stored index of 0 + // as "symbol not found", so no symbol may sit at index 0 or this would pass for the wrong reason + le.PutUint32(exports[0:4], va+52) // index 0: an unrelated name + le.PutUint32(exports[4:8], va+57) // index 1: "sbom" + le.PutUint32(exports[8:12], va+62) // index 2: "sbom_length" + le.PutUint32(exports[12:16], va+74) // index 3: "__svm_version_info" + + copy(exports[52:], "xxxx\x00") + copy(exports[57:], nativeImageSbomSymbol+"\x00") + copy(exports[62:], nativeImageSbomLengthSymbol+"\x00") + copy(exports[74:], nativeImageSbomVersionSymbol+"\x00") + + ni := nativeImagePE{exports: exports, exportSymbols: pe.DataDirectory{VirtualAddress: va}} + content := &exportContentPE{addressOfNames: va, numberOfNames: 4} + ni.fetchSbomSymbols(content) + + assert.EqualValues(t, 1, content.addressOfSbom) + assert.EqualValues(t, 2, content.addressOfSbomLength) + assert.EqualValues(t, 3, content.addressOfSvmVersion) +} + +// numberOfNames is read straight out of the file, so a hostile binary can claim ~4 billion names. The +// scan must stop when the name pointer table runs off the end of exports, not iterate that far. +func TestFetchSbomSymbols_HugeNameCountTerminatesQuickly(t *testing.T) { + exports := make([]byte, 16) // room for exactly 4 name pointer table entries + ni := nativeImagePE{exports: exports, exportSymbols: pe.DataDirectory{VirtualAddress: 0}} + content := &exportContentPE{addressOfNames: 0, numberOfNames: math.MaxUint32} + + start := time.Now() + require.NotPanics(t, func() { ni.fetchSbomSymbols(content) }) + assert.Less(t, time.Since(start), time.Second, + "the scan must stop at the end of exports, not loop numberOfNames times") +} + +// the one guard in fetchPkgs reachable without a real *pe.File: addressOfFunctions below the export +// directory base must error rather than underflow into a huge functionsBase. +func TestFetchPkgs_AddressOfFunctionsPrecedingDirectoryErrors(t *testing.T) { + const va = 0x2000 + exports := make([]byte, 96) + le := binary.LittleEndian + + // the four attributes fetchExportContent reads + le.PutUint32(exports[20:24], 10) // numberOfFunctions (unused before the guard under test) + le.PutUint32(exports[24:28], 4) // numberOfNames + le.PutUint32(exports[28:32], va-1) // addressOfFunctions: precedes the export directory + le.PutUint32(exports[32:36], va+36) + + // name pointer table at offset 36, resolving all three SBOM symbols so the guard under test is + // reached instead of the earlier "missing symbols" return + le.PutUint32(exports[36:40], va+52) + le.PutUint32(exports[40:44], va+57) + le.PutUint32(exports[44:48], va+62) + le.PutUint32(exports[48:52], va+74) + copy(exports[52:], "xxxx\x00") + copy(exports[57:], nativeImageSbomSymbol+"\x00") + copy(exports[62:], nativeImageSbomLengthSymbol+"\x00") + copy(exports[74:], nativeImageSbomVersionSymbol+"\x00") + + ni := nativeImagePE{exports: exports, exportSymbols: pe.DataDirectory{VirtualAddress: va}} + + var pkgs []pkg.Package + var rels []artifact.Relationship + var err error + require.NotPanics(t, func() { + pkgs, rels, err = ni.fetchPkgs() + }) + require.Error(t, err) + assert.NotContains(t, err.Error(), "recovered from panic", + "the guard must reject this cleanly, not rely on the recover") + assert.Contains(t, err.Error(), "exported function array precedes the export directory") + assert.Empty(t, pkgs) + assert.Empty(t, rels) +} + +// the three outcomes of weighing a declared size against both the cap and the bytes the file really has. +func TestReadExportDirectory(t *testing.T) { + t.Run("size over the cap is rejected", func(t *testing.T) { + dir := pe.DataDirectory{VirtualAddress: 0, Size: nativeImageMaxExportDirectorySize + 1} + _, err := readExportDirectory(bytes.NewReader(make([]byte, 1024)), dir) + require.Error(t, err) + assert.Contains(t, err.Error(), "byte limit") + }) + + t.Run("a size the file cannot satisfy is rejected", func(t *testing.T) { + dir := pe.DataDirectory{VirtualAddress: 0, Size: 600} + _, err := readExportDirectory(bytes.NewReader(make([]byte, 512)), dir) + require.Error(t, err) + assert.Contains(t, err.Error(), "remain") + }) + + t.Run("a size the file can exactly satisfy succeeds", func(t *testing.T) { + data := make([]byte, 200) + binary.LittleEndian.PutUint32(data[100:104], 0xABCDEF01) + dir := pe.DataDirectory{VirtualAddress: 100, Size: 100} // remaining bytes (200-100) exactly + + got, err := readExportDirectory(bytes.NewReader(data), dir) + require.NoError(t, err) + require.Len(t, got, 100) + assert.Equal(t, uint32(0xABCDEF01), binary.LittleEndian.Uint32(got[0:4])) + }) + + t.Run("a reader that cannot report its size is rejected", func(t *testing.T) { + data := make([]byte, 64) + r := struct{ io.ReaderAt }{bytes.NewReader(data)} // promotes ReadAt only, no Size() or Seek + dir := pe.DataDirectory{VirtualAddress: 0, Size: 32} + + _, err := readExportDirectory(r, dir) + require.ErrorContains(t, err, "unable to measure the binary") + }) +} + +// proves the remaining-bytes guard, not just the cap: a small, cheap file can still declare a size under +// the cap, and only the bytes really present can reject it before the read and its allocation. +func TestReadExportDirectory_DoesNotReadThePhantomBytes(t *testing.T) { + const realSize = 4096 + rec := &readSizeRecorder{Reader: bytes.NewReader(make([]byte, realSize))} + dir := pe.DataDirectory{VirtualAddress: 0, Size: nativeImageMaxExportDirectorySize} + + allocated := testutils.MeasureAlloc(t, func() { + _, err := readExportDirectory(rec, dir) + // a Size exactly at the cap must fall through to the remaining-bytes check, not be capped away; + // pins the `>` (not `>=`) in the cap comparison + require.ErrorContains(t, err, "remain") + }) + + t.Logf("allocated %d bytes for a directory declaring %d against a %d byte file", allocated, dir.Size, realSize) + assert.Less(t, allocated, uint64(intFile.MB), + "rejecting an oversized declaration must not first allocate against it") + assert.LessOrEqual(t, rec.maxRead, realSize, + "no read should ask for more than the file actually holds") +} + +// shortReader has 64 addressable bytes but hands back fewer than asked with a nil error, the +// non-conforming shape syft/internal/unionreader documents for squashfs +type shortReader struct { + b []byte + perRead int +} + +func (s shortReader) Size() int64 { return int64(len(s.b)) } + +func (s shortReader) ReadAt(p []byte, off int64) (int, error) { + if len(p) == 1 { // intFile.ReaderSize probes the last byte, and that probe has to succeed + p[0] = s.b[off] + return 1, nil + } + return copy(p[:s.perRead], s.b[off:off+int64(s.perRead)]), nil +} + +// regression for the real defect: a ReadAt handing back fewer bytes than asked with a nil error used to +// be accepted, and the zeroed tail of the buffer parsed as export directory data. +func TestReadExportDirectory_RejectsShortReadWithNilError(t *testing.T) { + tests := []struct { + name string + perRead int + }{ + {name: "reader answers with zero bytes and a nil error", perRead: 0}, + {name: "reader answers with a partial read and a nil error", perRead: 4}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := shortReader{b: make([]byte, 64), perRead: tt.perRead} + dir := pe.DataDirectory{VirtualAddress: 0, Size: 32} + + _, err := readExportDirectory(r, dir) + require.ErrorContains(t, err, "read") + }) + } +} diff --git a/syft/pkg/cataloger/java/internal/maven/resolver.go b/syft/pkg/cataloger/java/internal/maven/resolver.go index 56a92db20..b7d3d8ced 100644 --- a/syft/pkg/cataloger/java/internal/maven/resolver.go +++ b/syft/pkg/cataloger/java/internal/maven/resolver.go @@ -205,7 +205,7 @@ func (r *Resolver) resolveProjectProperty(ctx context.Context, resolutionContext pomValue = pomValue.Field(fieldNum) pomValueType = pomValue.Type() - if pomValueType.Kind() == reflect.Ptr { + if pomValueType.Kind() == reflect.Pointer { // we were recursing down the nested structs, but one of the steps // we need to take is a nil pointer, so give up if pomValue.IsNil() { diff --git a/syft/pkg/cataloger/javascript/license.go b/syft/pkg/cataloger/javascript/license.go new file mode 100644 index 000000000..bb3f5b70e --- /dev/null +++ b/syft/pkg/cataloger/javascript/license.go @@ -0,0 +1,53 @@ +package javascript + +import ( + "context" + "fmt" + + "github.com/anchore/syft/internal/cache" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" +) + +// javascriptLicenseResolver resolves licenses for a package from the npm registry, backed by syft's +// on-disk cache so that the same package/version is only fetched once -- both within a single run +// (the same dependency is commonly present in several lockfiles) and across runs. +// +// This mirrors the python cataloger's license resolver (see syft/pkg/cataloger/python/license.go) +// with one deliberate difference: errors are not cached (cache.GetResolver rather than +// cache.GetResolverCachingErrors). A missing license silently weakens downstream license policy +// checks, so a transient registry failure (a 429, an outage) or a private package name that 404s +// against the public registry must not be able to blank out licenses for the lifetime of the cache +// entry. A successful lookup that returns an empty license is a real answer and is still cached. +type javascriptLicenseResolver struct { + catalogerConfig CatalogerConfig + licenseCache cache.Resolver[string] +} + +func newJavascriptLicenseResolver(config CatalogerConfig) javascriptLicenseResolver { + return javascriptLicenseResolver{ + licenseCache: cache.GetResolver[string]("javascript", "v1"), + catalogerConfig: config, + } +} + +func (lr *javascriptLicenseResolver) getLicenses(ctx context.Context, packageName string, packageVersion string) pkg.LicenseSet { + var licenseSet pkg.LicenseSet + + if lr.catalogerConfig.SearchRemoteLicenses { + license, err := lr.getLicensesFromRemote(packageName, packageVersion) + if err == nil && license != "" { + licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, license)...) + } + if err != nil { + log.Debugf("unable to extract licenses from npm registry for package %s:%s: %+v", packageName, packageVersion, err) + } + } + return licenseSet +} + +func (lr *javascriptLicenseResolver) getLicensesFromRemote(packageName string, packageVersion string) (string, error) { + return lr.licenseCache.Resolve(fmt.Sprintf("%s/%s", packageName, packageVersion), func() (string, error) { + return getLicenseFromNpmRegistry(lr.catalogerConfig.NPMBaseURL, packageName, packageVersion) + }) +} diff --git a/syft/pkg/cataloger/javascript/license_test.go b/syft/pkg/cataloger/javascript/license_test.go new file mode 100644 index 000000000..2bd827d5e --- /dev/null +++ b/syft/pkg/cataloger/javascript/license_test.go @@ -0,0 +1,207 @@ +package javascript + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/cache" + "github.com/anchore/syft/syft/pkg" +) + +// useInMemoryCache installs a real (in-memory) cache manager for the duration of the test. By +// default syft's cache manager is bypassed, which is what keeps the rest of the cataloger tests +// isolated from one another; these tests need a functioning cache to observe caching behavior. +func useInMemoryCache(t *testing.T) { + t.Helper() + prev := cache.GetManager() + cache.SetManager(cache.NewInMemory(time.Hour)) + t.Cleanup(func() { cache.SetManager(prev) }) +} + +// npmRegistryStub is a stand-in for registry.npmjs.org that counts the requests it serves. +type npmRegistryStub struct { + *httptest.Server + requests atomic.Int64 +} + +func newNpmRegistryStub(t *testing.T, handler http.HandlerFunc) *npmRegistryStub { + t.Helper() + stub := &npmRegistryStub{} + stub.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + stub.requests.Add(1) + handler(w, r) + })) + t.Cleanup(stub.Close) + return stub +} + +func (s *npmRegistryStub) config() CatalogerConfig { + return CatalogerConfig{SearchRemoteLicenses: true, NPMBaseURL: s.URL} +} + +func TestJavascriptLicenseResolver_cachesSuccessfulLookups(t *testing.T) { + useInMemoryCache(t) + + stub := newNpmRegistryStub(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"license": "MIT"}`) + }) + + lr := newJavascriptLicenseResolver(stub.config()) + + for i := 0; i < 3; i++ { + license, err := lr.getLicensesFromRemote("@babel/code-frame", "7.10.4") + require.NoError(t, err) + assert.Equal(t, "MIT", license) + } + + assert.Equal(t, int64(1), stub.requests.Load(), "expected only the first lookup to reach the registry") +} + +func TestJavascriptLicenseResolver_cacheIsSharedAcrossResolvers(t *testing.T) { + useInMemoryCache(t) + + stub := newNpmRegistryStub(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"license": "MIT"}`) + }) + + // the same package/version is commonly present in more than one lockfile, each of which is + // parsed by a different adapter with its own resolver -- the cache is still shared + first := newJavascriptLicenseResolver(stub.config()) + second := newJavascriptLicenseResolver(stub.config()) + + license, err := first.getLicensesFromRemote("lodash", "4.17.21") + require.NoError(t, err) + assert.Equal(t, "MIT", license) + + license, err = second.getLicensesFromRemote("lodash", "4.17.21") + require.NoError(t, err) + assert.Equal(t, "MIT", license) + + assert.Equal(t, int64(1), stub.requests.Load()) +} + +func TestJavascriptLicenseResolver_cachesEmptyLicense(t *testing.T) { + useInMemoryCache(t) + + // a package that genuinely publishes no license field is a real answer, not a failure + stub := newNpmRegistryStub(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"name": "no-license-pkg"}`) + }) + + lr := newJavascriptLicenseResolver(stub.config()) + + for i := 0; i < 2; i++ { + license, err := lr.getLicensesFromRemote("no-license-pkg", "1.0.0") + require.NoError(t, err) + assert.Equal(t, "", license) + } + + assert.Equal(t, int64(1), stub.requests.Load()) +} + +func TestJavascriptLicenseResolver_doesNotCacheErrors(t *testing.T) { + useInMemoryCache(t) + + // unlike the python cataloger we intentionally do not cache failed lookups: a transient + // registry failure must not be able to blank out licenses for the lifetime of the cache entry + stub := newNpmRegistryStub(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + fmt.Fprintf(w, `too many requests`) // not valid JSON + }) + + lr := newJavascriptLicenseResolver(stub.config()) + + for i := 0; i < 3; i++ { + _, err := lr.getLicensesFromRemote("@babel/code-frame", "7.10.4") + require.Error(t, err) + } + + assert.Equal(t, int64(3), stub.requests.Load(), "expected every lookup to be retried against the registry") +} + +func TestJavascriptLicenseResolver_keyIncludesNameAndVersion(t *testing.T) { + useInMemoryCache(t) + + stub := newNpmRegistryStub(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"license": %q}`, r.URL.Path) + }) + + lr := newJavascriptLicenseResolver(stub.config()) + + tests := []struct { + name string + version string + expected string + }{ + {name: "lodash", version: "4.17.21", expected: "/lodash/4.17.21"}, + {name: "lodash", version: "4.17.20", expected: "/lodash/4.17.20"}, + {name: "@babel/code-frame", version: "7.10.4", expected: "/@babel/code-frame/7.10.4"}, + } + + for _, tt := range tests { + license, err := lr.getLicensesFromRemote(tt.name, tt.version) + require.NoError(t, err) + assert.Equal(t, tt.expected, license) + } + + assert.Equal(t, int64(len(tests)), stub.requests.Load()) +} + +func TestJavascriptLicenseResolver_getLicenses(t *testing.T) { + ctx := context.TODO() + + tests := []struct { + name string + searchRemote bool + response string + expected pkg.LicenseSet + expectedRequests int64 + }{ + { + name: "remote search disabled makes no request", + searchRemote: false, + response: `{"license": "MIT"}`, + expected: pkg.LicenseSet{}, + expectedRequests: 0, + }, + { + name: "remote search returns the license from the registry", + searchRemote: true, + response: `{"license": "MIT"}`, + expected: pkg.NewLicenseSet(pkg.NewLicenseWithContext(ctx, "MIT")), + expectedRequests: 1, + }, + { + name: "an empty license yields an empty license set", + searchRemote: true, + response: `{"name": "no-license-pkg"}`, + expected: pkg.LicenseSet{}, + expectedRequests: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + useInMemoryCache(t) + + stub := newNpmRegistryStub(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, tt.response) + }) + + cfg := stub.config() + cfg.SearchRemoteLicenses = tt.searchRemote + lr := newJavascriptLicenseResolver(cfg) + + assert.Equal(t, tt.expected, lr.getLicenses(ctx, "some-pkg", "1.0.0")) + assert.Equal(t, tt.expectedRequests, stub.requests.Load()) + }) + } +} diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go index 27b0590fb..085f186dd 100644 --- a/syft/pkg/cataloger/javascript/package.go +++ b/syft/pkg/cataloger/javascript/package.go @@ -85,7 +85,7 @@ func newPackageJSONPackage(ctx context.Context, resolver file.Resolver, u packag return p } -func newPackageLockV1Package(ctx context.Context, cfg CatalogerConfig, resolver file.Resolver, location file.Location, name string, u lockDependency) pkg.Package { +func newPackageLockV1Package(ctx context.Context, licenseResolver javascriptLicenseResolver, resolver file.Resolver, location file.Location, name string, u lockDependency) pkg.Package { version := u.Version const aliasPrefixPackageLockV1 = "npm:" @@ -101,17 +101,7 @@ func newPackageLockV1Package(ctx context.Context, cfg CatalogerConfig, resolver version = canonicalPackageAndVersion[versionSeparator+1:] } - var licenseSet pkg.LicenseSet - - if cfg.SearchRemoteLicenses { - license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, version) - if err == nil && license != "" { - licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, license)...) - } - if err != nil { - log.Debugf("unable to extract licenses from javascript package-lock.json for package %s:%s: %+v", name, version, err) - } - } + licenseSet := licenseResolver.getLicenses(ctx, name, version) return finalizeLockPkg( ctx, @@ -130,19 +120,13 @@ func newPackageLockV1Package(ctx context.Context, cfg CatalogerConfig, resolver ) } -func newPackageLockV2Package(ctx context.Context, cfg CatalogerConfig, resolver file.Resolver, location file.Location, name string, u lockPackage) pkg.Package { +func newPackageLockV2Package(ctx context.Context, licenseResolver javascriptLicenseResolver, resolver file.Resolver, location file.Location, name string, u lockPackage) pkg.Package { var licenseSet pkg.LicenseSet if u.License != nil { licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocationWithContext(ctx, location, u.License...)...) - } else if cfg.SearchRemoteLicenses { - license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, u.Version) - if err == nil && license != "" { - licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, license)...) - } - if err != nil { - log.Debugf("unable to extract licenses from javascript package-lock.json for package %s:%s: %+v", name, u.Version, err) - } + } else { + licenseSet = licenseResolver.getLicenses(ctx, name, u.Version) } return finalizeLockPkg( @@ -162,18 +146,9 @@ func newPackageLockV2Package(ctx context.Context, cfg CatalogerConfig, resolver ) } -func newPnpmPackage(ctx context.Context, cfg CatalogerConfig, resolver file.Resolver, location file.Location, name, version string, integrity string, dependencies map[string]string) pkg.Package { - var licenseSet pkg.LicenseSet +func newPnpmPackage(ctx context.Context, licenseResolver javascriptLicenseResolver, resolver file.Resolver, location file.Location, name, version string, integrity string, dependencies map[string]string) pkg.Package { + licenseSet := licenseResolver.getLicenses(ctx, name, version) - if cfg.SearchRemoteLicenses { - license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, version) - if err == nil && license != "" { - licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, license)...) - } - if err != nil { - log.Debugf("unable to extract licenses from javascript pnpm-lock.yaml for package %s:%s: %+v", name, version, err) - } - } return finalizeLockPkg( ctx, resolver, @@ -191,18 +166,9 @@ func newPnpmPackage(ctx context.Context, cfg CatalogerConfig, resolver file.Reso ) } -func newYarnLockPackage(ctx context.Context, cfg CatalogerConfig, resolver file.Resolver, location file.Location, name, version string, resolved string, integrity string, dependencies map[string]string) pkg.Package { - var licenseSet pkg.LicenseSet +func newYarnLockPackage(ctx context.Context, licenseResolver javascriptLicenseResolver, resolver file.Resolver, location file.Location, name, version string, resolved string, integrity string, dependencies map[string]string) pkg.Package { + licenseSet := licenseResolver.getLicenses(ctx, name, version) - if cfg.SearchRemoteLicenses { - license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, version) - if err == nil && license != "" { - licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, license)...) - } - if err != nil { - log.Debugf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, version, err) - } - } return finalizeLockPkg( ctx, resolver, @@ -220,18 +186,9 @@ func newYarnLockPackage(ctx context.Context, cfg CatalogerConfig, resolver file. ) } -func newBunPackage(ctx context.Context, cfg CatalogerConfig, resolver file.Resolver, location file.Location, name, version string, integrity string, metadata bunPackageMetadata) pkg.Package { - var licenseSet pkg.LicenseSet +func newBunPackage(ctx context.Context, licenseResolver javascriptLicenseResolver, resolver file.Resolver, location file.Location, name, version string, integrity string, metadata bunPackageMetadata) pkg.Package { + licenseSet := licenseResolver.getLicenses(ctx, name, version) - if cfg.SearchRemoteLicenses { - license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, version) - if err == nil && license != "" { - licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, license)...) - } - if err != nil { - log.Debugf("unable to extract licenses from javascript bun.lock for package %s:%s: %+v", name, version, err) - } - } return finalizeLockPkg( ctx, resolver, diff --git a/syft/pkg/cataloger/javascript/parse_bun_lock.go b/syft/pkg/cataloger/javascript/parse_bun_lock.go index 50afd59d7..1ef6c00ba 100644 --- a/syft/pkg/cataloger/javascript/parse_bun_lock.go +++ b/syft/pkg/cataloger/javascript/parse_bun_lock.go @@ -53,12 +53,14 @@ type bunWorkspace struct { } type genericBunLockAdapter struct { - cfg CatalogerConfig + cfg CatalogerConfig + licenseResolver javascriptLicenseResolver } func newGenericBunLockAdapter(cfg CatalogerConfig) genericBunLockAdapter { return genericBunLockAdapter{ - cfg: cfg, + cfg: cfg, + licenseResolver: newJavascriptLicenseResolver(cfg), } } @@ -106,7 +108,7 @@ func (a genericBunLockAdapter) parseBunLock(ctx context.Context, resolver file.R if devOnlyPkgs[p.Name] && !a.cfg.IncludeDevDependencies { continue } - packages = append(packages, newBunPackage(ctx, a.cfg, resolver, reader.Location, p.Name, p.Version, p.Integrity, p.Metadata)) + packages = append(packages, newBunPackage(ctx, a.licenseResolver, resolver, reader.Location, p.Name, p.Version, p.Integrity, p.Metadata)) } pkg.Sort(packages) diff --git a/syft/pkg/cataloger/javascript/parse_package_lock.go b/syft/pkg/cataloger/javascript/parse_package_lock.go index 4d12410ae..66c447d25 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock.go @@ -48,12 +48,14 @@ type lockPackage struct { type packageLockLicense []string type genericPackageLockAdapter struct { - cfg CatalogerConfig + cfg CatalogerConfig + licenseResolver javascriptLicenseResolver } func newGenericPackageLockAdapter(cfg CatalogerConfig) genericPackageLockAdapter { return genericPackageLockAdapter{ - cfg: cfg, + cfg: cfg, + licenseResolver: newJavascriptLicenseResolver(cfg), } } @@ -100,7 +102,7 @@ func (a genericPackageLockAdapter) parsePackageLock(ctx context.Context, resolve name = pkgMeta.Name } - newPkg := newPackageLockV2Package(ctx, a.cfg, resolver, reader.Location, getNameFromPath(name), pkgMeta) + newPkg := newPackageLockV2Package(ctx, a.licenseResolver, resolver, reader.Location, getNameFromPath(name), pkgMeta) pkgs = append(pkgs, newPkg) } } @@ -122,7 +124,7 @@ func (a genericPackageLockAdapter) packageLockV1Packages(ctx context.Context, re continue } - p := newPackageLockV1Package(ctx, a.cfg, resolver, location, name, pkgMeta) + p := newPackageLockV1Package(ctx, a.licenseResolver, resolver, location, name, pkgMeta) if _, exists := seen[p.PURL]; !exists { pkgs = append(pkgs, p) seen[p.PURL] = struct{}{} diff --git a/syft/pkg/cataloger/javascript/parse_pnpm_lock.go b/syft/pkg/cataloger/javascript/parse_pnpm_lock.go index eede2ccc8..707193957 100644 --- a/syft/pkg/cataloger/javascript/parse_pnpm_lock.go +++ b/syft/pkg/cataloger/javascript/parse_pnpm_lock.go @@ -74,12 +74,14 @@ type pnpmV9LockYaml struct { } type genericPnpmLockAdapter struct { - cfg CatalogerConfig + cfg CatalogerConfig + licenseResolver javascriptLicenseResolver } func newGenericPnpmLockAdapter(cfg CatalogerConfig) genericPnpmLockAdapter { return genericPnpmLockAdapter{ - cfg: cfg, + cfg: cfg, + licenseResolver: newJavascriptLicenseResolver(cfg), } } @@ -216,7 +218,7 @@ func (a genericPnpmLockAdapter) parsePnpmLock(ctx context.Context, resolver file if p.Dev && !a.cfg.IncludeDevDependencies { continue } - packages = append(packages, newPnpmPackage(ctx, a.cfg, resolver, reader.Location, p.Name, p.Version, p.Integrity, p.Dependencies)) + packages = append(packages, newPnpmPackage(ctx, a.licenseResolver, resolver, reader.Location, p.Name, p.Version, p.Integrity, p.Dependencies)) } errs = unknown.Join(errs, unknown.IfEmptyf(packages, "unable to determine packages")) diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index 7c53bc556..9213be760 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -67,12 +67,14 @@ type yarnV2PackageEntry struct { } type genericYarnLockAdapter struct { - cfg CatalogerConfig + cfg CatalogerConfig + licenseResolver javascriptLicenseResolver } func newGenericYarnLockAdapter(cfg CatalogerConfig) genericYarnLockAdapter { return genericYarnLockAdapter{ - cfg: cfg, + cfg: cfg, + licenseResolver: newJavascriptLicenseResolver(cfg), } } @@ -307,7 +309,7 @@ func (a genericYarnLockAdapter) parseYarnLock(ctx context.Context, resolver file if devOnlyPkgs[p.Name] && !a.cfg.IncludeDevDependencies { continue } - packages = append(packages, newYarnLockPackage(ctx, a.cfg, resolver, reader.Location, p.Name, p.Version, p.Resolved, p.Integrity, p.Dependencies)) + packages = append(packages, newYarnLockPackage(ctx, a.licenseResolver, resolver, reader.Location, p.Name, p.Version, p.Resolved, p.Integrity, p.Dependencies)) } pkg.Sort(packages) diff --git a/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go b/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go index ab7f7860e..de92128a2 100644 --- a/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go +++ b/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file.go @@ -22,6 +22,12 @@ import ( const modinfoName = ".modinfo" +// maxDecompressedModuleSize bounds what a compressed kernel module may expand to. The compressed size +// is bounded by the file in the image, what it expands to is not, and it lands in a temp file, so the +// exposure is disk. Real modules are tens of MB. This bounds one spill, not the process: peak disk is +// this times cataloger parallelism (NumCPU*4 by default, unbounded when --parallelism is negative). +const maxDecompressedModuleSize = 256 * 1024 * 1024 + func parseLinuxKernelModuleFile(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { unionReader, err := unionreader.GetUnionReader(reader) if err != nil { @@ -61,6 +67,11 @@ func parseLinuxKernelModuleFile(ctx context.Context, _ file.Resolver, _ *generic // the returned reader and must Close it; the underlying reader (r) is not closed by Close on the // passthrough path — its lifecycle is the caller's. func decompressedModuleReader(ctx context.Context, path string, r unionreader.UnionReader) (unionreader.UnionReader, error) { + return decompressedModuleReaderWithLimit(ctx, path, r, maxDecompressedModuleSize) +} + +// decompressedModuleReaderWithLimit takes the bound as a parameter so tests can exercise it cheaply. +func decompressedModuleReaderWithLimit(ctx context.Context, path string, r unionreader.UnionReader, maxDecompressed int64) (unionreader.UnionReader, error) { // fast path: plain .ko files don't need format sniffing if strings.HasSuffix(path, ".ko") { return &nopCloseUnionReader{UnionReader: r}, nil @@ -97,10 +108,17 @@ func decompressedModuleReader(ctx context.Context, path string, r unionreader.Un } tfr := &tempFileUnionReader{File: tempFile, cleanup: fileCleanup} - if _, err := io.Copy(tempFile, rc); err != nil { + // read one past the cap so hitting it is distinguishable from a module that ends there; truncating + // would hand the parser a partial ELF, which reads as corrupt rather than rejected. + written, err := io.Copy(tempFile, io.LimitReader(rc, maxDecompressed+1)) + if err != nil { _ = tfr.Close() return nil, fmt.Errorf("unable to write decompressed kernel module: %w", err) } + if written > maxDecompressed { + _ = tfr.Close() + return nil, fmt.Errorf("decompressed kernel module is larger than the max allowed size (%d bytes)", maxDecompressed) + } if _, err := tempFile.Seek(0, io.SeekStart); err != nil { _ = tfr.Close() return nil, fmt.Errorf("unable to rewind decompressed kernel module: %w", err) diff --git a/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file_test.go b/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file_test.go index cb99485bc..763feed1d 100644 --- a/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file_test.go +++ b/syft/pkg/cataloger/kernel/parse_linux_kernel_module_file_test.go @@ -17,6 +17,7 @@ import ( "github.com/anchore/syft/internal/tmpdir" "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/internal/unionreader" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" ) @@ -253,6 +254,58 @@ func TestDecompressedModuleReader(t *testing.T) { } } +// wrapUnionReader adapts a byte slice to the reader set decompressedModuleReader takes. +func wrapUnionReader(data []byte) unionreader.UnionReader { + br := bytes.NewReader(data) + return struct { + io.ReadCloser + io.ReaderAt + io.Seeker + }{ + ReadCloser: io.NopCloser(br), + ReaderAt: br, + Seeker: br, + } +} + +func TestDecompressedModuleReader_boundsDecompressedSize(t *testing.T) { + koBytes := minimalKOBytes([]string{"name=test", "vermagic=5.15.0 SMP mod_unload"}) + compressed := gzCompress(koBytes) + + t.Run("rejects a module that expands past the cap", func(t *testing.T) { + _, err := decompressedModuleReaderWithLimit(testContext(t), "/test.ko.gz", + wrapUnionReader(compressed), int64(len(koBytes)-1)) + + require.ErrorContains(t, err, "larger than the max allowed size") + }) + + t.Run("a module at exactly the cap is accepted whole", func(t *testing.T) { + // the byte-for-byte assert is the point: a "did not error" check would miss a truncating read + got, err := decompressedModuleReaderWithLimit(testContext(t), "/test.ko.gz", + wrapUnionReader(compressed), int64(len(koBytes))) + require.NoError(t, err) + t.Cleanup(func() { _ = got.Close() }) + + b, err := io.ReadAll(got) + require.NoError(t, err) + assert.Equal(t, koBytes, b, "at-cap module must come back byte for byte") + }) + + t.Run("the partial spill is cleaned up when the cap is hit", func(t *testing.T) { + // otherwise repeated hostile modules exhaust the disk anyway, which is what the cap prevents + dir := t.TempDir() + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(dir)) + + _, err := decompressedModuleReaderWithLimit(ctx, "/test.ko.gz", + wrapUnionReader(compressed), int64(len(koBytes)-1)) + require.Error(t, err) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Empty(t, entries, "temp spill should be removed when the module is rejected") + }) +} + func TestDecompressedModuleReader_TempFileRemovedOnClose(t *testing.T) { koBytes := minimalKOBytes([]string{"name=test", "vermagic=5.15.0 SMP"}) data := gzCompress(koBytes) diff --git a/syft/pkg/cataloger/php/interpreter_cataloger.go b/syft/pkg/cataloger/php/interpreter_cataloger.go index 4c1619eb3..80ae69d21 100644 --- a/syft/pkg/cataloger/php/interpreter_cataloger.go +++ b/syft/pkg/cataloger/php/interpreter_cataloger.go @@ -16,6 +16,9 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/internal/binutils" ) +// phpCPEName is the vendor and product used for every PHP interpreter CPE below. +const phpCPEName = "php" + type interpreterCataloger struct { name string extensionsGlob string @@ -49,8 +52,8 @@ func NewInterpreterCataloger() pkg.Cataloger { //nolint:funlen { Attributes: cpe.Attributes{ Part: "a", - Vendor: "php", - Product: "php", + Vendor: phpCPEName, + Product: phpCPEName, }, Source: cpe.NVDDictionaryLookupSource, }, @@ -71,8 +74,8 @@ func NewInterpreterCataloger() pkg.Cataloger { //nolint:funlen { Attributes: cpe.Attributes{ Part: "a", - Vendor: "php", - Product: "php", + Vendor: phpCPEName, + Product: phpCPEName, }, Source: cpe.NVDDictionaryLookupSource, }, @@ -86,15 +89,15 @@ func NewInterpreterCataloger() pkg.Cataloger { //nolint:funlen Package: "libphp", PURL: packageurl.PackageURL{ Type: packageurl.TypeGeneric, - Name: "php", + Name: phpCPEName, // the version will be filled in dynamically }, CPEs: []cpe.CPE{ { Attributes: cpe.Attributes{ Part: "a", - Vendor: "php", - Product: "php", + Vendor: phpCPEName, + Product: phpCPEName, }, Source: cpe.NVDDictionaryLookupSource, }, diff --git a/syft/pkg/cataloger/redhat/testdata/image-rpm-archive/Dockerfile b/syft/pkg/cataloger/redhat/testdata/image-rpm-archive/Dockerfile index b66f3d00f..bc328038e 100644 --- a/syft/pkg/cataloger/redhat/testdata/image-rpm-archive/Dockerfile +++ b/syft/pkg/cataloger/redhat/testdata/image-rpm-archive/Dockerfile @@ -17,7 +17,7 @@ FROM --platform=linux/amd64 rockylinux:9 AS rpm-downloader # $ rpm -qp --qf '%{NAME}-%{VERSION}-%{RELEASE} %{RSAHEADER:pgpsig}\n' basesystem-11-13.el9.0.1.noarch.rpm # basesystem-11-13.el9.0.1 RSA/SHA256, Thu Feb 29 17:37:22 2024, Key ID 702d426d350d275d -RUN curl -O https://dl.rockylinux.org/vault/rocky/9.3/BaseOS/x86_64/os/Packages/b/basesystem-11-13.el9.0.1.noarch.rpm +RUN curl -fO https://dl.rockylinux.org/vault/rocky/9.3/BaseOS/x86_64/os/Packages/b/basesystem-11-13.el9.0.1.noarch.rpm FROM scratch diff --git a/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go b/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go index df0c15767..063161d19 100644 --- a/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go +++ b/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/anchore/syft/internal/testutils" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" @@ -310,3 +311,47 @@ func TestParseKernelChangelog(t *testing.T) { }) } } + +// TestParseKernelChangelog_gzipBombStaysBounded pins a bound nothing in this file states out loud. +// parseKernelChangelog reads the decompressed changelog through a bufio.Scanner, and Scanner refuses a +// token past MaxScanTokenSize, so a gzip member expanding to gigabytes costs 64KB. That is the only +// thing holding this path, and it is incidental: switching to io.ReadAll or a bufio.Reader to pick up +// long lines would remove it silently, which is what this test is here to catch. +func TestParseKernelChangelog_gzipBombStaysBounded(t *testing.T) { + const expanded = 512 * 1024 * 1024 + + // one enormous line with no newline, the worst case for a line scanner + var gz bytes.Buffer + gw := gzip.NewWriter(&gz) + chunk := bytes.Repeat([]byte{'a'}, 32*1024) + for remaining := expanded; remaining > 0; { + n, err := gw.Write(chunk[:min(remaining, len(chunk))]) + require.NoError(t, err) + remaining -= n + } + require.NoError(t, gw.Close()) + require.Less(t, gz.Len(), 1024*1024, "compressed payload should be tiny relative to what it expands to") + + // prove the fixture is a bomb: the same stream read without a line bound costs its full size + unbounded := testutils.MeasureAlloc(t, func() { + zr, err := gzip.NewReader(bytes.NewReader(gz.Bytes())) + require.NoError(t, err) + defer zr.Close() + b, err := io.ReadAll(zr) + require.NoError(t, err) + require.Len(t, b, expanded) + }) + require.Greater(t, unbounded, uint64(expanded), + "fixture did not actually cost more than its own size; it is no longer a bomb") + + bounded := testutils.MeasureAlloc(t, func() { + _, _, err := parseKernelChangelog(context.Background(), nil, nil, + file.NewLocationReadCloser(file.NewLocation("/changelog.gz"), io.NopCloser(bytes.NewReader(gz.Bytes())))) + // the scanner refuses the oversized line, so this fails rather than parsing a version out of it + require.Error(t, err) + }) + + t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded) + assert.Less(t, bounded, uint64(8*1024*1024), + "the changelog scan must stay bounded by the line limit, not by what the member expands to") +} diff --git a/test/cli/convert_cmd_test.go b/test/cli/convert_cmd_test.go index 04df14910..402e120cf 100644 --- a/test/cli/convert_cmd_test.go +++ b/test/cli/convert_cmd_test.go @@ -12,6 +12,7 @@ import ( "github.com/anchore/syft/syft/format/cyclonedxxml" "github.com/anchore/syft/syft/format/spdxjson" "github.com/anchore/syft/syft/format/spdxtagvalue" + "github.com/anchore/syft/syft/format/syftjson" "github.com/anchore/syft/syft/sbom" ) @@ -64,6 +65,93 @@ func TestConvertCmd(t *testing.T) { } } +func TestConvertCmd_PassthroughExactFormat(t *testing.T) { + // pretty-printed input makes a verbatim copy distinguishable from a re-encode (which is compact) + sbomArgs := []string{"dir:./testdata/image-pkg-coverage", "-o", "syft-json"} + cmd, input, stderr := runSyft(t, map[string]string{"SYFT_FORMAT_PRETTY": "true"}, sbomArgs...) + if cmd.ProcessState.ExitCode() != 0 { + t.Log("STDOUT:\n", input) + t.Log("STDERR:\n", stderr) + t.Log("COMMAND:", strings.Join(cmd.Args, " ")) + t.Fatalf("failure executing syft creating an sbom") + return + } + + inputID, inputVersion := format.Identify(strings.NewReader(input)) + require.Equal(t, syftjson.ID, inputID) + require.NotEmpty(t, inputVersion) + + const ( + enabledWarning = "convert.passthrough-exact-format is enabled" + copiedWarning = "copying it unchanged instead of converting it" + ) + + tests := []struct { + name string + env map[string]string + to string + wantUnchanged bool + wantID sbom.FormatID + wantVersion string + wantInStderr []string + wantNotStderr []string + }{ + { + name: "matching syft-json is passed through unchanged", + env: map[string]string{"SYFT_CONVERT_PASSTHROUGH_EXACT_FORMAT": "true"}, + to: "syft-json", + wantUnchanged: true, + wantID: syftjson.ID, + wantVersion: inputVersion, + wantInStderr: []string{enabledWarning, copiedWarning}, + }, + { + name: "different format is converted", + env: map[string]string{"SYFT_CONVERT_PASSTHROUGH_EXACT_FORMAT": "true"}, + to: "cyclonedx-json", + wantID: cyclonedxjson.ID, + wantVersion: mustEncoder(cyclonedxjson.NewFormatEncoderWithConfig(cyclonedxjson.DefaultEncoderConfig())).Version(), + wantInStderr: []string{enabledWarning}, + wantNotStderr: []string{copiedWarning}, + }, + { + name: "option off by default, matching syft-json is re-encoded", + to: "syft-json", + wantID: syftjson.ID, + wantVersion: inputVersion, + wantNotStderr: []string{enabledWarning, copiedWarning}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := getSyftCommand(t, "convert", "-", "-o", test.to) + cmd.Stdin = strings.NewReader(input) + stdout, stderr := runCommandObj(t, cmd, test.env, false) + + assertSuccessfulReturnCode(t, stdout, stderr, cmd.ProcessState.ExitCode()) + logOutputOnFailure(t, cmd, stdout, stderr) + + foundID, foundVersion := format.Identify(strings.NewReader(stdout)) + require.Equal(t, test.wantID, foundID) + require.Equal(t, test.wantVersion, foundVersion) + + if test.wantUnchanged { + require.Equal(t, strings.TrimSpace(input), strings.TrimSpace(stdout)) + } else { + require.NotEqual(t, strings.TrimSpace(input), strings.TrimSpace(stdout)) + } + + for _, want := range test.wantInStderr { + require.Contains(t, stderr, want) + } + for _, notWant := range test.wantNotStderr { + require.NotContains(t, stderr, notWant) + } + }) + } +} + func mustEncoder(enc sbom.FormatEncoder, err error) sbom.FormatEncoder { if err != nil { panic(err) diff --git a/test/rules/rules.go b/test/rules/rules.go index fb68fb827..e001a12bd 100644 --- a/test/rules/rules.go +++ b/test/rules/rules.go @@ -38,6 +38,41 @@ func noUnboundedReads(m dsl.Matcher) { Report("do not use unbounded io.Copy to in-memory buffer; wrap the source reader with io.LimitReader") } +// nolint:unused +func noOverflowingBoundsCheck(m dsl.Matcher) { + // an offset+size bounds check on unsigned operands wraps rather than saturating, so a large + // offset out of a file header can produce a small sum and pass a check it should have failed. + // Written as a subtraction against the limit, there is nothing to wrap. + m.Match( + `$offset + $size > $limit`, + `$offset + $size >= $limit`, + `$offset + $size < $limit`, + `$offset + $size <= $limit`, + ). + Where((m["offset"].Type.OfKind("unsigned") || m["size"].Type.OfKind("unsigned")) && + // a constant addend cannot be attacker-controlled, which is the common safe shape of + // stepping a loop cursor (`pos+4 <= uint32(len(buf))`) + !m["offset"].Const && !m["size"].Const && + // same 32-bit floor as noUnboundedAllocations: a uint16 sum is a nibble or a count widened for + // the comparison, not an offset that can carry a file-controlled value up to the wrap point + m["offset"].Type.Size >= 4). + Report("bounds check can overflow: $offset + $size wraps on unsigned operands. write it as a subtraction against the limit instead, e.g. `$offset > $limit || $size > $limit-$offset`") +} + +// nolint:unused +func noSwallowedEOF(m dsl.Matcher) { + // treating EOF as success hands the caller a zero value it cannot distinguish from real data. In a + // loop that advances by however much was read, that is not just bad data but a cursor that never + // moves, so a truncated record spins forever. + m.Match( + `if errors.Is($err, io.EOF) { return nil }`, + `if errors.Is($err, io.EOF) { return nil, nil }`, + `if $err == io.EOF { return nil }`, + `if $err == io.EOF { return nil, nil }`, + ). + Report("EOF returned as success: the caller gets a zero value it cannot tell from real data. return the error, or return an explicit sentinel the caller checks") +} + // nolint:unused func noDirectTempFiles(m dsl.Matcher) { // catalogers must use tmpdir.FromContext(ctx) instead of creating temp files/dirs directly, @@ -103,3 +138,17 @@ func noDirectELFOpen(m dsl.Matcher) { Where(!m.File().PkgPath.Matches(`/syft/internal/elfutil$`)). Report("do not open ELF files with debug/elf directly; use elfutil.NewFile, which bounds declared decompressed section sizes") } + +// nolint:unused +func noUnboundedELFParsers(m dsl.Matcher) { + // buildinfo.Read and goversion's ReadExeFromReader each open debug/elf internally and expand sections + // sized by the file itself, the same unbounded allocation noDirectELFOpen guards against above. They + // are only safe in this repo because scan_binary.go gates the reader before calling them: readBuildInfo + // runs elfutil.CheckSectionNameTable first, getCryptoInformation runs elfutil.CheckAllSections first. + m.Match( + `buildinfo.Read($_)`, + `version.ReadExeFromReader($_)`, + ). + Where(!(m.File().Name.Matches(`scan_binary\.go$`) && m.File().PkgPath.Matches(`/syft/pkg/cataloger/golang$`))). + Report("do not call buildinfo.Read/goversion.ReadExeFromReader directly; route through the bounded wrappers in the golang cataloger (readBuildInfo, getCryptoInformation), or gate the reader with elfutil.CheckSectionNameTable/elfutil.CheckAllSections first") +}